From ec2daf6e33f9f9113ba085b6ff88592907b6f1ce Mon Sep 17 00:00:00 2001 From: Jon Flatley Date: Fri, 24 Jan 2020 15:18:32 -0800 Subject: [PATCH 001/635] platform: chrome: Add cros-usbpd-notify driver ChromiumOS uses ACPI device with HID "GOOG0003" for power delivery related events. The existing cros-usbpd-charger driver relies on these events without ever actually receiving them on ACPI platforms. This is because in the ChromeOS kernel trees, the GOOG0003 device is owned by an ACPI driver that offers firmware updates to USB-C chargers. Introduce a new platform driver under cros-ec, the ChromeOS embedded controller, that handles these PD events and dispatches them appropriately over a notifier chain to all drivers that use them. On platforms that don't have the ACPI device defined, the driver gets instantiated for ECs which support the EC_FEATURE_USB_PD feature bit, and the notification events will get delivered using the MKBP event handling mechanism. Co-Developed-by: Prashant Malani Reviewed-by: Gwendal Grignou Reviewed-by: Benson Leung Signed-off-by: Jon Flatley Signed-off-by: Prashant Malani Acked-By: Enric Balletbo i Serra Signed-off-by: Benson Leung --- drivers/platform/chrome/Kconfig | 14 ++ drivers/platform/chrome/Makefile | 1 + drivers/platform/chrome/cros_usbpd_notify.c | 169 ++++++++++++++++++ .../linux/platform_data/cros_usbpd_notify.h | 17 ++ 4 files changed, 201 insertions(+) create mode 100644 drivers/platform/chrome/cros_usbpd_notify.c create mode 100644 include/linux/platform_data/cros_usbpd_notify.h diff --git a/drivers/platform/chrome/Kconfig b/drivers/platform/chrome/Kconfig index 5f57282a28da..15fc8b8a2db8 100644 --- a/drivers/platform/chrome/Kconfig +++ b/drivers/platform/chrome/Kconfig @@ -226,6 +226,20 @@ config CROS_USBPD_LOGGER To compile this driver as a module, choose M here: the module will be called cros_usbpd_logger. +config CROS_USBPD_NOTIFY + tristate "ChromeOS Type-C power delivery event notifier" + depends on MFD_CROS_EC_DEV + default MFD_CROS_EC_DEV + help + If you say Y here, you get support for Type-C PD event notifications + from the ChromeOS EC. On ACPI platorms this driver will bind to the + GOOG0003 ACPI device, and on platforms which don't have this device it + will get initialized on ECs which support the feature + EC_FEATURE_USB_PD. + + To compile this driver as a module, choose M here: the + module will be called cros_usbpd_notify. + source "drivers/platform/chrome/wilco_ec/Kconfig" endif # CHROMEOS_PLATFORMS diff --git a/drivers/platform/chrome/Makefile b/drivers/platform/chrome/Makefile index aacd5920d8a1..f6465f8ef0b5 100644 --- a/drivers/platform/chrome/Makefile +++ b/drivers/platform/chrome/Makefile @@ -22,5 +22,6 @@ obj-$(CONFIG_CROS_EC_DEBUGFS) += cros_ec_debugfs.o obj-$(CONFIG_CROS_EC_SENSORHUB) += cros_ec_sensorhub.o obj-$(CONFIG_CROS_EC_SYSFS) += cros_ec_sysfs.o obj-$(CONFIG_CROS_USBPD_LOGGER) += cros_usbpd_logger.o +obj-$(CONFIG_CROS_USBPD_NOTIFY) += cros_usbpd_notify.o obj-$(CONFIG_WILCO_EC) += wilco_ec/ diff --git a/drivers/platform/chrome/cros_usbpd_notify.c b/drivers/platform/chrome/cros_usbpd_notify.c new file mode 100644 index 000000000000..3851bbd6e9a3 --- /dev/null +++ b/drivers/platform/chrome/cros_usbpd_notify.c @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2020 Google LLC + * + * This driver serves as the receiver of cros_ec PD host events. + */ + +#include +#include +#include +#include +#include + +#define DRV_NAME "cros-usbpd-notify" +#define ACPI_DRV_NAME "GOOG0003" + +static BLOCKING_NOTIFIER_HEAD(cros_usbpd_notifier_list); + +/** + * cros_usbpd_register_notify - Register a notifier callback for PD events. + * @nb: Notifier block pointer to register + * + * On ACPI platforms this corresponds to host events on the ECPD + * "GOOG0003" ACPI device. On non-ACPI platforms this will filter mkbp events + * for USB PD events. + * + * Return: 0 on success or negative error code. + */ +int cros_usbpd_register_notify(struct notifier_block *nb) +{ + return blocking_notifier_chain_register(&cros_usbpd_notifier_list, + nb); +} +EXPORT_SYMBOL_GPL(cros_usbpd_register_notify); + +/** + * cros_usbpd_unregister_notify - Unregister notifier callback for PD events. + * @nb: Notifier block pointer to unregister + * + * Unregister a notifier callback that was previously registered with + * cros_usbpd_register_notify(). + */ +void cros_usbpd_unregister_notify(struct notifier_block *nb) +{ + blocking_notifier_chain_unregister(&cros_usbpd_notifier_list, nb); +} +EXPORT_SYMBOL_GPL(cros_usbpd_unregister_notify); + +#ifdef CONFIG_ACPI + +static int cros_usbpd_notify_add_acpi(struct acpi_device *adev) +{ + return 0; +} + +static void cros_usbpd_notify_acpi(struct acpi_device *adev, u32 event) +{ + blocking_notifier_call_chain(&cros_usbpd_notifier_list, event, NULL); +} + +static const struct acpi_device_id cros_usbpd_notify_acpi_device_ids[] = { + { ACPI_DRV_NAME, 0 }, + { } +}; +MODULE_DEVICE_TABLE(acpi, cros_usbpd_notify_acpi_device_ids); + +static struct acpi_driver cros_usbpd_notify_acpi_driver = { + .name = DRV_NAME, + .class = DRV_NAME, + .ids = cros_usbpd_notify_acpi_device_ids, + .ops = { + .add = cros_usbpd_notify_add_acpi, + .notify = cros_usbpd_notify_acpi, + }, +}; + +#endif /* CONFIG_ACPI */ + +static int cros_usbpd_notify_plat(struct notifier_block *nb, + unsigned long queued_during_suspend, + void *data) +{ + struct cros_ec_device *ec_dev = (struct cros_ec_device *)data; + u32 host_event = cros_ec_get_host_event(ec_dev); + + if (!host_event) + return NOTIFY_BAD; + + if (host_event & EC_HOST_EVENT_MASK(EC_HOST_EVENT_PD_MCU)) { + blocking_notifier_call_chain(&cros_usbpd_notifier_list, + host_event, NULL); + return NOTIFY_OK; + } + return NOTIFY_DONE; +} + +static int cros_usbpd_notify_probe_plat(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct cros_ec_dev *ecdev = dev_get_drvdata(dev->parent); + struct notifier_block *nb; + int ret; + + nb = devm_kzalloc(dev, sizeof(*nb), GFP_KERNEL); + if (!nb) + return -ENOMEM; + + nb->notifier_call = cros_usbpd_notify_plat; + dev_set_drvdata(dev, nb); + + ret = blocking_notifier_chain_register(&ecdev->ec_dev->event_notifier, + nb); + if (ret < 0) { + dev_err(dev, "Failed to register notifier\n"); + return ret; + } + + return 0; +} + +static int cros_usbpd_notify_remove_plat(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct cros_ec_dev *ecdev = dev_get_drvdata(dev->parent); + struct notifier_block *nb = + (struct notifier_block *)dev_get_drvdata(dev); + + blocking_notifier_chain_unregister(&ecdev->ec_dev->event_notifier, nb); + + return 0; +} + +static struct platform_driver cros_usbpd_notify_plat_driver = { + .driver = { + .name = DRV_NAME, + }, + .probe = cros_usbpd_notify_probe_plat, + .remove = cros_usbpd_notify_remove_plat, +}; + +static int __init cros_usbpd_notify_init(void) +{ + int ret; + + ret = platform_driver_register(&cros_usbpd_notify_plat_driver); + if (ret < 0) + return ret; + +#ifdef CONFIG_ACPI + acpi_bus_register_driver(&cros_usbpd_notify_acpi_driver); +#endif + return 0; +} + +static void __exit cros_usbpd_notify_exit(void) +{ +#ifdef CONFIG_ACPI + acpi_bus_unregister_driver(&cros_usbpd_notify_acpi_driver); +#endif + platform_driver_unregister(&cros_usbpd_notify_plat_driver); +} + +module_init(cros_usbpd_notify_init); +module_exit(cros_usbpd_notify_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("ChromeOS power delivery notifier device"); +MODULE_AUTHOR("Jon Flatley "); +MODULE_ALIAS("platform:" DRV_NAME); diff --git a/include/linux/platform_data/cros_usbpd_notify.h b/include/linux/platform_data/cros_usbpd_notify.h new file mode 100644 index 000000000000..4f2791722b6d --- /dev/null +++ b/include/linux/platform_data/cros_usbpd_notify.h @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * ChromeOS EC Power Delivery Notifier Driver + * + * Copyright 2020 Google LLC + */ + +#ifndef __LINUX_PLATFORM_DATA_CROS_USBPD_NOTIFY_H +#define __LINUX_PLATFORM_DATA_CROS_USBPD_NOTIFY_H + +#include + +int cros_usbpd_register_notify(struct notifier_block *nb); + +void cros_usbpd_unregister_notify(struct notifier_block *nb); + +#endif /* __LINUX_PLATFORM_DATA_CROS_USBPD_NOTIFY_H */ From f2437e48ac7ae0e99b8e76d88aaf24a9b2e6d3e2 Mon Sep 17 00:00:00 2001 From: Jon Flatley Date: Fri, 24 Jan 2020 15:18:38 -0800 Subject: [PATCH 002/635] power: supply: cros-ec-usbpd-charger: Fix host events There's a bug on ACPI platforms where host events from the ECPD ACPI device never make their way to the cros-ec-usbpd-charger driver. This makes it so the only time the charger driver updates its state is when user space accesses its sysfs attributes. Now that these events have been unified into a single notifier chain on both ACPI and non-ACPI platforms, update the charger driver to use this new notifier. Reviewed-by: Benson Leung Co-Developed-by: Prashant Malani Signed-off-by: Jon Flatley Signed-off-by: Prashant Malani Reviewed-by: Sebastian Reichel Signed-off-by: Benson Leung --- drivers/power/supply/Kconfig | 2 +- drivers/power/supply/cros_usbpd-charger.c | 50 ++++++++--------------- 2 files changed, 19 insertions(+), 33 deletions(-) diff --git a/drivers/power/supply/Kconfig b/drivers/power/supply/Kconfig index 9a5591ab90d0..a6e191b4ff48 100644 --- a/drivers/power/supply/Kconfig +++ b/drivers/power/supply/Kconfig @@ -659,7 +659,7 @@ config CHARGER_RT9455 config CHARGER_CROS_USBPD tristate "ChromeOS EC based USBPD charger" - depends on CROS_EC + depends on CROS_USBPD_NOTIFY default n help Say Y here to enable ChromeOS EC based USBPD charger diff --git a/drivers/power/supply/cros_usbpd-charger.c b/drivers/power/supply/cros_usbpd-charger.c index 30c3d37511c9..2a45e84447fe 100644 --- a/drivers/power/supply/cros_usbpd-charger.c +++ b/drivers/power/supply/cros_usbpd-charger.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -517,32 +518,21 @@ static int cros_usbpd_charger_property_is_writeable(struct power_supply *psy, } static int cros_usbpd_charger_ec_event(struct notifier_block *nb, - unsigned long queued_during_suspend, + unsigned long host_event, void *_notify) { - struct cros_ec_device *ec_device; - struct charger_data *charger; - u32 host_event; + struct charger_data *charger = container_of(nb, struct charger_data, + notifier); - charger = container_of(nb, struct charger_data, notifier); - ec_device = charger->ec_device; - - host_event = cros_ec_get_host_event(ec_device); - if (host_event & EC_HOST_EVENT_MASK(EC_HOST_EVENT_PD_MCU)) { - cros_usbpd_charger_power_changed(charger->ports[0]->psy); - return NOTIFY_OK; - } else { - return NOTIFY_DONE; - } + cros_usbpd_charger_power_changed(charger->ports[0]->psy); + return NOTIFY_OK; } static void cros_usbpd_charger_unregister_notifier(void *data) { struct charger_data *charger = data; - struct cros_ec_device *ec_device = charger->ec_device; - blocking_notifier_chain_unregister(&ec_device->event_notifier, - &charger->notifier); + cros_usbpd_unregister_notify(&charger->notifier); } static int cros_usbpd_charger_probe(struct platform_device *pd) @@ -676,21 +666,17 @@ static int cros_usbpd_charger_probe(struct platform_device *pd) goto fail; } - if (ec_device->mkbp_event_supported) { - /* Get PD events from the EC */ - charger->notifier.notifier_call = cros_usbpd_charger_ec_event; - ret = blocking_notifier_chain_register( - &ec_device->event_notifier, - &charger->notifier); - if (ret < 0) { - dev_warn(dev, "failed to register notifier\n"); - } else { - ret = devm_add_action_or_reset(dev, - cros_usbpd_charger_unregister_notifier, - charger); - if (ret < 0) - goto fail; - } + /* Get PD events from the EC */ + charger->notifier.notifier_call = cros_usbpd_charger_ec_event; + ret = cros_usbpd_register_notify(&charger->notifier); + if (ret < 0) { + dev_warn(dev, "failed to register notifier\n"); + } else { + ret = devm_add_action_or_reset(dev, + cros_usbpd_charger_unregister_notifier, + charger); + if (ret < 0) + goto fail; } return 0; From 8673e944b50ec6e5afd4f599cf12b2798b629f3d Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 5 Feb 2020 11:48:28 +0200 Subject: [PATCH 003/635] platform/chrome: wilco_ec: Platform data shouldn't include kernel.h Replace with appropriate types.h. Also there is no need to include device.h, but mutex.h. For the pointers to unknown structures use forward declarations. In the *.c files we need to include all headers that provide APIs being used in the module. Signed-off-by: Andy Shevchenko Signed-off-by: Enric Balletbo i Serra --- drivers/platform/chrome/wilco_ec/properties.c | 3 +++ drivers/platform/chrome/wilco_ec/sysfs.c | 4 ++++ include/linux/platform_data/wilco-ec.h | 8 ++++++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/drivers/platform/chrome/wilco_ec/properties.c b/drivers/platform/chrome/wilco_ec/properties.c index 62f27610dd33..c2bf4c95c5d2 100644 --- a/drivers/platform/chrome/wilco_ec/properties.c +++ b/drivers/platform/chrome/wilco_ec/properties.c @@ -3,8 +3,11 @@ * Copyright 2019 Google LLC */ +#include +#include #include #include +#include #include /* Operation code; what the EC should do with the property */ diff --git a/drivers/platform/chrome/wilco_ec/sysfs.c b/drivers/platform/chrome/wilco_ec/sysfs.c index f0d174b6bb21..3c587b4054a5 100644 --- a/drivers/platform/chrome/wilco_ec/sysfs.c +++ b/drivers/platform/chrome/wilco_ec/sysfs.c @@ -8,8 +8,12 @@ * See Documentation/ABI/testing/sysfs-platform-wilco-ec for more information. */ +#include +#include #include +#include #include +#include #define CMD_KB_CMOS 0x7C #define SUB_CMD_KB_CMOS_AUTO_ON 0x03 diff --git a/include/linux/platform_data/wilco-ec.h b/include/linux/platform_data/wilco-ec.h index afede15a95bf..25f46a939637 100644 --- a/include/linux/platform_data/wilco-ec.h +++ b/include/linux/platform_data/wilco-ec.h @@ -8,8 +8,8 @@ #ifndef WILCO_EC_H #define WILCO_EC_H -#include -#include +#include +#include /* Message flags for using the mailbox() interface */ #define WILCO_EC_FLAG_NO_RESPONSE BIT(0) /* EC does not respond */ @@ -17,6 +17,10 @@ /* Normal commands have a maximum 32 bytes of data */ #define EC_MAILBOX_DATA_SIZE 32 +struct device; +struct resource; +struct platform_device; + /** * struct wilco_ec_device - Wilco Embedded Controller handle. * @dev: Device handle. From 42cd0ab476e2daffc23982c37822a78f9a53cdd5 Mon Sep 17 00:00:00 2001 From: Yicheng Li Date: Mon, 3 Feb 2020 14:53:56 -0800 Subject: [PATCH 004/635] platform/chrome: cros_ec: Query EC protocol version if EC transitions between RO/RW RO and RW of EC may have different EC protocol version. If EC transitions between RO and RW, but AP does not reboot (this is true for fingerprint microcontroller / cros_fp, but not true for main ec / cros_ec), the AP still uses the protocol version queried before transition, which can cause problems. In the case of fingerprint microcontroller, this causes AP to send the wrong version of EC_CMD_GET_NEXT_EVENT to RO in the interrupt handler, which in turn prevents RO to clear the interrupt line to AP, in an infinite loop. Once an EC_HOST_EVENT_INTERFACE_READY is received, we know that there might have been a transition between RO and RW, so re-query the protocol. Signed-off-by: Yicheng Li Tested-by: Marek Szyprowski Reviewed-by: Gwendal Grignou Signed-off-by: Enric Balletbo i Serra --- drivers/platform/chrome/cros_ec.c | 30 +++++++++++++++++++++ include/linux/platform_data/cros_ec_proto.h | 4 +++ 2 files changed, 34 insertions(+) diff --git a/drivers/platform/chrome/cros_ec.c b/drivers/platform/chrome/cros_ec.c index 6fc8f2c3ac51..7ee43b2e0654 100644 --- a/drivers/platform/chrome/cros_ec.c +++ b/drivers/platform/chrome/cros_ec.c @@ -138,6 +138,24 @@ static int cros_ec_sleep_event(struct cros_ec_device *ec_dev, u8 sleep_event) return ret; } +static int cros_ec_ready_event(struct notifier_block *nb, + unsigned long queued_during_suspend, + void *_notify) +{ + struct cros_ec_device *ec_dev = container_of(nb, struct cros_ec_device, + notifier_ready); + u32 host_event = cros_ec_get_host_event(ec_dev); + + if (host_event & EC_HOST_EVENT_MASK(EC_HOST_EVENT_INTERFACE_READY)) { + mutex_lock(&ec_dev->lock); + cros_ec_query_all(ec_dev); + mutex_unlock(&ec_dev->lock); + return NOTIFY_OK; + } + + return NOTIFY_DONE; +} + /** * cros_ec_register() - Register a new ChromeOS EC, using the provided info. * @ec_dev: Device to register. @@ -237,6 +255,18 @@ int cros_ec_register(struct cros_ec_device *ec_dev) dev_dbg(ec_dev->dev, "Error %d clearing sleep event to ec", err); + if (ec_dev->mkbp_event_supported) { + /* + * Register the notifier for EC_HOST_EVENT_INTERFACE_READY + * event. + */ + ec_dev->notifier_ready.notifier_call = cros_ec_ready_event; + err = blocking_notifier_chain_register(&ec_dev->event_notifier, + &ec_dev->notifier_ready); + if (err) + return err; + } + dev_info(dev, "Chrome EC device registered\n"); return 0; diff --git a/include/linux/platform_data/cros_ec_proto.h b/include/linux/platform_data/cros_ec_proto.h index ba5914770191..383243326676 100644 --- a/include/linux/platform_data/cros_ec_proto.h +++ b/include/linux/platform_data/cros_ec_proto.h @@ -125,6 +125,9 @@ struct cros_ec_command { * @host_event_wake_mask: Mask of host events that cause wake from suspend. * @last_event_time: exact time from the hard irq when we got notified of * a new event. + * @notifier_ready: The notifier_block to let the kernel re-query EC + * communication protocol when the EC sends + * EC_HOST_EVENT_INTERFACE_READY. * @ec: The platform_device used by the mfd driver to interface with the * main EC. * @pd: The platform_device used by the mfd driver to interface with the @@ -166,6 +169,7 @@ struct cros_ec_device { u32 host_event_wake_mask; u32 last_resume_result; ktime_t last_event_time; + struct notifier_block notifier_ready; /* The platform devices used by the mfd driver */ struct platform_device *ec; From b2ca916ce392a9d4cea3489a3efb2b627b839eaf Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Sun, 16 Feb 2020 12:00:48 -0800 Subject: [PATCH 005/635] ACPI: NUMA: Up-level "map to online node" functionality The acpi_map_pxm_to_online_node() helper is used to find the closest online node to a given proximity domain. This is used to map devices in a proximity domain with no online memory or cpus to the closest online node and populate a device's 'numa_node' property. The numa_node property allows applications to be migrated "close" to a resource. In preparation for providing a generic facility to optionally map an address range to its closest online node, or the node the range would represent were it to be onlined (target_node), up-level the core of acpi_map_pxm_to_online_node() to a generic mm/numa helper. Cc: Michal Hocko Acked-by: Rafael J. Wysocki Reviewed-by: Ingo Molnar Signed-off-by: Dan Williams Link: https://lore.kernel.org/r/158188324802.894464.13128795207831894206.stgit@dwillia2-desk3.amr.corp.intel.com --- drivers/acpi/numa/srat.c | 41 ---------------------------------------- include/linux/acpi.h | 23 +++++++++++++++++++++- include/linux/numa.h | 9 +++++++++ mm/mempolicy.c | 30 +++++++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 42 deletions(-) diff --git a/drivers/acpi/numa/srat.c b/drivers/acpi/numa/srat.c index eadbf90e65d1..47b4969d9b93 100644 --- a/drivers/acpi/numa/srat.c +++ b/drivers/acpi/numa/srat.c @@ -72,47 +72,6 @@ int acpi_map_pxm_to_node(int pxm) } EXPORT_SYMBOL(acpi_map_pxm_to_node); -/** - * acpi_map_pxm_to_online_node - Map proximity ID to online node - * @pxm: ACPI proximity ID - * - * This is similar to acpi_map_pxm_to_node(), but always returns an online - * node. When the mapped node from a given proximity ID is offline, it - * looks up the node distance table and returns the nearest online node. - * - * ACPI device drivers, which are called after the NUMA initialization has - * completed in the kernel, can call this interface to obtain their device - * NUMA topology from ACPI tables. Such drivers do not have to deal with - * offline nodes. A node may be offline when a device proximity ID is - * unique, SRAT memory entry does not exist, or NUMA is disabled, ex. - * "numa=off" on x86. - */ -int acpi_map_pxm_to_online_node(int pxm) -{ - int node, min_node; - - node = acpi_map_pxm_to_node(pxm); - - if (node == NUMA_NO_NODE) - node = 0; - - min_node = node; - if (!node_online(node)) { - int min_dist = INT_MAX, dist, n; - - for_each_online_node(n) { - dist = node_distance(node, n); - if (dist < min_dist) { - min_dist = dist; - min_node = n; - } - } - } - - return min_node; -} -EXPORT_SYMBOL(acpi_map_pxm_to_online_node); - static void __init acpi_table_print_srat_entry(struct acpi_subtable_header *header) { diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 0f24d701fbdc..3839363081f3 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -416,9 +416,30 @@ extern void acpi_osi_setup(char *str); extern bool acpi_osi_is_win8(void); #ifdef CONFIG_ACPI_NUMA -int acpi_map_pxm_to_online_node(int pxm); int acpi_map_pxm_to_node(int pxm); int acpi_get_node(acpi_handle handle); + +/** + * acpi_map_pxm_to_online_node - Map proximity ID to online node + * @pxm: ACPI proximity ID + * + * This is similar to acpi_map_pxm_to_node(), but always returns an online + * node. When the mapped node from a given proximity ID is offline, it + * looks up the node distance table and returns the nearest online node. + * + * ACPI device drivers, which are called after the NUMA initialization has + * completed in the kernel, can call this interface to obtain their device + * NUMA topology from ACPI tables. Such drivers do not have to deal with + * offline nodes. A node may be offline when a device proximity ID is + * unique, SRAT memory entry does not exist, or NUMA is disabled, ex. + * "numa=off" on x86. + */ +static inline int acpi_map_pxm_to_online_node(int pxm) +{ + int node = acpi_map_pxm_to_node(pxm); + + return numa_map_to_online_node(node); +} #else static inline int acpi_map_pxm_to_online_node(int pxm) { diff --git a/include/linux/numa.h b/include/linux/numa.h index 110b0e5d0fb0..20f4e44b186c 100644 --- a/include/linux/numa.h +++ b/include/linux/numa.h @@ -13,4 +13,13 @@ #define NUMA_NO_NODE (-1) +#ifdef CONFIG_NUMA +int numa_map_to_online_node(int node); +#else +static inline int numa_map_to_online_node(int node) +{ + return NUMA_NO_NODE; +} +#endif + #endif /* _LINUX_NUMA_H */ diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 977c641f78cf..756d6e5bb59f 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -127,6 +127,36 @@ static struct mempolicy default_policy = { static struct mempolicy preferred_node_policy[MAX_NUMNODES]; +/** + * numa_map_to_online_node - Find closest online node + * @nid: Node id to start the search + * + * Lookup the next closest node by distance if @nid is not online. + */ +int numa_map_to_online_node(int node) +{ + int min_node; + + if (node == NUMA_NO_NODE) + node = 0; + + min_node = node; + if (!node_online(node)) { + int min_dist = INT_MAX, dist, n; + + for_each_online_node(n) { + dist = node_distance(node, n); + if (dist < min_dist) { + min_dist = dist; + min_node = n; + } + } + } + + return min_node; +} +EXPORT_SYMBOL_GPL(numa_map_to_online_node); + struct mempolicy *get_task_policy(struct task_struct *p) { struct mempolicy *pol = p->mempolicy; From 4fcbe96e4d0bc4abe22ee10573ff663b9ebbcf17 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Sun, 16 Feb 2020 12:00:53 -0800 Subject: [PATCH 006/635] mm/numa: Skip NUMA_NO_NODE and online nodes in numa_map_to_online_node() Update numa_map_to_online_node() to stop falling back to numa node 0 when the input is NUMA_NO_NODE. Also, skip the lookup if @node is online. This makes the routine compatible with other arch node mapping routines. Reported-by: Aneesh Kumar K.V Reviewed-by: Aneesh Kumar K.V Link: https://lore.kernel.org/r/157401275716.43284.13185549705765009174.stgit@dwillia2-desk3.amr.corp.intel.com Reviewed-by: Ingo Molnar Signed-off-by: Dan Williams Link: https://lore.kernel.org/r/158188325316.894464.15650888748083329531.stgit@dwillia2-desk3.amr.corp.intel.com --- mm/mempolicy.c | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 756d6e5bb59f..19f7e71945a7 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -135,21 +135,17 @@ static struct mempolicy preferred_node_policy[MAX_NUMNODES]; */ int numa_map_to_online_node(int node) { - int min_node; + int min_dist = INT_MAX, dist, n, min_node; - if (node == NUMA_NO_NODE) - node = 0; + if (node == NUMA_NO_NODE || node_online(node)) + return node; min_node = node; - if (!node_online(node)) { - int min_dist = INT_MAX, dist, n; - - for_each_online_node(n) { - dist = node_distance(node, n); - if (dist < min_dist) { - min_dist = dist; - min_node = n; - } + for_each_online_node(n) { + dist = node_distance(node, n); + if (dist < min_dist) { + min_dist = dist; + min_node = n; } } From 575e23b6e13c5f575a65144579a233b76f5ca86b Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Sun, 16 Feb 2020 12:00:58 -0800 Subject: [PATCH 007/635] powerpc/papr_scm: Switch to numa_map_to_online_node() Now that the core exports numa_map_to_online_node() switch to that instead of the locally coded duplicate. Cc: Benjamin Herrenschmidt Cc: Paul Mackerras Cc: "Oliver O'Halloran" Acked-by: Michael Ellerman Reported-by: "Aneesh Kumar K.V" Reviewed-by: Aneesh Kumar K.V Link: https://lore.kernel.org/r/157401276263.43284.12616818803654229788.stgit@dwillia2-desk3.amr.corp.intel.com Reviewed-by: Ingo Molnar Signed-off-by: Dan Williams Link: https://lore.kernel.org/r/158188325830.894464.9454884523846454529.stgit@dwillia2-desk3.amr.corp.intel.com --- arch/powerpc/platforms/pseries/papr_scm.c | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/arch/powerpc/platforms/pseries/papr_scm.c b/arch/powerpc/platforms/pseries/papr_scm.c index 0b4467e378e5..3cc66224ec1f 100644 --- a/arch/powerpc/platforms/pseries/papr_scm.c +++ b/arch/powerpc/platforms/pseries/papr_scm.c @@ -285,25 +285,6 @@ int papr_scm_ndctl(struct nvdimm_bus_descriptor *nd_desc, struct nvdimm *nvdimm, return 0; } -static inline int papr_scm_node(int node) -{ - int min_dist = INT_MAX, dist; - int nid, min_node; - - if ((node == NUMA_NO_NODE) || node_online(node)) - return node; - - min_node = first_online_node; - for_each_online_node(nid) { - dist = node_distance(node, nid); - if (dist < min_dist) { - min_dist = dist; - min_node = nid; - } - } - return min_node; -} - static int papr_scm_nvdimm_init(struct papr_scm_priv *p) { struct device *dev = &p->pdev->dev; @@ -349,7 +330,7 @@ static int papr_scm_nvdimm_init(struct papr_scm_priv *p) memset(&ndr_desc, 0, sizeof(ndr_desc)); target_nid = dev_to_node(&p->pdev->dev); - online_nid = papr_scm_node(target_nid); + online_nid = numa_map_to_online_node(target_nid); ndr_desc.numa_node = online_nid; ndr_desc.target_node = target_nid; ndr_desc.res = &p->res; From 1e5d8e1e47afde23e3249aed25d7d124feff5c1c Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Sun, 16 Feb 2020 12:01:04 -0800 Subject: [PATCH 008/635] x86/mm: Introduce CONFIG_NUMA_KEEP_MEMINFO Currently x86 numa_meminfo is marked __initdata in the CONFIG_MEMORY_HOTPLUG=n case. In support of a new facility to allow drivers to map reserved memory to a 'target_node' (phys_to_target_node()), add support for removing the __initdata designation for those users. Both memory hotplug and phys_to_target_node() users select CONFIG_NUMA_KEEP_MEMINFO to tell the arch to maintain its physical address to NUMA mapping infrastructure post init. Cc: Dave Hansen Cc: Andy Lutomirski Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Borislav Petkov Cc: "H. Peter Anvin" Cc: Cc: Andrew Morton Cc: David Hildenbrand Cc: Michal Hocko Reviewed-by: Ingo Molnar Signed-off-by: Dan Williams Reviewed-by: Thomas Gleixner Link: https://lore.kernel.org/r/158188326422.894464.15742054998046628934.stgit@dwillia2-desk3.amr.corp.intel.com --- arch/x86/mm/numa.c | 6 +----- include/linux/numa.h | 7 +++++++ mm/Kconfig | 5 +++++ 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/arch/x86/mm/numa.c b/arch/x86/mm/numa.c index 99f7a68738f0..2450b21cc28a 100644 --- a/arch/x86/mm/numa.c +++ b/arch/x86/mm/numa.c @@ -25,11 +25,7 @@ nodemask_t numa_nodes_parsed __initdata; struct pglist_data *node_data[MAX_NUMNODES] __read_mostly; EXPORT_SYMBOL(node_data); -static struct numa_meminfo numa_meminfo -#ifndef CONFIG_MEMORY_HOTPLUG -__initdata -#endif -; +static struct numa_meminfo numa_meminfo __initdata_or_meminfo; static int numa_distance_cnt; static u8 *numa_distance; diff --git a/include/linux/numa.h b/include/linux/numa.h index 20f4e44b186c..5773cd2613fc 100644 --- a/include/linux/numa.h +++ b/include/linux/numa.h @@ -13,6 +13,13 @@ #define NUMA_NO_NODE (-1) +/* optionally keep NUMA memory info available post init */ +#ifdef CONFIG_NUMA_KEEP_MEMINFO +#define __initdata_or_meminfo +#else +#define __initdata_or_meminfo __initdata +#endif + #ifdef CONFIG_NUMA int numa_map_to_online_node(int node); #else diff --git a/mm/Kconfig b/mm/Kconfig index ab80933be65f..328268473fec 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -139,6 +139,10 @@ config HAVE_FAST_GUP config ARCH_KEEP_MEMBLOCK bool +# Keep arch NUMA mapping infrastructure post-init. +config NUMA_KEEP_MEMINFO + bool + config MEMORY_ISOLATION bool @@ -154,6 +158,7 @@ config MEMORY_HOTPLUG bool "Allow for memory hot-add" depends on SPARSEMEM || X86_64_ACPI_NUMA depends on ARCH_ENABLE_MEMORY_HOTPLUG + select NUMA_KEEP_MEMINFO if NUMA config MEMORY_HOTPLUG_SPARSE def_bool y From 5d30f92e7631286b8617777c5400c8eadcae50a1 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Sun, 16 Feb 2020 12:01:09 -0800 Subject: [PATCH 009/635] x86/NUMA: Provide a range-to-target_node lookup facility The DEV_DAX_KMEM facility is a generic mechanism to allow device-dax instances, fronting performance-differentiated-memory like pmem, to be added to the System RAM pool. The NUMA node for that hot-added memory is derived from the device-dax instance's 'target_node' attribute. Recall that the 'target_node' is the ACPI-PXM-to-node translation for memory when it comes online whereas the 'numa_node' attribute of the device represents the closest online cpu node. Presently useful target_node information from the ACPI SRAT is discarded with the expectation that "Reserved" memory will never be onlined. Now, DEV_DAX_KMEM violates that assumption, there is a need to retain the translation. Move, rather than discard, numa_memblk data to a secondary array that memory_add_physaddr_to_target_node() may consider at a later point in time. Cc: Dave Hansen Cc: Andy Lutomirski Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Borislav Petkov Cc: "H. Peter Anvin" Cc: Cc: Andrew Morton Cc: David Hildenbrand Cc: Michal Hocko Reported-by: kbuild test robot Reviewed-by: Ingo Molnar Signed-off-by: Dan Williams Reviewed-by: Thomas Gleixner Link: https://lore.kernel.org/r/158188326978.894464.217282995221175417.stgit@dwillia2-desk3.amr.corp.intel.com --- arch/x86/mm/numa.c | 61 ++++++++++++++++++++++++++++++++++++-------- include/linux/numa.h | 14 +++++++++- 2 files changed, 64 insertions(+), 11 deletions(-) diff --git a/arch/x86/mm/numa.c b/arch/x86/mm/numa.c index 2450b21cc28a..59ba008504dc 100644 --- a/arch/x86/mm/numa.c +++ b/arch/x86/mm/numa.c @@ -26,6 +26,7 @@ struct pglist_data *node_data[MAX_NUMNODES] __read_mostly; EXPORT_SYMBOL(node_data); static struct numa_meminfo numa_meminfo __initdata_or_meminfo; +static struct numa_meminfo numa_reserved_meminfo __initdata_or_meminfo; static int numa_distance_cnt; static u8 *numa_distance; @@ -164,6 +165,19 @@ void __init numa_remove_memblk_from(int idx, struct numa_meminfo *mi) (mi->nr_blks - idx) * sizeof(mi->blk[0])); } +/** + * numa_move_tail_memblk - Move a numa_memblk from one numa_meminfo to another + * @dst: numa_meminfo to append block to + * @idx: Index of memblk to remove + * @src: numa_meminfo to remove memblk from + */ +static void __init numa_move_tail_memblk(struct numa_meminfo *dst, int idx, + struct numa_meminfo *src) +{ + dst->blk[dst->nr_blks++] = src->blk[idx]; + numa_remove_memblk_from(idx, src); +} + /** * numa_add_memblk - Add one numa_memblk to numa_meminfo * @nid: NUMA node ID of the new memblk @@ -233,14 +247,19 @@ int __init numa_cleanup_meminfo(struct numa_meminfo *mi) for (i = 0; i < mi->nr_blks; i++) { struct numa_memblk *bi = &mi->blk[i]; - /* make sure all blocks are inside the limits */ + /* move / save reserved memory ranges */ + if (!memblock_overlaps_region(&memblock.memory, + bi->start, bi->end - bi->start)) { + numa_move_tail_memblk(&numa_reserved_meminfo, i--, mi); + continue; + } + + /* make sure all non-reserved blocks are inside the limits */ bi->start = max(bi->start, low); bi->end = min(bi->end, high); - /* and there's no empty or non-exist block */ - if (bi->start >= bi->end || - !memblock_overlaps_region(&memblock.memory, - bi->start, bi->end - bi->start)) + /* and there's no empty block */ + if (bi->start >= bi->end) numa_remove_memblk_from(i--, mi); } @@ -877,16 +896,38 @@ EXPORT_SYMBOL(cpumask_of_node); #endif /* !CONFIG_DEBUG_PER_CPU_MAPS */ -#ifdef CONFIG_MEMORY_HOTPLUG -int memory_add_physaddr_to_nid(u64 start) +#ifdef CONFIG_NUMA_KEEP_MEMINFO +static int meminfo_to_nid(struct numa_meminfo *mi, u64 start) { - struct numa_meminfo *mi = &numa_meminfo; - int nid = mi->blk[0].nid; int i; for (i = 0; i < mi->nr_blks; i++) if (mi->blk[i].start <= start && mi->blk[i].end > start) - nid = mi->blk[i].nid; + return mi->blk[i].nid; + return NUMA_NO_NODE; +} + +int phys_to_target_node(phys_addr_t start) +{ + int nid = meminfo_to_nid(&numa_meminfo, start); + + /* + * Prefer online nodes, but if reserved memory might be + * hot-added continue the search with reserved ranges. + */ + if (nid != NUMA_NO_NODE) + return nid; + + return meminfo_to_nid(&numa_reserved_meminfo, start); +} +EXPORT_SYMBOL_GPL(phys_to_target_node); + +int memory_add_physaddr_to_nid(u64 start) +{ + int nid = meminfo_to_nid(&numa_meminfo, start); + + if (nid == NUMA_NO_NODE) + nid = numa_meminfo.blk[0].nid; return nid; } EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid); diff --git a/include/linux/numa.h b/include/linux/numa.h index 5773cd2613fc..a42df804679e 100644 --- a/include/linux/numa.h +++ b/include/linux/numa.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _LINUX_NUMA_H #define _LINUX_NUMA_H - +#include #ifdef CONFIG_NODES_SHIFT #define NODES_SHIFT CONFIG_NODES_SHIFT @@ -21,12 +21,24 @@ #endif #ifdef CONFIG_NUMA +/* Generic implementation available */ int numa_map_to_online_node(int node); + +/* + * Optional architecture specific implementation, users need a "depends + * on $ARCH" + */ +int phys_to_target_node(phys_addr_t addr); #else static inline int numa_map_to_online_node(int node) { return NUMA_NO_NODE; } + +static inline int phys_to_target_node(phys_addr_t addr) +{ + return NUMA_NO_NODE; +} #endif #endif /* _LINUX_NUMA_H */ From 7b27a8622f802761d5c6abd6c37b22312a35343c Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Sun, 16 Feb 2020 12:01:16 -0800 Subject: [PATCH 010/635] libnvdimm/e820: Retrieve and populate correct 'target_node' info Use the new phys_to_target_node() and numa_map_to_online_node() helpers to retrieve the correct id for the 'numa_node' ("local" / online initiator node) and 'target_node' (offline target memory node) sysfs attributes. Below is an example from a 4 NUMA node system where all the memory on node2 is pmem / reserved. It should be noted that with the arrival of the ACPI HMAT table and EFI Specific Purpose Memory the kernel will start to see more platforms with reserved / performance differentiated memory in its own NUMA node. Hence all the stakeholders on the Cc for what is ostensibly a libnvdimm local patch. === Before === /* Notice no online memory on node2 at start */ # numactl --hardware available: 3 nodes (0-1,3) node 0 cpus: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 node 0 size: 3958 MB node 0 free: 3708 MB node 1 cpus: 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 node 1 size: 4027 MB node 1 free: 3871 MB node 3 cpus: node 3 size: 3994 MB node 3 free: 3971 MB node distances: node 0 1 3 0: 10 21 21 1: 21 10 21 3: 21 21 10 /* * Put the pmem namespace into devdax mode so it can be assigned to the * kmem driver */ # ndctl create-namespace -e namespace0.0 -m devdax -f { "dev":"namespace0.0", "mode":"devdax", "map":"dev", "size":"3.94 GiB (4.23 GB)", "uuid":"1650af9b-9ba3-4704-acd6-10178399d9a3", [..] } /* Online Persistent Memory as System RAM */ # daxctl reconfigure-device --mode=system-ram dax0.0 libdaxctl: memblock_in_dev: dax0.0: memory0: Unable to determine phys_index: Success libdaxctl: memblock_in_dev: dax0.0: memory0: Unable to determine phys_index: Success libdaxctl: memblock_in_dev: dax0.0: memory0: Unable to determine phys_index: Success libdaxctl: memblock_in_dev: dax0.0: memory0: Unable to determine phys_index: Success [ { "chardev":"dax0.0", "size":4225761280, "target_node":0, "mode":"system-ram" } ] reconfigured 1 device /* Note that the memory is onlined by default to the wrong node, node0 */ # numactl --hardware available: 3 nodes (0-1,3) node 0 cpus: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 node 0 size: 7926 MB node 0 free: 7655 MB node 1 cpus: 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 node 1 size: 4027 MB node 1 free: 3871 MB node 3 cpus: node 3 size: 3994 MB node 3 free: 3971 MB node distances: node 0 1 3 0: 10 21 21 1: 21 10 21 3: 21 21 10 === After === /* Notice that the "phys_index" error messages are gone */ # daxctl reconfigure-device --mode=system-ram dax0.0 [ { "chardev":"dax0.0", "size":4225761280, "target_node":2, "mode":"system-ram" } ] reconfigured 1 device /* Notice that node2 is now correctly populated */ # numactl --hardware available: 4 nodes (0-3) node 0 cpus: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 node 0 size: 3958 MB node 0 free: 3793 MB node 1 cpus: 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 node 1 size: 4027 MB node 1 free: 3851 MB node 2 cpus: node 2 size: 3968 MB node 2 free: 3968 MB node 3 cpus: node 3 size: 3994 MB node 3 free: 3908 MB node distances: node 0 1 2 3 0: 10 21 21 21 1: 21 10 21 21 2: 21 21 10 21 3: 21 21 21 10 Cc: Dave Hansen Cc: Andy Lutomirski Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Andrew Morton Cc: David Hildenbrand Cc: Michal Hocko Cc: Ira Weiny Cc: Vishal Verma Cc: Christoph Hellwig Reviewed-by: Ingo Molnar Signed-off-by: Dan Williams Link: https://lore.kernel.org/r/158188327614.894464.13122730362187722603.stgit@dwillia2-desk3.amr.corp.intel.com --- arch/x86/Kconfig | 1 + drivers/nvdimm/e820.c | 18 ++++-------------- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index beea77046f9b..d4e446daf457 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1664,6 +1664,7 @@ config X86_PMEM_LEGACY depends on PHYS_ADDR_T_64BIT depends on BLK_DEV select X86_PMEM_LEGACY_DEVICE + select NUMA_KEEP_MEMINFO if NUMA select LIBNVDIMM help Treat memory marked using the non-standard e820 type of 12 as used diff --git a/drivers/nvdimm/e820.c b/drivers/nvdimm/e820.c index e02f60ad6c99..4cd18be9d0e9 100644 --- a/drivers/nvdimm/e820.c +++ b/drivers/nvdimm/e820.c @@ -7,6 +7,7 @@ #include #include #include +#include static int e820_pmem_remove(struct platform_device *pdev) { @@ -16,27 +17,16 @@ static int e820_pmem_remove(struct platform_device *pdev) return 0; } -#ifdef CONFIG_MEMORY_HOTPLUG -static int e820_range_to_nid(resource_size_t addr) -{ - return memory_add_physaddr_to_nid(addr); -} -#else -static int e820_range_to_nid(resource_size_t addr) -{ - return NUMA_NO_NODE; -} -#endif - static int e820_register_one(struct resource *res, void *data) { struct nd_region_desc ndr_desc; struct nvdimm_bus *nvdimm_bus = data; + int nid = phys_to_target_node(res->start); memset(&ndr_desc, 0, sizeof(ndr_desc)); ndr_desc.res = res; - ndr_desc.numa_node = e820_range_to_nid(res->start); - ndr_desc.target_node = ndr_desc.numa_node; + ndr_desc.numa_node = numa_map_to_online_node(nid); + ndr_desc.target_node = nid; set_bit(ND_REGION_PAGEMAP, &ndr_desc.flags); if (!nvdimm_pmem_region_create(nvdimm_bus, &ndr_desc)) return -ENXIO; From b9bd80396a42d0335f6cf287d8b561c50b44a2d8 Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Sat, 1 Feb 2020 18:09:33 +0100 Subject: [PATCH 011/635] MAINTAINERS: clarify maintenance of nvdimm testing tool The git history shows that the files under ./tools/testing/nvdimm are being developed and maintained by the LIBNVDIMM maintainers. This was identified with a small script that finds all files only belonging to "THE REST" according to the current MAINTAINERS file, and I acted upon its output. Signed-off-by: Lukas Bulwahn Link: https://lore.kernel.org/r/20200201170933.924-1-lukas.bulwahn@gmail.com Signed-off-by: Dan Williams --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index a0d86490c2c6..6e65c79b4c07 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -9569,6 +9569,7 @@ F: drivers/acpi/nfit/* F: include/linux/nd.h F: include/linux/libnvdimm.h F: include/uapi/linux/ndctl.h +F: tools/testing/nvdimm/ LICENSES and SPDX stuff M: Thomas Gleixner From c0e71d602053e4e7637e4bc7d0bc9603ea77a33f Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Thu, 23 Jan 2020 16:47:20 +0100 Subject: [PATCH 012/635] tools/testing/nvdimm: Fix compilation failure without CONFIG_DEV_DAX_PMEM_COMPAT When a kernel is configured without CONFIG_DEV_DAX_PMEM_COMPAT, the compilation of tools/testing/nvdimm fails with: Building modules, stage 2. MODPOST 11 modules ERROR: "dax_pmem_compat_test" [tools/testing/nvdimm/test/nfit_test.ko] undefined! Fix the problem by calling dax_pmem_compat_test() only if the kernel has the required functionality. Signed-off-by: Jan Kara Link: https://lore.kernel.org/r/20200123154720.12097-1-jack@suse.cz Signed-off-by: Dan Williams --- tools/testing/nvdimm/test/nfit.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/testing/nvdimm/test/nfit.c b/tools/testing/nvdimm/test/nfit.c index bf6422a6af7f..a8ee5c4d41eb 100644 --- a/tools/testing/nvdimm/test/nfit.c +++ b/tools/testing/nvdimm/test/nfit.c @@ -3164,7 +3164,9 @@ static __init int nfit_test_init(void) mcsafe_test(); dax_pmem_test(); dax_pmem_core_test(); +#ifdef CONFIG_DEV_DAX_PMEM_COMPAT dax_pmem_compat_test(); +#endif nfit_test_setup(nfit_test_lookup, nfit_test_evaluate_dsm); From 1d0827b75ee7df497f611a2ac412a88135fb0ef5 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 30 Jan 2020 12:06:01 -0800 Subject: [PATCH 013/635] mm/memremap_pages: Kill unused __devm_memremap_pages() Kill this definition that was introduced in commit 41e94a851304 ("add devm_memremap_pages") add never used. Cc: Christoph Hellwig Reviewed-by: Aneesh Kumar K.V Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/158041476158.3889308.4221100673554151124.stgit@dwillia2-desk3.amr.corp.intel.com Signed-off-by: Dan Williams --- include/linux/io.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/include/linux/io.h b/include/linux/io.h index b1c44bb4b2d7..8394c56babc2 100644 --- a/include/linux/io.h +++ b/include/linux/io.h @@ -77,8 +77,6 @@ void *devm_memremap(struct device *dev, resource_size_t offset, size_t size, unsigned long flags); void devm_memunmap(struct device *dev, void *addr); -void *__devm_memremap_pages(struct device *dev, struct resource *res); - #ifdef CONFIG_PCI /* * The PCI specifications (Rev 3.0, 3.2.5 "Transaction Ordering and From 9ffc1d19fc4a6dfcfe06c91c2861ad6d44fdd92d Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 30 Jan 2020 12:06:07 -0800 Subject: [PATCH 014/635] mm/memremap_pages: Introduce memremap_compat_align() The "sub-section memory hotplug" facility allows memremap_pages() users like libnvdimm to compensate for hardware platforms like x86 that have a section size larger than their hardware memory mapping granularity. The compensation that sub-section support affords is being tolerant of physical memory resources shifting by units smaller (64MiB on x86) than the memory-hotplug section size (128 MiB). Where the platform physical-memory mapping granularity is limited by the number and capability of address-decode-registers in the memory controller. While the sub-section support allows memremap_pages() to operate on sub-section (2MiB) granularity, the Power architecture may still require 16MiB alignment on "!radix_enabled()" platforms. In order for libnvdimm to be able to detect and manage this per-arch limitation, introduce memremap_compat_align() as a common minimum alignment across all driver-facing memory-mapping interfaces, and let Power override it to 16MiB in the "!radix_enabled()" case. The assumption / requirement for 16MiB to be a viable memremap_compat_align() value is that Power does not have platforms where its equivalent of address-decode-registers never hardware remaps a persistent memory resource on smaller than 16MiB boundaries. Note that I tried my best to not add a new Kconfig symbol, but header include entanglements defeated the #ifndef memremap_compat_align design pattern and the need to export it defeats the __weak design pattern for arch overrides. Based on an initial patch by Aneesh. Link: http://lore.kernel.org/r/CAPcyv4gBGNP95APYaBcsocEa50tQj9b5h__83vgngjq3ouGX_Q@mail.gmail.com Reported-by: Aneesh Kumar K.V Reported-by: Jeff Moyer Cc: Benjamin Herrenschmidt Cc: Paul Mackerras Reviewed-by: Aneesh Kumar K.V Acked-by: Michael Ellerman (powerpc) Signed-off-by: Dan Williams --- arch/powerpc/Kconfig | 1 + arch/powerpc/mm/ioremap.c | 21 +++++++++++++++++++++ drivers/nvdimm/pfn_devs.c | 2 +- include/linux/memremap.h | 8 ++++++++ include/linux/mmzone.h | 1 + lib/Kconfig | 3 +++ mm/memremap.c | 23 +++++++++++++++++++++++ 7 files changed, 58 insertions(+), 1 deletion(-) diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 497b7d0b2d7e..e6ffe905e2b9 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -122,6 +122,7 @@ config PPC select ARCH_HAS_GCOV_PROFILE_ALL select ARCH_HAS_KCOV select ARCH_HAS_HUGEPD if HUGETLB_PAGE + select ARCH_HAS_MEMREMAP_COMPAT_ALIGN select ARCH_HAS_MMIOWB if PPC64 select ARCH_HAS_PHYS_TO_DMA select ARCH_HAS_PMEM_API diff --git a/arch/powerpc/mm/ioremap.c b/arch/powerpc/mm/ioremap.c index fc669643ce6a..b1a0aebe8c48 100644 --- a/arch/powerpc/mm/ioremap.c +++ b/arch/powerpc/mm/ioremap.c @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -97,3 +98,23 @@ void __iomem *do_ioremap(phys_addr_t pa, phys_addr_t offset, unsigned long size, return NULL; } + +#ifdef CONFIG_ZONE_DEVICE +/* + * Override the generic version in mm/memremap.c. + * + * With hash translation, the direct-map range is mapped with just one + * page size selected by htab_init_page_sizes(). Consult + * mmu_psize_defs[] to determine the minimum page size alignment. +*/ +unsigned long memremap_compat_align(void) +{ + unsigned int shift = mmu_psize_defs[mmu_linear_psize].shift; + + if (radix_enabled()) + return SUBSECTION_SIZE; + return max(SUBSECTION_SIZE, 1UL << shift); + +} +EXPORT_SYMBOL_GPL(memremap_compat_align); +#endif diff --git a/drivers/nvdimm/pfn_devs.c b/drivers/nvdimm/pfn_devs.c index b94f7a7e94b8..a5c25cb87116 100644 --- a/drivers/nvdimm/pfn_devs.c +++ b/drivers/nvdimm/pfn_devs.c @@ -750,7 +750,7 @@ static int nd_pfn_init(struct nd_pfn *nd_pfn) start = nsio->res.start; size = resource_size(&nsio->res); npfns = PHYS_PFN(size - SZ_8K); - align = max(nd_pfn->align, (1UL << SUBSECTION_SHIFT)); + align = max(nd_pfn->align, SUBSECTION_SIZE); end_trunc = start + size - ALIGN_DOWN(start + size, align); if (nd_pfn->mode == PFN_MODE_PMEM) { /* diff --git a/include/linux/memremap.h b/include/linux/memremap.h index 6fefb09af7c3..8af1cbd8f293 100644 --- a/include/linux/memremap.h +++ b/include/linux/memremap.h @@ -132,6 +132,7 @@ struct dev_pagemap *get_dev_pagemap(unsigned long pfn, unsigned long vmem_altmap_offset(struct vmem_altmap *altmap); void vmem_altmap_free(struct vmem_altmap *altmap, unsigned long nr_pfns); +unsigned long memremap_compat_align(void); #else static inline void *devm_memremap_pages(struct device *dev, struct dev_pagemap *pgmap) @@ -165,6 +166,12 @@ static inline void vmem_altmap_free(struct vmem_altmap *altmap, unsigned long nr_pfns) { } + +/* when memremap_pages() is disabled all archs can remap a single page */ +static inline unsigned long memremap_compat_align(void) +{ + return PAGE_SIZE; +} #endif /* CONFIG_ZONE_DEVICE */ static inline void put_dev_pagemap(struct dev_pagemap *pgmap) @@ -172,4 +179,5 @@ static inline void put_dev_pagemap(struct dev_pagemap *pgmap) if (pgmap) percpu_ref_put(pgmap->ref); } + #endif /* _LINUX_MEMREMAP_H_ */ diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 462f6873905a..6b77f7239af5 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -1170,6 +1170,7 @@ static inline unsigned long section_nr_to_pfn(unsigned long sec) #define SECTION_ALIGN_DOWN(pfn) ((pfn) & PAGE_SECTION_MASK) #define SUBSECTION_SHIFT 21 +#define SUBSECTION_SIZE (1UL << SUBSECTION_SHIFT) #define PFN_SUBSECTION_SHIFT (SUBSECTION_SHIFT - PAGE_SHIFT) #define PAGES_PER_SUBSECTION (1UL << PFN_SUBSECTION_SHIFT) diff --git a/lib/Kconfig b/lib/Kconfig index bc7e56370129..5d53f9609c25 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -615,6 +615,9 @@ config ARCH_HAS_PMEM_API config MEMREGION bool +config ARCH_HAS_MEMREMAP_COMPAT_ALIGN + bool + # use memcpy to implement user copies for nommu architectures config UACCESS_MEMCPY bool diff --git a/mm/memremap.c b/mm/memremap.c index 09b5b7adc773..3e7afaf05639 100644 --- a/mm/memremap.c +++ b/mm/memremap.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -14,6 +15,28 @@ static DEFINE_XARRAY(pgmap_array); +/* + * The memremap() and memremap_pages() interfaces are alternately used + * to map persistent memory namespaces. These interfaces place different + * constraints on the alignment and size of the mapping (namespace). + * memremap() can map individual PAGE_SIZE pages. memremap_pages() can + * only map subsections (2MB), and at least one architecture (PowerPC) + * the minimum mapping granularity of memremap_pages() is 16MB. + * + * The role of memremap_compat_align() is to communicate the minimum + * arch supported alignment of a namespace such that it can freely + * switch modes without violating the arch constraint. Namely, do not + * allow a namespace to be PAGE_SIZE aligned since that namespace may be + * reconfigured into a mode that requires SUBSECTION_SIZE alignment. + */ +#ifndef CONFIG_ARCH_HAS_MEMREMAP_COMPAT_ALIGN +unsigned long memremap_compat_align(void) +{ + return SUBSECTION_SIZE; +} +EXPORT_SYMBOL_GPL(memremap_compat_align); +#endif + #ifdef CONFIG_DEV_PAGEMAP_OPS DEFINE_STATIC_KEY_FALSE(devmap_managed_key); EXPORT_SYMBOL(devmap_managed_key); From c7896490dd1a4e6b346e8a475e4a433356362770 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Tue, 7 Jan 2020 15:10:28 +0100 Subject: [PATCH 015/635] leds: ns2: Absorb platform data Nothing in the kernel includes the external header so just push the contents into the ns2 leds driver. If someone wants to use platform data or board files to describe this device they should be able to do so using GPIO machine descriptors but in any case device tree should be the way forward for these systems in all cases I can think of, and the driver already supports that. Cc: Simon Guinot Cc: Vincent Donnefort Signed-off-by: Linus Walleij Tested-by: Simon Guinot Signed-off-by: Pavel Machek --- drivers/leds/leds-ns2.c | 30 +++++++++++++-- .../linux/platform_data/leds-kirkwood-ns2.h | 38 ------------------- 2 files changed, 27 insertions(+), 41 deletions(-) delete mode 100644 include/linux/platform_data/leds-kirkwood-ns2.h diff --git a/drivers/leds/leds-ns2.c b/drivers/leds/leds-ns2.c index 7c500dfdcfa3..6d37dda12c39 100644 --- a/drivers/leds/leds-ns2.c +++ b/drivers/leds/leds-ns2.c @@ -12,14 +12,38 @@ #include #include #include -#include +#include #include #include -#include #include -#include #include "leds.h" +enum ns2_led_modes { + NS_V2_LED_OFF, + NS_V2_LED_ON, + NS_V2_LED_SATA, +}; + +struct ns2_led_modval { + enum ns2_led_modes mode; + int cmd_level; + int slow_level; +}; + +struct ns2_led { + const char *name; + const char *default_trigger; + unsigned cmd; + unsigned slow; + int num_modes; + struct ns2_led_modval *modval; +}; + +struct ns2_led_platform_data { + int num_leds; + struct ns2_led *leds; +}; + /* * The Network Space v2 dual-GPIO LED is wired to a CPLD. Three different LED * modes are available: off, on and SATA activity blinking. The LED modes are diff --git a/include/linux/platform_data/leds-kirkwood-ns2.h b/include/linux/platform_data/leds-kirkwood-ns2.h deleted file mode 100644 index eb8a6860e816..000000000000 --- a/include/linux/platform_data/leds-kirkwood-ns2.h +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Platform data structure for Network Space v2 LED driver - * - * This file is licensed under the terms of the GNU General Public - * License version 2. This program is licensed "as is" without any - * warranty of any kind, whether express or implied. - */ - -#ifndef __LEDS_KIRKWOOD_NS2_H -#define __LEDS_KIRKWOOD_NS2_H - -enum ns2_led_modes { - NS_V2_LED_OFF, - NS_V2_LED_ON, - NS_V2_LED_SATA, -}; - -struct ns2_led_modval { - enum ns2_led_modes mode; - int cmd_level; - int slow_level; -}; - -struct ns2_led { - const char *name; - const char *default_trigger; - unsigned cmd; - unsigned slow; - int num_modes; - struct ns2_led_modval *modval; -}; - -struct ns2_led_platform_data { - int num_leds; - struct ns2_led *leds; -}; - -#endif /* __LEDS_KIRKWOOD_NS2_H */ From ccbbb117c17eaf108216830bdd1e85a551ccc031 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Tue, 7 Jan 2020 15:10:29 +0100 Subject: [PATCH 016/635] leds: ns2: Convert to GPIO descriptors This converts the NS2 LED driver to use GPIO descriptors. We take care to request the GPIOs "as is" which is what the current driver goes to lengths to achieve, then we use GPIOs throughout. As the nodes for each LED does not have any corresponding device, we need to use the DT-specific accessors to get these GPIO descriptors from the device tree. Cc: Vincent Donnefort Signed-off-by: Linus Walleij Tested-by: Simon Guinot Signed-off-by: Pavel Machek --- drivers/leds/leds-ns2.c | 73 +++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 42 deletions(-) diff --git a/drivers/leds/leds-ns2.c b/drivers/leds/leds-ns2.c index 6d37dda12c39..538ca5755602 100644 --- a/drivers/leds/leds-ns2.c +++ b/drivers/leds/leds-ns2.c @@ -33,8 +33,8 @@ struct ns2_led_modval { struct ns2_led { const char *name; const char *default_trigger; - unsigned cmd; - unsigned slow; + struct gpio_desc *cmd; + struct gpio_desc *slow; int num_modes; struct ns2_led_modval *modval; }; @@ -53,8 +53,8 @@ struct ns2_led_platform_data { struct ns2_led_data { struct led_classdev cdev; - unsigned int cmd; - unsigned int slow; + struct gpio_desc *cmd; + struct gpio_desc *slow; bool can_sleep; unsigned char sata; /* True when SATA mode active. */ rwlock_t rw_lock; /* Lock GPIOs. */ @@ -70,8 +70,8 @@ static int ns2_led_get_mode(struct ns2_led_data *led_dat, int cmd_level; int slow_level; - cmd_level = gpio_get_value_cansleep(led_dat->cmd); - slow_level = gpio_get_value_cansleep(led_dat->slow); + cmd_level = gpiod_get_value_cansleep(led_dat->cmd); + slow_level = gpiod_get_value_cansleep(led_dat->slow); for (i = 0; i < led_dat->num_modes; i++) { if (cmd_level == led_dat->modval[i].cmd_level && @@ -104,15 +104,15 @@ static void ns2_led_set_mode(struct ns2_led_data *led_dat, write_lock_irqsave(&led_dat->rw_lock, flags); if (!led_dat->can_sleep) { - gpio_set_value(led_dat->cmd, - led_dat->modval[i].cmd_level); - gpio_set_value(led_dat->slow, - led_dat->modval[i].slow_level); + gpiod_set_value(led_dat->cmd, + led_dat->modval[i].cmd_level); + gpiod_set_value(led_dat->slow, + led_dat->modval[i].slow_level); goto exit_unlock; } - gpio_set_value_cansleep(led_dat->cmd, led_dat->modval[i].cmd_level); - gpio_set_value_cansleep(led_dat->slow, led_dat->modval[i].slow_level); + gpiod_set_value_cansleep(led_dat->cmd, led_dat->modval[i].cmd_level); + gpiod_set_value_cansleep(led_dat->slow, led_dat->modval[i].slow_level); exit_unlock: write_unlock_irqrestore(&led_dat->rw_lock, flags); @@ -200,26 +200,6 @@ create_ns2_led(struct platform_device *pdev, struct ns2_led_data *led_dat, int ret; enum ns2_led_modes mode; - ret = devm_gpio_request_one(&pdev->dev, template->cmd, - gpio_get_value_cansleep(template->cmd) ? - GPIOF_OUT_INIT_HIGH : GPIOF_OUT_INIT_LOW, - template->name); - if (ret) { - dev_err(&pdev->dev, "%s: failed to setup command GPIO\n", - template->name); - return ret; - } - - ret = devm_gpio_request_one(&pdev->dev, template->slow, - gpio_get_value_cansleep(template->slow) ? - GPIOF_OUT_INIT_HIGH : GPIOF_OUT_INIT_LOW, - template->name); - if (ret) { - dev_err(&pdev->dev, "%s: failed to setup slow GPIO\n", - template->name); - return ret; - } - rwlock_init(&led_dat->rw_lock); led_dat->cdev.name = template->name; @@ -229,8 +209,8 @@ create_ns2_led(struct platform_device *pdev, struct ns2_led_data *led_dat, led_dat->cdev.groups = ns2_led_groups; led_dat->cmd = template->cmd; led_dat->slow = template->slow; - led_dat->can_sleep = gpio_cansleep(led_dat->cmd) | - gpio_cansleep(led_dat->slow); + led_dat->can_sleep = gpiod_cansleep(led_dat->cmd) | + gpiod_cansleep(led_dat->slow); if (led_dat->can_sleep) led_dat->cdev.brightness_set_blocking = ns2_led_set_blocking; else @@ -285,17 +265,26 @@ ns2_leds_get_of_pdata(struct device *dev, struct ns2_led_platform_data *pdata) const char *string; int i, num_modes; struct ns2_led_modval *modval; + struct gpio_desc *gd; - ret = of_get_named_gpio(child, "cmd-gpio", 0); - if (ret < 0) - goto err_node_put; - led->cmd = ret; - ret = of_get_named_gpio(child, "slow-gpio", 0); - if (ret < 0) - goto err_node_put; - led->slow = ret; ret = of_property_read_string(child, "label", &string); led->name = (ret == 0) ? string : child->name; + + gd = gpiod_get_from_of_node(child, "cmd-gpio", 0, + GPIOD_ASIS, led->name); + if (IS_ERR(gd)) { + ret = PTR_ERR(gd); + goto err_node_put; + } + led->cmd = gd; + gd = gpiod_get_from_of_node(child, "slow-gpio", 0, + GPIOD_ASIS, led->name); + if (IS_ERR(gd)) { + ret = PTR_ERR(gd); + goto err_node_put; + } + led->slow = gd; + ret = of_property_read_string(child, "linux,default-trigger", &string); if (ret == 0) From 877a50b8729994a701aac7b2092381bad2c6c145 Mon Sep 17 00:00:00 2001 From: YueHaibing Date: Wed, 8 Jan 2020 01:53:22 +0000 Subject: [PATCH 017/635] leds: leds-bd2802: remove set but not used variable 'pdata' Fixes gcc '-Wunused-but-set-variable' warning: drivers/leds/leds-bd2802.c: In function 'bd2802_probe': drivers/leds/leds-bd2802.c:663:35: warning: variable 'pdata' set but not used [-Wunused-but-set-variable] commit 4c3718f9d6a6 ("leds: bd2802: Convert to use GPIO descriptors") left behind this unused variable. Signed-off-by: YueHaibing Reviewed-by: Linus Walleij Signed-off-by: Pavel Machek --- drivers/leds/leds-bd2802.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/leds/leds-bd2802.c b/drivers/leds/leds-bd2802.c index bd61a823d0ca..8bbaef5a2986 100644 --- a/drivers/leds/leds-bd2802.c +++ b/drivers/leds/leds-bd2802.c @@ -660,7 +660,6 @@ static int bd2802_probe(struct i2c_client *client, const struct i2c_device_id *id) { struct bd2802_led *led; - struct bd2802_led_platform_data *pdata; int ret, i; led = devm_kzalloc(&client->dev, sizeof(struct bd2802_led), GFP_KERNEL); @@ -668,7 +667,6 @@ static int bd2802_probe(struct i2c_client *client, return -ENOMEM; led->client = client; - pdata = led->pdata = dev_get_platdata(&client->dev); i2c_set_clientdata(client, led); /* From cc9c077f6afe1c0e28bb33675bc7298db3a721d7 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Wed, 22 Jan 2020 23:37:49 +0000 Subject: [PATCH 018/635] leds: lm3697: fix spelling mistake "To" -> "Too" There is a spelling mistake in a dev_err message. Fix it. Signed-off-by: Colin Ian King Signed-off-by: Pavel Machek --- drivers/leds/leds-lm3697.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/leds/leds-lm3697.c b/drivers/leds/leds-lm3697.c index b71711aff8a3..872d26f9706a 100644 --- a/drivers/leds/leds-lm3697.c +++ b/drivers/leds/leds-lm3697.c @@ -246,7 +246,7 @@ static int lm3697_probe_dt(struct lm3697 *priv) led->num_leds = fwnode_property_count_u32(child, "led-sources"); if (led->num_leds > LM3697_MAX_LED_STRINGS) { - dev_err(&priv->client->dev, "To many LED strings defined\n"); + dev_err(&priv->client->dev, "Too many LED strings defined\n"); continue; } From 64d7e23115dd91997858eb0c370d798a0d6bf118 Mon Sep 17 00:00:00 2001 From: Thomas Bogendoerfer Date: Mon, 24 Feb 2020 12:17:33 +0100 Subject: [PATCH 019/635] leds: add SGI IP30 led support This patch implemenets a driver to support the front panel LEDs of SGI Octane (IP30) workstations. Reviewed-by: Dan Murphy Signed-off-by: Thomas Bogendoerfer Acked-by: Jacek Anaszewski Signed-off-by: Pavel Machek --- drivers/leds/Kconfig | 11 +++++ drivers/leds/Makefile | 1 + drivers/leds/leds-ip30.c | 86 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+) create mode 100644 drivers/leds/leds-ip30.c diff --git a/drivers/leds/Kconfig b/drivers/leds/Kconfig index d82f1dea3711..c664d84e1667 100644 --- a/drivers/leds/Kconfig +++ b/drivers/leds/Kconfig @@ -846,6 +846,17 @@ config LEDS_TPS6105X It is a single boost converter primarily for white LEDs and audio amplifiers. +config LEDS_IP30 + tristate "LED support for SGI Octane machines" + depends on LEDS_CLASS + depends on SGI_MFD_IOC3 + help + This option enables support for the Red and White LEDs of + SGI Octane machines. + + To compile this driver as a module, choose M here: the module + will be called leds-ip30. + comment "LED Triggers" source "drivers/leds/trigger/Kconfig" diff --git a/drivers/leds/Makefile b/drivers/leds/Makefile index d7e1107753fb..46bd611a03a9 100644 --- a/drivers/leds/Makefile +++ b/drivers/leds/Makefile @@ -86,6 +86,7 @@ obj-$(CONFIG_LEDS_TI_LMU_COMMON) += leds-ti-lmu-common.o obj-$(CONFIG_LEDS_LM3697) += leds-lm3697.o obj-$(CONFIG_LEDS_LM36274) += leds-lm36274.o obj-$(CONFIG_LEDS_TPS6105X) += leds-tps6105x.o +obj-$(CONFIG_LEDS_IP30) += leds-ip30.o # LED SPI Drivers obj-$(CONFIG_LEDS_CR0014114) += leds-cr0014114.o diff --git a/drivers/leds/leds-ip30.c b/drivers/leds/leds-ip30.c new file mode 100644 index 000000000000..e95ea786a43e --- /dev/null +++ b/drivers/leds/leds-ip30.c @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * LED Driver for SGI Octane machines + */ + +#include +#include +#include +#include + +#define IP30_LED_SYSTEM 0 +#define IP30_LED_FAULT 1 + +struct ip30_led { + struct led_classdev cdev; + u32 __iomem *reg; +}; + +static void ip30led_set(struct led_classdev *led_cdev, + enum led_brightness value) +{ + struct ip30_led *led = container_of(led_cdev, struct ip30_led, cdev); + + writel(value, led->reg); +} + +static int ip30led_create(struct platform_device *pdev, int num) +{ + struct resource *res; + struct ip30_led *data; + + res = platform_get_resource(pdev, IORESOURCE_MEM, num); + if (!res) + return -EBUSY; + + data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + data->reg = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(data->reg)) + return PTR_ERR(data->reg); + + + switch (num) { + case IP30_LED_SYSTEM: + data->cdev.name = "white:system"; + break; + case IP30_LED_FAULT: + data->cdev.name = "red:fault"; + break; + default: + return -EINVAL; + } + + data->cdev.brightness = readl(data->reg); + data->cdev.max_brightness = 1; + data->cdev.brightness_set = ip30led_set; + + return devm_led_classdev_register(&pdev->dev, &data->cdev); +} + +static int ip30led_probe(struct platform_device *pdev) +{ + int ret; + + ret = ip30led_create(pdev, IP30_LED_SYSTEM); + if (ret < 0) + return ret; + + return ip30led_create(pdev, IP30_LED_FAULT); +} + +static struct platform_driver ip30led_driver = { + .probe = ip30led_probe, + .driver = { + .name = "ip30-leds", + }, +}; + +module_platform_driver(ip30led_driver); + +MODULE_AUTHOR("Thomas Bogendoerfer "); +MODULE_DESCRIPTION("SGI Octane LED driver"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:ip30-leds"); From b43a8f01fccbfdddbc7f9b2bbad11b7db3fda4e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Fri, 24 Jan 2020 17:54:07 +0100 Subject: [PATCH 020/635] leds: pwm: simplify if condition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit .pwm_period_ns is an unsigned integer. So when led->pwm_period_ns > 0 is false, we now assign 0 to a value that is already 0, so it doesn't hurt and we can skip checking the actual value. Signed-off-by: Uwe Kleine-König Tested-by: Jeff LaBundy Signed-off-by: Pavel Machek --- drivers/leds/leds-pwm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/leds/leds-pwm.c b/drivers/leds/leds-pwm.c index 8b6965a563e9..b72fd89ff390 100644 --- a/drivers/leds/leds-pwm.c +++ b/drivers/leds/leds-pwm.c @@ -102,7 +102,7 @@ static int led_pwm_add(struct device *dev, struct led_pwm_priv *priv, pwm_get_args(led_data->pwm, &pargs); led_data->period = pargs.period; - if (!led_data->period && (led->pwm_period_ns > 0)) + if (!led_data->period) led_data->period = led->pwm_period_ns; ret = devm_led_classdev_register(dev, &led_data->cdev); From dd47a83453e4a5b0d6a91fe702b7fbc1984fb610 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Fri, 24 Jan 2020 17:54:08 +0100 Subject: [PATCH 021/635] leds: pwm: convert to atomic PWM API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pwm_config(), pwm_enable() and pwm_disable() should get removed in the long run. So update the driver to use the atomic API that is here to stay. A few side effects: - led_pwm_set() now returns an error when setting the PWM fails. - During .probe() the PWM isn't disabled implicitly by pwm_apply_args() any more. Signed-off-by: Uwe Kleine-König Tested-by: Jeff LaBundy Signed-off-by: Pavel Machek --- drivers/leds/leds-pwm.c | 41 +++++++++-------------------------------- 1 file changed, 9 insertions(+), 32 deletions(-) diff --git a/drivers/leds/leds-pwm.c b/drivers/leds/leds-pwm.c index b72fd89ff390..9111cdede0ee 100644 --- a/drivers/leds/leds-pwm.c +++ b/drivers/leds/leds-pwm.c @@ -22,9 +22,8 @@ struct led_pwm_data { struct led_classdev cdev; struct pwm_device *pwm; + struct pwm_state pwmstate; unsigned int active_low; - unsigned int period; - int duty; }; struct led_pwm_priv { @@ -32,44 +31,29 @@ struct led_pwm_priv { struct led_pwm_data leds[0]; }; -static void __led_pwm_set(struct led_pwm_data *led_dat) -{ - int new_duty = led_dat->duty; - - pwm_config(led_dat->pwm, new_duty, led_dat->period); - - if (new_duty == 0) - pwm_disable(led_dat->pwm); - else - pwm_enable(led_dat->pwm); -} - static int led_pwm_set(struct led_classdev *led_cdev, enum led_brightness brightness) { struct led_pwm_data *led_dat = container_of(led_cdev, struct led_pwm_data, cdev); unsigned int max = led_dat->cdev.max_brightness; - unsigned long long duty = led_dat->period; + unsigned long long duty = led_dat->pwmstate.period; duty *= brightness; do_div(duty, max); if (led_dat->active_low) - duty = led_dat->period - duty; + duty = led_dat->pwmstate.period - duty; - led_dat->duty = duty; - - __led_pwm_set(led_dat); - - return 0; + led_dat->pwmstate.duty_cycle = duty; + led_dat->pwmstate.enabled = duty > 0; + return pwm_apply_state(led_dat->pwm, &led_dat->pwmstate); } static int led_pwm_add(struct device *dev, struct led_pwm_priv *priv, struct led_pwm *led, struct fwnode_handle *fwnode) { struct led_pwm_data *led_data = &priv->leds[priv->num_leds]; - struct pwm_args pargs; int ret; led_data->active_low = led->active_low; @@ -93,17 +77,10 @@ static int led_pwm_add(struct device *dev, struct led_pwm_priv *priv, led_data->cdev.brightness_set_blocking = led_pwm_set; - /* - * FIXME: pwm_apply_args() should be removed when switching to the - * atomic PWM API. - */ - pwm_apply_args(led_data->pwm); + pwm_init_state(led_data->pwm, &led_data->pwmstate); - pwm_get_args(led_data->pwm, &pargs); - - led_data->period = pargs.period; - if (!led_data->period) - led_data->period = led->pwm_period_ns; + if (!led_data->pwmstate.period) + led_data->pwmstate.period = led->pwm_period_ns; ret = devm_led_classdev_register(dev, &led_data->cdev); if (ret == 0) { From e26060e1d47a535e0bc41a00708b79bc52dc9fb3 Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Wed, 12 Feb 2020 18:14:01 -0600 Subject: [PATCH 022/635] iommu/qcom: Replace zero-length array with flexible-array member The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertently introduced[3] to the codebase from now on. Also, notice that, dynamic memory allocations won't be affected by this change: "Flexible array members have incomplete type, and so the sizeof operator may not be applied. As a quirk of the original implementation of zero-length arrays, sizeof evaluates to zero."[1] This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Signed-off-by: Joerg Roedel --- drivers/iommu/qcom_iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/qcom_iommu.c b/drivers/iommu/qcom_iommu.c index 4328da0b0a9f..f6117726da99 100644 --- a/drivers/iommu/qcom_iommu.c +++ b/drivers/iommu/qcom_iommu.c @@ -48,7 +48,7 @@ struct qcom_iommu_dev { void __iomem *local_base; u32 sec_id; u8 num_ctxs; - struct qcom_iommu_ctx *ctxs[0]; /* indexed by asid-1 */ + struct qcom_iommu_ctx *ctxs[]; /* indexed by asid-1 */ }; struct qcom_iommu_ctx { From c20f36534666e37858a14e591114d93cc1be0d34 Mon Sep 17 00:00:00 2001 From: Adrian Huang Date: Fri, 14 Feb 2020 18:44:51 +0800 Subject: [PATCH 023/635] iommu/amd: Fix the configuration of GCR3 table root pointer The SPA of the GCR3 table root pointer[51:31] masks 20 bits. However, this requires 21 bits (Please see the AMD IOMMU specification). This leads to the potential failure when the bit 51 of SPA of the GCR3 table root pointer is 1'. Signed-off-by: Adrian Huang Fixes: 52815b75682e2 ("iommu/amd: Add support for IOMMUv2 domain mode") Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu_types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h index f8d01d6b00da..ca8c4522045b 100644 --- a/drivers/iommu/amd_iommu_types.h +++ b/drivers/iommu/amd_iommu_types.h @@ -348,7 +348,7 @@ #define DTE_GCR3_VAL_A(x) (((x) >> 12) & 0x00007ULL) #define DTE_GCR3_VAL_B(x) (((x) >> 15) & 0x0ffffULL) -#define DTE_GCR3_VAL_C(x) (((x) >> 31) & 0xfffffULL) +#define DTE_GCR3_VAL_C(x) (((x) >> 31) & 0x1fffffULL) #define DTE_GCR3_INDEX_A 0 #define DTE_GCR3_INDEX_B 1 From fa4afd78ea12cf31113f8b146b696c500d6a9dc3 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Fri, 14 Feb 2020 17:38:27 +0100 Subject: [PATCH 024/635] iommu/virtio: Build virtio-iommu as module Now that the infrastructure changes are in place, enable virtio-iommu to be built as a module. Remove the redundant pci_request_acs() call, since it's not exported but is already invoked during DMA setup. Signed-off-by: Jean-Philippe Brucker Signed-off-by: Joerg Roedel --- drivers/iommu/Kconfig | 4 ++-- drivers/iommu/virtio-iommu.c | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index d2fade984999..c5df570ef84a 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -506,8 +506,8 @@ config HYPERV_IOMMU guests to run with x2APIC mode enabled. config VIRTIO_IOMMU - bool "Virtio IOMMU driver" - depends on VIRTIO=y + tristate "Virtio IOMMU driver" + depends on VIRTIO depends on ARM64 select IOMMU_API select INTERVAL_TREE diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c index cce329d71fba..93ff58632452 100644 --- a/drivers/iommu/virtio-iommu.c +++ b/drivers/iommu/virtio-iommu.c @@ -1082,7 +1082,6 @@ static int viommu_probe(struct virtio_device *vdev) #ifdef CONFIG_PCI if (pci_bus_type.iommu_ops != &viommu_ops) { - pci_request_acs(); ret = bus_set_iommu(&pci_bus_type, &viommu_ops); if (ret) goto err_unregister; From 098accf2da940189f4d62d3514d17f8bb05dc6e1 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 13 Feb 2020 14:00:21 +0000 Subject: [PATCH 025/635] iommu: Use C99 flexible array in fwspec Although the 1-element array was a typical pre-C99 way to implement variable-length structures, and indeed is a fundamental construct in the APIs of certain other popular platforms, there's no good reason for it here (and in particular the sizeof() trick is far too "clever" for its own good). We can just as easily implement iommu_fwspec's preallocation behaviour using a standard flexible array member, so let's make it look the way most readers would expect. Signed-off-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 15 ++++++++------- include/linux/iommu.h | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 3e3528436e0b..660eea8d1d2f 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2405,7 +2405,8 @@ int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode, if (fwspec) return ops == fwspec->ops ? 0 : -EINVAL; - fwspec = kzalloc(sizeof(*fwspec), GFP_KERNEL); + /* Preallocate for the overwhelmingly common case of 1 ID */ + fwspec = kzalloc(struct_size(fwspec, ids, 1), GFP_KERNEL); if (!fwspec) return -ENOMEM; @@ -2432,15 +2433,15 @@ EXPORT_SYMBOL_GPL(iommu_fwspec_free); int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids) { struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); - size_t size; - int i; + int i, new_num; if (!fwspec) return -EINVAL; - size = offsetof(struct iommu_fwspec, ids[fwspec->num_ids + num_ids]); - if (size > sizeof(*fwspec)) { - fwspec = krealloc(fwspec, size, GFP_KERNEL); + new_num = fwspec->num_ids + num_ids; + if (new_num > 1) { + fwspec = krealloc(fwspec, struct_size(fwspec, ids, new_num), + GFP_KERNEL); if (!fwspec) return -ENOMEM; @@ -2450,7 +2451,7 @@ int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids) for (i = 0; i < num_ids; i++) fwspec->ids[fwspec->num_ids + i] = ids[i]; - fwspec->num_ids += num_ids; + fwspec->num_ids = new_num; return 0; } EXPORT_SYMBOL_GPL(iommu_fwspec_add_ids); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index d1b5f4d98569..4d1ba76c9a64 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -592,7 +592,7 @@ struct iommu_fwspec { u32 flags; u32 num_pasid_bits; unsigned int num_ids; - u32 ids[1]; + u32 ids[]; }; /* ATS is supported */ From 01091c496f920e634ea84b689f480c39016752a8 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Tue, 25 Feb 2020 19:20:06 +0300 Subject: [PATCH 026/635] acpi/nfit: improve bounds checking for 'func' The 'func' variable can come from the user in the __nd_ioctl(). If it's too high then the (1 << func) shift in acpi_nfit_clear_to_send() is undefined. In acpi_nfit_ctl() we pass 'func' to test_bit(func, &dsm_mask) which could result in an out of bounds access. To fix these issues, I introduced the NVDIMM_CMD_MAX (31) define and updated nfit_dsm_revid() to use that define as well instead of magic numbers. Fixes: 11189c1089da ("acpi/nfit: Fix command-supported detection") Signed-off-by: Dan Carpenter Reviewed-by: Dan Williams Link: https://lore.kernel.org/r/20200225161927.hvftuq7kjn547fyj@kili.mountain Signed-off-by: Dan Williams --- drivers/acpi/nfit/core.c | 10 ++++++---- drivers/acpi/nfit/nfit.h | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/acpi/nfit/core.c b/drivers/acpi/nfit/core.c index a3320f93616d..d0090f71585c 100644 --- a/drivers/acpi/nfit/core.c +++ b/drivers/acpi/nfit/core.c @@ -360,7 +360,7 @@ static union acpi_object *acpi_label_info(acpi_handle handle) static u8 nfit_dsm_revid(unsigned family, unsigned func) { - static const u8 revid_table[NVDIMM_FAMILY_MAX+1][32] = { + static const u8 revid_table[NVDIMM_FAMILY_MAX+1][NVDIMM_CMD_MAX+1] = { [NVDIMM_FAMILY_INTEL] = { [NVDIMM_INTEL_GET_MODES] = 2, [NVDIMM_INTEL_GET_FWINFO] = 2, @@ -386,7 +386,7 @@ static u8 nfit_dsm_revid(unsigned family, unsigned func) if (family > NVDIMM_FAMILY_MAX) return 0; - if (func > 31) + if (func > NVDIMM_CMD_MAX) return 0; id = revid_table[family][func]; if (id == 0) @@ -492,7 +492,8 @@ int acpi_nfit_ctl(struct nvdimm_bus_descriptor *nd_desc, struct nvdimm *nvdimm, * Check for a valid command. For ND_CMD_CALL, we also have to * make sure that the DSM function is supported. */ - if (cmd == ND_CMD_CALL && !test_bit(func, &dsm_mask)) + if (cmd == ND_CMD_CALL && + (func > NVDIMM_CMD_MAX || !test_bit(func, &dsm_mask))) return -ENOTTY; else if (!test_bit(cmd, &cmd_mask)) return -ENOTTY; @@ -3492,7 +3493,8 @@ static int acpi_nfit_clear_to_send(struct nvdimm_bus_descriptor *nd_desc, if (nvdimm && cmd == ND_CMD_CALL && call_pkg->nd_family == NVDIMM_FAMILY_INTEL) { func = call_pkg->nd_command; - if ((1 << func) & NVDIMM_INTEL_SECURITY_CMDMASK) + if (func > NVDIMM_CMD_MAX || + (1 << func) & NVDIMM_INTEL_SECURITY_CMDMASK) return -EOPNOTSUPP; } diff --git a/drivers/acpi/nfit/nfit.h b/drivers/acpi/nfit/nfit.h index 24241941181c..b317f4043705 100644 --- a/drivers/acpi/nfit/nfit.h +++ b/drivers/acpi/nfit/nfit.h @@ -34,6 +34,7 @@ | ACPI_NFIT_MEM_NOT_ARMED | ACPI_NFIT_MEM_MAP_FAILED) #define NVDIMM_FAMILY_MAX NVDIMM_FAMILY_HYPERV +#define NVDIMM_CMD_MAX 31 #define NVDIMM_STANDARD_CMDMASK \ (1 << ND_CMD_SMART | 1 << ND_CMD_SMART_THRESHOLD | 1 << ND_CMD_DIMM_FLAGS \ From f84afbdd3a9e5e10633695677b95422572f920dc Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Tue, 25 Feb 2020 19:20:56 +0300 Subject: [PATCH 027/635] libnvdimm: Out of bounds read in __nd_ioctl() The "cmd" comes from the user and it can be up to 255. It it's more than the number of bits in long, it results out of bounds read when we check test_bit(cmd, &cmd_mask). The highest valid value for "cmd" is ND_CMD_CALL (10) so I added a compare against that. Fixes: 62232e45f4a2 ("libnvdimm: control (ioctl) messages for nvdimm_bus and nvdimm devices") Signed-off-by: Dan Carpenter Link: https://lore.kernel.org/r/20200225162055.amtosfy7m35aivxg@kili.mountain Signed-off-by: Dan Williams --- drivers/nvdimm/bus.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/nvdimm/bus.c b/drivers/nvdimm/bus.c index a8b515968569..09087c38fabd 100644 --- a/drivers/nvdimm/bus.c +++ b/drivers/nvdimm/bus.c @@ -1042,8 +1042,10 @@ static int __nd_ioctl(struct nvdimm_bus *nvdimm_bus, struct nvdimm *nvdimm, return -EFAULT; } - if (!desc || (desc->out_num + desc->in_num == 0) || - !test_bit(cmd, &cmd_mask)) + if (!desc || + (desc->out_num + desc->in_num == 0) || + cmd > ND_CMD_CALL || + !test_bit(cmd, &cmd_mask)) return -ENOTTY; /* fail write commands (when read-only) */ From 2f8caa9f8f3254cd4527ec6a71428107623b3fa8 Mon Sep 17 00:00:00 2001 From: Rajeshwari Date: Mon, 6 Jan 2020 18:59:29 +0530 Subject: [PATCH 028/635] dt-bindings: thermal: tsens: Add configuration in yaml Added configuration in dt-bindings for SC7180. Signed-off-by: Rajeshwari Reviewed-by: Bjorn Andersson Acked-by: Rob Herring Reviewed-by: Amit Kucheria Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/1578317369-16045-3-git-send-email-rkambl@codeaurora.org --- Documentation/devicetree/bindings/thermal/qcom-tsens.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/thermal/qcom-tsens.yaml b/Documentation/devicetree/bindings/thermal/qcom-tsens.yaml index eef13b9446a8..c0ed030d0960 100644 --- a/Documentation/devicetree/bindings/thermal/qcom-tsens.yaml +++ b/Documentation/devicetree/bindings/thermal/qcom-tsens.yaml @@ -38,6 +38,7 @@ properties: - enum: - qcom,msm8996-tsens - qcom,msm8998-tsens + - qcom,sc7180-tsens - qcom,sdm845-tsens - const: qcom,tsens-v2 From f775ac78fcfc6bdc96bdda07029d11f2a5e84869 Mon Sep 17 00:00:00 2001 From: Pi-Hsun Shih Date: Fri, 14 Feb 2020 16:26:38 +0800 Subject: [PATCH 029/635] platform/chrome: cros_ec_rpmsg: Fix race with host event Host event can be sent by remoteproc by any time, and cros_ec_rpmsg_callback would be called after cros_ec_rpmsg_create_ept. But the cros_ec_device is initialized after that, which cause host event handler to use cros_ec_device that are not initialized properly yet. Fix this by don't schedule host event handler before cros_ec_register returns. Instead, remember that we have a pending host event, and schedule host event handler after cros_ec_register. Fixes: 71cddb7097e2 ("platform/chrome: cros_ec_rpmsg: Fix race with host command when probe failed.") Signed-off-by: Pi-Hsun Shih Signed-off-by: Enric Balletbo i Serra --- drivers/platform/chrome/cros_ec_rpmsg.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/drivers/platform/chrome/cros_ec_rpmsg.c b/drivers/platform/chrome/cros_ec_rpmsg.c index dbc3f5523b83..7e8629e3db74 100644 --- a/drivers/platform/chrome/cros_ec_rpmsg.c +++ b/drivers/platform/chrome/cros_ec_rpmsg.c @@ -44,6 +44,8 @@ struct cros_ec_rpmsg { struct completion xfer_ack; struct work_struct host_event_work; struct rpmsg_endpoint *ept; + bool has_pending_host_event; + bool probe_done; }; /** @@ -177,7 +179,14 @@ static int cros_ec_rpmsg_callback(struct rpmsg_device *rpdev, void *data, memcpy(ec_dev->din, resp->data, len); complete(&ec_rpmsg->xfer_ack); } else if (resp->type == HOST_EVENT_MARK) { - schedule_work(&ec_rpmsg->host_event_work); + /* + * If the host event is sent before cros_ec_register is + * finished, queue the host event. + */ + if (ec_rpmsg->probe_done) + schedule_work(&ec_rpmsg->host_event_work); + else + ec_rpmsg->has_pending_host_event = true; } else { dev_warn(ec_dev->dev, "rpmsg received invalid type = %d", resp->type); @@ -240,6 +249,11 @@ static int cros_ec_rpmsg_probe(struct rpmsg_device *rpdev) return ret; } + ec_rpmsg->probe_done = true; + + if (ec_rpmsg->has_pending_host_event) + schedule_work(&ec_rpmsg->host_event_work); + return 0; } From 7d3ca507fda9b60544bb86b9ef89cadc14755d70 Mon Sep 17 00:00:00 2001 From: Sergiu Cuciurean Date: Thu, 27 Feb 2020 16:09:17 +0200 Subject: [PATCH 030/635] platform/chrome: cros_ec_spi: Use new structure for SPI transfer delays In a recent change to the SPI subsystem [1], a new `delay` struct was added to replace the `delay_usecs`. This change replaces the current `delay_usecs` with `delay` for this driver. The `spi_transfer_delay_exec()` function [in the SPI framework] makes sure that both `delay_usecs` & `delay` are used (in this order to preserve backwards compatibility). [1] commit bebcfd272df6 ("spi: introduce `delay` field for `spi_transfer` + spi_transfer_delay_exec()") Signed-off-by: Sergiu Cuciurean Signed-off-by: Enric Balletbo i Serra --- drivers/platform/chrome/cros_ec_spi.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/platform/chrome/cros_ec_spi.c b/drivers/platform/chrome/cros_ec_spi.c index 46786d2d679a..665ab154bb4f 100644 --- a/drivers/platform/chrome/cros_ec_spi.c +++ b/drivers/platform/chrome/cros_ec_spi.c @@ -127,7 +127,8 @@ static int terminate_request(struct cros_ec_device *ec_dev) */ spi_message_init(&msg); memset(&trans, 0, sizeof(trans)); - trans.delay_usecs = ec_spi->end_of_msg_delay; + trans.delay.value = ec_spi->end_of_msg_delay; + trans.delay.unit = SPI_DELAY_UNIT_NSECS; spi_message_add_tail(&trans, &msg); ret = spi_sync_locked(ec_spi->spi, &msg); @@ -416,7 +417,8 @@ static int do_cros_ec_pkt_xfer_spi(struct cros_ec_device *ec_dev, spi_message_init(&msg); if (ec_spi->start_of_msg_delay) { memset(&trans_delay, 0, sizeof(trans_delay)); - trans_delay.delay_usecs = ec_spi->start_of_msg_delay; + trans_delay.delay.value = ec_spi->start_of_msg_delay; + trans_delay.delay.unit = SPI_DELAY_UNIT_USECS; spi_message_add_tail(&trans_delay, &msg); } From c5cd2b47b203f63682778c2a1783198e6b644294 Mon Sep 17 00:00:00 2001 From: Enric Balletbo i Serra Date: Thu, 20 Feb 2020 16:58:52 +0100 Subject: [PATCH 031/635] platform/chrome: cros_ec_proto: Report command not supported In practice most drivers that use the EC protocol what really care is if the result was successful or not, hence, we introduced a cros_ec_cmd_xfer_status() function that converts EC errors to standard Linux error codes. On some few cases, though, we are interested on know if the command is supported or not, and in such cases, just ignore the error. To achieve this, return a -ENOTSUPP error when the command is not supported. This will allow us to finish the conversion of all users to use the cros_ec_cmd_xfer_status() function instead of cros_ec_cmd_xfer() and make the latest private to the protocol driver, so users of the protocol are not confused in which function they should use. Signed-off-by: Enric Balletbo i Serra Tested-by: Prashant Malani --- drivers/platform/chrome/cros_ec_proto.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/platform/chrome/cros_ec_proto.c b/drivers/platform/chrome/cros_ec_proto.c index 3cfa643f1d07..3e745e0fe092 100644 --- a/drivers/platform/chrome/cros_ec_proto.c +++ b/drivers/platform/chrome/cros_ec_proto.c @@ -553,7 +553,10 @@ EXPORT_SYMBOL(cros_ec_cmd_xfer); * replied with success status. It's not necessary to check msg->result when * using this function. * - * Return: The number of bytes transferred on success or negative error code. + * Return: + * >=0 - The number of bytes transferred + * -ENOTSUPP - Operation not supported + * -EPROTO - Protocol error */ int cros_ec_cmd_xfer_status(struct cros_ec_device *ec_dev, struct cros_ec_command *msg) @@ -563,6 +566,10 @@ int cros_ec_cmd_xfer_status(struct cros_ec_device *ec_dev, ret = cros_ec_cmd_xfer(ec_dev, msg); if (ret < 0) { dev_err(ec_dev->dev, "Command xfer error (err:%d)\n", ret); + } else if (msg->result == EC_RES_INVALID_VERSION) { + dev_dbg(ec_dev->dev, "Command invalid version (err:%d)\n", + msg->result); + return -ENOTSUPP; } else if (msg->result != EC_RES_SUCCESS) { dev_dbg(ec_dev->dev, "Command result (err: %d)\n", msg->result); return -EPROTO; From 6bc15ad784ed9607ecf06e4fa3451383d2778d47 Mon Sep 17 00:00:00 2001 From: Enric Balletbo i Serra Date: Thu, 20 Feb 2020 16:58:54 +0100 Subject: [PATCH 032/635] platform/chrome: cros_ec_vbc: Use cros_ec_cmd_xfer_status helper This patch makes use of cros_ec_cmd_xfer_status() instead of cros_ec_cmd_xfer(). In this case the change is trivial and the only reason to do it is because we want to make cros_ec_cmd_xfer() a private function for the EC protocol and let people only use the cros_ec_cmd_xfer_status() to return Linux standard error codes. Signed-off-by: Enric Balletbo i Serra Tested-by: Prashant Malani --- drivers/platform/chrome/cros_ec_vbc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/platform/chrome/cros_ec_vbc.c b/drivers/platform/chrome/cros_ec_vbc.c index 8edae465105c..46482d12cffe 100644 --- a/drivers/platform/chrome/cros_ec_vbc.c +++ b/drivers/platform/chrome/cros_ec_vbc.c @@ -40,7 +40,7 @@ static ssize_t vboot_context_read(struct file *filp, struct kobject *kobj, msg->outsize = para_sz; msg->insize = resp_sz; - err = cros_ec_cmd_xfer(ecdev, msg); + err = cros_ec_cmd_xfer_status(ecdev, msg); if (err < 0) { dev_err(dev, "Error sending read request: %d\n", err); kfree(msg); @@ -83,7 +83,7 @@ static ssize_t vboot_context_write(struct file *filp, struct kobject *kobj, msg->outsize = para_sz; msg->insize = 0; - err = cros_ec_cmd_xfer(ecdev, msg); + err = cros_ec_cmd_xfer_status(ecdev, msg); if (err < 0) { dev_err(dev, "Error sending write request: %d\n", err); kfree(msg); From 413dda8f2c6f1ba90e93ba7b4214b1d8cf8ec171 Mon Sep 17 00:00:00 2001 From: Enric Balletbo i Serra Date: Thu, 20 Feb 2020 16:58:55 +0100 Subject: [PATCH 033/635] platform/chrome: cros_ec_chardev: Use cros_ec_cmd_xfer_status helper This patch makes use of cros_ec_cmd_xfer_status() instead of cros_ec_cmd_xfer(). In this case the change is trivial and the only reason to do it is because we want to make cros_ec_cmd_xfer() a private function for the EC protocol and let people only use the cros_ec_cmd_xfer_status() to return Linux standard error codes. Signed-off-by: Enric Balletbo i Serra Tested-by: Prashant Malani --- drivers/platform/chrome/cros_ec_chardev.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/platform/chrome/cros_ec_chardev.c b/drivers/platform/chrome/cros_ec_chardev.c index c65e70bc168d..b51ab24055f3 100644 --- a/drivers/platform/chrome/cros_ec_chardev.c +++ b/drivers/platform/chrome/cros_ec_chardev.c @@ -301,7 +301,7 @@ static long cros_ec_chardev_ioctl_xcmd(struct cros_ec_dev *ec, void __user *arg) } s_cmd->command += ec->cmd_offset; - ret = cros_ec_cmd_xfer(ec->ec_dev, s_cmd); + ret = cros_ec_cmd_xfer_status(ec->ec_dev, s_cmd); /* Only copy data to userland if data was received. */ if (ret < 0) goto exit; From 72540a57165392b8d4eea6c076886d3128849604 Mon Sep 17 00:00:00 2001 From: Enric Balletbo i Serra Date: Thu, 20 Feb 2020 16:58:56 +0100 Subject: [PATCH 034/635] platform/chrome: cros_ec_sysfs: Use cros_ec_cmd_xfer_status helper This patch makes use of cros_ec_cmd_xfer_status() instead of cros_ec_cmd_xfer(). In this case the change is trivial and the only reason to do it is because we want to make cros_ec_cmd_xfer() a private function for the EC protocol and let people only use the cros_ec_cmd_xfer_status() to return Linux standard error codes. Looking at the code I am even unsure that makes sense differentiate these two errors but let's not change the behaviour for now. Signed-off-by: Enric Balletbo i Serra Tested-by: Prashant Malani --- drivers/platform/chrome/cros_ec_sysfs.c | 36 ++++++++++++------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/drivers/platform/chrome/cros_ec_sysfs.c b/drivers/platform/chrome/cros_ec_sysfs.c index 07dac97ad57c..d45ea5d5bfa4 100644 --- a/drivers/platform/chrome/cros_ec_sysfs.c +++ b/drivers/platform/chrome/cros_ec_sysfs.c @@ -149,14 +149,14 @@ static ssize_t version_show(struct device *dev, /* Get build info. */ msg->command = EC_CMD_GET_BUILD_INFO + ec->cmd_offset; msg->insize = EC_HOST_PARAM_SIZE; - ret = cros_ec_cmd_xfer(ec->ec_dev, msg); - if (ret < 0) - count += scnprintf(buf + count, PAGE_SIZE - count, - "Build info: XFER ERROR %d\n", ret); - else if (msg->result != EC_RES_SUCCESS) + ret = cros_ec_cmd_xfer_status(ec->ec_dev, msg); + if (ret == -EPROTO) { count += scnprintf(buf + count, PAGE_SIZE - count, "Build info: EC error %d\n", msg->result); - else { + } else if (ret < 0) { + count += scnprintf(buf + count, PAGE_SIZE - count, + "Build info: XFER ERROR %d\n", ret); + } else { msg->data[EC_HOST_PARAM_SIZE - 1] = '\0'; count += scnprintf(buf + count, PAGE_SIZE - count, "Build info: %s\n", msg->data); @@ -165,14 +165,14 @@ static ssize_t version_show(struct device *dev, /* Get chip info. */ msg->command = EC_CMD_GET_CHIP_INFO + ec->cmd_offset; msg->insize = sizeof(*r_chip); - ret = cros_ec_cmd_xfer(ec->ec_dev, msg); - if (ret < 0) - count += scnprintf(buf + count, PAGE_SIZE - count, - "Chip info: XFER ERROR %d\n", ret); - else if (msg->result != EC_RES_SUCCESS) + ret = cros_ec_cmd_xfer_status(ec->ec_dev, msg); + if (ret == -EPROTO) { count += scnprintf(buf + count, PAGE_SIZE - count, "Chip info: EC error %d\n", msg->result); - else { + } else if (ret < 0) { + count += scnprintf(buf + count, PAGE_SIZE - count, + "Chip info: XFER ERROR %d\n", ret); + } else { r_chip = (struct ec_response_get_chip_info *)msg->data; r_chip->vendor[sizeof(r_chip->vendor) - 1] = '\0'; @@ -189,14 +189,14 @@ static ssize_t version_show(struct device *dev, /* Get board version */ msg->command = EC_CMD_GET_BOARD_VERSION + ec->cmd_offset; msg->insize = sizeof(*r_board); - ret = cros_ec_cmd_xfer(ec->ec_dev, msg); - if (ret < 0) - count += scnprintf(buf + count, PAGE_SIZE - count, - "Board version: XFER ERROR %d\n", ret); - else if (msg->result != EC_RES_SUCCESS) + ret = cros_ec_cmd_xfer_status(ec->ec_dev, msg); + if (ret == -EPROTO) { count += scnprintf(buf + count, PAGE_SIZE - count, "Board version: EC error %d\n", msg->result); - else { + } else if (ret < 0) { + count += scnprintf(buf + count, PAGE_SIZE - count, + "Board version: XFER ERROR %d\n", ret); + } else { r_board = (struct ec_response_board_version *)msg->data; count += scnprintf(buf + count, PAGE_SIZE - count, From cff38b0c58fc738ad559582c13878aa882da6a56 Mon Sep 17 00:00:00 2001 From: Enric Balletbo i Serra Date: Thu, 20 Feb 2020 16:58:57 +0100 Subject: [PATCH 035/635] platform/chrome: cros_ec_lightbar: Use cros_ec_cmd_xfer_status helper This patch makes use of cros_ec_cmd_xfer_status() instead of cros_ec_cmd_xfer(). It allows us to remove some redundand code. In this case, though, we are changing a bit the behaviour because of returning -EINVAL on protocol error we propagate the error return for cros_ec_cmd_xfer_status() function, but I think it will be fine, even more clear as we don't mask the Linux error code. Signed-off-by: Enric Balletbo i Serra Tested-by: Prashant Malani --- drivers/platform/chrome/cros_ec_lightbar.c | 50 ++++++---------------- 1 file changed, 13 insertions(+), 37 deletions(-) diff --git a/drivers/platform/chrome/cros_ec_lightbar.c b/drivers/platform/chrome/cros_ec_lightbar.c index b4c110c5fee0..b59180bff5a3 100644 --- a/drivers/platform/chrome/cros_ec_lightbar.c +++ b/drivers/platform/chrome/cros_ec_lightbar.c @@ -116,7 +116,7 @@ static int get_lightbar_version(struct cros_ec_dev *ec, param = (struct ec_params_lightbar *)msg->data; param->cmd = LIGHTBAR_CMD_VERSION; - ret = cros_ec_cmd_xfer(ec->ec_dev, msg); + ret = cros_ec_cmd_xfer_status(ec->ec_dev, msg); if (ret < 0) { ret = 0; goto exit; @@ -193,15 +193,10 @@ static ssize_t brightness_store(struct device *dev, if (ret) goto exit; - ret = cros_ec_cmd_xfer(ec->ec_dev, msg); + ret = cros_ec_cmd_xfer_status(ec->ec_dev, msg); if (ret < 0) goto exit; - if (msg->result != EC_RES_SUCCESS) { - ret = -EINVAL; - goto exit; - } - ret = count; exit: kfree(msg); @@ -258,13 +253,10 @@ static ssize_t led_rgb_store(struct device *dev, struct device_attribute *attr, goto exit; } - ret = cros_ec_cmd_xfer(ec->ec_dev, msg); + ret = cros_ec_cmd_xfer_status(ec->ec_dev, msg); if (ret < 0) goto exit; - if (msg->result != EC_RES_SUCCESS) - goto exit; - i = 0; ok = 1; } @@ -305,14 +297,13 @@ static ssize_t sequence_show(struct device *dev, if (ret) goto exit; - ret = cros_ec_cmd_xfer(ec->ec_dev, msg); - if (ret < 0) - goto exit; - - if (msg->result != EC_RES_SUCCESS) { + ret = cros_ec_cmd_xfer_status(ec->ec_dev, msg); + if (ret == -EPROTO) { ret = scnprintf(buf, PAGE_SIZE, "ERROR: EC returned %d\n", msg->result); goto exit; + } else if (ret < 0) { + goto exit; } resp = (struct ec_response_lightbar *)msg->data; @@ -344,13 +335,10 @@ static int lb_send_empty_cmd(struct cros_ec_dev *ec, uint8_t cmd) if (ret) goto error; - ret = cros_ec_cmd_xfer(ec->ec_dev, msg); + ret = cros_ec_cmd_xfer_status(ec->ec_dev, msg); if (ret < 0) goto error; - if (msg->result != EC_RES_SUCCESS) { - ret = -EINVAL; - goto error; - } + ret = 0; error: kfree(msg); @@ -377,13 +365,10 @@ static int lb_manual_suspend_ctrl(struct cros_ec_dev *ec, uint8_t enable) if (ret) goto error; - ret = cros_ec_cmd_xfer(ec->ec_dev, msg); + ret = cros_ec_cmd_xfer_status(ec->ec_dev, msg); if (ret < 0) goto error; - if (msg->result != EC_RES_SUCCESS) { - ret = -EINVAL; - goto error; - } + ret = 0; error: kfree(msg); @@ -425,15 +410,10 @@ static ssize_t sequence_store(struct device *dev, struct device_attribute *attr, if (ret) goto exit; - ret = cros_ec_cmd_xfer(ec->ec_dev, msg); + ret = cros_ec_cmd_xfer_status(ec->ec_dev, msg); if (ret < 0) goto exit; - if (msg->result != EC_RES_SUCCESS) { - ret = -EINVAL; - goto exit; - } - ret = count; exit: kfree(msg); @@ -487,13 +467,9 @@ static ssize_t program_store(struct device *dev, struct device_attribute *attr, */ msg->outsize = count + extra_bytes; - ret = cros_ec_cmd_xfer(ec->ec_dev, msg); + ret = cros_ec_cmd_xfer_status(ec->ec_dev, msg); if (ret < 0) goto exit; - if (msg->result != EC_RES_SUCCESS) { - ret = -EINVAL; - goto exit; - } ret = count; exit: From c06e58b6b7336d717b1ec9a5c7f825a94b67386e Mon Sep 17 00:00:00 2001 From: Enric Balletbo i Serra Date: Thu, 20 Feb 2020 16:58:58 +0100 Subject: [PATCH 036/635] platform/chrome: cros_ec: Use cros_ec_cmd_xfer_status helper This patch makes use of cros_ec_cmd_xfer_status() instead of cros_ec_cmd_xfer(). In this case the change is trivial and the only reason to do it is because we want to make cros_ec_cmd_xfer() a private function for the EC protocol and let people only use the cros_ec_cmd_xfer_status() to return Linux standard error codes. Signed-off-by: Enric Balletbo i Serra Tested-by: Prashant Malani --- drivers/platform/chrome/cros_ec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/platform/chrome/cros_ec.c b/drivers/platform/chrome/cros_ec.c index 7ee43b2e0654..3104680b7485 100644 --- a/drivers/platform/chrome/cros_ec.c +++ b/drivers/platform/chrome/cros_ec.c @@ -120,7 +120,7 @@ static int cros_ec_sleep_event(struct cros_ec_device *ec_dev, u8 sleep_event) buf.msg.command = EC_CMD_HOST_SLEEP_EVENT; - ret = cros_ec_cmd_xfer(ec_dev, &buf.msg); + ret = cros_ec_cmd_xfer_status(ec_dev, &buf.msg); /* For now, report failure to transition to S0ix with a warning. */ if (ret >= 0 && ec_dev->host_sleep_v1 && From fa1c76df5dcd15a839bda750248ddb1a9c438f77 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 26 Feb 2020 13:54:00 +0000 Subject: [PATCH 037/635] MAINTAINERS: Cover Arm SMMU DT bindings We'd like to be aware of proposed DT binding changes even when they don't directly touch the drivers themselves. Signed-off-by: Robin Murphy Signed-off-by: Will Deacon --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index fcd79fc38928..ac6f6e9e1c6f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1426,6 +1426,7 @@ M: Will Deacon R: Robin Murphy L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) S: Maintained +F: Documentation/devicetree/bindings/iommu/arm,smmu* F: drivers/iommu/arm-smmu* F: drivers/iommu/io-pgtable-arm.c F: drivers/iommu/io-pgtable-arm-v7s.c From f2ce16c3c15e90f8279ccda8deeb24682a4a3706 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Tue, 3 Mar 2020 21:27:48 +0100 Subject: [PATCH 038/635] iommu/omap: Fix pointer cast -Wpointer-to-int-cast warnings on 64 bit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pointers should be casted to unsigned long to avoid -Wpointer-to-int-cast warnings when compiling on 64-bit platform (e.g. with COMPILE_TEST): drivers/iommu/omap-iommu.c: In function ‘omap2_iommu_enable’: drivers/iommu/omap-iommu.c:170:25: warning: cast from pointer to integer of different size [-Wpointer-to-int-cast] if (!obj->iopgd || !IS_ALIGNED((u32)obj->iopgd, SZ_16K)) ^ Signed-off-by: Krzysztof Kozlowski Signed-off-by: Joerg Roedel --- drivers/iommu/omap-iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c index be551cc34be4..50e8acf88ec4 100644 --- a/drivers/iommu/omap-iommu.c +++ b/drivers/iommu/omap-iommu.c @@ -167,7 +167,7 @@ static int omap2_iommu_enable(struct omap_iommu *obj) { u32 l, pa; - if (!obj->iopgd || !IS_ALIGNED((u32)obj->iopgd, SZ_16K)) + if (!obj->iopgd || !IS_ALIGNED((unsigned long)obj->iopgd, SZ_16K)) return -EINVAL; pa = virt_to_phys(obj->iopgd); From 6135a891dc0bc43265cd583614419e04f3fd42b4 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Tue, 3 Mar 2020 21:27:49 +0100 Subject: [PATCH 039/635] iommu/omap: Fix printing format for size_t on 64-bit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Print size_t as %zu or %zx to fix -Wformat warnings when compiling on 64-bit platform (e.g. with COMPILE_TEST): drivers/iommu/omap-iommu.c: In function ‘flush_iotlb_page’: drivers/iommu/omap-iommu.c:437:47: warning: format ‘%x’ expects argument of type ‘unsigned int’, but argument 7 has type ‘size_t {aka long unsigned int}’ [-Wformat=] Acked-by: Suman Anna Signed-off-by: Krzysztof Kozlowski Signed-off-by: Joerg Roedel --- drivers/iommu/omap-iommu.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c index 50e8acf88ec4..887fefcb03b4 100644 --- a/drivers/iommu/omap-iommu.c +++ b/drivers/iommu/omap-iommu.c @@ -434,7 +434,7 @@ static void flush_iotlb_page(struct omap_iommu *obj, u32 da) bytes = iopgsz_to_bytes(cr.cam & 3); if ((start <= da) && (da < start + bytes)) { - dev_dbg(obj->dev, "%s: %08x<=%08x(%x)\n", + dev_dbg(obj->dev, "%s: %08x<=%08x(%zx)\n", __func__, start, da, bytes); iotlb_load_cr(obj, &cr); iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY); @@ -1352,11 +1352,11 @@ static int omap_iommu_map(struct iommu_domain *domain, unsigned long da, omap_pgsz = bytes_to_iopgsz(bytes); if (omap_pgsz < 0) { - dev_err(dev, "invalid size to map: %d\n", bytes); + dev_err(dev, "invalid size to map: %zu\n", bytes); return -EINVAL; } - dev_dbg(dev, "mapping da 0x%lx to pa %pa size 0x%x\n", da, &pa, bytes); + dev_dbg(dev, "mapping da 0x%lx to pa %pa size 0x%zx\n", da, &pa, bytes); iotlb_init_entry(&e, da, pa, omap_pgsz); @@ -1393,7 +1393,7 @@ static size_t omap_iommu_unmap(struct iommu_domain *domain, unsigned long da, size_t bytes = 0; int i; - dev_dbg(dev, "unmapping da 0x%lx size %u\n", da, size); + dev_dbg(dev, "unmapping da 0x%lx size %zu\n", da, size); iommu = omap_domain->iommus; for (i = 0; i < omap_domain->num_iommus; i++, iommu++) { From d84edddc447df2d87e6ce9fbf94d95225504ab32 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Tue, 3 Mar 2020 21:27:50 +0100 Subject: [PATCH 040/635] iommu/omap: Fix -Woverflow warnings when compiling on 64-bit architectures Although the OMAP IOMMU driver supports only ARMv7 (32-bit) platforms, it can be compile tested for other architectures, including 64-bit ones. In such case the warning appears: In file included from drivers/iommu/omap-iommu.c:33:0: drivers/iommu/omap-iommu.c: In function 'omap_iommu_iova_to_phys': >> drivers/iommu/omap-iopgtable.h:44:21: warning: large integer implicitly truncated to unsigned type [-Woverflow] #define IOPTE_MASK (~(IOPTE_SIZE - 1)) ^ >> drivers/iommu/omap-iommu.c:1641:41: note: in expansion of macro 'IOPTE_MASK' ret = omap_iommu_translate(*pte, da, IOPTE_MASK); ^~~~~~~~~~ Fix this by using architecture-depending types in omap_iommu_translate(): 1. Pointer should be cast to unsigned long, 2. Virtual addresses should be cast to dma_addr_t. On 32-bit this will be the same as original code (using u32). On 64-bit it should produce meaningful result, although it does not really matter. Reported-by: kbuild test robot Signed-off-by: Krzysztof Kozlowski Signed-off-by: Joerg Roedel --- drivers/iommu/omap-iopgtable.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/omap-iopgtable.h b/drivers/iommu/omap-iopgtable.h index 1a4adb59a859..51d74002cc30 100644 --- a/drivers/iommu/omap-iopgtable.h +++ b/drivers/iommu/omap-iopgtable.h @@ -63,7 +63,8 @@ * * va to pa translation */ -static inline phys_addr_t omap_iommu_translate(u32 d, u32 va, u32 mask) +static inline phys_addr_t omap_iommu_translate(unsigned long d, dma_addr_t va, + dma_addr_t mask) { return (d & mask) | (va & (~mask)); } From e93a1695d7fb551376b1c1220a267d032b6ad159 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Tue, 3 Mar 2020 21:27:51 +0100 Subject: [PATCH 041/635] iommu: Enable compile testing for some of drivers Some of the IOMMU drivers can be compile tested to increase build coverage. The OMAP, Rockchip and Exynos drivers use device.dev_archdata.iommu field which does not exist on all platforms. The sPAPR TCE and ARM SMMU have also restrictions where they can be built. Signed-off-by: Krzysztof Kozlowski Signed-off-by: Joerg Roedel --- drivers/iommu/Kconfig | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index d2fade984999..ff41c3d9e11f 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -273,7 +273,7 @@ config IRQ_REMAP # OMAP IOMMU support config OMAP_IOMMU bool "OMAP IOMMU Support" - depends on ARM && MMU + depends on ARM && MMU || (COMPILE_TEST && (ARM || ARM64 || IA64 || SPARC)) depends on ARCH_OMAP2PLUS || COMPILE_TEST select IOMMU_API ---help--- @@ -291,7 +291,7 @@ config OMAP_IOMMU_DEBUG config ROCKCHIP_IOMMU bool "Rockchip IOMMU Support" - depends on ARM || ARM64 + depends on ARM || ARM64 || (COMPILE_TEST && (ARM64 || IA64 || SPARC)) depends on ARCH_ROCKCHIP || COMPILE_TEST select IOMMU_API select ARM_DMA_USE_IOMMU @@ -325,7 +325,7 @@ config TEGRA_IOMMU_SMMU config EXYNOS_IOMMU bool "Exynos IOMMU Support" - depends on ARCH_EXYNOS && MMU + depends on ARCH_EXYNOS && MMU || (COMPILE_TEST && (ARM || ARM64 || IA64 || SPARC)) depends on !CPU_BIG_ENDIAN # revisit driver if we can enable big-endian ptes select IOMMU_API select ARM_DMA_USE_IOMMU @@ -361,7 +361,7 @@ config IPMMU_VMSA config SPAPR_TCE_IOMMU bool "sPAPR TCE IOMMU Support" - depends on PPC_POWERNV || PPC_PSERIES + depends on PPC_POWERNV || PPC_PSERIES || (PPC && COMPILE_TEST) select IOMMU_API help Enables bits of IOMMU API required by VFIO. The iommu_ops @@ -370,7 +370,7 @@ config SPAPR_TCE_IOMMU # ARM IOMMU support config ARM_SMMU tristate "ARM Ltd. System MMU (SMMU) Support" - depends on (ARM64 || ARM) && MMU + depends on (ARM64 || ARM || (COMPILE_TEST && !GENERIC_ATOMIC64)) && MMU select IOMMU_API select IOMMU_IO_PGTABLE_LPAE select ARM_DMA_USE_IOMMU if ARM @@ -440,7 +440,7 @@ config S390_IOMMU config S390_CCW_IOMMU bool "S390 CCW IOMMU Support" - depends on S390 && CCW + depends on S390 && CCW || COMPILE_TEST select IOMMU_API help Enables bits of IOMMU API required by VFIO. The iommu_ops @@ -448,7 +448,7 @@ config S390_CCW_IOMMU config S390_AP_IOMMU bool "S390 AP IOMMU Support" - depends on S390 && ZCRYPT + depends on S390 && ZCRYPT || COMPILE_TEST select IOMMU_API help Enables bits of IOMMU API required by VFIO. The iommu_ops @@ -456,7 +456,7 @@ config S390_AP_IOMMU config MTK_IOMMU bool "MTK IOMMU Support" - depends on ARM || ARM64 + depends on ARM || ARM64 || COMPILE_TEST depends on ARCH_MEDIATEK || COMPILE_TEST select ARM_DMA_USE_IOMMU select IOMMU_API From 7e11f4c0c5ccaac5e8ee1b2e46b3c2cbc47d199e Mon Sep 17 00:00:00 2001 From: Enric Balletbo i Serra Date: Thu, 5 Mar 2020 11:28:38 +0100 Subject: [PATCH 042/635] platform/chrome: Kconfig: Remove CONFIG_ prefix from MFD_CROS_EC section Remove the CONFIG_ prefix from the select statement for MFD_CROS_EC. Fixes: 2fa2b980e3fe1 ("mfd / platform: cros_ec: Rename config to a better name") Reported-by: Randy Dunlap Signed-off-by: Enric Balletbo i Serra Acked-by: Andy Shevchenko Acked-by: Randy Dunlap --- drivers/platform/chrome/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/platform/chrome/Kconfig b/drivers/platform/chrome/Kconfig index 15fc8b8a2db8..5ae6c49f553d 100644 --- a/drivers/platform/chrome/Kconfig +++ b/drivers/platform/chrome/Kconfig @@ -7,7 +7,7 @@ config MFD_CROS_EC tristate "Platform support for Chrome hardware (transitional)" select CHROME_PLATFORMS select CROS_EC - select CONFIG_MFD_CROS_EC_DEV + select MFD_CROS_EC_DEV depends on X86 || ARM || ARM64 || COMPILE_TEST help This is a transitional Kconfig option and will be removed after From 1dd5ddc125b4625c3beb8e644ae872445d739bbc Mon Sep 17 00:00:00 2001 From: "Sergei A. Trusov" Date: Thu, 5 Mar 2020 19:53:06 -0800 Subject: [PATCH 043/635] Input: goodix - fix touch coordinates on Cube I15-TC The touchscreen on the Cube I15-TC don't match the default display, with 0,0 touches being reported when touching at the top-right of the screen. Add a quirk to invert the x coordinate. Reported-and-tested-by: Arkadiy Signed-off-by: Sergei A. Trusov Reviewed-by: Bastien Nocera Signed-off-by: Dmitry Torokhov --- drivers/input/touchscreen/goodix.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c index 0403102e807e..37b35ab97beb 100644 --- a/drivers/input/touchscreen/goodix.c +++ b/drivers/input/touchscreen/goodix.c @@ -168,6 +168,22 @@ static const struct dmi_system_id nine_bytes_report[] = { {} }; +/* + * Those tablets have their x coordinate inverted + */ +static const struct dmi_system_id inverted_x_screen[] = { +#if defined(CONFIG_DMI) && defined(CONFIG_X86) + { + .ident = "Cube I15-TC", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Cube"), + DMI_MATCH(DMI_PRODUCT_NAME, "I15-TC") + }, + }, +#endif + {} +}; + /** * goodix_i2c_read - read data from a register of the i2c slave device. * @@ -780,6 +796,12 @@ static int goodix_configure_dev(struct goodix_ts_data *ts) "Non-standard 9-bytes report format quirk\n"); } + if (dmi_check_system(inverted_x_screen)) { + ts->prop.invert_x = true; + dev_dbg(&ts->client->dev, + "Applying 'inverted x screen' quirk\n"); + } + error = input_mt_init_slots(ts->input_dev, ts->max_touch_num, INPUT_MT_DIRECT | INPUT_MT_DROP_UNUSED); if (error) { From 061706716384f1633d3d5090b22a99f33f1fcf2f Mon Sep 17 00:00:00 2001 From: Johnny Chuang Date: Fri, 6 Mar 2020 10:05:20 -0800 Subject: [PATCH 044/635] Input: elants_i2c - report resolution information for touch major This patch supports reporting resolution for ABS_MT_TOUCH_MAJOR event. This information is needed in showing pressure/width radius. Signed-off-by: Johnny Chuang Reviewed-by: Harry Cutts Reviewed-by: Peter Hutterer Acked-by: Benjamin Tissoires Link: https://lore.kernel.org/r/1582766000-23023-1-git-send-email-johnny.chuang.emc@gmail.com Signed-off-by: Dmitry Torokhov --- drivers/input/touchscreen/elants_i2c.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/input/touchscreen/elants_i2c.c b/drivers/input/touchscreen/elants_i2c.c index 491179967b29..14c577c16b16 100644 --- a/drivers/input/touchscreen/elants_i2c.c +++ b/drivers/input/touchscreen/elants_i2c.c @@ -1309,6 +1309,7 @@ static int elants_i2c_probe(struct i2c_client *client, input_set_abs_params(ts->input, ABS_MT_PRESSURE, 0, 255, 0, 0); input_abs_set_res(ts->input, ABS_MT_POSITION_X, ts->x_res); input_abs_set_res(ts->input, ABS_MT_POSITION_Y, ts->y_res); + input_abs_set_res(ts->input, ABS_MT_TOUCH_MAJOR, 1); error = input_register_device(ts->input); if (error) { From d543c8424c116549c4322ff0341e388452aca0ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20S=C3=B6derlund?= Date: Wed, 12 Feb 2020 23:47:32 +0100 Subject: [PATCH 045/635] thermal: rcar_gen3_thermal: Remove unneeded curly brackets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When devm_add_action() was turned into devm_add_action_or_reset() the curly brackets for the error case where kept but are not needed, remove them to match the style of the driver. Signed-off-by: Niklas Söderlund Reviewed-by: Yoshihiro Shimoda Reviewed-by: Geert Uytterhoeven Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200212224732.736785-1-niklas.soderlund+renesas@ragnatech.se --- drivers/thermal/rcar_gen3_thermal.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/thermal/rcar_gen3_thermal.c b/drivers/thermal/rcar_gen3_thermal.c index 72877bdc072d..e877c90c5c8b 100644 --- a/drivers/thermal/rcar_gen3_thermal.c +++ b/drivers/thermal/rcar_gen3_thermal.c @@ -446,9 +446,8 @@ static int rcar_gen3_thermal_probe(struct platform_device *pdev) goto error_unregister; ret = devm_add_action_or_reset(dev, rcar_gen3_hwmon_action, zone); - if (ret) { + if (ret) goto error_unregister; - } ret = of_thermal_get_ntrips(tsc->zone); if (ret < 0) From 267c4d8d132c8646fc90a58d40e3117656a4e9b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20S=C3=B6derlund?= Date: Wed, 12 Feb 2020 23:49:17 +0100 Subject: [PATCH 046/635] thermal: rcar_gen3_thermal: Generate interrupt when temperature changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The desired behavior of the driver is to generate an interrupt and call thermal_zone_device_update() as soon as the temperature have changed more then one degree. When the set_trips operation was implemented it was believed that the trip window set by the framework would move around the current temperature and the hysteresis value described in devicetree. The behavior of the framework is however to set a window based on the trip points described in devicetree. Remove the set_trips operation which was not used correctly and update the temperatures that triggers interrupts directly from the interrupt handler. Signed-off-by: Niklas Söderlund Reviewed-by: Yoshihiro Shimoda Reviewed-by: Kieran Bingham Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200212224917.737314-1-niklas.soderlund+renesas@ragnatech.se --- drivers/thermal/rcar_gen3_thermal.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/drivers/thermal/rcar_gen3_thermal.c b/drivers/thermal/rcar_gen3_thermal.c index e877c90c5c8b..a2bf9d8074cc 100644 --- a/drivers/thermal/rcar_gen3_thermal.c +++ b/drivers/thermal/rcar_gen3_thermal.c @@ -81,8 +81,6 @@ struct rcar_gen3_thermal_tsc { void __iomem *base; struct thermal_zone_device *zone; struct equation_coefs coef; - int low; - int high; int tj_t; int id; /* thermal channel id */ }; @@ -204,12 +202,14 @@ static int rcar_gen3_thermal_mcelsius_to_temp(struct rcar_gen3_thermal_tsc *tsc, return INT_FIXPT(val); } -static int rcar_gen3_thermal_set_trips(void *devdata, int low, int high) +static int rcar_gen3_thermal_update_range(struct rcar_gen3_thermal_tsc *tsc) { - struct rcar_gen3_thermal_tsc *tsc = devdata; + int temperature, low, high; - low = clamp_val(low, -40000, 120000); - high = clamp_val(high, -40000, 120000); + rcar_gen3_thermal_get_temp(tsc, &temperature); + + low = temperature - MCELSIUS(1); + high = temperature + MCELSIUS(1); rcar_gen3_thermal_write(tsc, REG_GEN3_IRQTEMP1, rcar_gen3_thermal_mcelsius_to_temp(tsc, low)); @@ -217,15 +217,11 @@ static int rcar_gen3_thermal_set_trips(void *devdata, int low, int high) rcar_gen3_thermal_write(tsc, REG_GEN3_IRQTEMP2, rcar_gen3_thermal_mcelsius_to_temp(tsc, high)); - tsc->low = low; - tsc->high = high; - return 0; } static const struct thermal_zone_of_device_ops rcar_gen3_tz_of_ops = { .get_temp = rcar_gen3_thermal_get_temp, - .set_trips = rcar_gen3_thermal_set_trips, }; static void rcar_thermal_irq_set(struct rcar_gen3_thermal_priv *priv, bool on) @@ -246,9 +242,11 @@ static irqreturn_t rcar_gen3_thermal_irq(int irq, void *data) for (i = 0; i < priv->num_tscs; i++) { status = rcar_gen3_thermal_read(priv->tscs[i], REG_GEN3_IRQSTR); rcar_gen3_thermal_write(priv->tscs[i], REG_GEN3_IRQSTR, 0); - if (status) + if (status) { + rcar_gen3_thermal_update_range(priv->tscs[i]); thermal_zone_device_update(priv->tscs[i]->zone, THERMAL_EVENT_UNSPECIFIED); + } } return IRQ_HANDLED; @@ -453,6 +451,8 @@ static int rcar_gen3_thermal_probe(struct platform_device *pdev) if (ret < 0) goto error_unregister; + rcar_gen3_thermal_update_range(tsc); + dev_info(dev, "TSC%d: Loaded %d trip points\n", i, ret); } @@ -491,7 +491,7 @@ static int __maybe_unused rcar_gen3_thermal_resume(struct device *dev) struct rcar_gen3_thermal_tsc *tsc = priv->tscs[i]; priv->thermal_init(tsc); - rcar_gen3_thermal_set_trips(tsc, tsc->low, tsc->high); + rcar_gen3_thermal_update_range(tsc); } rcar_thermal_irq_set(priv, true); From f2ccf917ad33d464bf7a455c64543fcd3b5512ea Mon Sep 17 00:00:00 2001 From: Baolin Wang Date: Tue, 18 Feb 2020 16:10:27 +0800 Subject: [PATCH 047/635] dt-bindings: thermal: sprd: Add the Spreadtrum thermal documentation Add the Spreadtrum thermal documentation. Signed-off-by: Baolin Wang Signed-off-by: Baolin Wang Reviewed-by: Rob Herring Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/444e45ce0a9b390b2502dfcefd1ddb36948fa8e1.1582013101.git.baolin.wang7@gmail.com --- .../bindings/thermal/sprd-thermal.yaml | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 Documentation/devicetree/bindings/thermal/sprd-thermal.yaml diff --git a/Documentation/devicetree/bindings/thermal/sprd-thermal.yaml b/Documentation/devicetree/bindings/thermal/sprd-thermal.yaml new file mode 100644 index 000000000000..3bfe05504456 --- /dev/null +++ b/Documentation/devicetree/bindings/thermal/sprd-thermal.yaml @@ -0,0 +1,108 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/thermal/sprd-thermal.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Spreadtrum thermal sensor controller bindings + +maintainers: + - Orson Zhai + - Baolin Wang + - Chunyan Zhang + +properties: + compatible: + const: sprd,ums512-thermal + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + + clock-names: + items: + - const: enable + + nvmem-cells: + maxItems: 2 + description: + Reference to nvmem nodes for the calibration data. + + nvmem-cell-names: + maxItems: 2 + items: + - const: thm_sign_cal + - const: thm_ratio_cal + + "#thermal-sensor-cells": + const: 1 + + "#address-cells": + const: 1 + + "#size-cells": + const: 0 + +patternProperties: + "^([a-z]*-)?sensor(-section)?@[0-9]+$": + type: object + description: + Represent one thermal sensor. + + properties: + reg: + description: Specify the sensor id. + maxItems: 1 + + nvmem-cells: + maxItems: 1 + description: + Reference to an nvmem node for the calibration data. + + nvmem-cell-names: + const: sen_delta_cal + + required: + - reg + - nvmem-cells + - nvmem-cell-names + +required: + - compatible + - reg + - clocks + - clock-names + - nvmem-cells + - nvmem-cell-names + - "#thermal-sensor-cells" + - "#address-cells" + - "#size-cells" + +examples: + - | + ap_thm0: thermal@32200000 { + compatible = "sprd,ums512-thermal"; + reg = <0 0x32200000 0 0x10000>; + clock-names = "enable"; + clocks = <&aonapb_gate 32>; + #thermal-sensor-cells = <1>; + nvmem-cells = <&thm0_sign>, <&thm0_ratio>; + nvmem-cell-names = "thm_sign_cal", "thm_ratio_cal"; + #address-cells = <1>; + #size-cells = <0>; + + prometheus-sensor@0 { + reg = <0>; + nvmem-cells = <&thm0_sen0>; + nvmem-cell-names = "sen_delta_cal"; + }; + + ank-sensor@1 { + reg = <1>; + nvmem-cells = <&thm0_sen1>; + nvmem-cell-names = "sen_delta_cal"; + }; + }; +... From 554fdbaf19b188224d52d2fa80c049e4d42002e8 Mon Sep 17 00:00:00 2001 From: Freeman Liu Date: Tue, 18 Feb 2020 16:10:28 +0800 Subject: [PATCH 048/635] thermal: sprd: Add Spreadtrum thermal driver support This patch adds the support for Spreadtrum thermal sensor controller, which can support maximum 8 sensors. Signed-off-by: Freeman Liu Co-developed-with: Baolin Wang Signed-off-by: Baolin Wang Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/ebeb2839cff4d4027b37e787427c5af0e11880c8.1582013101.git.baolin.wang7@gmail.com --- drivers/thermal/Kconfig | 7 + drivers/thermal/Makefile | 1 + drivers/thermal/sprd_thermal.c | 552 +++++++++++++++++++++++++++++++++ 3 files changed, 560 insertions(+) create mode 100644 drivers/thermal/sprd_thermal.c diff --git a/drivers/thermal/Kconfig b/drivers/thermal/Kconfig index 5a05db5438d6..a2a940e8a8e3 100644 --- a/drivers/thermal/Kconfig +++ b/drivers/thermal/Kconfig @@ -460,4 +460,11 @@ config UNIPHIER_THERMAL Enable this to plug in UniPhier on-chip PVT thermal driver into the thermal framework. The driver supports CPU thermal zone temperature reporting and a couple of trip points. + +config SPRD_THERMAL + tristate "Temperature sensor on Spreadtrum SoCs" + depends on ARCH_SPRD || COMPILE_TEST + help + Support for the Spreadtrum thermal sensor driver in the Linux thermal + framework. endif diff --git a/drivers/thermal/Makefile b/drivers/thermal/Makefile index 9fb88e26fb10..dbdd98c117f5 100644 --- a/drivers/thermal/Makefile +++ b/drivers/thermal/Makefile @@ -57,3 +57,4 @@ obj-$(CONFIG_GENERIC_ADC_THERMAL) += thermal-generic-adc.o obj-$(CONFIG_ZX2967_THERMAL) += zx2967_thermal.o obj-$(CONFIG_UNIPHIER_THERMAL) += uniphier_thermal.o obj-$(CONFIG_AMLOGIC_THERMAL) += amlogic_thermal.o +obj-$(CONFIG_SPRD_THERMAL) += sprd_thermal.o diff --git a/drivers/thermal/sprd_thermal.c b/drivers/thermal/sprd_thermal.c new file mode 100644 index 000000000000..a340374e8c51 --- /dev/null +++ b/drivers/thermal/sprd_thermal.c @@ -0,0 +1,552 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2020 Spreadtrum Communications Inc. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SPRD_THM_CTL 0x0 +#define SPRD_THM_INT_EN 0x4 +#define SPRD_THM_INT_STS 0x8 +#define SPRD_THM_INT_RAW_STS 0xc +#define SPRD_THM_DET_PERIOD 0x10 +#define SPRD_THM_INT_CLR 0x14 +#define SPRD_THM_INT_CLR_ST 0x18 +#define SPRD_THM_MON_PERIOD 0x4c +#define SPRD_THM_MON_CTL 0x50 +#define SPRD_THM_INTERNAL_STS1 0x54 +#define SPRD_THM_RAW_READ_MSK 0x3ff + +#define SPRD_THM_OFFSET(id) ((id) * 0x4) +#define SPRD_THM_TEMP(id) (SPRD_THM_OFFSET(id) + 0x5c) +#define SPRD_THM_THRES(id) (SPRD_THM_OFFSET(id) + 0x2c) + +#define SPRD_THM_SEN(id) BIT((id) + 2) +#define SPRD_THM_SEN_OVERHEAT_EN(id) BIT((id) + 8) +#define SPRD_THM_SEN_OVERHEAT_ALARM_EN(id) BIT((id) + 0) + +/* bits definitions for register THM_CTL */ +#define SPRD_THM_SET_RDY_ST BIT(13) +#define SPRD_THM_SET_RDY BIT(12) +#define SPRD_THM_MON_EN BIT(1) +#define SPRD_THM_EN BIT(0) + +/* bits definitions for register THM_INT_CTL */ +#define SPRD_THM_BIT_INT_EN BIT(26) +#define SPRD_THM_OVERHEAT_EN BIT(25) +#define SPRD_THM_OTP_TRIP_SHIFT 10 + +/* bits definitions for register SPRD_THM_INTERNAL_STS1 */ +#define SPRD_THM_TEMPER_RDY BIT(0) + +#define SPRD_THM_DET_PERIOD_DATA 0x800 +#define SPRD_THM_DET_PERIOD_MASK GENMASK(19, 0) +#define SPRD_THM_MON_MODE 0x7 +#define SPRD_THM_MON_MODE_MASK GENMASK(3, 0) +#define SPRD_THM_MON_PERIOD_DATA 0x10 +#define SPRD_THM_MON_PERIOD_MASK GENMASK(15, 0) +#define SPRD_THM_THRES_MASK GENMASK(19, 0) +#define SPRD_THM_INT_CLR_MASK GENMASK(24, 0) + +/* thermal sensor calibration parameters */ +#define SPRD_THM_TEMP_LOW -40000 +#define SPRD_THM_TEMP_HIGH 120000 +#define SPRD_THM_OTP_TEMP 120000 +#define SPRD_THM_HOT_TEMP 75000 +#define SPRD_THM_RAW_DATA_LOW 0 +#define SPRD_THM_RAW_DATA_HIGH 1000 +#define SPRD_THM_SEN_NUM 8 +#define SPRD_THM_DT_OFFSET 24 +#define SPRD_THM_RATION_OFFSET 17 +#define SPRD_THM_RATION_SIGN 16 + +#define SPRD_THM_RDYST_POLLING_TIME 10 +#define SPRD_THM_RDYST_TIMEOUT 700 +#define SPRD_THM_TEMP_READY_POLL_TIME 10000 +#define SPRD_THM_TEMP_READY_TIMEOUT 600000 +#define SPRD_THM_MAX_SENSOR 8 + +struct sprd_thermal_sensor { + struct thermal_zone_device *tzd; + struct sprd_thermal_data *data; + struct device *dev; + int cal_slope; + int cal_offset; + int id; +}; + +struct sprd_thermal_data { + const struct sprd_thm_variant_data *var_data; + struct sprd_thermal_sensor *sensor[SPRD_THM_MAX_SENSOR]; + struct clk *clk; + void __iomem *base; + u32 ratio_off; + int ratio_sign; + int nr_sensors; +}; + +/* + * The conversion between ADC and temperature is based on linear relationship, + * and use idea_k to specify the slope and ideal_b to specify the offset. + * + * Since different Spreadtrum SoCs have different ideal_k and ideal_b, + * we should save ideal_k and ideal_b in the device data structure. + */ +struct sprd_thm_variant_data { + u32 ideal_k; + u32 ideal_b; +}; + +static const struct sprd_thm_variant_data ums512_data = { + .ideal_k = 262, + .ideal_b = 66400, +}; + +static inline void sprd_thm_update_bits(void __iomem *reg, u32 mask, u32 val) +{ + u32 tmp, orig; + + orig = readl(reg); + tmp = orig & ~mask; + tmp |= val & mask; + writel(tmp, reg); +} + +static int sprd_thm_cal_read(struct device_node *np, const char *cell_id, + u32 *val) +{ + struct nvmem_cell *cell; + void *buf; + size_t len; + + cell = of_nvmem_cell_get(np, cell_id); + if (IS_ERR(cell)) + return PTR_ERR(cell); + + buf = nvmem_cell_read(cell, &len); + nvmem_cell_put(cell); + if (IS_ERR(buf)) + return PTR_ERR(buf); + + if (len > sizeof(u32)) { + kfree(buf); + return -EINVAL; + } + + memcpy(val, buf, len); + + kfree(buf); + return 0; +} + +static int sprd_thm_sensor_calibration(struct device_node *np, + struct sprd_thermal_data *thm, + struct sprd_thermal_sensor *sen) +{ + int ret; + /* + * According to thermal datasheet, the default calibration offset is 64, + * and the default ratio is 1000. + */ + int dt_offset = 64, ratio = 1000; + + ret = sprd_thm_cal_read(np, "sen_delta_cal", &dt_offset); + if (ret) + return ret; + + ratio += thm->ratio_sign * thm->ratio_off; + + /* + * According to the ideal slope K and ideal offset B, combined with + * calibration value of thermal from efuse, then calibrate the real + * slope k and offset b: + * k_cal = (k * ratio) / 1000. + * b_cal = b + (dt_offset - 64) * 500. + */ + sen->cal_slope = (thm->var_data->ideal_k * ratio) / 1000; + sen->cal_offset = thm->var_data->ideal_b + (dt_offset - 128) * 250; + + return 0; +} + +static int sprd_thm_rawdata_to_temp(struct sprd_thermal_sensor *sen, + u32 rawdata) +{ + clamp(rawdata, (u32)SPRD_THM_RAW_DATA_LOW, (u32)SPRD_THM_RAW_DATA_HIGH); + + /* + * According to the thermal datasheet, the formula of converting + * adc value to the temperature value should be: + * T_final = k_cal * x - b_cal. + */ + return sen->cal_slope * rawdata - sen->cal_offset; +} + +static int sprd_thm_temp_to_rawdata(int temp, struct sprd_thermal_sensor *sen) +{ + u32 val; + + clamp(temp, (int)SPRD_THM_TEMP_LOW, (int)SPRD_THM_TEMP_HIGH); + + /* + * According to the thermal datasheet, the formula of converting + * adc value to the temperature value should be: + * T_final = k_cal * x - b_cal. + */ + val = (temp + sen->cal_offset) / sen->cal_slope; + + return clamp(val, val, (u32)(SPRD_THM_RAW_DATA_HIGH - 1)); +} + +static int sprd_thm_read_temp(void *devdata, int *temp) +{ + struct sprd_thermal_sensor *sen = devdata; + u32 data; + + data = readl(sen->data->base + SPRD_THM_TEMP(sen->id)) & + SPRD_THM_RAW_READ_MSK; + + *temp = sprd_thm_rawdata_to_temp(sen, data); + + return 0; +} + +static const struct thermal_zone_of_device_ops sprd_thm_ops = { + .get_temp = sprd_thm_read_temp, +}; + +static int sprd_thm_poll_ready_status(struct sprd_thermal_data *thm) +{ + u32 val; + int ret; + + /* + * Wait for thermal ready status before configuring thermal parameters. + */ + ret = readl_poll_timeout(thm->base + SPRD_THM_CTL, val, + !(val & SPRD_THM_SET_RDY_ST), + SPRD_THM_RDYST_POLLING_TIME, + SPRD_THM_RDYST_TIMEOUT); + if (ret) + return ret; + + sprd_thm_update_bits(thm->base + SPRD_THM_CTL, SPRD_THM_MON_EN, + SPRD_THM_MON_EN); + sprd_thm_update_bits(thm->base + SPRD_THM_CTL, SPRD_THM_SET_RDY, + SPRD_THM_SET_RDY); + return 0; +} + +static int sprd_thm_wait_temp_ready(struct sprd_thermal_data *thm) +{ + u32 val; + + /* Wait for first temperature data ready before reading temperature */ + return readl_poll_timeout(thm->base + SPRD_THM_INTERNAL_STS1, val, + !(val & SPRD_THM_TEMPER_RDY), + SPRD_THM_TEMP_READY_POLL_TIME, + SPRD_THM_TEMP_READY_TIMEOUT); +} + +static int sprd_thm_set_ready(struct sprd_thermal_data *thm) +{ + int ret; + + ret = sprd_thm_poll_ready_status(thm); + if (ret) + return ret; + + /* + * Clear interrupt status, enable thermal interrupt and enable thermal. + * + * The SPRD thermal controller integrates a hardware interrupt signal, + * which means if the temperature is overheat, it will generate an + * interrupt and notify the event to PMIC automatically to shutdown the + * system. So here we should enable the interrupt bits, though we have + * not registered an irq handler. + */ + writel(SPRD_THM_INT_CLR_MASK, thm->base + SPRD_THM_INT_CLR); + sprd_thm_update_bits(thm->base + SPRD_THM_INT_EN, + SPRD_THM_BIT_INT_EN, SPRD_THM_BIT_INT_EN); + sprd_thm_update_bits(thm->base + SPRD_THM_CTL, + SPRD_THM_EN, SPRD_THM_EN); + return 0; +} + +static void sprd_thm_sensor_init(struct sprd_thermal_data *thm, + struct sprd_thermal_sensor *sen) +{ + u32 otp_rawdata, hot_rawdata; + + otp_rawdata = sprd_thm_temp_to_rawdata(SPRD_THM_OTP_TEMP, sen); + hot_rawdata = sprd_thm_temp_to_rawdata(SPRD_THM_HOT_TEMP, sen); + + /* Enable the sensor' overheat temperature protection interrupt */ + sprd_thm_update_bits(thm->base + SPRD_THM_INT_EN, + SPRD_THM_SEN_OVERHEAT_ALARM_EN(sen->id), + SPRD_THM_SEN_OVERHEAT_ALARM_EN(sen->id)); + + /* Set the sensor' overheat and hot threshold temperature */ + sprd_thm_update_bits(thm->base + SPRD_THM_THRES(sen->id), + SPRD_THM_THRES_MASK, + (otp_rawdata << SPRD_THM_OTP_TRIP_SHIFT) | + hot_rawdata); + + /* Enable the corresponding sensor */ + sprd_thm_update_bits(thm->base + SPRD_THM_CTL, SPRD_THM_SEN(sen->id), + SPRD_THM_SEN(sen->id)); +} + +static void sprd_thm_para_config(struct sprd_thermal_data *thm) +{ + /* Set the period of two valid temperature detection action */ + sprd_thm_update_bits(thm->base + SPRD_THM_DET_PERIOD, + SPRD_THM_DET_PERIOD_MASK, SPRD_THM_DET_PERIOD); + + /* Set the sensors' monitor mode */ + sprd_thm_update_bits(thm->base + SPRD_THM_MON_CTL, + SPRD_THM_MON_MODE_MASK, SPRD_THM_MON_MODE); + + /* Set the sensors' monitor period */ + sprd_thm_update_bits(thm->base + SPRD_THM_MON_PERIOD, + SPRD_THM_MON_PERIOD_MASK, SPRD_THM_MON_PERIOD); +} + +static void sprd_thm_toggle_sensor(struct sprd_thermal_sensor *sen, bool on) +{ + struct thermal_zone_device *tzd = sen->tzd; + + tzd->ops->set_mode(tzd, + on ? THERMAL_DEVICE_ENABLED : THERMAL_DEVICE_DISABLED); +} + +static int sprd_thm_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct device_node *sen_child; + struct sprd_thermal_data *thm; + struct sprd_thermal_sensor *sen; + const struct sprd_thm_variant_data *pdata; + int ret, i; + u32 val; + + pdata = of_device_get_match_data(&pdev->dev); + if (!pdata) { + dev_err(&pdev->dev, "No matching driver data found\n"); + return -EINVAL; + } + + thm = devm_kzalloc(&pdev->dev, sizeof(*thm), GFP_KERNEL); + if (!thm) + return -ENOMEM; + + thm->var_data = pdata; + thm->base = devm_platform_ioremap_resource(pdev, 0); + if (!thm->base) + return -ENOMEM; + + thm->nr_sensors = of_get_child_count(np); + if (thm->nr_sensors == 0 || thm->nr_sensors > SPRD_THM_MAX_SENSOR) { + dev_err(&pdev->dev, "incorrect sensor count\n"); + return -EINVAL; + } + + thm->clk = devm_clk_get(&pdev->dev, "enable"); + if (IS_ERR(thm->clk)) { + dev_err(&pdev->dev, "failed to get enable clock\n"); + return PTR_ERR(thm->clk); + } + + ret = clk_prepare_enable(thm->clk); + if (ret) + return ret; + + sprd_thm_para_config(thm); + + ret = sprd_thm_cal_read(np, "thm_sign_cal", &val); + if (ret) + goto disable_clk; + + if (val > 0) + thm->ratio_sign = -1; + else + thm->ratio_sign = 1; + + ret = sprd_thm_cal_read(np, "thm_ratio_cal", &thm->ratio_off); + if (ret) + goto disable_clk; + + for_each_child_of_node(np, sen_child) { + sen = devm_kzalloc(&pdev->dev, sizeof(*sen), GFP_KERNEL); + if (!sen) { + ret = -ENOMEM; + goto disable_clk; + } + + sen->data = thm; + sen->dev = &pdev->dev; + + ret = of_property_read_u32(sen_child, "reg", &sen->id); + if (ret) { + dev_err(&pdev->dev, "get sensor reg failed"); + goto disable_clk; + } + + ret = sprd_thm_sensor_calibration(sen_child, thm, sen); + if (ret) { + dev_err(&pdev->dev, "efuse cal analysis failed"); + goto disable_clk; + } + + sprd_thm_sensor_init(thm, sen); + + sen->tzd = devm_thermal_zone_of_sensor_register(sen->dev, + sen->id, + sen, + &sprd_thm_ops); + if (IS_ERR(sen->tzd)) { + dev_err(&pdev->dev, "register thermal zone failed %d\n", + sen->id); + ret = PTR_ERR(sen->tzd); + goto disable_clk; + } + + thm->sensor[sen->id] = sen; + } + + ret = sprd_thm_set_ready(thm); + if (ret) + goto disable_clk; + + ret = sprd_thm_wait_temp_ready(thm); + if (ret) + goto disable_clk; + + for (i = 0; i < thm->nr_sensors; i++) + sprd_thm_toggle_sensor(thm->sensor[i], true); + + platform_set_drvdata(pdev, thm); + return 0; + +disable_clk: + clk_disable_unprepare(thm->clk); + return ret; +} + +#ifdef CONFIG_PM_SLEEP +static void sprd_thm_hw_suspend(struct sprd_thermal_data *thm) +{ + int i; + + for (i = 0; i < thm->nr_sensors; i++) { + sprd_thm_update_bits(thm->base + SPRD_THM_CTL, + SPRD_THM_SEN(thm->sensor[i]->id), 0); + } + + sprd_thm_update_bits(thm->base + SPRD_THM_CTL, + SPRD_THM_EN, 0x0); +} + +static int sprd_thm_suspend(struct device *dev) +{ + struct sprd_thermal_data *thm = dev_get_drvdata(dev); + int i; + + for (i = 0; i < thm->nr_sensors; i++) + sprd_thm_toggle_sensor(thm->sensor[i], false); + + sprd_thm_hw_suspend(thm); + clk_disable_unprepare(thm->clk); + + return 0; +} + +static int sprd_thm_hw_resume(struct sprd_thermal_data *thm) +{ + int ret, i; + + for (i = 0; i < thm->nr_sensors; i++) { + sprd_thm_update_bits(thm->base + SPRD_THM_CTL, + SPRD_THM_SEN(thm->sensor[i]->id), + SPRD_THM_SEN(thm->sensor[i]->id)); + } + + ret = sprd_thm_poll_ready_status(thm); + if (ret) + return ret; + + writel(SPRD_THM_INT_CLR_MASK, thm->base + SPRD_THM_INT_CLR); + sprd_thm_update_bits(thm->base + SPRD_THM_CTL, + SPRD_THM_EN, SPRD_THM_EN); + return sprd_thm_wait_temp_ready(thm); +} + +static int sprd_thm_resume(struct device *dev) +{ + struct sprd_thermal_data *thm = dev_get_drvdata(dev); + int ret, i; + + ret = clk_prepare_enable(thm->clk); + if (ret) + return ret; + + ret = sprd_thm_hw_resume(thm); + if (ret) + goto disable_clk; + + for (i = 0; i < thm->nr_sensors; i++) + sprd_thm_toggle_sensor(thm->sensor[i], true); + + return 0; + +disable_clk: + clk_disable_unprepare(thm->clk); + return ret; +} +#endif + +static int sprd_thm_remove(struct platform_device *pdev) +{ + struct sprd_thermal_data *thm = platform_get_drvdata(pdev); + int i; + + for (i = 0; i < thm->nr_sensors; i++) { + sprd_thm_toggle_sensor(thm->sensor[i], false); + devm_thermal_zone_of_sensor_unregister(&pdev->dev, + thm->sensor[i]->tzd); + } + + clk_disable_unprepare(thm->clk); + return 0; +} + +static const struct of_device_id sprd_thermal_of_match[] = { + { .compatible = "sprd,ums512-thermal", .data = &ums512_data }, + { }, +}; + +static const struct dev_pm_ops sprd_thermal_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(sprd_thm_suspend, sprd_thm_resume) +}; + +static struct platform_driver sprd_thermal_driver = { + .probe = sprd_thm_probe, + .remove = sprd_thm_remove, + .driver = { + .name = "sprd-thermal", + .pm = &sprd_thermal_pm_ops, + .of_match_table = sprd_thermal_of_match, + }, +}; + +module_platform_driver(sprd_thermal_driver); + +MODULE_AUTHOR("Freeman Liu "); +MODULE_DESCRIPTION("Spreadtrum thermal driver"); +MODULE_LICENSE("GPL v2"); From 7dc6ffbd79c22b3a8433290a692c57e1917575a4 Mon Sep 17 00:00:00 2001 From: Anson Huang Date: Sat, 22 Feb 2020 08:08:48 +0800 Subject: [PATCH 049/635] dt-bindings: fsl: scu: add thermal binding NXP i.MX8QXP is an ARMv8 SoC with a Cortex-M4 core inside as system controller, the system controller is in charge of system power, clock and thermal sensors etc. management, Linux kernel has to communicate with system controller via MU (message unit) IPC to get temperature from thermal sensors, this patch adds binding doc for i.MX system controller thermal driver. Signed-off-by: Anson Huang Reviewed-by: Rob Herring Reviewed-by: Dong Aisheng Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/1582330132-13461-1-git-send-email-Anson.Huang@nxp.com --- .../bindings/arm/freescale/fsl,scu.txt | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Documentation/devicetree/bindings/arm/freescale/fsl,scu.txt b/Documentation/devicetree/bindings/arm/freescale/fsl,scu.txt index e07735a8c2c7..7f42cc37038e 100644 --- a/Documentation/devicetree/bindings/arm/freescale/fsl,scu.txt +++ b/Documentation/devicetree/bindings/arm/freescale/fsl,scu.txt @@ -166,6 +166,17 @@ Required properties: followed by "fsl,imx-sc-key"; - linux,keycodes: See Documentation/devicetree/bindings/input/keys.txt +Thermal bindings based on SCU Message Protocol +------------------------------------------------------------ + +Required properties: +- compatible: Should be : + "fsl,imx8qxp-sc-thermal" + followed by "fsl,imx-sc-thermal"; + +- #thermal-sensor-cells: See Documentation/devicetree/bindings/thermal/thermal.txt + for a description. + Example (imx8qxp): ------------- aliases { @@ -238,6 +249,11 @@ firmware { compatible = "fsl,imx8qxp-sc-wdt", "fsl,imx-sc-wdt"; timeout-sec = <60>; }; + + tsens: thermal-sensor { + compatible = "fsl,imx8qxp-sc-thermal", "fsl,imx-sc-thermal"; + #thermal-sensor-cells = <1>; + }; }; }; From 34471abfc8fedcebf7c45a5e37ef383b8bb16de3 Mon Sep 17 00:00:00 2001 From: Anson Huang Date: Sat, 22 Feb 2020 08:08:49 +0800 Subject: [PATCH 050/635] thermal: of-thermal: add API for getting sensor ID from DT This patch adds new API thermal_zone_of_get_sensor_id() to provide the feature of getting sensor ID from DT thermal zone's node. It's useful for thermal driver to register the specific thermal zone devices from DT in a common way. Signed-off-by: Anson Huang Reviewed-by: Dong Aisheng Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/1582330132-13461-2-git-send-email-Anson.Huang@nxp.com --- drivers/thermal/of-thermal.c | 62 ++++++++++++++++++++++++++---------- include/linux/thermal.h | 10 ++++++ 2 files changed, 56 insertions(+), 16 deletions(-) diff --git a/drivers/thermal/of-thermal.c b/drivers/thermal/of-thermal.c index ef0baa954ff0..874a47d6923f 100644 --- a/drivers/thermal/of-thermal.c +++ b/drivers/thermal/of-thermal.c @@ -448,6 +448,50 @@ thermal_zone_of_add_sensor(struct device_node *zone, return tzd; } +/** + * thermal_zone_of_get_sensor_id - get sensor ID from a DT thermal zone + * @tz_np: a valid thermal zone device node. + * @sensor_np: a sensor node of a valid sensor device. + * @id: the sensor ID returned if success. + * + * This function will get sensor ID from a given thermal zone node and + * the sensor node must match the temperature provider @sensor_np. + * + * Return: 0 on success, proper error code otherwise. + */ + +int thermal_zone_of_get_sensor_id(struct device_node *tz_np, + struct device_node *sensor_np, + u32 *id) +{ + struct of_phandle_args sensor_specs; + int ret; + + ret = of_parse_phandle_with_args(tz_np, + "thermal-sensors", + "#thermal-sensor-cells", + 0, + &sensor_specs); + if (ret) + return ret; + + if (sensor_specs.np != sensor_np) { + of_node_put(sensor_specs.np); + return -ENODEV; + } + + if (sensor_specs.args_count > 1) + pr_warn("%pOFn: too many cells in sensor specifier %d\n", + sensor_specs.np, sensor_specs.args_count); + + *id = sensor_specs.args_count ? sensor_specs.args[0] : 0; + + of_node_put(sensor_specs.np); + + return 0; +} +EXPORT_SYMBOL_GPL(thermal_zone_of_get_sensor_id); + /** * thermal_zone_of_sensor_register - registers a sensor to a DT thermal zone * @dev: a valid struct device pointer of a sensor device. Must contain @@ -499,36 +543,22 @@ thermal_zone_of_sensor_register(struct device *dev, int sensor_id, void *data, sensor_np = of_node_get(dev->of_node); for_each_available_child_of_node(np, child) { - struct of_phandle_args sensor_specs; int ret, id; /* For now, thermal framework supports only 1 sensor per zone */ - ret = of_parse_phandle_with_args(child, "thermal-sensors", - "#thermal-sensor-cells", - 0, &sensor_specs); + ret = thermal_zone_of_get_sensor_id(child, sensor_np, &id); if (ret) continue; - if (sensor_specs.args_count >= 1) { - id = sensor_specs.args[0]; - WARN(sensor_specs.args_count > 1, - "%pOFn: too many cells in sensor specifier %d\n", - sensor_specs.np, sensor_specs.args_count); - } else { - id = 0; - } - - if (sensor_specs.np == sensor_np && id == sensor_id) { + if (id == sensor_id) { tzd = thermal_zone_of_add_sensor(child, sensor_np, data, ops); if (!IS_ERR(tzd)) tzd->ops->set_mode(tzd, THERMAL_DEVICE_ENABLED); - of_node_put(sensor_specs.np); of_node_put(child); goto exit; } - of_node_put(sensor_specs.np); } exit: of_node_put(sensor_np); diff --git a/include/linux/thermal.h b/include/linux/thermal.h index 126913c6a53b..53e6f677761f 100644 --- a/include/linux/thermal.h +++ b/include/linux/thermal.h @@ -364,6 +364,9 @@ struct thermal_trip { /* Function declarations */ #ifdef CONFIG_THERMAL_OF +int thermal_zone_of_get_sensor_id(struct device_node *tz_np, + struct device_node *sensor_np, + u32 *id); struct thermal_zone_device * thermal_zone_of_sensor_register(struct device *dev, int id, void *data, const struct thermal_zone_of_device_ops *ops); @@ -375,6 +378,13 @@ struct thermal_zone_device *devm_thermal_zone_of_sensor_register( void devm_thermal_zone_of_sensor_unregister(struct device *dev, struct thermal_zone_device *tz); #else + +static int thermal_zone_of_get_sensor_id(struct device_node *tz_np, + struct device_node *sensor_np, + u32 *id) +{ + return -ENOENT; +} static inline struct thermal_zone_device * thermal_zone_of_sensor_register(struct device *dev, int id, void *data, const struct thermal_zone_of_device_ops *ops) From e20db70dba1c0783b9878ce37171ad560b1ebaf3 Mon Sep 17 00:00:00 2001 From: Anson Huang Date: Sat, 22 Feb 2020 08:08:50 +0800 Subject: [PATCH 051/635] thermal: imx_sc: add i.MX system controller thermal support i.MX8QXP is an ARMv8 SoC which has a Cortex-M4 system controller inside, the system controller is in charge of controlling power, clock and thermal sensors etc.. This patch adds i.MX system controller thermal driver support, Linux kernel has to communicate with system controller via MU (message unit) IPC to get each thermal sensor's temperature, it supports multiple sensors which are passed from device tree, please see the binding doc for details. Signed-off-by: Anson Huang Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/1582330132-13461-3-git-send-email-Anson.Huang@nxp.com --- drivers/thermal/Kconfig | 11 +++ drivers/thermal/Makefile | 1 + drivers/thermal/imx_sc_thermal.c | 148 +++++++++++++++++++++++++++++++ 3 files changed, 160 insertions(+) create mode 100644 drivers/thermal/imx_sc_thermal.c diff --git a/drivers/thermal/Kconfig b/drivers/thermal/Kconfig index a2a940e8a8e3..bb7b010f1ca7 100644 --- a/drivers/thermal/Kconfig +++ b/drivers/thermal/Kconfig @@ -251,6 +251,17 @@ config IMX_THERMAL cpufreq is used as the cooling device to throttle CPUs when the passive trip is crossed. +config IMX_SC_THERMAL + tristate "Temperature sensor driver for NXP i.MX SoCs with System Controller" + depends on IMX_SCU || COMPILE_TEST + depends on OF + help + Support for Temperature Monitor (TEMPMON) found on NXP i.MX SoCs with + system controller inside, Linux kernel has to communicate with system + controller via MU (message unit) IPC to get temperature from thermal + sensor. It supports one critical trip point and one + passive trip point for each thermal sensor. + config MAX77620_THERMAL tristate "Temperature sensor driver for Maxim MAX77620 PMIC" depends on MFD_MAX77620 diff --git a/drivers/thermal/Makefile b/drivers/thermal/Makefile index dbdd98c117f5..4d1b4e71b3e8 100644 --- a/drivers/thermal/Makefile +++ b/drivers/thermal/Makefile @@ -43,6 +43,7 @@ obj-$(CONFIG_DB8500_THERMAL) += db8500_thermal.o obj-$(CONFIG_ARMADA_THERMAL) += armada_thermal.o obj-$(CONFIG_TANGO_THERMAL) += tango_thermal.o obj-$(CONFIG_IMX_THERMAL) += imx_thermal.o +obj-$(CONFIG_IMX_SC_THERMAL) += imx_sc_thermal.o obj-$(CONFIG_MAX77620_THERMAL) += max77620_thermal.o obj-$(CONFIG_QORIQ_THERMAL) += qoriq_thermal.o obj-$(CONFIG_DA9062_THERMAL) += da9062-thermal.o diff --git a/drivers/thermal/imx_sc_thermal.c b/drivers/thermal/imx_sc_thermal.c new file mode 100644 index 000000000000..4955dfd285ce --- /dev/null +++ b/drivers/thermal/imx_sc_thermal.c @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright 2018-2020 NXP. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "thermal_core.h" + +#define IMX_SC_MISC_FUNC_GET_TEMP 13 + +static struct imx_sc_ipc *thermal_ipc_handle; + +struct imx_sc_sensor { + struct thermal_zone_device *tzd; + u32 resource_id; +}; + +struct req_get_temp { + u16 resource_id; + u8 type; +} __packed; + +struct resp_get_temp { + u16 celsius; + u8 tenths; +} __packed; + +struct imx_sc_msg_misc_get_temp { + struct imx_sc_rpc_msg hdr; + union { + struct req_get_temp req; + struct resp_get_temp resp; + } data; +}; + +static int imx_sc_thermal_get_temp(void *data, int *temp) +{ + struct imx_sc_msg_misc_get_temp msg; + struct imx_sc_rpc_msg *hdr = &msg.hdr; + struct imx_sc_sensor *sensor = data; + int ret; + + msg.data.req.resource_id = sensor->resource_id; + msg.data.req.type = IMX_SC_C_TEMP; + + hdr->ver = IMX_SC_RPC_VERSION; + hdr->svc = IMX_SC_RPC_SVC_MISC; + hdr->func = IMX_SC_MISC_FUNC_GET_TEMP; + hdr->size = 2; + + ret = imx_scu_call_rpc(thermal_ipc_handle, &msg, true); + if (ret) { + dev_err(&sensor->tzd->device, "read temp sensor %d failed, ret %d\n", + sensor->resource_id, ret); + return ret; + } + + *temp = msg.data.resp.celsius * 1000 + msg.data.resp.tenths * 100; + + return 0; +} + +static const struct thermal_zone_of_device_ops imx_sc_thermal_ops = { + .get_temp = imx_sc_thermal_get_temp, +}; + +static int imx_sc_thermal_probe(struct platform_device *pdev) +{ + struct device_node *np, *child, *sensor_np; + struct imx_sc_sensor *sensor; + int ret; + + ret = imx_scu_get_handle(&thermal_ipc_handle); + if (ret) + return ret; + + np = of_find_node_by_name(NULL, "thermal-zones"); + if (!np) + return -ENODEV; + + sensor_np = of_node_get(pdev->dev.of_node); + + for_each_available_child_of_node(np, child) { + sensor = devm_kzalloc(&pdev->dev, sizeof(*sensor), GFP_KERNEL); + if (!sensor) { + of_node_put(sensor_np); + return -ENOMEM; + } + + ret = thermal_zone_of_get_sensor_id(child, + sensor_np, + &sensor->resource_id); + if (ret < 0) { + dev_err(&pdev->dev, + "failed to get valid sensor resource id: %d\n", + ret); + break; + } + + sensor->tzd = devm_thermal_zone_of_sensor_register(&pdev->dev, + sensor->resource_id, + sensor, + &imx_sc_thermal_ops); + if (IS_ERR(sensor->tzd)) { + dev_err(&pdev->dev, "failed to register thermal zone\n"); + ret = PTR_ERR(sensor->tzd); + break; + } + } + + of_node_put(sensor_np); + + return ret; +} + +static int imx_sc_thermal_remove(struct platform_device *pdev) +{ + return 0; +} + +static const struct of_device_id imx_sc_thermal_table[] = { + { .compatible = "fsl,imx-sc-thermal", }, + {} +}; +MODULE_DEVICE_TABLE(of, imx_sc_thermal_table); + +static struct platform_driver imx_sc_thermal_driver = { + .probe = imx_sc_thermal_probe, + .remove = imx_sc_thermal_remove, + .driver = { + .name = "imx-sc-thermal", + .of_match_table = imx_sc_thermal_table, + }, +}; +module_platform_driver(imx_sc_thermal_driver); + +MODULE_AUTHOR("Anson Huang "); +MODULE_DESCRIPTION("Thermal driver for NXP i.MX SoCs with system controller"); +MODULE_LICENSE("GPL v2"); From 82bdde8ecdd7e593c08f456dbb7e2e08b9b01bb5 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Fri, 28 Feb 2020 10:23:31 +0100 Subject: [PATCH 052/635] thermal: exynos: Silence warning during deferred probe Don't confuse user with meaningless warning about the failure of registering sensors in case of deferred probe. Signed-off-by: Marek Szyprowski Reviewed-by: Amit Kucheria Acked-by: Bartlomiej Zolnierkiewicz Reviewed-by: Krzysztof Kozlowski Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200228092331.21548-1-m.szyprowski@samsung.com --- drivers/thermal/samsung/exynos_tmu.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/thermal/samsung/exynos_tmu.c b/drivers/thermal/samsung/exynos_tmu.c index fd4a17812f33..e9a90bc23b11 100644 --- a/drivers/thermal/samsung/exynos_tmu.c +++ b/drivers/thermal/samsung/exynos_tmu.c @@ -1094,7 +1094,9 @@ static int exynos_tmu_probe(struct platform_device *pdev) &exynos_sensor_ops); if (IS_ERR(data->tzd)) { ret = PTR_ERR(data->tzd); - dev_err(&pdev->dev, "Failed to register sensor: %d\n", ret); + if (ret != -EPROBE_DEFER) + dev_err(&pdev->dev, "Failed to register sensor: %d\n", + ret); goto err_sclk; } From 444eb18d091ace343edd65dce5c30e851c911fef Mon Sep 17 00:00:00 2001 From: Anson Huang Date: Sat, 29 Feb 2020 11:44:19 +0800 Subject: [PATCH 053/635] dt-bindings: thermal: imx8mm-thermal: Add binding doc for i.MX8MM Add thermal binding doc for Freescale's i.MX8MM Thermal Monitoring Unit. Signed-off-by: Anson Huang Reviewed-by: Rob Herring Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/1582947862-11073-1-git-send-email-Anson.Huang@nxp.com --- .../bindings/thermal/imx8mm-thermal.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 Documentation/devicetree/bindings/thermal/imx8mm-thermal.txt diff --git a/Documentation/devicetree/bindings/thermal/imx8mm-thermal.txt b/Documentation/devicetree/bindings/thermal/imx8mm-thermal.txt new file mode 100644 index 000000000000..d09ae82b49ee --- /dev/null +++ b/Documentation/devicetree/bindings/thermal/imx8mm-thermal.txt @@ -0,0 +1,15 @@ +* Thermal Monitoring Unit (TMU) on Freescale i.MX8MM SoC + +Required properties: +- compatible : Must be "fsl,imx8mm-tmu". +- reg : Address range of TMU registers. +- clocks : TMU's clock source. +- #thermal-sensor-cells : Should be 0. See ./thermal.txt for a description. + +Example: +tmu: tmu@30260000 { + compatible = "fsl,imx8mm-tmu"; + reg = <0x30260000 0x10000>; + clocks = <&clk IMX8MM_CLK_TMU_ROOT>; + #thermal-sensor-cells = <0>; +}; From 5eed800a68116a3cc66731162f7ea431d2732d4c Mon Sep 17 00:00:00 2001 From: Anson Huang Date: Sat, 29 Feb 2020 11:44:20 +0800 Subject: [PATCH 054/635] thermal: imx8mm: Add support for i.MX8MM thermal monitoring unit i.MX8MM has a thermal monitoring unit(TMU) inside, it ONLY has one sensor for CPU, add support for reading immediate temperature of this sensor. Signed-off-by: Anson Huang Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/1582947862-11073-2-git-send-email-Anson.Huang@nxp.com --- drivers/thermal/Kconfig | 10 +++ drivers/thermal/Makefile | 1 + drivers/thermal/imx8mm_thermal.c | 131 +++++++++++++++++++++++++++++++ 3 files changed, 142 insertions(+) create mode 100644 drivers/thermal/imx8mm_thermal.c diff --git a/drivers/thermal/Kconfig b/drivers/thermal/Kconfig index bb7b010f1ca7..770e0bd9535d 100644 --- a/drivers/thermal/Kconfig +++ b/drivers/thermal/Kconfig @@ -262,6 +262,16 @@ config IMX_SC_THERMAL sensor. It supports one critical trip point and one passive trip point for each thermal sensor. +config IMX8MM_THERMAL + tristate "Temperature sensor driver for Freescale i.MX8MM SoC" + depends on ARCH_MXC + depends on OF + help + Support for Thermal Monitoring Unit (TMU) found on Freescale i.MX8MM SoC. + It supports one critical trip point and one passive trip point. The + cpufreq is used as the cooling device to throttle CPUs when the passive + trip is crossed. + config MAX77620_THERMAL tristate "Temperature sensor driver for Maxim MAX77620 PMIC" depends on MFD_MAX77620 diff --git a/drivers/thermal/Makefile b/drivers/thermal/Makefile index 4d1b4e71b3e8..8c8ed7b79915 100644 --- a/drivers/thermal/Makefile +++ b/drivers/thermal/Makefile @@ -44,6 +44,7 @@ obj-$(CONFIG_ARMADA_THERMAL) += armada_thermal.o obj-$(CONFIG_TANGO_THERMAL) += tango_thermal.o obj-$(CONFIG_IMX_THERMAL) += imx_thermal.o obj-$(CONFIG_IMX_SC_THERMAL) += imx_sc_thermal.o +obj-$(CONFIG_IMX8MM_THERMAL) += imx8mm_thermal.o obj-$(CONFIG_MAX77620_THERMAL) += max77620_thermal.o obj-$(CONFIG_QORIQ_THERMAL) += qoriq_thermal.o obj-$(CONFIG_DA9062_THERMAL) += da9062-thermal.o diff --git a/drivers/thermal/imx8mm_thermal.c b/drivers/thermal/imx8mm_thermal.c new file mode 100644 index 000000000000..d597cebb0545 --- /dev/null +++ b/drivers/thermal/imx8mm_thermal.c @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2020 NXP. + * + * Author: Anson Huang + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "thermal_core.h" + +#define TER 0x0 /* TMU enable */ +#define TRITSR 0x20 /* TMU immediate temp */ + +#define TER_EN BIT(31) +#define TRITSR_VAL_MASK 0xff + +#define TEMP_LOW_LIMIT 10 + +struct imx8mm_tmu { + struct thermal_zone_device *tzd; + void __iomem *base; + struct clk *clk; +}; + +static int tmu_get_temp(void *data, int *temp) +{ + struct imx8mm_tmu *tmu = data; + u32 val; + + val = readl_relaxed(tmu->base + TRITSR) & TRITSR_VAL_MASK; + if (val < TEMP_LOW_LIMIT) + return -EAGAIN; + + *temp = val * 1000; + + return 0; +} + +static struct thermal_zone_of_device_ops tmu_tz_ops = { + .get_temp = tmu_get_temp, +}; + +static int imx8mm_tmu_probe(struct platform_device *pdev) +{ + struct imx8mm_tmu *tmu; + u32 val; + int ret; + + tmu = devm_kzalloc(&pdev->dev, sizeof(struct imx8mm_tmu), GFP_KERNEL); + if (!tmu) + return -ENOMEM; + + tmu->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(tmu->base)) + return PTR_ERR(tmu->base); + + tmu->clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(tmu->clk)) { + ret = PTR_ERR(tmu->clk); + if (ret != -EPROBE_DEFER) + dev_err(&pdev->dev, + "failed to get tmu clock: %d\n", ret); + return ret; + } + + ret = clk_prepare_enable(tmu->clk); + if (ret) { + dev_err(&pdev->dev, "failed to enable tmu clock: %d\n", ret); + return ret; + } + + tmu->tzd = devm_thermal_zone_of_sensor_register(&pdev->dev, 0, + tmu, &tmu_tz_ops); + if (IS_ERR(tmu->tzd)) { + dev_err(&pdev->dev, + "failed to register thermal zone sensor: %d\n", ret); + return PTR_ERR(tmu->tzd); + } + + platform_set_drvdata(pdev, tmu); + + /* enable the monitor */ + val = readl_relaxed(tmu->base + TER); + val |= TER_EN; + writel_relaxed(val, tmu->base + TER); + + return 0; +} + +static int imx8mm_tmu_remove(struct platform_device *pdev) +{ + struct imx8mm_tmu *tmu = platform_get_drvdata(pdev); + u32 val; + + /* disable TMU */ + val = readl_relaxed(tmu->base + TER); + val &= ~TER_EN; + writel_relaxed(val, tmu->base + TER); + + clk_disable_unprepare(tmu->clk); + platform_set_drvdata(pdev, NULL); + + return 0; +} + +static const struct of_device_id imx8mm_tmu_table[] = { + { .compatible = "fsl,imx8mm-tmu", }, + { }, +}; + +static struct platform_driver imx8mm_tmu = { + .driver = { + .name = "i.mx8mm_thermal", + .of_match_table = imx8mm_tmu_table, + }, + .probe = imx8mm_tmu_probe, + .remove = imx8mm_tmu_remove, +}; +module_platform_driver(imx8mm_tmu); + +MODULE_AUTHOR("Anson Huang "); +MODULE_DESCRIPTION("i.MX8MM Thermal Monitor Unit driver"); +MODULE_LICENSE("GPL v2"); From eb8504620381f457dd84d00604772a960c21edd4 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Sat, 29 Feb 2020 21:45:27 +0100 Subject: [PATCH 055/635] thermal: Rephrase the Kconfig text for thermal The thermal subsystem may have relied on sysfs in 2008 when it was introduced, but these days the thermal zones will more often than not come from the hardware descriptions and not from sysfs. Drop the "Generic" phrases as well: there are no non-generic drivers that I know of, the thermal framework is by definition generic. Reword a bit and fix some grammar. [ Daniel Lezcano ] : fixed Randy's comment s/offers/offer/ Signed-off-by: Linus Walleij Acked-by: Daniel Lezcano Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200229204527.143796-1-linus.walleij@linaro.org --- drivers/thermal/Kconfig | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/thermal/Kconfig b/drivers/thermal/Kconfig index 770e0bd9535d..478eda79520d 100644 --- a/drivers/thermal/Kconfig +++ b/drivers/thermal/Kconfig @@ -1,17 +1,18 @@ # SPDX-License-Identifier: GPL-2.0-only # -# Generic thermal sysfs drivers configuration +# Generic thermal drivers configuration # menuconfig THERMAL - bool "Generic Thermal sysfs driver" + bool "Thermal drivers" help - Generic Thermal Sysfs driver offers a generic mechanism for + Thermal drivers offer a generic mechanism for thermal management. Usually it's made up of one or more thermal - zone and cooling device. + zones and cooling devices. Each thermal zone contains its own temperature, trip points, - cooling devices. - All platforms with ACPI thermal support can use this driver. + and cooling devices. + All platforms with ACPI or Open Firmware thermal support can use + this driver. If you want this support, you should say Y here. if THERMAL From 7375f2ac6b2d5f42ebc42e25f5251aa3284f30ed Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Sat, 29 Feb 2020 22:05:32 +0100 Subject: [PATCH 056/635] thermal: ti-soc-thermal: Use GPIO descriptors This switches the TI SoC thermal driver to use GPIO descriptors instead of retrieveing a GPIO number from the device tree and requesting the GPIO separately. Cc: Keerthy Signed-off-by: Linus Walleij Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200229210532.173430-1-linus.walleij@linaro.org --- drivers/thermal/ti-soc-thermal/ti-bandgap.c | 44 ++++++--------------- drivers/thermal/ti-soc-thermal/ti-bandgap.h | 4 +- 2 files changed, 15 insertions(+), 33 deletions(-) diff --git a/drivers/thermal/ti-soc-thermal/ti-bandgap.c b/drivers/thermal/ti-soc-thermal/ti-bandgap.c index 2fa78f738568..263b0420fbe4 100644 --- a/drivers/thermal/ti-soc-thermal/ti-bandgap.c +++ b/drivers/thermal/ti-soc-thermal/ti-bandgap.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include #include @@ -24,7 +24,6 @@ #include #include #include -#include #include #include "ti-bandgap.h" @@ -743,27 +742,13 @@ exit: static int ti_bandgap_tshut_init(struct ti_bandgap *bgp, struct platform_device *pdev) { - int gpio_nr = bgp->tshut_gpio; int status; - /* Request for gpio_86 line */ - status = gpio_request(gpio_nr, "tshut"); - if (status < 0) { - dev_err(bgp->dev, "Could not request for TSHUT GPIO:%i\n", 86); - return status; - } - status = gpio_direction_input(gpio_nr); - if (status) { - dev_err(bgp->dev, "Cannot set input TSHUT GPIO %d\n", gpio_nr); - return status; - } - - status = request_irq(gpio_to_irq(gpio_nr), ti_bandgap_tshut_irq_handler, + status = request_irq(gpiod_to_irq(bgp->tshut_gpiod), + ti_bandgap_tshut_irq_handler, IRQF_TRIGGER_RISING, "tshut", NULL); - if (status) { - gpio_free(gpio_nr); + if (status) dev_err(bgp->dev, "request irq failed for TSHUT"); - } return 0; } @@ -860,11 +845,10 @@ static struct ti_bandgap *ti_bandgap_build(struct platform_device *pdev) } while (res); if (TI_BANDGAP_HAS(bgp, TSHUT)) { - bgp->tshut_gpio = of_get_gpio(node, 0); - if (!gpio_is_valid(bgp->tshut_gpio)) { - dev_err(&pdev->dev, "invalid gpio for tshut (%d)\n", - bgp->tshut_gpio); - return ERR_PTR(-EINVAL); + bgp->tshut_gpiod = devm_gpiod_get(&pdev->dev, NULL, GPIOD_IN); + if (IS_ERR(bgp->tshut_gpiod)) { + dev_err(&pdev->dev, "invalid gpio for tshut\n"); + return ERR_CAST(bgp->tshut_gpiod); } } @@ -1046,10 +1030,8 @@ put_clks: put_fclock: clk_put(bgp->fclock); free_irqs: - if (TI_BANDGAP_HAS(bgp, TSHUT)) { - free_irq(gpio_to_irq(bgp->tshut_gpio), NULL); - gpio_free(bgp->tshut_gpio); - } + if (TI_BANDGAP_HAS(bgp, TSHUT)) + free_irq(gpiod_to_irq(bgp->tshut_gpiod), NULL); return ret; } @@ -1079,10 +1061,8 @@ int ti_bandgap_remove(struct platform_device *pdev) if (TI_BANDGAP_HAS(bgp, TALERT)) free_irq(bgp->irq, bgp); - if (TI_BANDGAP_HAS(bgp, TSHUT)) { - free_irq(gpio_to_irq(bgp->tshut_gpio), NULL); - gpio_free(bgp->tshut_gpio); - } + if (TI_BANDGAP_HAS(bgp, TSHUT)) + free_irq(gpiod_to_irq(bgp->tshut_gpiod), NULL); return 0; } diff --git a/drivers/thermal/ti-soc-thermal/ti-bandgap.h b/drivers/thermal/ti-soc-thermal/ti-bandgap.h index bb9b0f7faf99..fce4657e9486 100644 --- a/drivers/thermal/ti-soc-thermal/ti-bandgap.h +++ b/drivers/thermal/ti-soc-thermal/ti-bandgap.h @@ -13,6 +13,8 @@ #include #include +struct gpio_desc; + /** * DOC: bandgap driver data structure * ================================== @@ -199,7 +201,7 @@ struct ti_bandgap { struct clk *div_clk; spinlock_t lock; /* shields this struct */ int irq; - int tshut_gpio; + struct gpio_desc *tshut_gpiod; u32 clk_rate; }; From 1fd213f38b999b95777d4dddc7e232e08f2294c8 Mon Sep 17 00:00:00 2001 From: Anson Huang Date: Mon, 2 Mar 2020 10:51:25 +0800 Subject: [PATCH 057/635] thermal: imx_sc_thermal: Align imx sc thermal msg structs to 4 The i.MX SCU API strongly assumes that messages are composed out of 4-bytes words but some of our message structs have odd sizeofs, use __packed __aligned(4) to avoid potential oopses. Signed-off-by: Anson Huang Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/1583117485-30922-1-git-send-email-Anson.Huang@nxp.com --- drivers/thermal/imx_sc_thermal.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/thermal/imx_sc_thermal.c b/drivers/thermal/imx_sc_thermal.c index 4955dfd285ce..dbb277af232d 100644 --- a/drivers/thermal/imx_sc_thermal.c +++ b/drivers/thermal/imx_sc_thermal.c @@ -27,12 +27,12 @@ struct imx_sc_sensor { struct req_get_temp { u16 resource_id; u8 type; -} __packed; +} __packed __aligned(4); struct resp_get_temp { u16 celsius; u8 tenths; -} __packed; +} __packed __aligned(4); struct imx_sc_msg_misc_get_temp { struct imx_sc_rpc_msg hdr; @@ -40,7 +40,7 @@ struct imx_sc_msg_misc_get_temp { struct req_get_temp req; struct resp_get_temp resp; } data; -}; +} __packed __aligned(4); static int imx_sc_thermal_get_temp(void *data, int *temp) { From b009514f9786e50578c52eeef6d2020f84630773 Mon Sep 17 00:00:00 2001 From: Anson Huang Date: Mon, 2 Mar 2020 16:54:26 +0800 Subject: [PATCH 058/635] thermal: imx_thermal: Use __maybe_unused instead of CONFIG_PM_SLEEP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use __maybe_unused for power management related functions instead of #if CONFIG_PM_SLEEP to simplify the code. Signed-off-by: Anson Huang Acked-by: Uwe Kleine-König Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/1583139266-23615-1-git-send-email-Anson.Huang@nxp.com --- drivers/thermal/imx_thermal.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/thermal/imx_thermal.c b/drivers/thermal/imx_thermal.c index bb6754a5342c..e75dda53e71b 100644 --- a/drivers/thermal/imx_thermal.c +++ b/drivers/thermal/imx_thermal.c @@ -878,8 +878,7 @@ static int imx_thermal_remove(struct platform_device *pdev) return 0; } -#ifdef CONFIG_PM_SLEEP -static int imx_thermal_suspend(struct device *dev) +static int __maybe_unused imx_thermal_suspend(struct device *dev) { struct imx_thermal_data *data = dev_get_drvdata(dev); struct regmap *map = data->tempmon; @@ -900,7 +899,7 @@ static int imx_thermal_suspend(struct device *dev) return 0; } -static int imx_thermal_resume(struct device *dev) +static int __maybe_unused imx_thermal_resume(struct device *dev) { struct imx_thermal_data *data = dev_get_drvdata(dev); struct regmap *map = data->tempmon; @@ -918,7 +917,6 @@ static int imx_thermal_resume(struct device *dev) return 0; } -#endif static SIMPLE_DEV_PM_OPS(imx_thermal_pm_ops, imx_thermal_suspend, imx_thermal_resume); From 15a26319c41962b7cf87603bc51d667f43d2ca67 Mon Sep 17 00:00:00 2001 From: Anson Huang Date: Tue, 3 Mar 2020 16:04:43 +0800 Subject: [PATCH 059/635] thermal: Fix build warning of !defined(CONFIG_THERMAL_OF) Add "inline" to thermal_zone_of_get_sensor_id() function to avoid below build warning of !defined(CONFIG_THERMAL_OF). In file included from drivers/hwmon/hwmon.c:22: include/linux/thermal.h:382:12: warning: 'thermal_zone_of_get_sensor_id' defined but not used [-Wunused-function] 382 | static int thermal_zone_of_get_sensor_id(struct device_node *tz_np, Signed-off-by: Anson Huang Reported-by: Stephen Rothwell Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/1583222684-10229-1-git-send-email-Anson.Huang@nxp.com --- include/linux/thermal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/thermal.h b/include/linux/thermal.h index 53e6f677761f..c91b1e344d56 100644 --- a/include/linux/thermal.h +++ b/include/linux/thermal.h @@ -379,7 +379,7 @@ void devm_thermal_zone_of_sensor_unregister(struct device *dev, struct thermal_zone_device *tz); #else -static int thermal_zone_of_get_sensor_id(struct device_node *tz_np, +static inline int thermal_zone_of_get_sensor_id(struct device_node *tz_np, struct device_node *sensor_np, u32 *id) { From ef502fc1f0ddac2c98aca58b255d9603cd20aa3d Mon Sep 17 00:00:00 2001 From: Anson Huang Date: Tue, 3 Mar 2020 16:04:44 +0800 Subject: [PATCH 060/635] thermal: Remove COMPILE_TEST for IMX_SC_THERMAL i.MX SCU thermal driver depends on IMX_SCU which does NOT have COMPILE_TEST enabled, so need to remove COMPILE_TEST for i.MX SCU thermal as well. Signed-off-by: Anson Huang Reported-by: Stephen Rothwell Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/1583222684-10229-2-git-send-email-Anson.Huang@nxp.com --- drivers/thermal/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/thermal/Kconfig b/drivers/thermal/Kconfig index 478eda79520d..f3f70503df04 100644 --- a/drivers/thermal/Kconfig +++ b/drivers/thermal/Kconfig @@ -254,7 +254,7 @@ config IMX_THERMAL config IMX_SC_THERMAL tristate "Temperature sensor driver for NXP i.MX SoCs with System Controller" - depends on IMX_SCU || COMPILE_TEST + depends on IMX_SCU depends on OF help Support for Temperature Monitor (TEMPMON) found on NXP i.MX SoCs with From cbe259fd80b7b02fba0dad79d8fdda8b70a8b963 Mon Sep 17 00:00:00 2001 From: Yuantian Tang Date: Tue, 3 Mar 2020 16:46:41 +0800 Subject: [PATCH 061/635] thermal: qoriq: Fix a compiling issue Qoriq thermal driver is used by both PowerPC and ARM architecture. When built for PowerPC architecture, it reports error: undefined reference to `.__devm_regmap_init_mmio_clk' To fix it, select config REGMAP_MMIO. Fixes: 4316237bd627 (thermal: qoriq: Convert driver to use regmap API) Signed-off-by: Yuantian Tang Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200303084641.35687-1-andy.tang@nxp.com --- drivers/thermal/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/thermal/Kconfig b/drivers/thermal/Kconfig index f3f70503df04..4d6753f2b18f 100644 --- a/drivers/thermal/Kconfig +++ b/drivers/thermal/Kconfig @@ -287,6 +287,7 @@ config QORIQ_THERMAL tristate "QorIQ Thermal Monitoring Unit" depends on THERMAL_OF depends on HAS_IOMEM + select REGMAP_MMIO help Support for Thermal Monitoring Unit (TMU) found on QorIQ platforms. It supports one critical trip point and one passive trip point. The From 52cbc58b3bad2d37286110ce3adc7441c2d3de81 Mon Sep 17 00:00:00 2001 From: Anson Huang Date: Fri, 6 Mar 2020 23:37:37 +0800 Subject: [PATCH 062/635] thermal: Add COMPILE_TEST support for i.MX8MM Add COMPILE_TEST support to i.MX8MM thermal driver for better compile testing coverage. Signed-off-by: Anson Huang Reviewed-by: Amit Kucheria Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/1583509057-8197-1-git-send-email-Anson.Huang@nxp.com --- drivers/thermal/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/thermal/Kconfig b/drivers/thermal/Kconfig index 4d6753f2b18f..91af271e9bb0 100644 --- a/drivers/thermal/Kconfig +++ b/drivers/thermal/Kconfig @@ -265,7 +265,7 @@ config IMX_SC_THERMAL config IMX8MM_THERMAL tristate "Temperature sensor driver for Freescale i.MX8MM SoC" - depends on ARCH_MXC + depends on ARCH_MXC || COMPILE_TEST depends on OF help Support for Thermal Monitoring Unit (TMU) found on Freescale i.MX8MM SoC. From 59e356a967c403b6b8dcb0035edbe0c45a84ac85 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 3 Feb 2020 14:49:33 -0500 Subject: [PATCH 063/635] NFS: Use the 64-bit server readdir cookies when possible When we're running as a 64-bit architecture and are not running in 32-bit compatibility mode, it is better to use the 64-bit readdir cookies that supplied by the server. Doing so improves the accuracy of telldir()/seekdir(), particularly when the directory is changing, for instance, when doing 'rm -rf'. We still fall back to using the 32-bit offsets on 32-bit architectures and when in compatibility mode. Signed-off-by: Trond Myklebust --- fs/nfs/dir.c | 61 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index 193d6fb363b7..5e23855e8097 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -144,7 +144,6 @@ struct nfs_cache_array { struct nfs_cache_array_entry array[0]; }; -typedef int (*decode_dirent_t)(struct xdr_stream *, struct nfs_entry *, bool); typedef struct { struct file *file; struct page *page; @@ -153,7 +152,7 @@ typedef struct { u64 *dir_cookie; u64 last_cookie; loff_t current_index; - decode_dirent_t decode; + loff_t prev_index; unsigned long dir_verifier; unsigned long timestamp; @@ -240,6 +239,25 @@ out: return ret; } +static inline +int is_32bit_api(void) +{ +#ifdef CONFIG_COMPAT + return in_compat_syscall(); +#else + return (BITS_PER_LONG == 32); +#endif +} + +static +bool nfs_readdir_use_cookie(const struct file *filp) +{ + if ((filp->f_mode & FMODE_32BITHASH) || + (!(filp->f_mode & FMODE_64BITHASH) && is_32bit_api())) + return false; + return true; +} + static int nfs_readdir_search_for_pos(struct nfs_cache_array *array, nfs_readdir_descriptor_t *desc) { @@ -289,7 +307,7 @@ int nfs_readdir_search_for_cookie(struct nfs_cache_array *array, nfs_readdir_des !nfs_readdir_inode_mapping_valid(nfsi)) { ctx->duped = 0; ctx->attr_gencount = nfsi->attr_gencount; - } else if (new_pos < desc->ctx->pos) { + } else if (new_pos < desc->prev_index) { if (ctx->duped > 0 && ctx->dup_cookie == *desc->dir_cookie) { if (printk_ratelimit()) { @@ -305,7 +323,11 @@ int nfs_readdir_search_for_cookie(struct nfs_cache_array *array, nfs_readdir_des ctx->dup_cookie = *desc->dir_cookie; ctx->duped = -1; } - desc->ctx->pos = new_pos; + if (nfs_readdir_use_cookie(desc->file)) + desc->ctx->pos = *desc->dir_cookie; + else + desc->ctx->pos = new_pos; + desc->prev_index = new_pos; desc->cache_entry_index = i; return 0; } @@ -376,9 +398,10 @@ error: static int xdr_decode(nfs_readdir_descriptor_t *desc, struct nfs_entry *entry, struct xdr_stream *xdr) { + struct inode *inode = file_inode(desc->file); int error; - error = desc->decode(xdr, entry, desc->plus); + error = NFS_PROTO(inode)->decode_dirent(xdr, entry, desc->plus); if (error) return error; entry->fattr->time_start = desc->timestamp; @@ -756,6 +779,7 @@ int readdir_search_pagecache(nfs_readdir_descriptor_t *desc) if (desc->page_index == 0) { desc->current_index = 0; + desc->prev_index = 0; desc->last_cookie = 0; } do { @@ -786,11 +810,14 @@ int nfs_do_filldir(nfs_readdir_descriptor_t *desc) desc->eof = true; break; } - desc->ctx->pos++; if (i < (array->size-1)) *desc->dir_cookie = array->array[i+1].cookie; else *desc->dir_cookie = array->last_cookie; + if (nfs_readdir_use_cookie(file)) + desc->ctx->pos = *desc->dir_cookie; + else + desc->ctx->pos++; if (ctx->duped != 0) ctx->duped = 1; } @@ -860,9 +887,14 @@ static int nfs_readdir(struct file *file, struct dir_context *ctx) { struct dentry *dentry = file_dentry(file); struct inode *inode = d_inode(dentry); - nfs_readdir_descriptor_t my_desc, - *desc = &my_desc; struct nfs_open_dir_context *dir_ctx = file->private_data; + nfs_readdir_descriptor_t my_desc = { + .file = file, + .ctx = ctx, + .dir_cookie = &dir_ctx->dir_cookie, + .plus = nfs_use_readdirplus(inode, ctx), + }, + *desc = &my_desc; int res = 0; dfprintk(FILE, "NFS: readdir(%pD2) starting at cookie %llu\n", @@ -875,14 +907,6 @@ static int nfs_readdir(struct file *file, struct dir_context *ctx) * to either find the entry with the appropriate number or * revalidate the cookie. */ - memset(desc, 0, sizeof(*desc)); - - desc->file = file; - desc->ctx = ctx; - desc->dir_cookie = &dir_ctx->dir_cookie; - desc->decode = NFS_PROTO(inode)->decode_dirent; - desc->plus = nfs_use_readdirplus(inode, ctx); - if (ctx->pos == 0 || nfs_attribute_cache_expired(inode)) res = nfs_revalidate_mapping(inode, file->f_mapping); if (res < 0) @@ -954,7 +978,10 @@ static loff_t nfs_llseek_dir(struct file *filp, loff_t offset, int whence) } if (offset != filp->f_pos) { filp->f_pos = offset; - dir_ctx->dir_cookie = 0; + if (nfs_readdir_use_cookie(filp)) + dir_ctx->dir_cookie = offset; + else + dir_ctx->dir_cookie = 0; dir_ctx->duped = 0; } inode_unlock(inode); From 57f188e047731e50cb6af94d473c3c70d3bce7f9 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 27 Jan 2020 12:44:41 -0500 Subject: [PATCH 064/635] NFSv4: nfs_update_inplace_delegation() should update delegation cred If the cred assigned to the delegation that we're updating differs from the one we're updating too, then we need to update that field too. Signed-off-by: Trond Myklebust --- fs/nfs/delegation.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/fs/nfs/delegation.c b/fs/nfs/delegation.c index 1865322de142..509b7235b132 100644 --- a/fs/nfs/delegation.c +++ b/fs/nfs/delegation.c @@ -377,6 +377,18 @@ nfs_inode_detach_delegation(struct inode *inode) return delegation; } +static void +nfs_update_delegation_cred(struct nfs_delegation *delegation, + const struct cred *cred) +{ + const struct cred *old; + + if (cred_fscmp(delegation->cred, cred) != 0) { + old = xchg(&delegation->cred, get_cred(cred)); + put_cred(old); + } +} + static void nfs_update_inplace_delegation(struct nfs_delegation *delegation, const struct nfs_delegation *update) @@ -385,8 +397,14 @@ nfs_update_inplace_delegation(struct nfs_delegation *delegation, delegation->stateid.seqid = update->stateid.seqid; smp_wmb(); delegation->type = update->type; - if (test_and_clear_bit(NFS_DELEGATION_REVOKED, &delegation->flags)) + delegation->pagemod_limit = update->pagemod_limit; + if (test_bit(NFS_DELEGATION_REVOKED, &delegation->flags)) { + delegation->change_attr = update->change_attr; + nfs_update_delegation_cred(delegation, update->cred); + /* smp_mb__before_atomic() is implicit due to xchg() */ + clear_bit(NFS_DELEGATION_REVOKED, &delegation->flags); atomic_long_inc(&nfs_active_delegations); + } } } From 59b5639490f51aa604d18064dcf0c2d72eb1decf Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 27 Jan 2020 13:07:26 -0500 Subject: [PATCH 065/635] NFSv4/pnfs: pnfs_set_layout_stateid() should update the layout cred If the cred assigned to the layout that we're updating differs from the one used to retrieve the new layout segment, then we need to update the layout plh_lc_cred field. Signed-off-by: Trond Myklebust --- fs/nfs/callback_proc.c | 2 +- fs/nfs/pnfs.c | 20 ++++++++++++++++---- fs/nfs/pnfs.h | 1 + 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c index cd4c6bc81cae..b6ffac9963c8 100644 --- a/fs/nfs/callback_proc.c +++ b/fs/nfs/callback_proc.c @@ -280,7 +280,7 @@ static u32 initiate_file_draining(struct nfs_client *clp, goto unlock; } - pnfs_set_layout_stateid(lo, &args->cbl_stateid, true); + pnfs_set_layout_stateid(lo, &args->cbl_stateid, NULL, true); switch (pnfs_mark_matching_lsegs_return(lo, &free_me_list, &args->cbl_range, be32_to_cpu(args->cbl_stateid.seqid))) { diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index 542ea8dfd1bc..b21eb4882846 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -903,10 +903,21 @@ pnfs_destroy_all_layouts(struct nfs_client *clp) pnfs_destroy_layouts_byclid(clp, false); } +static void +pnfs_set_layout_cred(struct pnfs_layout_hdr *lo, const struct cred *cred) +{ + const struct cred *old; + + if (cred && cred_fscmp(lo->plh_lc_cred, cred) != 0) { + old = xchg(&lo->plh_lc_cred, get_cred(cred)); + put_cred(old); + } +} + /* update lo->plh_stateid with new if is more recent */ void pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new, - bool update_barrier) + const struct cred *cred, bool update_barrier) { u32 oldseq, newseq, new_barrier = 0; @@ -914,6 +925,7 @@ pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new, newseq = be32_to_cpu(new->seqid); if (!pnfs_layout_is_valid(lo)) { + pnfs_set_layout_cred(lo, cred); nfs4_stateid_copy(&lo->plh_stateid, new); lo->plh_barrier = newseq; pnfs_clear_layoutreturn_info(lo); @@ -1109,7 +1121,7 @@ void pnfs_layoutreturn_free_lsegs(struct pnfs_layout_hdr *lo, pnfs_mark_matching_lsegs_invalid(lo, &freeme, range, seq); pnfs_free_returned_lsegs(lo, &freeme, range, seq); - pnfs_set_layout_stateid(lo, stateid, true); + pnfs_set_layout_stateid(lo, stateid, NULL, true); } else pnfs_mark_layout_stateid_invalid(lo, &freeme); out_unlock: @@ -2323,14 +2335,14 @@ pnfs_layout_process(struct nfs4_layoutget *lgp) if (!pnfs_layout_is_valid(lo)) { /* We have a completely new layout */ - pnfs_set_layout_stateid(lo, &res->stateid, true); + pnfs_set_layout_stateid(lo, &res->stateid, lgp->cred, true); } else if (nfs4_stateid_match_other(&lo->plh_stateid, &res->stateid)) { /* existing state ID, make sure the sequence number matches. */ if (pnfs_layout_stateid_blocked(lo, &res->stateid)) { dprintk("%s forget reply due to sequence\n", __func__); goto out_forget; } - pnfs_set_layout_stateid(lo, &res->stateid, false); + pnfs_set_layout_stateid(lo, &res->stateid, lgp->cred, false); } else { /* * We got an entirely new state ID. Mark all segments for the diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h index 0fafdadc9c8d..cfb89d47c79d 100644 --- a/fs/nfs/pnfs.h +++ b/fs/nfs/pnfs.h @@ -267,6 +267,7 @@ bool nfs4_layout_refresh_old_stateid(nfs4_stateid *dst, void pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo); void pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new, + const struct cred *cred, bool update_barrier); int pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo, struct list_head *tmp_list, From 244fcd2f9a904523f1b8c1a6c94749e230ac053a Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Fri, 20 Dec 2019 10:43:37 -0500 Subject: [PATCH 066/635] NFS: Ensure we time out if a delegreturn does not complete We can't allow delegreturn to hold up nfs4_evict_inode() forever, since that can cause the memory shrinkers to block. This patch therefore ensures that we eventually time out, and complete the reclaim of the inode. Signed-off-by: Trond Myklebust --- fs/nfs/nfs4proc.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 69b7ab7a5815..692868dedb46 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -6259,6 +6259,7 @@ static void nfs4_delegreturn_done(struct rpc_task *task, void *calldata) /* Fallthrough */ case -NFS4ERR_BAD_STATEID: case -NFS4ERR_STALE_STATEID: + case -ETIMEDOUT: task->tk_status = 0; break; case -NFS4ERR_OLD_STATEID: @@ -6349,7 +6350,7 @@ static int _nfs4_proc_delegreturn(struct inode *inode, const struct cred *cred, .rpc_client = server->client, .rpc_message = &msg, .callback_ops = &nfs4_delegreturn_ops, - .flags = RPC_TASK_ASYNC, + .flags = RPC_TASK_ASYNC | RPC_TASK_TIMEOUT, }; int status = 0; From 1d179d6bd67369a52edea8562154b31ee20be1cc Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Fri, 7 Feb 2020 19:32:49 -0500 Subject: [PATCH 067/635] NFS: alloc_nfs_open_context() must use the file cred when available If we're creating a nfs_open_context() for a specific file pointer, we must use the cred assigned to that file. Fixes: a52458b48af1 ("NFS/NFSD/SUNRPC: replace generic creds with 'struct cred'.") Signed-off-by: Trond Myklebust --- fs/nfs/inode.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index 11bf15800ac9..a10fb87c6ac3 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -959,16 +959,16 @@ struct nfs_open_context *alloc_nfs_open_context(struct dentry *dentry, struct file *filp) { struct nfs_open_context *ctx; - const struct cred *cred = get_current_cred(); ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); - if (!ctx) { - put_cred(cred); + if (!ctx) return ERR_PTR(-ENOMEM); - } nfs_sb_active(dentry->d_sb); ctx->dentry = dget(dentry); - ctx->cred = cred; + if (filp) + ctx->cred = get_cred(filp->f_cred); + else + ctx->cred = get_current_cred(); ctx->ll_cred = NULL; ctx->state = NULL; ctx->mode = f_mode; From 7eac52648a4c24ad23a05f62db97867c92a5747b Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Fri, 7 Feb 2020 19:11:12 -0500 Subject: [PATCH 068/635] SUNRPC: Add a flag to avoid reference counts on credentials Add a flag to signal to the RPC layer that the credential is already pinned for the duration of the RPC call. Signed-off-by: Trond Myklebust --- include/linux/sunrpc/sched.h | 1 + net/sunrpc/clnt.c | 5 +++-- net/sunrpc/sched.c | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/include/linux/sunrpc/sched.h b/include/linux/sunrpc/sched.h index a6ef35184ef1..df696efdd675 100644 --- a/include/linux/sunrpc/sched.h +++ b/include/linux/sunrpc/sched.h @@ -132,6 +132,7 @@ struct rpc_task_setup { #define RPC_TASK_TIMEOUT 0x1000 /* fail with ETIMEDOUT on timeout */ #define RPC_TASK_NOCONNECT 0x2000 /* return ENOTCONN if not connected */ #define RPC_TASK_NO_RETRANS_TIMEOUT 0x4000 /* wait forever for a reply */ +#define RPC_TASK_CRED_NOREF 0x8000 /* No refcount on the credential */ #define RPC_IS_ASYNC(t) ((t)->tk_flags & RPC_TASK_ASYNC) #define RPC_IS_SWAPPER(t) ((t)->tk_flags & RPC_TASK_SWAPPER) diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index 7324b21f923e..2345e563c2f4 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -1099,8 +1099,9 @@ rpc_task_set_rpc_message(struct rpc_task *task, const struct rpc_message *msg) task->tk_msg.rpc_proc = msg->rpc_proc; task->tk_msg.rpc_argp = msg->rpc_argp; task->tk_msg.rpc_resp = msg->rpc_resp; - if (msg->rpc_cred != NULL) - task->tk_msg.rpc_cred = get_cred(msg->rpc_cred); + task->tk_msg.rpc_cred = msg->rpc_cred; + if (!(task->tk_flags & RPC_TASK_CRED_NOREF)) + get_cred(task->tk_msg.rpc_cred); } } diff --git a/net/sunrpc/sched.c b/net/sunrpc/sched.c index 55e900255b0c..6eff14119a88 100644 --- a/net/sunrpc/sched.c +++ b/net/sunrpc/sched.c @@ -1162,7 +1162,8 @@ static void rpc_release_resources_task(struct rpc_task *task) { xprt_release(task); if (task->tk_msg.rpc_cred) { - put_cred(task->tk_msg.rpc_cred); + if (!(task->tk_flags & RPC_TASK_CRED_NOREF)) + put_cred(task->tk_msg.rpc_cred); task->tk_msg.rpc_cred = NULL; } rpc_task_release_client(task); From 263fb9c21e7a86e8e373a23edfa82700422f5e6a Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Fri, 7 Feb 2020 19:16:34 -0500 Subject: [PATCH 069/635] SUNRPC: Don't take a reference to the cred on synchronous tasks If the RPC call is synchronous, assume the cred is already pinned by the caller. Signed-off-by: Trond Myklebust --- net/sunrpc/clnt.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index 2345e563c2f4..252b044cdcdf 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -1127,6 +1127,9 @@ struct rpc_task *rpc_run_task(const struct rpc_task_setup *task_setup_data) task = rpc_new_task(task_setup_data); + if (!RPC_IS_ASYNC(task)) + task->tk_flags |= RPC_TASK_CRED_NOREF; + rpc_task_set_client(task, task_setup_data->rpc_client); rpc_task_set_rpc_message(task, task_setup_data->rpc_message); From 542b994bdb2662d38f1a400adf3e5da3adceb50d Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Fri, 7 Feb 2020 19:25:56 -0500 Subject: [PATCH 070/635] NFS: Assume cred is pinned by open context in I/O requests In read/write/commit, we should be able to assume that the cred is pinned by the open context. Signed-off-by: Trond Myklebust --- fs/nfs/pagelist.c | 2 +- fs/nfs/write.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/nfs/pagelist.c b/fs/nfs/pagelist.c index 20b3717cd7ca..c9c3edefc5be 100644 --- a/fs/nfs/pagelist.c +++ b/fs/nfs/pagelist.c @@ -627,7 +627,7 @@ int nfs_initiate_pgio(struct rpc_clnt *clnt, struct nfs_pgio_header *hdr, .callback_ops = call_ops, .callback_data = hdr, .workqueue = nfsiod_workqueue, - .flags = RPC_TASK_ASYNC | flags, + .flags = RPC_TASK_ASYNC | RPC_TASK_CRED_NOREF | flags, }; int ret = 0; diff --git a/fs/nfs/write.c b/fs/nfs/write.c index c478b772cc49..5544ee6cfda8 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -1707,7 +1707,7 @@ int nfs_initiate_commit(struct rpc_clnt *clnt, struct nfs_commit_data *data, .callback_ops = call_ops, .callback_data = data, .workqueue = nfsiod_workqueue, - .flags = RPC_TASK_ASYNC | flags, + .flags = RPC_TASK_ASYNC | RPC_TASK_CRED_NOREF | flags, .priority = priority, }; /* Set up the initial task struct. */ From 6129650720207feb768f2161d58f3e398ca2805d Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Fri, 7 Feb 2020 19:38:12 -0500 Subject: [PATCH 071/635] NFSv4: Avoid referencing the cred unnecessarily during NFSv4 I/O Avoid unnecessary references to the cred when we have already referenced it through the open context or the open owner. Signed-off-by: Trond Myklebust --- fs/nfs/nfs4proc.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 692868dedb46..a7fe64b93852 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -2346,7 +2346,7 @@ static int _nfs4_proc_open_confirm(struct nfs4_opendata *data) .callback_ops = &nfs4_open_confirm_ops, .callback_data = data, .workqueue = nfsiod_workqueue, - .flags = RPC_TASK_ASYNC, + .flags = RPC_TASK_ASYNC | RPC_TASK_CRED_NOREF, }; int status; @@ -2511,7 +2511,7 @@ static int nfs4_run_open_task(struct nfs4_opendata *data, .callback_ops = &nfs4_open_ops, .callback_data = data, .workqueue = nfsiod_workqueue, - .flags = RPC_TASK_ASYNC, + .flags = RPC_TASK_ASYNC | RPC_TASK_CRED_NOREF, }; int status; @@ -3651,7 +3651,7 @@ int nfs4_do_close(struct nfs4_state *state, gfp_t gfp_mask, int wait) .rpc_message = &msg, .callback_ops = &nfs4_close_ops, .workqueue = nfsiod_workqueue, - .flags = RPC_TASK_ASYNC, + .flags = RPC_TASK_ASYNC | RPC_TASK_CRED_NOREF, }; int status = -ENOMEM; @@ -6350,7 +6350,7 @@ static int _nfs4_proc_delegreturn(struct inode *inode, const struct cred *cred, .rpc_client = server->client, .rpc_message = &msg, .callback_ops = &nfs4_delegreturn_ops, - .flags = RPC_TASK_ASYNC | RPC_TASK_TIMEOUT, + .flags = RPC_TASK_ASYNC | RPC_TASK_CRED_NOREF | RPC_TASK_TIMEOUT, }; int status = 0; @@ -6933,7 +6933,7 @@ static int _nfs4_do_setlk(struct nfs4_state *state, int cmd, struct file_lock *f .rpc_message = &msg, .callback_ops = &nfs4_lock_ops, .workqueue = nfsiod_workqueue, - .flags = RPC_TASK_ASYNC, + .flags = RPC_TASK_ASYNC | RPC_TASK_CRED_NOREF, }; int ret; From 63ec2b69e9556fc73f34e7e36bd880caf1db1fe6 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Fri, 7 Feb 2020 19:40:14 -0500 Subject: [PATCH 072/635] NFSv4: Avoid unnecessary credential references in layoutget Layoutget is just using the credential attached to the open context. Signed-off-by: Trond Myklebust --- fs/nfs/nfs4proc.c | 2 +- fs/nfs/pnfs.c | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index a7fe64b93852..22dca497c5de 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -9177,7 +9177,7 @@ nfs4_proc_layoutget(struct nfs4_layoutget *lgp, long *timeout) .rpc_message = &msg, .callback_ops = &nfs4_layoutget_call_ops, .callback_data = lgp, - .flags = RPC_TASK_ASYNC, + .flags = RPC_TASK_ASYNC | RPC_TASK_CRED_NOREF, }; struct pnfs_layout_segment *lseg = NULL; struct nfs4_exception exception = { diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index b21eb4882846..cb99ac954688 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -1073,7 +1073,7 @@ pnfs_alloc_init_layoutget_args(struct inode *ino, lgp->args.ctx = get_nfs_open_context(ctx); nfs4_stateid_copy(&lgp->args.stateid, stateid); lgp->gfp_flags = gfp_flags; - lgp->cred = get_cred(ctx->cred); + lgp->cred = ctx->cred; return lgp; } @@ -1084,7 +1084,6 @@ void pnfs_layoutget_free(struct nfs4_layoutget *lgp) nfs4_free_pages(lgp->args.layout.pages, max_pages); if (lgp->args.inode) pnfs_put_layout_hdr(NFS_I(lgp->args.inode)->layout); - put_cred(lgp->cred); put_nfs_open_context(lgp->args.ctx); kfree(lgp); } From 49cd32543fa68889111c66c79cc0a6ed2ec02019 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Fri, 7 Feb 2020 19:44:33 -0500 Subject: [PATCH 073/635] NFS: Avoid referencing the cred twice in async rename/unlink In both async rename and rename, we take a reference to the cred in the call arguments. Signed-off-by: Trond Myklebust --- fs/nfs/unlink.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/nfs/unlink.c b/fs/nfs/unlink.c index 0effeee28352..b27ebdccef70 100644 --- a/fs/nfs/unlink.c +++ b/fs/nfs/unlink.c @@ -98,7 +98,7 @@ static void nfs_do_call_unlink(struct inode *inode, struct nfs_unlinkdata *data) .callback_ops = &nfs_unlink_ops, .callback_data = data, .workqueue = nfsiod_workqueue, - .flags = RPC_TASK_ASYNC, + .flags = RPC_TASK_ASYNC | RPC_TASK_CRED_NOREF, }; struct rpc_task *task; struct inode *dir = d_inode(data->dentry->d_parent); @@ -341,7 +341,7 @@ nfs_async_rename(struct inode *old_dir, struct inode *new_dir, .callback_ops = &nfs_rename_ops, .workqueue = nfsiod_workqueue, .rpc_client = NFS_CLIENT(old_dir), - .flags = RPC_TASK_ASYNC, + .flags = RPC_TASK_ASYNC | RPC_TASK_CRED_NOREF, }; data = kzalloc(sizeof(*data), GFP_KERNEL); From a8b373eefc82094ba84bf35a770cdb21196224e4 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Sat, 8 Feb 2020 09:14:11 -0500 Subject: [PATCH 074/635] NFS: Limit the size of the access cache by default Currently, we have no real limit on the access cache size (we set it to ULONG_MAX). That can lead to credentials getting pinned for a very long time on lots of files if you have a system with a lot of memory. Signed-off-by: Trond Myklebust --- fs/nfs/dir.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index 5e23855e8097..f22366f350a9 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -2309,7 +2309,7 @@ static DEFINE_SPINLOCK(nfs_access_lru_lock); static LIST_HEAD(nfs_access_lru_list); static atomic_long_t nfs_access_nr_entries; -static unsigned long nfs_access_max_cachesize = ULONG_MAX; +static unsigned long nfs_access_max_cachesize = 4*1024*1024; module_param(nfs_access_max_cachesize, ulong, 0644); MODULE_PARM_DESC(nfs_access_max_cachesize, "NFS access maximum total cache length"); From 194a0dc8e2c00c2b35da323639db8d96375490f9 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 10 Feb 2020 14:45:34 -0500 Subject: [PATCH 075/635] pNFS/flexfiles: Report DELAY and GRACE errors from the DS to the server Ensure that if the DS is returning too many DELAY and GRACE errors, we also report that to the MDS through the layouterror mechanism. Signed-off-by: Trond Myklebust --- fs/nfs/flexfilelayout/flexfilelayout.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c index bb9148b83166..8b8171b48893 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.c +++ b/fs/nfs/flexfilelayout/flexfilelayout.c @@ -1297,21 +1297,23 @@ static void ff_layout_io_track_ds_error(struct pnfs_layout_segment *lseg, } } - switch (status) { - case NFS4ERR_DELAY: - case NFS4ERR_GRACE: - return; - default: - break; - } - mirror = FF_LAYOUT_COMP(lseg, idx); err = ff_layout_track_ds_error(FF_LAYOUT_FROM_HDR(lseg->pls_layout), mirror, offset, length, status, opnum, GFP_NOIO); - if (status == NFS4ERR_NXIO) + + switch (status) { + case NFS4ERR_DELAY: + case NFS4ERR_GRACE: + break; + case NFS4ERR_NXIO: ff_layout_mark_ds_unreachable(lseg, idx); - pnfs_error_mark_layout_for_return(lseg->pls_layout->plh_inode, lseg); + /* Fallthrough */ + default: + pnfs_error_mark_layout_for_return(lseg->pls_layout->plh_inode, + lseg); + } + dprintk("%s: err %d op %d status %u\n", __func__, err, opnum, status); } From d911c57a19551c6bef116a3b55c6b089901aacb0 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Thu, 27 Feb 2020 11:01:12 -0500 Subject: [PATCH 076/635] NFSv4/pnfs: Return valid stateids in nfs_layout_find_inode_by_stateid() Make sure to test the stateid for validity so that we catch instances where the server may have been reusing stateids in nfs_layout_find_inode_by_stateid(). Fixes: 7b410d9ce460 ("pNFS: Delay getting the layout header in CB_LAYOUTRECALL handlers") Signed-off-by: Trond Myklebust --- fs/nfs/callback_proc.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c index b6ffac9963c8..eb9d035451a2 100644 --- a/fs/nfs/callback_proc.c +++ b/fs/nfs/callback_proc.c @@ -128,6 +128,8 @@ static struct inode *nfs_layout_find_inode_by_stateid(struct nfs_client *clp, list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { list_for_each_entry(lo, &server->layouts, plh_layouts) { + if (!pnfs_layout_is_valid(lo)) + continue; if (stateid != NULL && !nfs4_stateid_match_other(stateid, &lo->plh_stateid)) continue; From cf6605d1940a5ead7f2de78b5926f9c3179cda41 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Tue, 18 Feb 2020 17:14:40 -0500 Subject: [PATCH 077/635] NFSv4: Ensure layout headers are RCU safe Signed-off-by: Trond Myklebust --- fs/nfs/blocklayout/blocklayout.c | 2 +- fs/nfs/filelayout/filelayout.c | 2 +- fs/nfs/flexfilelayout/flexfilelayout.c | 6 +++--- fs/nfs/pnfs.c | 12 ++++++------ fs/nfs/pnfs.h | 2 ++ 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/fs/nfs/blocklayout/blocklayout.c b/fs/nfs/blocklayout/blocklayout.c index 690221747b47..d1a0e2c8b1b4 100644 --- a/fs/nfs/blocklayout/blocklayout.c +++ b/fs/nfs/blocklayout/blocklayout.c @@ -476,7 +476,7 @@ static void bl_free_layout_hdr(struct pnfs_layout_hdr *lo) err = ext_tree_remove(bl, true, 0, LLONG_MAX); WARN_ON(err); - kfree(bl); + kfree_rcu(bl, bl_layout.plh_rcu); } static struct pnfs_layout_hdr *__bl_alloc_layout_hdr(struct inode *inode, diff --git a/fs/nfs/filelayout/filelayout.c b/fs/nfs/filelayout/filelayout.c index c9b605f6c9cb..bd234394a87c 100644 --- a/fs/nfs/filelayout/filelayout.c +++ b/fs/nfs/filelayout/filelayout.c @@ -1146,7 +1146,7 @@ filelayout_alloc_layout_hdr(struct inode *inode, gfp_t gfp_flags) static void filelayout_free_layout_hdr(struct pnfs_layout_hdr *lo) { - kfree(FILELAYOUT_FROM_HDR(lo)); + kfree_rcu(FILELAYOUT_FROM_HDR(lo), generic_hdr.plh_rcu); } static struct pnfs_ds_commit_info * diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c index 8b8171b48893..e7d8ae4d0cc5 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.c +++ b/fs/nfs/flexfilelayout/flexfilelayout.c @@ -59,14 +59,14 @@ ff_layout_alloc_layout_hdr(struct inode *inode, gfp_t gfp_flags) static void ff_layout_free_layout_hdr(struct pnfs_layout_hdr *lo) { + struct nfs4_flexfile_layout *ffl = FF_LAYOUT_FROM_HDR(lo); struct nfs4_ff_layout_ds_err *err, *n; - list_for_each_entry_safe(err, n, &FF_LAYOUT_FROM_HDR(lo)->error_list, - list) { + list_for_each_entry_safe(err, n, &ffl->error_list, list) { list_del(&err->list); kfree(err); } - kfree(FF_LAYOUT_FROM_HDR(lo)); + kfree_rcu(ffl, generic_hdr.plh_rcu); } static int decode_pnfs_stateid(struct xdr_stream *xdr, nfs4_stateid *stateid) diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index cb99ac954688..268e7b9ff54e 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -268,11 +268,11 @@ pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo) struct nfs_server *server = NFS_SERVER(lo->plh_inode); struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld; - if (!list_empty(&lo->plh_layouts)) { + if (test_and_clear_bit(NFS_LAYOUT_HASHED, &lo->plh_flags)) { struct nfs_client *clp = server->nfs_client; spin_lock(&clp->cl_lock); - list_del_init(&lo->plh_layouts); + list_del_rcu(&lo->plh_layouts); spin_unlock(&clp->cl_lock); } put_cred(lo->plh_lc_cred); @@ -784,7 +784,8 @@ pnfs_layout_bulk_destroy_byserver_locked(struct nfs_client *clp, break; inode = igrab(lo->plh_inode); if (inode != NULL) { - list_del_init(&lo->plh_layouts); + if (test_and_clear_bit(NFS_LAYOUT_HASHED, &lo->plh_flags)) + list_del_rcu(&lo->plh_layouts); if (pnfs_layout_add_bulk_destroy_list(inode, layout_list)) continue; @@ -1870,15 +1871,14 @@ static void pnfs_clear_first_layoutget(struct pnfs_layout_hdr *lo) static void _add_to_server_list(struct pnfs_layout_hdr *lo, struct nfs_server *server) { - if (list_empty(&lo->plh_layouts)) { + if (!test_and_set_bit(NFS_LAYOUT_HASHED, &lo->plh_flags)) { struct nfs_client *clp = server->nfs_client; /* The lo must be on the clp list if there is any * chance of a CB_LAYOUTRECALL(FILE) coming in. */ spin_lock(&clp->cl_lock); - if (list_empty(&lo->plh_layouts)) - list_add_tail(&lo->plh_layouts, &server->layouts); + list_add_tail_rcu(&lo->plh_layouts, &server->layouts); spin_unlock(&clp->cl_lock); } } diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h index cfb89d47c79d..8df9aa02d336 100644 --- a/fs/nfs/pnfs.h +++ b/fs/nfs/pnfs.h @@ -105,6 +105,7 @@ enum { NFS_LAYOUT_INVALID_STID, /* layout stateid id is invalid */ NFS_LAYOUT_FIRST_LAYOUTGET, /* Serialize first layoutget */ NFS_LAYOUT_INODE_FREEING, /* The inode is being freed */ + NFS_LAYOUT_HASHED, /* The layout visible */ }; enum layoutdriver_policy_flags { @@ -203,6 +204,7 @@ struct pnfs_layout_hdr { loff_t plh_lwb; /* last write byte for layoutcommit */ const struct cred *plh_lc_cred; /* layoutcommit cred */ struct inode *plh_inode; + struct rcu_head plh_rcu; }; struct pnfs_device { From 58ac3e59235f1fa174c6e9c5e69111a7b2fa2652 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Thu, 27 Feb 2020 11:24:06 -0500 Subject: [PATCH 078/635] NFSv4/pnfs: Clean up nfs_layout_find_inode() Now that we can rely on just the rcu_read_lock(), remove the clp->cl_lock and clean up. Signed-off-by: Trond Myklebust --- fs/nfs/callback_proc.c | 52 +++++++++++++++++------------------------- 1 file changed, 21 insertions(+), 31 deletions(-) diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c index eb9d035451a2..97084804a953 100644 --- a/fs/nfs/callback_proc.c +++ b/fs/nfs/callback_proc.c @@ -121,33 +121,31 @@ out: */ static struct inode *nfs_layout_find_inode_by_stateid(struct nfs_client *clp, const nfs4_stateid *stateid) + __must_hold(RCU) { struct nfs_server *server; struct inode *inode; struct pnfs_layout_hdr *lo; + rcu_read_lock(); list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { - list_for_each_entry(lo, &server->layouts, plh_layouts) { + list_for_each_entry_rcu(lo, &server->layouts, plh_layouts) { if (!pnfs_layout_is_valid(lo)) continue; if (stateid != NULL && !nfs4_stateid_match_other(stateid, &lo->plh_stateid)) continue; + if (!nfs_sb_active(server->super)) + continue; inode = igrab(lo->plh_inode); - if (!inode) - return ERR_PTR(-EAGAIN); - if (!nfs_sb_active(inode->i_sb)) { - rcu_read_unlock(); - spin_unlock(&clp->cl_lock); - iput(inode); - spin_lock(&clp->cl_lock); - rcu_read_lock(); - return ERR_PTR(-EAGAIN); - } - return inode; + rcu_read_unlock(); + if (inode) + return inode; + nfs_sb_deactive(server->super); + return ERR_PTR(-EAGAIN); } } - + rcu_read_unlock(); return ERR_PTR(-ENOENT); } @@ -165,28 +163,25 @@ static struct inode *nfs_layout_find_inode_by_fh(struct nfs_client *clp, struct inode *inode; struct pnfs_layout_hdr *lo; + rcu_read_lock(); list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { - list_for_each_entry(lo, &server->layouts, plh_layouts) { + list_for_each_entry_rcu(lo, &server->layouts, plh_layouts) { nfsi = NFS_I(lo->plh_inode); if (nfs_compare_fh(fh, &nfsi->fh)) continue; if (nfsi->layout != lo) continue; + if (!nfs_sb_active(server->super)) + continue; inode = igrab(lo->plh_inode); - if (!inode) - return ERR_PTR(-EAGAIN); - if (!nfs_sb_active(inode->i_sb)) { - rcu_read_unlock(); - spin_unlock(&clp->cl_lock); - iput(inode); - spin_lock(&clp->cl_lock); - rcu_read_lock(); - return ERR_PTR(-EAGAIN); - } - return inode; + rcu_read_unlock(); + if (inode) + return inode; + nfs_sb_deactive(server->super); + return ERR_PTR(-EAGAIN); } } - + rcu_read_unlock(); return ERR_PTR(-ENOENT); } @@ -196,14 +191,9 @@ static struct inode *nfs_layout_find_inode(struct nfs_client *clp, { struct inode *inode; - spin_lock(&clp->cl_lock); - rcu_read_lock(); inode = nfs_layout_find_inode_by_stateid(clp, stateid); if (inode == ERR_PTR(-ENOENT)) inode = nfs_layout_find_inode_by_fh(clp, fh); - rcu_read_unlock(); - spin_unlock(&clp->cl_lock); - return inode; } From 3c9e502b59fbd243cfac7cc6c875e432d285102a Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Wed, 26 Feb 2020 19:16:09 -0500 Subject: [PATCH 079/635] NFS: Add a helper nfs_client_for_each_server() Add a helper nfs_client_for_each_server() to iterate through all the filesystems that are attached to a struct nfs_client, and apply a function to all the active ones. Signed-off-by: Trond Myklebust --- fs/nfs/internal.h | 4 +++- fs/nfs/super.c | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h index f80c47d5ff27..3b6fa9edc9b5 100644 --- a/fs/nfs/internal.h +++ b/fs/nfs/internal.h @@ -417,7 +417,9 @@ extern int __init register_nfs_fs(void); extern void __exit unregister_nfs_fs(void); extern bool nfs_sb_active(struct super_block *sb); extern void nfs_sb_deactive(struct super_block *sb); - +extern int nfs_client_for_each_server(struct nfs_client *clp, + int (*fn)(struct nfs_server *, void *), + void *data); /* io.c */ extern void nfs_start_io_read(struct inode *inode); extern void nfs_end_io_read(struct inode *inode); diff --git a/fs/nfs/super.c b/fs/nfs/super.c index dada09b391c6..eb3a85492396 100644 --- a/fs/nfs/super.c +++ b/fs/nfs/super.c @@ -176,6 +176,41 @@ void nfs_sb_deactive(struct super_block *sb) } EXPORT_SYMBOL_GPL(nfs_sb_deactive); +static int __nfs_list_for_each_server(struct list_head *head, + int (*fn)(struct nfs_server *, void *), + void *data) +{ + struct nfs_server *server, *last = NULL; + int ret = 0; + + rcu_read_lock(); + list_for_each_entry_rcu(server, head, client_link) { + if (!nfs_sb_active(server->super)) + continue; + rcu_read_unlock(); + if (last) + nfs_sb_deactive(last->super); + last = server; + ret = fn(server, data); + if (ret) + goto out; + rcu_read_lock(); + } + rcu_read_unlock(); +out: + if (last) + nfs_sb_deactive(last->super); + return ret; +} + +int nfs_client_for_each_server(struct nfs_client *clp, + int (*fn)(struct nfs_server *, void *), + void *data) +{ + return __nfs_list_for_each_server(&clp->cl_superblocks, fn, data); +} +EXPORT_SYMBOL_GPL(nfs_client_for_each_server); + /* * Deliver file system statistics to userspace */ From af3b61bf61319a4808f310d04ef3ff28c224cb1e Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Thu, 27 Feb 2020 08:29:02 -0500 Subject: [PATCH 080/635] NFSv4: Clean up nfs_client_return_marked_delegations() Convert it to use the nfs_client_for_each_server() helper, and make it more efficient by skipping delegations for inodes we know are in the process of being freed. Also improve the efficiency of the cursor by skipping delegations that are being freed. Signed-off-by: Trond Myklebust --- fs/nfs/delegation.c | 159 +++++++++++++++++++++----------------------- 1 file changed, 75 insertions(+), 84 deletions(-) diff --git a/fs/nfs/delegation.c b/fs/nfs/delegation.c index 509b7235b132..19f66d3e58e8 100644 --- a/fs/nfs/delegation.c +++ b/fs/nfs/delegation.c @@ -563,6 +563,79 @@ static bool nfs_delegation_need_return(struct nfs_delegation *delegation) return ret; } +static int nfs_server_return_marked_delegations(struct nfs_server *server, + void __always_unused *data) +{ + struct nfs_delegation *delegation; + struct nfs_delegation *prev; + struct inode *inode; + struct inode *place_holder = NULL; + struct nfs_delegation *place_holder_deleg = NULL; + int err = 0; + +restart: + /* + * To avoid quadratic looping we hold a reference + * to an inode place_holder. Each time we restart, we + * list delegation in the server from the delegations + * of that inode. + * prev is an RCU-protected pointer to a delegation which + * wasn't marked for return and might be a good choice for + * the next place_holder. + */ + prev = NULL; + delegation = NULL; + rcu_read_lock(); + if (place_holder) + delegation = rcu_dereference(NFS_I(place_holder)->delegation); + if (!delegation || delegation != place_holder_deleg) + delegation = list_entry_rcu(server->delegations.next, + struct nfs_delegation, super_list); + list_for_each_entry_from_rcu(delegation, &server->delegations, super_list) { + struct inode *to_put = NULL; + + if (test_bit(NFS_DELEGATION_INODE_FREEING, &delegation->flags)) + continue; + if (!nfs_delegation_need_return(delegation)) { + if (nfs4_is_valid_delegation(delegation, 0)) + prev = delegation; + continue; + } + + if (prev) { + struct inode *tmp = nfs_delegation_grab_inode(prev); + if (tmp) { + to_put = place_holder; + place_holder = tmp; + place_holder_deleg = prev; + } + } + + inode = nfs_delegation_grab_inode(delegation); + if (inode == NULL) { + rcu_read_unlock(); + iput(to_put); + goto restart; + } + delegation = nfs_start_delegation_return_locked(NFS_I(inode)); + rcu_read_unlock(); + + iput(to_put); + + err = nfs_end_delegation_return(inode, delegation, 0); + iput(inode); + cond_resched(); + if (!err) + goto restart; + set_bit(NFS4CLNT_DELEGRETURN, &server->nfs_client->cl_state); + goto out; + } + rcu_read_unlock(); +out: + iput(place_holder); + return err; +} + /** * nfs_client_return_marked_delegations - return previously marked delegations * @clp: nfs_client to process @@ -575,90 +648,8 @@ static bool nfs_delegation_need_return(struct nfs_delegation *delegation) */ int nfs_client_return_marked_delegations(struct nfs_client *clp) { - struct nfs_delegation *delegation; - struct nfs_delegation *prev; - struct nfs_server *server; - struct inode *inode; - struct inode *place_holder = NULL; - struct nfs_delegation *place_holder_deleg = NULL; - int err = 0; - -restart: - /* - * To avoid quadratic looping we hold a reference - * to an inode place_holder. Each time we restart, we - * list nfs_servers from the server of that inode, and - * delegation in the server from the delegations of that - * inode. - * prev is an RCU-protected pointer to a delegation which - * wasn't marked for return and might be a good choice for - * the next place_holder. - */ - rcu_read_lock(); - prev = NULL; - if (place_holder) - server = NFS_SERVER(place_holder); - else - server = list_entry_rcu(clp->cl_superblocks.next, - struct nfs_server, client_link); - list_for_each_entry_from_rcu(server, &clp->cl_superblocks, client_link) { - delegation = NULL; - if (place_holder && server == NFS_SERVER(place_holder)) - delegation = rcu_dereference(NFS_I(place_holder)->delegation); - if (!delegation || delegation != place_holder_deleg) - delegation = list_entry_rcu(server->delegations.next, - struct nfs_delegation, super_list); - list_for_each_entry_from_rcu(delegation, &server->delegations, super_list) { - struct inode *to_put = NULL; - - if (!nfs_delegation_need_return(delegation)) { - prev = delegation; - continue; - } - if (!nfs_sb_active(server->super)) - break; /* continue in outer loop */ - - if (prev) { - struct inode *tmp; - - tmp = nfs_delegation_grab_inode(prev); - if (tmp) { - to_put = place_holder; - place_holder = tmp; - place_holder_deleg = prev; - } - } - - inode = nfs_delegation_grab_inode(delegation); - if (inode == NULL) { - rcu_read_unlock(); - if (to_put) - iput(to_put); - nfs_sb_deactive(server->super); - goto restart; - } - delegation = nfs_start_delegation_return_locked(NFS_I(inode)); - rcu_read_unlock(); - - if (to_put) - iput(to_put); - - err = nfs_end_delegation_return(inode, delegation, 0); - iput(inode); - nfs_sb_deactive(server->super); - cond_resched(); - if (!err) - goto restart; - set_bit(NFS4CLNT_DELEGRETURN, &clp->cl_state); - if (place_holder) - iput(place_holder); - return err; - } - } - rcu_read_unlock(); - if (place_holder) - iput(place_holder); - return 0; + return nfs_client_for_each_server(clp, + nfs_server_return_marked_delegations, NULL); } /** From 1bba38b28344a4126c65886d7337f74ae8f9fe86 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Thu, 27 Feb 2020 09:08:25 -0500 Subject: [PATCH 081/635] NFSv4: Clean up nfs_delegation_reap_unclaimed() Convert nfs_delegation_reap_unclaimed() to use nfs_client_for_each_server() for efficiency. Signed-off-by: Trond Myklebust --- fs/nfs/delegation.c | 78 ++++++++++++++++++++++----------------------- 1 file changed, 38 insertions(+), 40 deletions(-) diff --git a/fs/nfs/delegation.c b/fs/nfs/delegation.c index 19f66d3e58e8..cb03ba99ae51 100644 --- a/fs/nfs/delegation.c +++ b/fs/nfs/delegation.c @@ -1092,6 +1092,42 @@ void nfs_delegation_mark_reclaim(struct nfs_client *clp) rcu_read_unlock(); } +static int nfs_server_reap_unclaimed_delegations(struct nfs_server *server, + void __always_unused *data) +{ + struct nfs_delegation *delegation; + struct inode *inode; +restart: + rcu_read_lock(); +restart_locked: + list_for_each_entry_rcu(delegation, &server->delegations, super_list) { + if (test_bit(NFS_DELEGATION_INODE_FREEING, + &delegation->flags) || + test_bit(NFS_DELEGATION_RETURNING, + &delegation->flags) || + test_bit(NFS_DELEGATION_NEED_RECLAIM, + &delegation->flags) == 0) + continue; + inode = nfs_delegation_grab_inode(delegation); + if (inode == NULL) + goto restart_locked; + delegation = nfs_start_delegation_return_locked(NFS_I(inode)); + rcu_read_unlock(); + if (delegation != NULL) { + if (nfs_detach_delegation(NFS_I(inode), delegation, + server) != NULL) + nfs_free_delegation(delegation); + /* Match nfs_start_delegation_return_locked */ + nfs_put_delegation(delegation); + } + iput(inode); + cond_resched(); + goto restart; + } + rcu_read_unlock(); + return 0; +} + /** * nfs_delegation_reap_unclaimed - reap unclaimed delegations after reboot recovery is done * @clp: nfs_client to process @@ -1099,46 +1135,8 @@ void nfs_delegation_mark_reclaim(struct nfs_client *clp) */ void nfs_delegation_reap_unclaimed(struct nfs_client *clp) { - struct nfs_delegation *delegation; - struct nfs_server *server; - struct inode *inode; - -restart: - rcu_read_lock(); - list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { - list_for_each_entry_rcu(delegation, &server->delegations, - super_list) { - if (test_bit(NFS_DELEGATION_INODE_FREEING, - &delegation->flags) || - test_bit(NFS_DELEGATION_RETURNING, - &delegation->flags) || - test_bit(NFS_DELEGATION_NEED_RECLAIM, - &delegation->flags) == 0) - continue; - if (!nfs_sb_active(server->super)) - break; /* continue in outer loop */ - inode = nfs_delegation_grab_inode(delegation); - if (inode == NULL) { - rcu_read_unlock(); - nfs_sb_deactive(server->super); - goto restart; - } - delegation = nfs_start_delegation_return_locked(NFS_I(inode)); - rcu_read_unlock(); - if (delegation != NULL) { - if (nfs_detach_delegation(NFS_I(inode), delegation, - server) != NULL) - nfs_free_delegation(delegation); - /* Match nfs_start_delegation_return_locked */ - nfs_put_delegation(delegation); - } - iput(inode); - nfs_sb_deactive(server->super); - cond_resched(); - goto restart; - } - } - rcu_read_unlock(); + nfs_client_for_each_server(clp, nfs_server_reap_unclaimed_delegations, + NULL); } static inline bool nfs4_server_rebooted(const struct nfs_client *clp) From 7f156ef0bf45b61385711c5fabf5c4f155a0bb29 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Thu, 27 Feb 2020 09:15:19 -0500 Subject: [PATCH 082/635] NFSv4: Clean up nfs_delegation_reap_expired() Convert to use nfs_client_for_each_server() for efficiency. Signed-off-by: Trond Myklebust --- fs/nfs/delegation.c | 87 ++++++++++++++++++++++----------------------- 1 file changed, 42 insertions(+), 45 deletions(-) diff --git a/fs/nfs/delegation.c b/fs/nfs/delegation.c index cb03ba99ae51..01974f17afc9 100644 --- a/fs/nfs/delegation.c +++ b/fs/nfs/delegation.c @@ -1222,6 +1222,46 @@ nfs_delegation_test_free_expired(struct inode *inode, nfs_remove_bad_delegation(inode, stateid); } +static int nfs_server_reap_expired_delegations(struct nfs_server *server, + void __always_unused *data) +{ + struct nfs_delegation *delegation; + struct inode *inode; + const struct cred *cred; + nfs4_stateid stateid; +restart: + rcu_read_lock(); +restart_locked: + list_for_each_entry_rcu(delegation, &server->delegations, super_list) { + if (test_bit(NFS_DELEGATION_INODE_FREEING, + &delegation->flags) || + test_bit(NFS_DELEGATION_RETURNING, + &delegation->flags) || + test_bit(NFS_DELEGATION_TEST_EXPIRED, + &delegation->flags) == 0) + continue; + inode = nfs_delegation_grab_inode(delegation); + if (inode == NULL) + goto restart_locked; + cred = get_cred_rcu(delegation->cred); + nfs4_stateid_copy(&stateid, &delegation->stateid); + clear_bit(NFS_DELEGATION_TEST_EXPIRED, &delegation->flags); + rcu_read_unlock(); + nfs_delegation_test_free_expired(inode, &stateid, cred); + put_cred(cred); + if (!nfs4_server_rebooted(server->nfs_client)) { + iput(inode); + cond_resched(); + goto restart; + } + nfs_inode_mark_test_expired_delegation(server,inode); + iput(inode); + return -EAGAIN; + } + rcu_read_unlock(); + return 0; +} + /** * nfs_reap_expired_delegations - reap expired delegations * @clp: nfs_client to process @@ -1233,51 +1273,8 @@ nfs_delegation_test_free_expired(struct inode *inode, */ void nfs_reap_expired_delegations(struct nfs_client *clp) { - struct nfs_delegation *delegation; - struct nfs_server *server; - struct inode *inode; - const struct cred *cred; - nfs4_stateid stateid; - -restart: - rcu_read_lock(); - list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { - list_for_each_entry_rcu(delegation, &server->delegations, - super_list) { - if (test_bit(NFS_DELEGATION_INODE_FREEING, - &delegation->flags) || - test_bit(NFS_DELEGATION_RETURNING, - &delegation->flags) || - test_bit(NFS_DELEGATION_TEST_EXPIRED, - &delegation->flags) == 0) - continue; - if (!nfs_sb_active(server->super)) - break; /* continue in outer loop */ - inode = nfs_delegation_grab_inode(delegation); - if (inode == NULL) { - rcu_read_unlock(); - nfs_sb_deactive(server->super); - goto restart; - } - cred = get_cred_rcu(delegation->cred); - nfs4_stateid_copy(&stateid, &delegation->stateid); - clear_bit(NFS_DELEGATION_TEST_EXPIRED, &delegation->flags); - rcu_read_unlock(); - nfs_delegation_test_free_expired(inode, &stateid, cred); - put_cred(cred); - if (nfs4_server_rebooted(clp)) { - nfs_inode_mark_test_expired_delegation(server,inode); - iput(inode); - nfs_sb_deactive(server->super); - return; - } - iput(inode); - nfs_sb_deactive(server->super); - cond_resched(); - goto restart; - } - } - rcu_read_unlock(); + nfs_client_for_each_server(clp, nfs_server_reap_expired_delegations, + NULL); } void nfs_inode_find_delegation_state_and_recover(struct inode *inode, From b5fdf8418c370d69e8b2d3588e0cf2a375ab26c1 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Tue, 18 Feb 2020 15:58:31 -0500 Subject: [PATCH 083/635] NFSv4: Add support for CB_RECALL_ANY for flexfiles layouts When we receive a CB_RECALL_ANY that asks us to return flexfiles layouts, we iterate through all the layouts and look at whether or not there are active open file descriptors that might need them for I/O. If there are no such descriptors, we return the layouts. Signed-off-by: Trond Myklebust --- fs/nfs/callback.h | 4 +- fs/nfs/callback_proc.c | 13 ++++ fs/nfs/nfs4_fs.h | 4 +- fs/nfs/nfs4state.c | 24 ++++++- fs/nfs/nfs4trace.h | 8 ++- fs/nfs/pnfs.c | 148 +++++++++++++++++++++++++++++++++++++---- fs/nfs/pnfs.h | 3 + 7 files changed, 186 insertions(+), 18 deletions(-) diff --git a/fs/nfs/callback.h b/fs/nfs/callback.h index 549350259840..6a2033131c06 100644 --- a/fs/nfs/callback.h +++ b/fs/nfs/callback.h @@ -127,7 +127,9 @@ extern __be32 nfs4_callback_sequence(void *argp, void *resp, #define RCA4_TYPE_MASK_OBJ_LAYOUT_MAX 9 #define RCA4_TYPE_MASK_OTHER_LAYOUT_MIN 12 #define RCA4_TYPE_MASK_OTHER_LAYOUT_MAX 15 -#define RCA4_TYPE_MASK_ALL 0xf31f +#define PNFS_FF_RCA4_TYPE_MASK_READ 16 +#define PNFS_FF_RCA4_TYPE_MASK_RW 17 +#define RCA4_TYPE_MASK_ALL 0x3f31f struct cb_recallanyargs { uint32_t craa_objs_to_keep; diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c index 97084804a953..e61dbc9b86ae 100644 --- a/fs/nfs/callback_proc.c +++ b/fs/nfs/callback_proc.c @@ -597,6 +597,7 @@ __be32 nfs4_callback_recallany(void *argp, void *resp, struct cb_recallanyargs *args = argp; __be32 status; fmode_t flags = 0; + bool schedule_manager = false; status = cpu_to_be32(NFS4ERR_OP_NOT_IN_SESSION); if (!cps->clp) /* set in cb_sequence */ @@ -619,6 +620,18 @@ __be32 nfs4_callback_recallany(void *argp, void *resp, if (args->craa_type_mask & BIT(RCA4_TYPE_MASK_FILE_LAYOUT)) pnfs_recall_all_layouts(cps->clp); + + if (args->craa_type_mask & BIT(PNFS_FF_RCA4_TYPE_MASK_READ)) { + set_bit(NFS4CLNT_RECALL_ANY_LAYOUT_READ, &cps->clp->cl_state); + schedule_manager = true; + } + if (args->craa_type_mask & BIT(PNFS_FF_RCA4_TYPE_MASK_RW)) { + set_bit(NFS4CLNT_RECALL_ANY_LAYOUT_RW, &cps->clp->cl_state); + schedule_manager = true; + } + if (schedule_manager) + nfs4_schedule_state_manager(cps->clp); + out: dprintk("%s: exit with status = %d\n", __func__, ntohl(status)); return status; diff --git a/fs/nfs/nfs4_fs.h b/fs/nfs/nfs4_fs.h index 8be1ba7c62bb..2b7f6dcd2eb8 100644 --- a/fs/nfs/nfs4_fs.h +++ b/fs/nfs/nfs4_fs.h @@ -42,7 +42,9 @@ enum nfs4_client_state { NFS4CLNT_LEASE_MOVED, NFS4CLNT_DELEGATION_EXPIRED, NFS4CLNT_RUN_MANAGER, - NFS4CLNT_DELEGRETURN_RUNNING, + NFS4CLNT_RECALL_RUNNING, + NFS4CLNT_RECALL_ANY_LAYOUT_READ, + NFS4CLNT_RECALL_ANY_LAYOUT_RW, }; #define NFS4_RENEW_TIMEOUT 0x01 diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c index f7723d221945..ac93715c05a4 100644 --- a/fs/nfs/nfs4state.c +++ b/fs/nfs/nfs4state.c @@ -2524,6 +2524,21 @@ static int nfs4_bind_conn_to_session(struct nfs_client *clp) } return 0; } + +static void nfs4_layoutreturn_any_run(struct nfs_client *clp) +{ + int iomode = 0; + + if (test_and_clear_bit(NFS4CLNT_RECALL_ANY_LAYOUT_READ, &clp->cl_state)) + iomode += IOMODE_READ; + if (test_and_clear_bit(NFS4CLNT_RECALL_ANY_LAYOUT_RW, &clp->cl_state)) + iomode += IOMODE_RW; + /* Note: IOMODE_READ + IOMODE_RW == IOMODE_ANY */ + if (iomode) { + pnfs_layout_return_unused_byclid(clp, iomode); + set_bit(NFS4CLNT_RUN_MANAGER, &clp->cl_state); + } +} #else /* CONFIG_NFS_V4_1 */ static int nfs4_reset_session(struct nfs_client *clp) { return 0; } @@ -2531,6 +2546,10 @@ static int nfs4_bind_conn_to_session(struct nfs_client *clp) { return 0; } + +static void nfs4_layoutreturn_any_run(struct nfs_client *clp) +{ +} #endif /* CONFIG_NFS_V4_1 */ static void nfs4_state_manager(struct nfs_client *clp) @@ -2635,12 +2654,13 @@ static void nfs4_state_manager(struct nfs_client *clp) nfs4_end_drain_session(clp); nfs4_clear_state_manager_bit(clp); - if (!test_and_set_bit(NFS4CLNT_DELEGRETURN_RUNNING, &clp->cl_state)) { + if (!test_and_set_bit(NFS4CLNT_RECALL_RUNNING, &clp->cl_state)) { if (test_and_clear_bit(NFS4CLNT_DELEGRETURN, &clp->cl_state)) { nfs_client_return_marked_delegations(clp); set_bit(NFS4CLNT_RUN_MANAGER, &clp->cl_state); } - clear_bit(NFS4CLNT_DELEGRETURN_RUNNING, &clp->cl_state); + nfs4_layoutreturn_any_run(clp); + clear_bit(NFS4CLNT_RECALL_RUNNING, &clp->cl_state); } /* Did we race with an attempt to give us more work? */ diff --git a/fs/nfs/nfs4trace.h b/fs/nfs/nfs4trace.h index 1e97e5e04cb4..543541173a3d 100644 --- a/fs/nfs/nfs4trace.h +++ b/fs/nfs/nfs4trace.h @@ -584,7 +584,9 @@ TRACE_DEFINE_ENUM(NFS4CLNT_MOVED); TRACE_DEFINE_ENUM(NFS4CLNT_LEASE_MOVED); TRACE_DEFINE_ENUM(NFS4CLNT_DELEGATION_EXPIRED); TRACE_DEFINE_ENUM(NFS4CLNT_RUN_MANAGER); -TRACE_DEFINE_ENUM(NFS4CLNT_DELEGRETURN_RUNNING); +TRACE_DEFINE_ENUM(NFS4CLNT_RECALL_RUNNING); +TRACE_DEFINE_ENUM(NFS4CLNT_RECALL_ANY_LAYOUT_READ); +TRACE_DEFINE_ENUM(NFS4CLNT_RECALL_ANY_LAYOUT_RW); #define show_nfs4_clp_state(state) \ __print_flags(state, "|", \ @@ -605,7 +607,9 @@ TRACE_DEFINE_ENUM(NFS4CLNT_DELEGRETURN_RUNNING); { NFS4CLNT_LEASE_MOVED, "LEASE_MOVED" }, \ { NFS4CLNT_DELEGATION_EXPIRED, "DELEGATION_EXPIRED" }, \ { NFS4CLNT_RUN_MANAGER, "RUN_MANAGER" }, \ - { NFS4CLNT_DELEGRETURN_RUNNING, "DELEGRETURN_RUNNING" }) + { NFS4CLNT_RECALL_RUNNING, "RECALL_RUNNING" }, \ + { NFS4CLNT_RECALL_ANY_LAYOUT_READ, "RECALL_ANY_LAYOUT_READ" }, \ + { NFS4CLNT_RECALL_ANY_LAYOUT_RW, "RECALL_ANY_LAYOUT_RW" }) TRACE_EVENT(nfs4_state_mgr, TP_PROTO( diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index 268e7b9ff54e..6b25117fca5f 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -309,6 +309,16 @@ pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo) } } +static struct inode * +pnfs_grab_inode_layout_hdr(struct pnfs_layout_hdr *lo) +{ + struct inode *inode = igrab(lo->plh_inode); + if (inode) + return inode; + set_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags); + return NULL; +} + static void pnfs_set_plh_return_info(struct pnfs_layout_hdr *lo, enum pnfs_iomode iomode, u32 seq) @@ -782,7 +792,7 @@ pnfs_layout_bulk_destroy_byserver_locked(struct nfs_client *clp, /* If the sb is being destroyed, just bail */ if (!nfs_sb_active(server->super)) break; - inode = igrab(lo->plh_inode); + inode = pnfs_grab_inode_layout_hdr(lo); if (inode != NULL) { if (test_and_clear_bit(NFS_LAYOUT_HASHED, &lo->plh_flags)) list_del_rcu(&lo->plh_layouts); @@ -795,7 +805,6 @@ pnfs_layout_bulk_destroy_byserver_locked(struct nfs_client *clp, } else { rcu_read_unlock(); spin_unlock(&clp->cl_lock); - set_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags); } nfs_sb_deactive(server->super); spin_lock(&clp->cl_lock); @@ -2434,29 +2443,26 @@ pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo, return -ENOENT; } -void pnfs_error_mark_layout_for_return(struct inode *inode, - struct pnfs_layout_segment *lseg) +static void +pnfs_mark_layout_for_return(struct inode *inode, + const struct pnfs_layout_range *range) { - struct pnfs_layout_hdr *lo = NFS_I(inode)->layout; - struct pnfs_layout_range range = { - .iomode = lseg->pls_range.iomode, - .offset = 0, - .length = NFS4_MAX_UINT64, - }; + struct pnfs_layout_hdr *lo; bool return_now = false; spin_lock(&inode->i_lock); + lo = NFS_I(inode)->layout; if (!pnfs_layout_is_valid(lo)) { spin_unlock(&inode->i_lock); return; } - pnfs_set_plh_return_info(lo, range.iomode, 0); + pnfs_set_plh_return_info(lo, range->iomode, 0); /* * mark all matching lsegs so that we are sure to have no live * segments at hand when sending layoutreturn. See pnfs_put_lseg() * for how it works. */ - if (pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, &range, 0) != -EBUSY) { + if (pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, range, 0) != -EBUSY) { nfs4_stateid stateid; enum pnfs_iomode iomode; @@ -2469,8 +2475,126 @@ void pnfs_error_mark_layout_for_return(struct inode *inode, nfs_commit_inode(inode, 0); } } + +void pnfs_error_mark_layout_for_return(struct inode *inode, + struct pnfs_layout_segment *lseg) +{ + struct pnfs_layout_range range = { + .iomode = lseg->pls_range.iomode, + .offset = 0, + .length = NFS4_MAX_UINT64, + }; + + pnfs_mark_layout_for_return(inode, &range); +} EXPORT_SYMBOL_GPL(pnfs_error_mark_layout_for_return); +static bool +pnfs_layout_can_be_returned(struct pnfs_layout_hdr *lo) +{ + return pnfs_layout_is_valid(lo) && + !test_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags) && + !test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags); +} + +static struct pnfs_layout_segment * +pnfs_find_first_lseg(struct pnfs_layout_hdr *lo, + const struct pnfs_layout_range *range, + enum pnfs_iomode iomode) +{ + struct pnfs_layout_segment *lseg; + + list_for_each_entry(lseg, &lo->plh_segs, pls_list) { + if (!test_bit(NFS_LSEG_VALID, &lseg->pls_flags)) + continue; + if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) + continue; + if (lseg->pls_range.iomode != iomode && iomode != IOMODE_ANY) + continue; + if (pnfs_lseg_range_intersecting(&lseg->pls_range, range)) + return lseg; + } + return NULL; +} + +/* Find open file states whose mode matches that of the range */ +static bool +pnfs_should_return_unused_layout(struct pnfs_layout_hdr *lo, + const struct pnfs_layout_range *range) +{ + struct list_head *head; + struct nfs_open_context *ctx; + fmode_t mode = 0; + + if (!pnfs_layout_can_be_returned(lo) || + !pnfs_find_first_lseg(lo, range, range->iomode)) + return false; + + head = &NFS_I(lo->plh_inode)->open_files; + list_for_each_entry_rcu(ctx, head, list) { + if (ctx->state) + mode |= ctx->state->state & (FMODE_READ|FMODE_WRITE); + } + + switch (range->iomode) { + default: + break; + case IOMODE_READ: + mode &= ~FMODE_WRITE; + break; + case IOMODE_RW: + if (pnfs_find_first_lseg(lo, range, IOMODE_READ)) + mode &= ~FMODE_READ; + } + return mode == 0; +} + +static int +pnfs_layout_return_unused_byserver(struct nfs_server *server, void *data) +{ + const struct pnfs_layout_range *range = data; + struct pnfs_layout_hdr *lo; + struct inode *inode; +restart: + rcu_read_lock(); + list_for_each_entry_rcu(lo, &server->layouts, plh_layouts) { + if (!pnfs_layout_can_be_returned(lo) || + test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) + continue; + inode = lo->plh_inode; + spin_lock(&inode->i_lock); + if (!pnfs_should_return_unused_layout(lo, range)) { + spin_unlock(&inode->i_lock); + continue; + } + spin_unlock(&inode->i_lock); + inode = pnfs_grab_inode_layout_hdr(lo); + if (!inode) + continue; + rcu_read_unlock(); + pnfs_mark_layout_for_return(inode, range); + iput(inode); + cond_resched(); + goto restart; + } + rcu_read_unlock(); + return 0; +} + +void +pnfs_layout_return_unused_byclid(struct nfs_client *clp, + enum pnfs_iomode iomode) +{ + struct pnfs_layout_range range = { + .iomode = iomode, + .offset = 0, + .length = NFS4_MAX_UINT64, + }; + + nfs_client_for_each_server(clp, pnfs_layout_return_unused_byserver, + &range); +} + void pnfs_generic_pg_check_layout(struct nfs_pageio_descriptor *pgio) { diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h index 8df9aa02d336..7bfb6970134a 100644 --- a/fs/nfs/pnfs.h +++ b/fs/nfs/pnfs.h @@ -329,6 +329,9 @@ int pnfs_write_done_resend_to_mds(struct nfs_pgio_header *); struct nfs4_threshold *pnfs_mdsthreshold_alloc(void); void pnfs_error_mark_layout_for_return(struct inode *inode, struct pnfs_layout_segment *lseg); +void pnfs_layout_return_unused_byclid(struct nfs_client *clp, + enum pnfs_iomode iomode); + /* nfs4_deviceid_flags */ enum { NFS_DEVICEID_INVALID = 0, /* set when MDS clientid recalled */ From 68e9a2463d08f870abf8edfe9125b7738f7a63ce Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Fri, 28 Feb 2020 23:44:14 +0000 Subject: [PATCH 084/635] SUNRPC: remove redundant assignments to variable status The variable status is being initialized with a value that is never read and it is being updated later with a new value. The initialization is redundant and can be removed. Addresses-Coverity: ("Unused value") Signed-off-by: Colin Ian King Signed-off-by: Trond Myklebust --- net/sunrpc/xprtsock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c index d86c664ea6af..c9e039961e29 100644 --- a/net/sunrpc/xprtsock.c +++ b/net/sunrpc/xprtsock.c @@ -1970,7 +1970,7 @@ static int xs_local_setup_socket(struct sock_xprt *transport) struct rpc_xprt *xprt = &transport->xprt; struct file *filp; struct socket *sock; - int status = -EIO; + int status; status = __sock_create(xprt->xprt_net, AF_LOCAL, SOCK_STREAM, 0, &sock, 1); From eb095c14030fbb07fcc61c64b6b39cc297a429c6 Mon Sep 17 00:00:00 2001 From: Zhouyi Zhou Date: Fri, 6 Mar 2020 03:45:26 +0000 Subject: [PATCH 085/635] NFS:remove redundant call to nfs_do_access In function nfs_permission: 1. the rcu_read_lock and rcu_read_unlock around nfs_do_access is unnecessary because the rcu critical data structure is already protected in subsidiary function nfs_access_get_cached_rcu. No other data structure needs rcu_read_lock in nfs_do_access. 2. call nfs_do_access once is enough, because: 2-1. when mask has MAY_NOT_BLOCK bit The second call to nfs_do_access will not happen. 2-2. when mask has no MAY_NOT_BLOCK bit The second call to nfs_do_access will happen if res == -ECHILD, which means the first nfs_do_access goes out after statement if (!may_block). The second call to nfs_do_access will go through this procedure once again except continue the work after if (!may_block). But above work can be performed by only one call to nfs_do_access without mangling the mask flag. Tested in x86_64 Signed-off-by: Zhouyi Zhou Signed-off-by: Trond Myklebust --- fs/nfs/dir.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index f22366f350a9..b3a675da84a2 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -2759,14 +2759,7 @@ force_lookup: if (!NFS_PROTO(inode)->access) goto out_notsup; - /* Always try fast lookups first */ - rcu_read_lock(); - res = nfs_do_access(inode, cred, mask|MAY_NOT_BLOCK); - rcu_read_unlock(); - if (res == -ECHILD && !(mask & MAY_NOT_BLOCK)) { - /* Fast lookup failed, try the slow way */ - res = nfs_do_access(inode, cred, mask); - } + res = nfs_do_access(inode, cred, mask); out: if (!res && (mask & MAY_EXEC)) res = nfs_execute_ok(inode, mask); From f5fdf1243fb750598b46305dd03c553949cfa14f Mon Sep 17 00:00:00 2001 From: Murphy Zhou Date: Fri, 14 Feb 2020 22:34:09 +0800 Subject: [PATCH 086/635] NFSv4.2: error out when relink swapfile This fixes xfstests generic/356 failure on NFSv4.2. Signed-off-by: Murphy Zhou Signed-off-by: Trond Myklebust --- fs/nfs/nfs4file.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fs/nfs/nfs4file.c b/fs/nfs/nfs4file.c index 1297919e0fce..8e5d6223ddd3 100644 --- a/fs/nfs/nfs4file.c +++ b/fs/nfs/nfs4file.c @@ -252,6 +252,9 @@ static loff_t nfs42_remap_file_range(struct file *src_file, loff_t src_off, if (remap_flags & ~REMAP_FILE_ADVISORY) return -EINVAL; + if (IS_SWAPFILE(dst_inode) || IS_SWAPFILE(src_inode)) + return -ETXTBSY; + /* check alignment w.r.t. clone_blksize */ ret = -EINVAL; if (bs) { From 5601cda82b0cc88c7d87b612c8b8b920de801be3 Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Mon, 9 Mar 2020 13:24:42 -0500 Subject: [PATCH 087/635] nfs: Replace zero-length array with flexible-array member The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertently introduced[3] to the codebase from now on. Also, notice that, dynamic memory allocations won't be affected by this change: "Flexible array members have incomplete type, and so the sizeof operator may not be applied. As a quirk of the original implementation of zero-length arrays, sizeof evaluates to zero."[1] This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Signed-off-by: Trond Myklebust --- fs/nfs/dir.c | 2 +- fs/nfs/nfs4proc.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index b3a675da84a2..f14184d0ba82 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -141,7 +141,7 @@ struct nfs_cache_array { int size; int eof_index; u64 last_cookie; - struct nfs_cache_array_entry array[0]; + struct nfs_cache_array_entry array[]; }; typedef struct { diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 22dca497c5de..905c7d1bc277 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -5550,7 +5550,7 @@ unwind: struct nfs4_cached_acl { int cached; size_t len; - char data[0]; + char data[]; }; static void nfs4_set_cached_acl(struct inode *inode, struct nfs4_cached_acl *acl) From 4047aa909c4a40fceebc36fff708d465a4d3c6e2 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Wed, 11 Mar 2020 11:21:07 -0400 Subject: [PATCH 088/635] sunrpc: Fix gss_unwrap_resp_integ() again xdr_buf_read_mic() tries to find unused contiguous space in a received xdr_buf in order to linearize the checksum for the call to gss_verify_mic. However, the corner cases in this code are numerous and we seem to keep missing them. I've just hit yet another buffer overrun related to it. This overrun is at the end of xdr_buf_read_mic(): 1284 if (buf->tail[0].iov_len != 0) 1285 mic->data = buf->tail[0].iov_base + buf->tail[0].iov_len; 1286 else 1287 mic->data = buf->head[0].iov_base + buf->head[0].iov_len; 1288 __read_bytes_from_xdr_buf(&subbuf, mic->data, mic->len); 1289 return 0; This logic assumes the transport has set the length of the tail based on the size of the received message. base + len is then supposed to be off the end of the message but still within the actual buffer. In fact, the length of the tail is set by the upper layer when the Call is encoded so that the end of the tail is actually the end of the allocated buffer itself. This causes the logic above to set mic->data to point past the end of the receive buffer. The "mic->data = head" arm of this if statement is no less fragile. As near as I can tell, this has been a problem forever. I'm not sure that minimizing au_rslack recently changed this pathology much. So instead, let's use a more straightforward approach: kmalloc a separate buffer to linearize the checksum. This is similar to how gss_validate() currently works. Coming back to this code, I had some trouble understanding what was going on. So I've cleaned up the variable naming and added a few comments that point back to the XDR definition in RFC 2203 to help guide future spelunkers, including myself. As an added clean up, the functionality that was in xdr_buf_read_mic() is folded directly into gss_unwrap_resp_integ(), as that is its only caller. Signed-off-by: Chuck Lever Reviewed-by: Benjamin Coddington Signed-off-by: Trond Myklebust --- net/sunrpc/auth_gss/auth_gss.c | 77 +++++++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 19 deletions(-) diff --git a/net/sunrpc/auth_gss/auth_gss.c b/net/sunrpc/auth_gss/auth_gss.c index 24ca861815b1..98b2c8bc8f40 100644 --- a/net/sunrpc/auth_gss/auth_gss.c +++ b/net/sunrpc/auth_gss/auth_gss.c @@ -1934,35 +1934,69 @@ gss_unwrap_resp_auth(struct rpc_cred *cred) return 0; } +/* + * RFC 2203, Section 5.3.2.2 + * + * struct rpc_gss_integ_data { + * opaque databody_integ<>; + * opaque checksum<>; + * }; + * + * struct rpc_gss_data_t { + * unsigned int seq_num; + * proc_req_arg_t arg; + * }; + */ static int gss_unwrap_resp_integ(struct rpc_task *task, struct rpc_cred *cred, struct gss_cl_ctx *ctx, struct rpc_rqst *rqstp, struct xdr_stream *xdr) { - struct xdr_buf integ_buf, *rcv_buf = &rqstp->rq_rcv_buf; - u32 data_offset, mic_offset, integ_len, maj_stat; + struct xdr_buf gss_data, *rcv_buf = &rqstp->rq_rcv_buf; struct rpc_auth *auth = cred->cr_auth; + u32 len, offset, seqno, maj_stat; struct xdr_netobj mic; - __be32 *p; + int ret; - p = xdr_inline_decode(xdr, 2 * sizeof(*p)); - if (unlikely(!p)) + ret = -EIO; + mic.data = NULL; + + /* opaque databody_integ<>; */ + if (xdr_stream_decode_u32(xdr, &len)) goto unwrap_failed; - integ_len = be32_to_cpup(p++); - if (integ_len & 3) + if (len & 3) goto unwrap_failed; - data_offset = (u8 *)(p) - (u8 *)rcv_buf->head[0].iov_base; - mic_offset = integ_len + data_offset; - if (mic_offset > rcv_buf->len) + offset = rcv_buf->len - xdr_stream_remaining(xdr); + if (xdr_stream_decode_u32(xdr, &seqno)) goto unwrap_failed; - if (be32_to_cpup(p) != rqstp->rq_seqno) + if (seqno != rqstp->rq_seqno) goto bad_seqno; + if (xdr_buf_subsegment(rcv_buf, &gss_data, offset, len)) + goto unwrap_failed; - if (xdr_buf_subsegment(rcv_buf, &integ_buf, data_offset, integ_len)) + /* + * The xdr_stream now points to the beginning of the + * upper layer payload, to be passed below to + * rpcauth_unwrap_resp_decode(). The checksum, which + * follows the upper layer payload in @rcv_buf, is + * located and parsed without updating the xdr_stream. + */ + + /* opaque checksum<>; */ + offset += len; + if (xdr_decode_word(rcv_buf, offset, &len)) goto unwrap_failed; - if (xdr_buf_read_mic(rcv_buf, &mic, mic_offset)) + offset += sizeof(__be32); + if (offset + len > rcv_buf->len) goto unwrap_failed; - maj_stat = gss_verify_mic(ctx->gc_gss_ctx, &integ_buf, &mic); + mic.len = len; + mic.data = kmalloc(len, GFP_NOFS); + if (!mic.data) + goto unwrap_failed; + if (read_bytes_from_xdr_buf(rcv_buf, offset, mic.data, mic.len)) + goto unwrap_failed; + + maj_stat = gss_verify_mic(ctx->gc_gss_ctx, &gss_data, &mic); if (maj_stat == GSS_S_CONTEXT_EXPIRED) clear_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags); if (maj_stat != GSS_S_COMPLETE) @@ -1970,16 +2004,21 @@ gss_unwrap_resp_integ(struct rpc_task *task, struct rpc_cred *cred, auth->au_rslack = auth->au_verfsize + 2 + 1 + XDR_QUADLEN(mic.len); auth->au_ralign = auth->au_verfsize + 2; - return 0; + ret = 0; + +out: + kfree(mic.data); + return ret; + unwrap_failed: trace_rpcgss_unwrap_failed(task); - return -EIO; + goto out; bad_seqno: - trace_rpcgss_bad_seqno(task, rqstp->rq_seqno, be32_to_cpup(p)); - return -EIO; + trace_rpcgss_bad_seqno(task, rqstp->rq_seqno, seqno); + goto out; bad_mic: trace_rpcgss_verify_mic(task, maj_stat); - return -EIO; + goto out; } static int From 8d6bda7f23a9b3ef1d7e386f01924c37f18fe771 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Wed, 11 Mar 2020 11:21:12 -0400 Subject: [PATCH 089/635] SUNRPC: Remove xdr_buf_read_mic() Clean up: this function is no longer used. Signed-off-by: Chuck Lever Reviewed-by: Benjamin Coddington Signed-off-by: Trond Myklebust --- include/linux/sunrpc/xdr.h | 1 - net/sunrpc/xdr.c | 55 -------------------------------------- 2 files changed, 56 deletions(-) diff --git a/include/linux/sunrpc/xdr.h b/include/linux/sunrpc/xdr.h index b41f34977995..8a6dd5bd6748 100644 --- a/include/linux/sunrpc/xdr.h +++ b/include/linux/sunrpc/xdr.h @@ -184,7 +184,6 @@ xdr_adjust_iovec(struct kvec *iov, __be32 *p) extern void xdr_shift_buf(struct xdr_buf *, size_t); extern void xdr_buf_from_iov(struct kvec *, struct xdr_buf *); extern int xdr_buf_subsegment(struct xdr_buf *, struct xdr_buf *, unsigned int, unsigned int); -extern int xdr_buf_read_mic(struct xdr_buf *, struct xdr_netobj *, unsigned int); extern int read_bytes_from_xdr_buf(struct xdr_buf *, unsigned int, void *, unsigned int); extern int write_bytes_to_xdr_buf(struct xdr_buf *, unsigned int, void *, unsigned int); diff --git a/net/sunrpc/xdr.c b/net/sunrpc/xdr.c index e5497dc2475b..15b58c5144f9 100644 --- a/net/sunrpc/xdr.c +++ b/net/sunrpc/xdr.c @@ -1235,61 +1235,6 @@ xdr_encode_word(struct xdr_buf *buf, unsigned int base, u32 obj) } EXPORT_SYMBOL_GPL(xdr_encode_word); -/** - * xdr_buf_read_mic() - obtain the address of the GSS mic from xdr buf - * @buf: pointer to buffer containing a mic - * @mic: on success, returns the address of the mic - * @offset: the offset in buf where mic may be found - * - * This function may modify the xdr buf if the mic is found to be straddling - * a boundary between head, pages, and tail. On success the mic can be read - * from the address returned. There is no need to free the mic. - * - * Return: Success returns 0, otherwise an integer error. - */ -int xdr_buf_read_mic(struct xdr_buf *buf, struct xdr_netobj *mic, unsigned int offset) -{ - struct xdr_buf subbuf; - unsigned int boundary; - - if (xdr_decode_word(buf, offset, &mic->len)) - return -EFAULT; - offset += 4; - - /* Is the mic partially in the head? */ - boundary = buf->head[0].iov_len; - if (offset < boundary && (offset + mic->len) > boundary) - xdr_shift_buf(buf, boundary - offset); - - /* Is the mic partially in the pages? */ - boundary += buf->page_len; - if (offset < boundary && (offset + mic->len) > boundary) - xdr_shrink_pagelen(buf, boundary - offset); - - if (xdr_buf_subsegment(buf, &subbuf, offset, mic->len)) - return -EFAULT; - - /* Is the mic contained entirely in the head? */ - mic->data = subbuf.head[0].iov_base; - if (subbuf.head[0].iov_len == mic->len) - return 0; - /* ..or is the mic contained entirely in the tail? */ - mic->data = subbuf.tail[0].iov_base; - if (subbuf.tail[0].iov_len == mic->len) - return 0; - - /* Find a contiguous area in @buf to hold all of @mic */ - if (mic->len > buf->buflen - buf->len) - return -ENOMEM; - if (buf->tail[0].iov_len != 0) - mic->data = buf->tail[0].iov_base + buf->tail[0].iov_len; - else - mic->data = buf->head[0].iov_base + buf->head[0].iov_len; - __read_bytes_from_xdr_buf(&subbuf, mic->data, mic->len); - return 0; -} -EXPORT_SYMBOL_GPL(xdr_buf_read_mic); - /* Returns 0 on success, or else a negative error code. */ static int xdr_xcode_array2(struct xdr_buf *buf, unsigned int base, From d162372af306d22ee57d8a723dd4552e932304a9 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Wed, 11 Mar 2020 11:21:17 -0400 Subject: [PATCH 090/635] SUNRPC: Trim stack utilization in the wrap and unwrap paths By preventing compiler inlining of the integrity and privacy helpers, stack utilization for the common case (authentication only) goes way down. Signed-off-by: Chuck Lever Signed-off-by: Trond Myklebust --- net/sunrpc/auth_gss/auth_gss.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/net/sunrpc/auth_gss/auth_gss.c b/net/sunrpc/auth_gss/auth_gss.c index 98b2c8bc8f40..45707a306f20 100644 --- a/net/sunrpc/auth_gss/auth_gss.c +++ b/net/sunrpc/auth_gss/auth_gss.c @@ -1724,8 +1724,9 @@ bad_mic: goto out; } -static int gss_wrap_req_integ(struct rpc_cred *cred, struct gss_cl_ctx *ctx, - struct rpc_task *task, struct xdr_stream *xdr) +static noinline_for_stack int +gss_wrap_req_integ(struct rpc_cred *cred, struct gss_cl_ctx *ctx, + struct rpc_task *task, struct xdr_stream *xdr) { struct rpc_rqst *rqstp = task->tk_rqstp; struct xdr_buf integ_buf, *snd_buf = &rqstp->rq_snd_buf; @@ -1816,8 +1817,9 @@ out: return -EAGAIN; } -static int gss_wrap_req_priv(struct rpc_cred *cred, struct gss_cl_ctx *ctx, - struct rpc_task *task, struct xdr_stream *xdr) +static noinline_for_stack int +gss_wrap_req_priv(struct rpc_cred *cred, struct gss_cl_ctx *ctx, + struct rpc_task *task, struct xdr_stream *xdr) { struct rpc_rqst *rqstp = task->tk_rqstp; struct xdr_buf *snd_buf = &rqstp->rq_snd_buf; @@ -1947,7 +1949,7 @@ gss_unwrap_resp_auth(struct rpc_cred *cred) * proc_req_arg_t arg; * }; */ -static int +static noinline_for_stack int gss_unwrap_resp_integ(struct rpc_task *task, struct rpc_cred *cred, struct gss_cl_ctx *ctx, struct rpc_rqst *rqstp, struct xdr_stream *xdr) @@ -2021,7 +2023,7 @@ bad_mic: goto out; } -static int +static noinline_for_stack int gss_unwrap_resp_priv(struct rpc_task *task, struct rpc_cred *cred, struct gss_cl_ctx *ctx, struct rpc_rqst *rqstp, struct xdr_stream *xdr) From 3faae9813b9c831356d457c0a5d847b7528bf97a Mon Sep 17 00:00:00 2001 From: Tina Zhang Date: Mon, 2 Mar 2020 16:31:30 +0800 Subject: [PATCH 091/635] drm/i915/gvt: Add some regs to force-to-nonpriv whitelist Those regs are added in order to slove the following complains: [70811.201818] gvt: vgpu(1) Invalid FORCE_NONPRIV write 2341 at offset 24d8 [70811.201825] gvt: vgpu(1) Invalid FORCE_NONPRIV write 2351 at offset 24dc [70811.201831] gvt: vgpu(1) Invalid FORCE_NONPRIV write 10000d82 at offset 24e0 [70811.201837] gvt: vgpu(1) Invalid FORCE_NONPRIV write 10064844 at offset 24e4 So solve them by adding the required regs to the whitelist. Signed-off-by: Tina Zhang Acked-by: Zhenyu Wang Signed-off-by: Zhenyu Wang Link: http://patchwork.freedesktop.org/patch/msgid/20200302083130.17831-1-tina.zhang@intel.com --- drivers/gpu/drm/i915/gvt/handlers.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c index 0182e2a5acff..2faf50e1b051 100644 --- a/drivers/gpu/drm/i915/gvt/handlers.c +++ b/drivers/gpu/drm/i915/gvt/handlers.c @@ -462,11 +462,14 @@ static int pipeconf_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, return 0; } -/* ascendingly sorted */ +/* sorted in ascending order */ static i915_reg_t force_nonpriv_white_list[] = { + _MMIO(0xd80), GEN9_CS_DEBUG_MODE1, //_MMIO(0x20ec) GEN9_CTX_PREEMPT_REG,//_MMIO(0x2248) - PS_INVOCATION_COUNT,//_MMIO(0x2348) + CL_PRIMITIVES_COUNT, //_MMIO(0x2340) + PS_INVOCATION_COUNT, //_MMIO(0x2348) + PS_DEPTH_COUNT, //_MMIO(0x2350) GEN8_CS_CHICKEN1,//_MMIO(0x2580) _MMIO(0x2690), _MMIO(0x2694), @@ -491,6 +494,7 @@ static i915_reg_t force_nonpriv_white_list[] = { _MMIO(0xe18c), _MMIO(0xe48c), _MMIO(0xe5f4), + _MMIO(0x64844), }; /* a simple bsearch */ From 75db1a5b2aea2a7528bd4a99882d59dd4cdb36cc Mon Sep 17 00:00:00 2001 From: Tina Zhang Date: Tue, 17 Mar 2020 17:36:15 +0800 Subject: [PATCH 092/635] drm/i915/gvt: Fix display port type issue According to the vbt provided by GVT-g, the display output type should be display port, instead of DVI. Currently, GVT-g display model only supports display port emulation, not DVI or HDMI. This patch fixes this by telling guest i915 the right output type supported by GVT-g. v3: make port_b and port_c use dp type connector too. (Zhenyu) v2: refine the commit message. (Zhenyu) Signed-off-by: Tina Zhang Acked-by: Zhenyu Wang Signed-off-by: Zhenyu Wang Link: http://patchwork.freedesktop.org/patch/msgid/20200317093615.10538-1-tina.zhang@intel.com --- drivers/gpu/drm/i915/gvt/display.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/i915/gvt/display.c b/drivers/gpu/drm/i915/gvt/display.c index 6e5c9885d9fe..a83df2f84eb9 100644 --- a/drivers/gpu/drm/i915/gvt/display.c +++ b/drivers/gpu/drm/i915/gvt/display.c @@ -221,7 +221,7 @@ static void emulate_monitor_status_change(struct intel_vgpu *vgpu) ~(TRANS_DDI_BPC_MASK | TRANS_DDI_MODE_SELECT_MASK | TRANS_DDI_PORT_MASK); vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) |= - (TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DVI | + (TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DP_SST | (PORT_B << TRANS_DDI_PORT_SHIFT) | TRANS_DDI_FUNC_ENABLE); if (IS_BROADWELL(dev_priv)) { @@ -241,7 +241,7 @@ static void emulate_monitor_status_change(struct intel_vgpu *vgpu) ~(TRANS_DDI_BPC_MASK | TRANS_DDI_MODE_SELECT_MASK | TRANS_DDI_PORT_MASK); vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) |= - (TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DVI | + (TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DP_SST | (PORT_C << TRANS_DDI_PORT_SHIFT) | TRANS_DDI_FUNC_ENABLE); if (IS_BROADWELL(dev_priv)) { @@ -261,7 +261,7 @@ static void emulate_monitor_status_change(struct intel_vgpu *vgpu) ~(TRANS_DDI_BPC_MASK | TRANS_DDI_MODE_SELECT_MASK | TRANS_DDI_PORT_MASK); vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) |= - (TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DVI | + (TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DP_SST | (PORT_D << TRANS_DDI_PORT_SHIFT) | TRANS_DDI_FUNC_ENABLE); if (IS_BROADWELL(dev_priv)) { From 300b124fcf6ad2cd99a7b721e0f096785e0a3134 Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Tue, 19 Nov 2019 15:36:14 +0200 Subject: [PATCH 093/635] ovl: fix value of i_ino for lower hardlink corner case Commit 6dde1e42f497 ("ovl: make i_ino consistent with st_ino in more cases"), relaxed the condition nfs_export=on in order to set the value of i_ino to xino map of real ino. Specifically, it also relaxed the pre-condition that index=on for consistent i_ino. This opened the corner case of lower hardlink in ovl_get_inode(), which calls ovl_fill_inode() with ino=0 and then ovl_init_inode() is called to set i_ino to lower real ino without the xino mapping. Pass the correct values of ino;fsid in this case to ovl_fill_inode(), so it can initialize i_ino correctly. Fixes: 6dde1e42f497 ("ovl: make i_ino consistent with st_ino in more ...") Signed-off-by: Amir Goldstein Signed-off-by: Miklos Szeredi --- fs/overlayfs/inode.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fs/overlayfs/inode.c b/fs/overlayfs/inode.c index 79e8994e3bc1..3f993c114829 100644 --- a/fs/overlayfs/inode.c +++ b/fs/overlayfs/inode.c @@ -891,7 +891,7 @@ struct inode *ovl_get_inode(struct super_block *sb, struct dentry *lowerdentry = lowerpath ? lowerpath->dentry : NULL; bool bylower = ovl_hash_bylower(sb, upperdentry, lowerdentry, oip->index); - int fsid = bylower ? oip->lowerpath->layer->fsid : 0; + int fsid = bylower ? lowerpath->layer->fsid : 0; bool is_dir, metacopy = false; unsigned long ino = 0; int err = oip->newinode ? -EEXIST : -ENOMEM; @@ -941,6 +941,8 @@ struct inode *ovl_get_inode(struct super_block *sb, err = -ENOMEM; goto out_err; } + ino = realinode->i_ino; + fsid = lowerpath->layer->fsid; } ovl_fill_inode(inode, realinode->i_mode, realinode->i_rdev, ino, fsid); ovl_inode_init(inode, upperdentry, lowerdentry, oip->lowerdata); From 735c907d7b7df501e951ba07134b9f6f989a94e4 Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Tue, 19 Nov 2019 17:14:55 +0200 Subject: [PATCH 094/635] ovl: fix out of date comment and unreachable code ovl_inode_update() is no longer called from create object code path. Fixes: 01b39dcc9568 ("ovl: use inode_insert5() to hash a newly...") Signed-off-by: Amir Goldstein Signed-off-by: Miklos Szeredi --- fs/overlayfs/inode.c | 8 +++++--- fs/overlayfs/util.c | 2 -- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/fs/overlayfs/inode.c b/fs/overlayfs/inode.c index 3f993c114829..98a605b54e2d 100644 --- a/fs/overlayfs/inode.c +++ b/fs/overlayfs/inode.c @@ -571,9 +571,11 @@ static void ovl_fill_inode(struct inode *inode, umode_t mode, dev_t rdev, * bits to encode layer), set the same value used for st_ino to i_ino, * so inode number exposed via /proc/locks and a like will be * consistent with d_ino and st_ino values. An i_ino value inconsistent - * with d_ino also causes nfsd readdirplus to fail. When called from - * ovl_new_inode(), ino arg is 0, so i_ino will be updated to real - * upper inode i_ino on ovl_inode_init() or ovl_inode_update(). + * with d_ino also causes nfsd readdirplus to fail. + * + * When called from ovl_create_object() => ovl_new_inode(), with + * ino = 0, i_ino will be updated to consistent value later on in + * ovl_get_inode() => ovl_fill_inode(). */ if (ovl_same_dev(inode->i_sb)) { inode->i_ino = ino; diff --git a/fs/overlayfs/util.c b/fs/overlayfs/util.c index 042f7eb4f7f4..6584e5800a59 100644 --- a/fs/overlayfs/util.c +++ b/fs/overlayfs/util.c @@ -416,8 +416,6 @@ void ovl_inode_update(struct inode *inode, struct dentry *upperdentry) smp_wmb(); OVL_I(inode)->__upperdentry = upperdentry; if (inode_unhashed(inode)) { - if (!inode->i_ino) - inode->i_ino = upperinode->i_ino; inode->i_private = upperinode; __insert_inode_hash(inode, (unsigned long) upperinode); } From 2effc5c25d55b92b89303292128540618f182671 Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Tue, 19 Nov 2019 17:49:17 +0200 Subject: [PATCH 095/635] ovl: factor out helper ovl_get_root() Allocates and initializes the root dentry and inode. Signed-off-by: Amir Goldstein Signed-off-by: Miklos Szeredi --- fs/overlayfs/super.c | 44 +++++++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index ac967f1cb6e5..ae9e8b0877f3 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -1589,6 +1589,34 @@ static int ovl_check_overlapping_layers(struct super_block *sb, return 0; } +static struct dentry *ovl_get_root(struct super_block *sb, + struct dentry *upperdentry, + struct ovl_entry *oe) +{ + struct dentry *root; + + root = d_make_root(ovl_new_inode(sb, S_IFDIR, 0)); + if (!root) + return NULL; + + root->d_fsdata = oe; + + if (upperdentry) { + ovl_dentry_set_upper_alias(root); + if (ovl_is_impuredir(upperdentry)) + ovl_set_flag(OVL_IMPURE, d_inode(root)); + } + + /* Root is always merge -> can have whiteouts */ + ovl_set_flag(OVL_WHITEOUTS, d_inode(root)); + ovl_dentry_set_flag(OVL_E_CONNECTED, root); + ovl_set_upperdata(d_inode(root)); + ovl_inode_init(d_inode(root), upperdentry, ovl_dentry_lower(root), + NULL); + + return root; +} + static int ovl_fill_super(struct super_block *sb, void *data, int silent) { struct path upperpath = { }; @@ -1710,25 +1738,11 @@ static int ovl_fill_super(struct super_block *sb, void *data, int silent) sb->s_flags |= SB_POSIXACL; err = -ENOMEM; - root_dentry = d_make_root(ovl_new_inode(sb, S_IFDIR, 0)); + root_dentry = ovl_get_root(sb, upperpath.dentry, oe); if (!root_dentry) goto out_free_oe; - root_dentry->d_fsdata = oe; - mntput(upperpath.mnt); - if (upperpath.dentry) { - ovl_dentry_set_upper_alias(root_dentry); - if (ovl_is_impuredir(upperpath.dentry)) - ovl_set_flag(OVL_IMPURE, d_inode(root_dentry)); - } - - /* Root is always merge -> can have whiteouts */ - ovl_set_flag(OVL_WHITEOUTS, d_inode(root_dentry)); - ovl_dentry_set_flag(OVL_E_CONNECTED, root_dentry); - ovl_set_upperdata(d_inode(root_dentry)); - ovl_inode_init(d_inode(root_dentry), upperpath.dentry, - ovl_dentry_lower(root_dentry), NULL); sb->s_root = root_dentry; From 62c832ed4e1b67cc65d909005aae5ce7031a2b38 Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Tue, 19 Nov 2019 15:31:46 +0200 Subject: [PATCH 096/635] ovl: simplify i_ino initialization Move i_ino initialization to ovl_inode_init() to avoid the dance of setting i_ino in ovl_fill_inode() sometimes on the first call and sometimes on the seconds call. Signed-off-by: Amir Goldstein Signed-off-by: Miklos Szeredi --- fs/overlayfs/inode.c | 35 ++++++++++++++++++++++++++--------- fs/overlayfs/overlayfs.h | 4 ++-- fs/overlayfs/super.c | 13 +++++++++++-- fs/overlayfs/util.c | 18 ------------------ 4 files changed, 39 insertions(+), 31 deletions(-) diff --git a/fs/overlayfs/inode.c b/fs/overlayfs/inode.c index 98a605b54e2d..13219a5864c4 100644 --- a/fs/overlayfs/inode.c +++ b/fs/overlayfs/inode.c @@ -561,8 +561,7 @@ static inline void ovl_lockdep_annotate_inode_mutex_key(struct inode *inode) #endif } -static void ovl_fill_inode(struct inode *inode, umode_t mode, dev_t rdev, - unsigned long ino, int fsid) +static void ovl_map_ino(struct inode *inode, unsigned long ino, int fsid) { int xinobits = ovl_xino_bits(inode->i_sb); @@ -572,10 +571,6 @@ static void ovl_fill_inode(struct inode *inode, umode_t mode, dev_t rdev, * so inode number exposed via /proc/locks and a like will be * consistent with d_ino and st_ino values. An i_ino value inconsistent * with d_ino also causes nfsd readdirplus to fail. - * - * When called from ovl_create_object() => ovl_new_inode(), with - * ino = 0, i_ino will be updated to consistent value later on in - * ovl_get_inode() => ovl_fill_inode(). */ if (ovl_same_dev(inode->i_sb)) { inode->i_ino = ino; @@ -584,6 +579,28 @@ static void ovl_fill_inode(struct inode *inode, umode_t mode, dev_t rdev, } else { inode->i_ino = get_next_ino(); } +} + +void ovl_inode_init(struct inode *inode, struct ovl_inode_params *oip, + unsigned long ino, int fsid) +{ + struct inode *realinode; + + if (oip->upperdentry) + OVL_I(inode)->__upperdentry = oip->upperdentry; + if (oip->lowerpath && oip->lowerpath->dentry) + OVL_I(inode)->lower = igrab(d_inode(oip->lowerpath->dentry)); + if (oip->lowerdata) + OVL_I(inode)->lowerdata = igrab(d_inode(oip->lowerdata)); + + realinode = ovl_inode_real(inode); + ovl_copyattr(realinode, inode); + ovl_copyflags(realinode, inode); + ovl_map_ino(inode, ino, fsid); +} + +static void ovl_fill_inode(struct inode *inode, umode_t mode, dev_t rdev) +{ inode->i_mode = mode; inode->i_flags |= S_NOCMTIME; #ifdef CONFIG_FS_POSIX_ACL @@ -721,7 +738,7 @@ struct inode *ovl_new_inode(struct super_block *sb, umode_t mode, dev_t rdev) inode = new_inode(sb); if (inode) - ovl_fill_inode(inode, mode, rdev, 0, 0); + ovl_fill_inode(inode, mode, rdev); return inode; } @@ -946,8 +963,8 @@ struct inode *ovl_get_inode(struct super_block *sb, ino = realinode->i_ino; fsid = lowerpath->layer->fsid; } - ovl_fill_inode(inode, realinode->i_mode, realinode->i_rdev, ino, fsid); - ovl_inode_init(inode, upperdentry, lowerdentry, oip->lowerdata); + ovl_fill_inode(inode, realinode->i_mode, realinode->i_rdev); + ovl_inode_init(inode, oip, ino, fsid); if (upperdentry && ovl_is_impuredir(upperdentry)) ovl_set_flag(OVL_IMPURE, inode); diff --git a/fs/overlayfs/overlayfs.h b/fs/overlayfs/overlayfs.h index 3d3f2b8bdae5..442fbffe9d7e 100644 --- a/fs/overlayfs/overlayfs.h +++ b/fs/overlayfs/overlayfs.h @@ -264,8 +264,6 @@ void ovl_set_upperdata(struct inode *inode); bool ovl_redirect_dir(struct super_block *sb); const char *ovl_dentry_get_redirect(struct dentry *dentry); void ovl_dentry_set_redirect(struct dentry *dentry, const char *redirect); -void ovl_inode_init(struct inode *inode, struct dentry *upperdentry, - struct dentry *lowerdentry, struct dentry *lowerdata); void ovl_inode_update(struct inode *inode, struct dentry *upperdentry); void ovl_dir_modified(struct dentry *dentry, bool impurity); u64 ovl_dentry_version_get(struct dentry *dentry); @@ -410,6 +408,8 @@ struct ovl_inode_params { char *redirect; struct dentry *lowerdata; }; +void ovl_inode_init(struct inode *inode, struct ovl_inode_params *oip, + unsigned long ino, int fsid); struct inode *ovl_new_inode(struct super_block *sb, umode_t mode, dev_t rdev); struct inode *ovl_lookup_inode(struct super_block *sb, struct dentry *real, bool is_upper); diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index ae9e8b0877f3..4db51c7f48e1 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -1594,6 +1594,13 @@ static struct dentry *ovl_get_root(struct super_block *sb, struct ovl_entry *oe) { struct dentry *root; + struct ovl_path *lowerpath = &oe->lowerstack[0]; + unsigned long ino = d_inode(lowerpath->dentry)->i_ino; + int fsid = lowerpath->layer->fsid; + struct ovl_inode_params oip = { + .upperdentry = upperdentry, + .lowerpath = lowerpath, + }; root = d_make_root(ovl_new_inode(sb, S_IFDIR, 0)); if (!root) @@ -1602,6 +1609,9 @@ static struct dentry *ovl_get_root(struct super_block *sb, root->d_fsdata = oe; if (upperdentry) { + /* Root inode uses upper st_ino/i_ino */ + ino = d_inode(upperdentry)->i_ino; + fsid = 0; ovl_dentry_set_upper_alias(root); if (ovl_is_impuredir(upperdentry)) ovl_set_flag(OVL_IMPURE, d_inode(root)); @@ -1611,8 +1621,7 @@ static struct dentry *ovl_get_root(struct super_block *sb, ovl_set_flag(OVL_WHITEOUTS, d_inode(root)); ovl_dentry_set_flag(OVL_E_CONNECTED, root); ovl_set_upperdata(d_inode(root)); - ovl_inode_init(d_inode(root), upperdentry, ovl_dentry_lower(root), - NULL); + ovl_inode_init(d_inode(root), &oip, ino, fsid); return root; } diff --git a/fs/overlayfs/util.c b/fs/overlayfs/util.c index 6584e5800a59..f318e4811e32 100644 --- a/fs/overlayfs/util.c +++ b/fs/overlayfs/util.c @@ -386,24 +386,6 @@ void ovl_dentry_set_redirect(struct dentry *dentry, const char *redirect) oi->redirect = redirect; } -void ovl_inode_init(struct inode *inode, struct dentry *upperdentry, - struct dentry *lowerdentry, struct dentry *lowerdata) -{ - struct inode *realinode = d_inode(upperdentry ?: lowerdentry); - - if (upperdentry) - OVL_I(inode)->__upperdentry = upperdentry; - if (lowerdentry) - OVL_I(inode)->lower = igrab(d_inode(lowerdentry)); - if (lowerdata) - OVL_I(inode)->lowerdata = igrab(d_inode(lowerdata)); - - ovl_copyattr(realinode, inode); - ovl_copyflags(realinode, inode); - if (!inode->i_ino) - inode->i_ino = realinode->i_ino; -} - void ovl_inode_update(struct inode *inode, struct dentry *upperdentry) { struct inode *upperinode = d_inode(upperdentry); From 4c494bd582fa23d2d70851e958da6cee1dc161cb Mon Sep 17 00:00:00 2001 From: Miklos Szeredi Date: Tue, 17 Mar 2020 15:04:22 +0100 Subject: [PATCH 097/635] ovl: document permission model Add missing piece of documentation regarding how permissions are checked in overlayfs. Signed-off-by: Miklos Szeredi --- Documentation/filesystems/overlayfs.rst | 44 +++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/Documentation/filesystems/overlayfs.rst b/Documentation/filesystems/overlayfs.rst index e443be7928db..e398fdf7353e 100644 --- a/Documentation/filesystems/overlayfs.rst +++ b/Documentation/filesystems/overlayfs.rst @@ -248,6 +248,50 @@ overlay filesystem (though an operation on the name of the file such as rename or unlink will of course be noticed and handled). +Permission model +---------------- + +Permission checking in the overlay filesystem follows these principles: + + 1) permission check SHOULD return the same result before and after copy up + + 2) task creating the overlay mount MUST NOT gain additional privileges + + 3) non-mounting task MAY gain additional privileges through the overlay, + compared to direct access on underlying lower or upper filesystems + +This is achieved by performing two permission checks on each access + + a) check if current task is allowed access based on local DAC (owner, + group, mode and posix acl), as well as MAC checks + + b) check if mounting task would be allowed real operation on lower or + upper layer based on underlying filesystem permissions, again including + MAC checks + +Check (a) ensures consistency (1) since owner, group, mode and posix acls +are copied up. On the other hand it can result in server enforced +permissions (used by NFS, for example) being ignored (3). + +Check (b) ensures that no task gains permissions to underlying layers that +the mounting task does not have (2). This also means that it is possible +to create setups where the consistency rule (1) does not hold; normally, +however, the mounting task will have sufficient privileges to perform all +operations. + +Another way to demonstrate this model is drawing parallels between + + mount -t overlay overlay -olowerdir=/lower,upperdir=/upper,... /merged + +and + + cp -a /lower /upper + mount --bind /upper /merged + +The resulting access permissions should be the same. The difference is in +the time of copy (on-demand vs. up-front). + + Multiple lower layers --------------------- From c61ca5572508516b41039aecb23d936466076950 Mon Sep 17 00:00:00 2001 From: Miklos Szeredi Date: Tue, 17 Mar 2020 15:04:22 +0100 Subject: [PATCH 098/635] ovl: ignore failure to copy up unknown xattrs This issue came up with NFSv4 as the lower layer, which generates "system.nfs4_acl" xattrs (even for plain old unix permissions). Prior to this patch this prevented copy-up from succeeding. The overlayfs permission model mandates that permissions are checked locally for the task and remotely for the mounter(*). NFS4 ACLs are not supported by the Linux kernel currently, hence they cannot be enforced locally. Which means it is indifferent whether this attribute is copied or not. Generalize this to any xattr that is not used in access checking (i.e. it's not a POSIX ACL and not in the "security." namespace). Incidentally, best effort copying of xattrs seems to also be the behavior of "cp -a", which is what overlayfs tries to mimic. (*) Documentation/filesystems/overlayfs.txt#Permission model Signed-off-by: Miklos Szeredi --- fs/overlayfs/copy_up.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/fs/overlayfs/copy_up.c b/fs/overlayfs/copy_up.c index 9fc47c2e078d..9709cf22cab3 100644 --- a/fs/overlayfs/copy_up.c +++ b/fs/overlayfs/copy_up.c @@ -36,6 +36,13 @@ static int ovl_ccup_get(char *buf, const struct kernel_param *param) module_param_call(check_copy_up, ovl_ccup_set, ovl_ccup_get, NULL, 0644); MODULE_PARM_DESC(check_copy_up, "Obsolete; does nothing"); +static bool ovl_must_copy_xattr(const char *name) +{ + return !strcmp(name, XATTR_POSIX_ACL_ACCESS) || + !strcmp(name, XATTR_POSIX_ACL_DEFAULT) || + !strncmp(name, XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN); +} + int ovl_copy_xattr(struct dentry *old, struct dentry *new) { ssize_t list_size, size, value_size = 0; @@ -107,8 +114,13 @@ retry: continue; /* Discard */ } error = vfs_setxattr(new, name, value, size, 0); - if (error) - break; + if (error) { + if (error != -EOPNOTSUPP || ovl_must_copy_xattr(name)) + break; + + /* Ignore failure to copy unknown xattrs */ + error = 0; + } } kfree(value); out: From 3bb7df928a7dca0ab5ae143ecffbd8859850cd08 Mon Sep 17 00:00:00 2001 From: Miklos Szeredi Date: Tue, 17 Mar 2020 15:04:22 +0100 Subject: [PATCH 099/635] ovl: restructure dentry revalidation Use a common loop for plain and weak revalidation. This will aid doing revalidation on upper layer. This patch doesn't change behavior. Signed-off-by: Miklos Szeredi --- fs/overlayfs/super.c | 59 ++++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index 4db51c7f48e1..a05bf1179ce0 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -113,45 +113,46 @@ bug: return dentry; } -static int ovl_dentry_revalidate(struct dentry *dentry, unsigned int flags) +static int ovl_revalidate_real(struct dentry *d, unsigned int flags, bool weak) +{ + int ret = 1; + + if (weak) { + if (d->d_flags & DCACHE_OP_WEAK_REVALIDATE) + ret = d->d_op->d_weak_revalidate(d, flags); + } else if (d->d_flags & DCACHE_OP_REVALIDATE) { + ret = d->d_op->d_revalidate(d, flags); + if (!ret) { + if (!(flags & LOOKUP_RCU)) + d_invalidate(d); + ret = -ESTALE; + } + } + return ret; +} + +static int ovl_dentry_revalidate_common(struct dentry *dentry, + unsigned int flags, bool weak) { struct ovl_entry *oe = dentry->d_fsdata; unsigned int i; int ret = 1; - for (i = 0; i < oe->numlower; i++) { - struct dentry *d = oe->lowerstack[i].dentry; - - if (d->d_flags & DCACHE_OP_REVALIDATE) { - ret = d->d_op->d_revalidate(d, flags); - if (ret < 0) - return ret; - if (!ret) { - if (!(flags & LOOKUP_RCU)) - d_invalidate(d); - return -ESTALE; - } - } + for (i = 0; ret > 0 && i < oe->numlower; i++) { + ret = ovl_revalidate_real(oe->lowerstack[i].dentry, flags, + weak); } - return 1; + return ret; +} + +static int ovl_dentry_revalidate(struct dentry *dentry, unsigned int flags) +{ + return ovl_dentry_revalidate_common(dentry, flags, false); } static int ovl_dentry_weak_revalidate(struct dentry *dentry, unsigned int flags) { - struct ovl_entry *oe = dentry->d_fsdata; - unsigned int i; - int ret = 1; - - for (i = 0; i < oe->numlower; i++) { - struct dentry *d = oe->lowerstack[i].dentry; - - if (d->d_flags & DCACHE_OP_WEAK_REVALIDATE) { - ret = d->d_op->d_weak_revalidate(d, flags); - if (ret <= 0) - break; - } - } - return ret; + return ovl_dentry_revalidate_common(dentry, flags, true); } static const struct dentry_operations ovl_dentry_operations = { From 7925dad839e70ab43140e26d6b93e807b5969f05 Mon Sep 17 00:00:00 2001 From: Miklos Szeredi Date: Tue, 17 Mar 2020 15:04:22 +0100 Subject: [PATCH 100/635] ovl: separate detection of remote upper layer from stacked overlay Following patch will allow remote as upper layer, but not overlay stacked on upper layer. Separate the two concepts. This patch is doesn't change behavior. Signed-off-by: Miklos Szeredi --- fs/overlayfs/namei.c | 3 ++- fs/overlayfs/super.c | 14 +++++++------- fs/overlayfs/util.c | 3 +-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/fs/overlayfs/namei.c b/fs/overlayfs/namei.c index ed9e129fae04..a5b998a93a24 100644 --- a/fs/overlayfs/namei.c +++ b/fs/overlayfs/namei.c @@ -845,7 +845,8 @@ struct dentry *ovl_lookup(struct inode *dir, struct dentry *dentry, if (err) goto out; - if (upperdentry && unlikely(ovl_dentry_remote(upperdentry))) { + if (upperdentry && (upperdentry->d_flags & DCACHE_OP_REAL || + unlikely(ovl_dentry_remote(upperdentry)))) { dput(upperdentry); err = -EREMOTE; goto out; diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index a05bf1179ce0..bc35cd6233ee 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -752,13 +752,13 @@ static int ovl_mount_dir(const char *name, struct path *path) ovl_unescape(tmp); err = ovl_mount_dir_noesc(tmp, path); - if (!err) - if (ovl_dentry_remote(path->dentry)) { - pr_err("filesystem on '%s' not supported as upperdir\n", - tmp); - path_put_init(path); - err = -EINVAL; - } + if (!err && (ovl_dentry_remote(path->dentry) || + path->dentry->d_flags & DCACHE_OP_REAL)) { + pr_err("filesystem on '%s' not supported as upperdir\n", + tmp); + path_put_init(path); + err = -EINVAL; + } kfree(tmp); } return err; diff --git a/fs/overlayfs/util.c b/fs/overlayfs/util.c index f318e4811e32..997c31509c91 100644 --- a/fs/overlayfs/util.c +++ b/fs/overlayfs/util.c @@ -93,8 +93,7 @@ struct ovl_entry *ovl_alloc_entry(unsigned int numlower) bool ovl_dentry_remote(struct dentry *dentry) { return dentry->d_flags & - (DCACHE_OP_REVALIDATE | DCACHE_OP_WEAK_REVALIDATE | - DCACHE_OP_REAL); + (DCACHE_OP_REVALIDATE | DCACHE_OP_WEAK_REVALIDATE); } bool ovl_dentry_weird(struct dentry *dentry) From f428884456192230ba1573e4028fb9282ffa2bb7 Mon Sep 17 00:00:00 2001 From: Miklos Szeredi Date: Tue, 17 Mar 2020 15:04:22 +0100 Subject: [PATCH 101/635] ovl: decide if revalidate needed on a per-dentry basis Allow completely skipping ->revalidate() on a per-dentry basis, in case the underlying layers used for a dentry do not themselves have ->revalidate(). E.g. negative overlay dentry has no underlying layers, hence revalidate is unnecessary. Or if lower layer is remote but overlay dentry is pure-upper, then can skip revalidate. The following places need to update whether the dentry needs revalidate or not: - fill-super (root dentry) - lookup - create - fh_to_dentry Signed-off-by: Miklos Szeredi --- fs/overlayfs/dir.c | 3 +++ fs/overlayfs/export.c | 2 ++ fs/overlayfs/namei.c | 3 +++ fs/overlayfs/overlayfs.h | 2 ++ fs/overlayfs/super.c | 21 +++++---------------- fs/overlayfs/util.c | 15 +++++++++++++++ 6 files changed, 30 insertions(+), 16 deletions(-) diff --git a/fs/overlayfs/dir.c b/fs/overlayfs/dir.c index 8e57d5372b8f..b3471ef51440 100644 --- a/fs/overlayfs/dir.c +++ b/fs/overlayfs/dir.c @@ -243,6 +243,9 @@ static int ovl_instantiate(struct dentry *dentry, struct inode *inode, ovl_dir_modified(dentry->d_parent, false); ovl_dentry_set_upper_alias(dentry); + ovl_dentry_update_reval(dentry, newdentry, + DCACHE_OP_REVALIDATE | DCACHE_OP_WEAK_REVALIDATE); + if (!hardlink) { /* * ovl_obtain_alias() can be called after ovl_create_real() diff --git a/fs/overlayfs/export.c b/fs/overlayfs/export.c index 6f54d70cef27..a58b3d9b06b9 100644 --- a/fs/overlayfs/export.c +++ b/fs/overlayfs/export.c @@ -324,6 +324,8 @@ static struct dentry *ovl_obtain_alias(struct super_block *sb, if (upper_alias) ovl_dentry_set_upper_alias(dentry); } + ovl_dentry_update_reval(dentry, upper, + DCACHE_OP_REVALIDATE | DCACHE_OP_WEAK_REVALIDATE); return d_instantiate_anon(dentry, inode); diff --git a/fs/overlayfs/namei.c b/fs/overlayfs/namei.c index a5b998a93a24..76e61cc27822 100644 --- a/fs/overlayfs/namei.c +++ b/fs/overlayfs/namei.c @@ -1077,6 +1077,9 @@ struct dentry *ovl_lookup(struct inode *dir, struct dentry *dentry, goto out_free_oe; } + ovl_dentry_update_reval(dentry, upperdentry, + DCACHE_OP_REVALIDATE | DCACHE_OP_WEAK_REVALIDATE); + revert_creds(old_cred); if (origin_path) { dput(origin_path->dentry); diff --git a/fs/overlayfs/overlayfs.h b/fs/overlayfs/overlayfs.h index 442fbffe9d7e..70d17040ab4d 100644 --- a/fs/overlayfs/overlayfs.h +++ b/fs/overlayfs/overlayfs.h @@ -230,6 +230,8 @@ bool ovl_index_all(struct super_block *sb); bool ovl_verify_lower(struct super_block *sb); struct ovl_entry *ovl_alloc_entry(unsigned int numlower); bool ovl_dentry_remote(struct dentry *dentry); +void ovl_dentry_update_reval(struct dentry *dentry, struct dentry *upperdentry, + unsigned int mask); bool ovl_dentry_weird(struct dentry *dentry); enum ovl_path_type ovl_path_type(struct dentry *dentry); void ovl_path_upper(struct dentry *dentry, struct path *path); diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index bc35cd6233ee..d944ab46b69e 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -158,11 +158,6 @@ static int ovl_dentry_weak_revalidate(struct dentry *dentry, unsigned int flags) static const struct dentry_operations ovl_dentry_operations = { .d_release = ovl_dentry_release, .d_real = ovl_d_real, -}; - -static const struct dentry_operations ovl_reval_dentry_operations = { - .d_release = ovl_dentry_release, - .d_real = ovl_d_real, .d_revalidate = ovl_dentry_revalidate, .d_weak_revalidate = ovl_dentry_weak_revalidate, }; @@ -779,7 +774,7 @@ static int ovl_check_namelen(struct path *path, struct ovl_fs *ofs, } static int ovl_lower_dir(const char *name, struct path *path, - struct ovl_fs *ofs, int *stack_depth, bool *remote) + struct ovl_fs *ofs, int *stack_depth) { int fh_type; int err; @@ -794,9 +789,6 @@ static int ovl_lower_dir(const char *name, struct path *path, *stack_depth = max(*stack_depth, path->mnt->mnt_sb->s_stack_depth); - if (ovl_dentry_remote(path->dentry)) - *remote = true; - /* * The inodes index feature and NFS export need to encode and decode * file handles, so they require that all layers support them. @@ -1441,7 +1433,6 @@ static struct ovl_entry *ovl_get_lowerstack(struct super_block *sb, char *lowertmp, *lower; struct path *stack = NULL; unsigned int stacklen, numlower = 0, i; - bool remote = false; struct ovl_entry *oe; err = -ENOMEM; @@ -1473,7 +1464,7 @@ static struct ovl_entry *ovl_get_lowerstack(struct super_block *sb, lower = lowertmp; for (numlower = 0; numlower < stacklen; numlower++) { err = ovl_lower_dir(lower, &stack[numlower], ofs, - &sb->s_stack_depth, &remote); + &sb->s_stack_depth); if (err) goto out_err; @@ -1501,11 +1492,6 @@ static struct ovl_entry *ovl_get_lowerstack(struct super_block *sb, oe->lowerstack[i].layer = &ofs->layers[i+1]; } - if (remote) - sb->s_d_op = &ovl_reval_dentry_operations; - else - sb->s_d_op = &ovl_dentry_operations; - out: for (i = 0; i < numlower; i++) path_put(&stack[i]); @@ -1623,6 +1609,7 @@ static struct dentry *ovl_get_root(struct super_block *sb, ovl_dentry_set_flag(OVL_E_CONNECTED, root); ovl_set_upperdata(d_inode(root)); ovl_inode_init(d_inode(root), &oip, ino, fsid); + ovl_dentry_update_reval(root, upperdentry, DCACHE_OP_WEAK_REVALIDATE); return root; } @@ -1636,6 +1623,8 @@ static int ovl_fill_super(struct super_block *sb, void *data, int silent) struct cred *cred; int err; + sb->s_d_op = &ovl_dentry_operations; + err = -ENOMEM; ofs = kzalloc(sizeof(struct ovl_fs), GFP_KERNEL); if (!ofs) diff --git a/fs/overlayfs/util.c b/fs/overlayfs/util.c index 997c31509c91..1cd805ba1eeb 100644 --- a/fs/overlayfs/util.c +++ b/fs/overlayfs/util.c @@ -96,6 +96,21 @@ bool ovl_dentry_remote(struct dentry *dentry) (DCACHE_OP_REVALIDATE | DCACHE_OP_WEAK_REVALIDATE); } +void ovl_dentry_update_reval(struct dentry *dentry, struct dentry *upperdentry, + unsigned int mask) +{ + struct ovl_entry *oe = OVL_E(dentry); + unsigned int i, flags = 0; + + for (i = 0; i < oe->numlower; i++) + flags |= oe->lowerstack[i].dentry->d_flags; + + spin_lock(&dentry->d_lock); + dentry->d_flags &= ~mask; + dentry->d_flags |= flags & mask; + spin_unlock(&dentry->d_lock); +} + bool ovl_dentry_weird(struct dentry *dentry) { return dentry->d_flags & (DCACHE_NEED_AUTOMOUNT | From bccece1ead368149f90e05ad104dfbfda4422f5d Mon Sep 17 00:00:00 2001 From: Miklos Szeredi Date: Tue, 17 Mar 2020 15:04:22 +0100 Subject: [PATCH 102/635] ovl: allow remote upper No reason to prevent upper layer being a remote filesystem. Do the revalidation in that case, just as we already do for lower layers. This lets virtiofs be used as upper layer, which appears to be a real use case. Signed-off-by: Miklos Szeredi --- fs/overlayfs/namei.c | 3 +-- fs/overlayfs/super.c | 8 ++++++-- fs/overlayfs/util.c | 2 ++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/fs/overlayfs/namei.c b/fs/overlayfs/namei.c index 76e61cc27822..0db23baf98e7 100644 --- a/fs/overlayfs/namei.c +++ b/fs/overlayfs/namei.c @@ -845,8 +845,7 @@ struct dentry *ovl_lookup(struct inode *dir, struct dentry *dentry, if (err) goto out; - if (upperdentry && (upperdentry->d_flags & DCACHE_OP_REAL || - unlikely(ovl_dentry_remote(upperdentry)))) { + if (upperdentry && upperdentry->d_flags & DCACHE_OP_REAL) { dput(upperdentry); err = -EREMOTE; goto out; diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index d944ab46b69e..e8965fd1a608 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -135,9 +135,14 @@ static int ovl_dentry_revalidate_common(struct dentry *dentry, unsigned int flags, bool weak) { struct ovl_entry *oe = dentry->d_fsdata; + struct dentry *upper; unsigned int i; int ret = 1; + upper = ovl_dentry_upper(dentry); + if (upper) + ret = ovl_revalidate_real(upper, flags, weak); + for (i = 0; ret > 0 && i < oe->numlower; i++) { ret = ovl_revalidate_real(oe->lowerstack[i].dentry, flags, weak); @@ -747,8 +752,7 @@ static int ovl_mount_dir(const char *name, struct path *path) ovl_unescape(tmp); err = ovl_mount_dir_noesc(tmp, path); - if (!err && (ovl_dentry_remote(path->dentry) || - path->dentry->d_flags & DCACHE_OP_REAL)) { + if (!err && path->dentry->d_flags & DCACHE_OP_REAL) { pr_err("filesystem on '%s' not supported as upperdir\n", tmp); path_put_init(path); diff --git a/fs/overlayfs/util.c b/fs/overlayfs/util.c index 1cd805ba1eeb..36b60788ee47 100644 --- a/fs/overlayfs/util.c +++ b/fs/overlayfs/util.c @@ -102,6 +102,8 @@ void ovl_dentry_update_reval(struct dentry *dentry, struct dentry *upperdentry, struct ovl_entry *oe = OVL_E(dentry); unsigned int i, flags = 0; + if (upperdentry) + flags |= upperdentry->d_flags; for (i = 0; i < oe->numlower; i++) flags |= oe->lowerstack[i].dentry->d_flags; From cad218ab3320789ea4621c15e56b8114a4160bab Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Thu, 20 Feb 2020 09:00:19 +0200 Subject: [PATCH 103/635] ovl: check if upper fs supports RENAME_WHITEOUT As with other required upper fs features, we only warn if support is missing to avoid breaking existing sub-optimal setups. Signed-off-by: Amir Goldstein Signed-off-by: Miklos Szeredi --- fs/overlayfs/dir.c | 2 +- fs/overlayfs/overlayfs.h | 1 + fs/overlayfs/super.c | 69 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 1 deletion(-) diff --git a/fs/overlayfs/dir.c b/fs/overlayfs/dir.c index b3471ef51440..c91b5aae8e32 100644 --- a/fs/overlayfs/dir.c +++ b/fs/overlayfs/dir.c @@ -42,7 +42,7 @@ int ovl_cleanup(struct inode *wdir, struct dentry *wdentry) return err; } -static struct dentry *ovl_lookup_temp(struct dentry *workdir) +struct dentry *ovl_lookup_temp(struct dentry *workdir) { struct dentry *temp; char name[20]; diff --git a/fs/overlayfs/overlayfs.h b/fs/overlayfs/overlayfs.h index 70d17040ab4d..ea531b1075f2 100644 --- a/fs/overlayfs/overlayfs.h +++ b/fs/overlayfs/overlayfs.h @@ -453,6 +453,7 @@ struct ovl_cattr { struct dentry *ovl_create_real(struct inode *dir, struct dentry *newdentry, struct ovl_cattr *attr); int ovl_cleanup(struct inode *dir, struct dentry *dentry); +struct dentry *ovl_lookup_temp(struct dentry *workdir); struct dentry *ovl_create_temp(struct dentry *workdir, struct ovl_cattr *attr); /* file.c */ diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index e8965fd1a608..b6ecaf95db9f 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -1071,6 +1071,66 @@ out: return err; } +/* + * Returns 1 if RENAME_WHITEOUT is supported, 0 if not supported and + * negative values if error is encountered. + */ +static int ovl_check_rename_whiteout(struct dentry *workdir) +{ + struct inode *dir = d_inode(workdir); + struct dentry *temp; + struct dentry *dest; + struct dentry *whiteout; + struct name_snapshot name; + int err; + + inode_lock_nested(dir, I_MUTEX_PARENT); + + temp = ovl_create_temp(workdir, OVL_CATTR(S_IFREG | 0)); + err = PTR_ERR(temp); + if (IS_ERR(temp)) + goto out_unlock; + + dest = ovl_lookup_temp(workdir); + err = PTR_ERR(dest); + if (IS_ERR(dest)) { + dput(temp); + goto out_unlock; + } + + /* Name is inline and stable - using snapshot as a copy helper */ + take_dentry_name_snapshot(&name, temp); + err = ovl_do_rename(dir, temp, dir, dest, RENAME_WHITEOUT); + if (err) { + if (err == -EINVAL) + err = 0; + goto cleanup_temp; + } + + whiteout = lookup_one_len(name.name.name, workdir, name.name.len); + err = PTR_ERR(whiteout); + if (IS_ERR(whiteout)) + goto cleanup_temp; + + err = ovl_is_whiteout(whiteout); + + /* Best effort cleanup of whiteout and temp file */ + if (err) + ovl_cleanup(dir, whiteout); + dput(whiteout); + +cleanup_temp: + ovl_cleanup(dir, temp); + release_dentry_name_snapshot(&name); + dput(temp); + dput(dest); + +out_unlock: + inode_unlock(dir); + + return err; +} + static int ovl_make_workdir(struct super_block *sb, struct ovl_fs *ofs, struct path *workpath) { @@ -1116,6 +1176,15 @@ static int ovl_make_workdir(struct super_block *sb, struct ovl_fs *ofs, else pr_warn("upper fs does not support tmpfile.\n"); + + /* Check if upper/work fs supports RENAME_WHITEOUT */ + err = ovl_check_rename_whiteout(ofs->workdir); + if (err < 0) + goto out; + + if (!err) + pr_warn("upper fs does not support RENAME_WHITEOUT.\n"); + /* * Check if upper/work fs supports trusted.overlay.* xattr */ From d80172c2d85947efe285693c5cea9c8976f478bf Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Thu, 20 Feb 2020 09:03:08 +0200 Subject: [PATCH 104/635] ovl: strict upper fs requirements for remote upper fs Overlayfs works sub-optimally with upper fs that has no xattr/d_type/ RENAME_WHITEOUT support. We should basically deprecate support for those filesystems, but so far, we only issue a warning and don't fail the mount for the sake of backward compat. Some features are already being disabled with no xattr support. For newly supported remote upper fs, we do not need to worry about backward compatibility, so we can fail the mount if upper fs is a sub-optimal filesystem. This reduces the in-tree remote filesystems supported as upper to just FUSE, for which the remote upper fs support was added. Signed-off-by: Amir Goldstein Signed-off-by: Miklos Szeredi --- fs/overlayfs/super.c | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index b6ecaf95db9f..ac013ed97a53 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -1136,6 +1136,8 @@ static int ovl_make_workdir(struct super_block *sb, struct ovl_fs *ofs, { struct vfsmount *mnt = ofs->upper_mnt; struct dentry *temp; + bool rename_whiteout; + bool d_type; int fh_type; int err; @@ -1161,11 +1163,8 @@ static int ovl_make_workdir(struct super_block *sb, struct ovl_fs *ofs, if (err < 0) goto out; - /* - * We allowed this configuration and don't want to break users over - * kernel upgrade. So warn instead of erroring out. - */ - if (!err) + d_type = err; + if (!d_type) pr_warn("upper fs needs to support d_type.\n"); /* Check if upper/work fs supports O_TMPFILE */ @@ -1182,7 +1181,8 @@ static int ovl_make_workdir(struct super_block *sb, struct ovl_fs *ofs, if (err < 0) goto out; - if (!err) + rename_whiteout = err; + if (!rename_whiteout) pr_warn("upper fs does not support RENAME_WHITEOUT.\n"); /* @@ -1199,6 +1199,18 @@ static int ovl_make_workdir(struct super_block *sb, struct ovl_fs *ofs, vfs_removexattr(ofs->workdir, OVL_XATTR_OPAQUE); } + /* + * We allowed sub-optimal upper fs configuration and don't want to break + * users over kernel upgrade, but we never allowed remote upper fs, so + * we can enforce strict requirements for remote upper fs. + */ + if (ovl_dentry_remote(ofs->workdir) && + (!d_type || !rename_whiteout || ofs->noxattr)) { + pr_err("upper fs missing required features.\n"); + err = -EINVAL; + goto out; + } + /* Check if upper/work fs supports file handles */ fh_type = ovl_can_decode_fh(ofs->workdir->d_sb); if (ofs->config.index && !fh_type) { From 504f38410a51f513606c3c18c7f6a32c1ca2afb5 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Thu, 24 Oct 2019 02:06:45 +0100 Subject: [PATCH 105/635] ovl: ovl_obtain_alias(): don't call d_instantiate_anon() for old The situation is the same as for __d_obtain_alias() (which is what that thing is parallel to) - if we find a preexisting alias, we want to grab it, drop the inode and return the alias we'd found. The only thing d_instantiate_anon() does compared to that is spurious security_d_instiate() that has already been done to that dentry with exact same arguments. Signed-off-by: Al Viro Signed-off-by: Miklos Szeredi --- fs/overlayfs/export.c | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/fs/overlayfs/export.c b/fs/overlayfs/export.c index a58b3d9b06b9..475c61f53f0f 100644 --- a/fs/overlayfs/export.c +++ b/fs/overlayfs/export.c @@ -308,31 +308,35 @@ static struct dentry *ovl_obtain_alias(struct super_block *sb, ovl_set_flag(OVL_UPPERDATA, inode); dentry = d_find_any_alias(inode); - if (!dentry) { - dentry = d_alloc_anon(inode->i_sb); - if (!dentry) - goto nomem; - oe = ovl_alloc_entry(lower ? 1 : 0); - if (!oe) - goto nomem; + if (dentry) + goto out_iput; - if (lower) { - oe->lowerstack->dentry = dget(lower); - oe->lowerstack->layer = lowerpath->layer; - } - dentry->d_fsdata = oe; - if (upper_alias) - ovl_dentry_set_upper_alias(dentry); + dentry = d_alloc_anon(inode->i_sb); + if (unlikely(!dentry)) + goto nomem; + oe = ovl_alloc_entry(lower ? 1 : 0); + if (!oe) + goto nomem; + + if (lower) { + oe->lowerstack->dentry = dget(lower); + oe->lowerstack->layer = lowerpath->layer; } + dentry->d_fsdata = oe; + if (upper_alias) + ovl_dentry_set_upper_alias(dentry); + ovl_dentry_update_reval(dentry, upper, DCACHE_OP_REVALIDATE | DCACHE_OP_WEAK_REVALIDATE); return d_instantiate_anon(dentry, inode); nomem: - iput(inode); dput(dentry); - return ERR_PTR(-ENOMEM); + dentry = ERR_PTR(-ENOMEM); +out_iput: + iput(inode); + return dentry; } /* Get the upper or lower dentry in stach whose on layer @idx */ From 0efbe7c4f948eb1233ef52c1cdba0b4ff973df36 Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Mon, 9 Mar 2020 15:22:33 -0500 Subject: [PATCH 106/635] ovl: replace zero-length array with flexible-array member The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertently introduced[3] to the codebase from now on. Also, notice that, dynamic memory allocations won't be affected by this change: "Flexible array members have incomplete type, and so the sizeof operator may not be applied. As a quirk of the original implementation of zero-length arrays, sizeof evaluates to zero."[1] This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Fixes: cbe7fba8edfc ("ovl: make sure that real fid is 32bit aligned in memory") Signed-off-by: Miklos Szeredi --- fs/overlayfs/overlayfs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/overlayfs/overlayfs.h b/fs/overlayfs/overlayfs.h index ea531b1075f2..e81fea86139e 100644 --- a/fs/overlayfs/overlayfs.h +++ b/fs/overlayfs/overlayfs.h @@ -87,7 +87,7 @@ struct ovl_fb { u8 flags; /* OVL_FH_FLAG_* */ u8 type; /* fid_type of fid */ uuid_t uuid; /* uuid of filesystem */ - u32 fid[0]; /* file identifier should be 32bit aligned in-memory */ + u32 fid[]; /* file identifier should be 32bit aligned in-memory */ } __packed; /* In-memory and on-wire format for overlay file handle */ From a5a84682ecb67e74d1f265d1634c146c990e0e4f Mon Sep 17 00:00:00 2001 From: Chengguang Xu Date: Mon, 10 Feb 2020 11:11:14 +0800 Subject: [PATCH 107/635] ovl: fix a typo in comment Fix a typo in comment. (annonate -> annotate) Signed-off-by: Chengguang Xu Signed-off-by: Miklos Szeredi --- fs/overlayfs/inode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/overlayfs/inode.c b/fs/overlayfs/inode.c index 13219a5864c4..956c6f9c9c11 100644 --- a/fs/overlayfs/inode.c +++ b/fs/overlayfs/inode.c @@ -504,7 +504,7 @@ static const struct address_space_operations ovl_aops = { /* * It is possible to stack overlayfs instance on top of another - * overlayfs instance as lower layer. We need to annonate the + * overlayfs instance as lower layer. We need to annotate the * stackable i_mutex locks according to stack level of the super * block instance. An overlayfs instance can never be in stack * depth 0 (there is always a real fs below it). An overlayfs From b2ba7e91fa81bec9b64c47ab852145559cad2b68 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 27 Feb 2020 21:31:45 -0800 Subject: [PATCH 108/635] libnvdimm/pfn: Prevent raw mode fallback if pfn-infoblock valid The EOPNOTSUPP return code from the pmem driver indicates that the namespace has a configuration that may be valid, but the current kernel does not support it. Expand this to all of the nd_pfn_validate() error conditions after the infoblock has been verified as self consistent. This prevents exposing the namespace to I/O when the infoblock needs to be corrected, or the system needs to be put into a different configuration (like changing the page size on PowerPC). Cc: Aneesh Kumar K.V Cc: Jeff Moyer Reviewed-by: Aneesh Kumar K.V Signed-off-by: Dan Williams --- drivers/nvdimm/pfn_devs.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/nvdimm/pfn_devs.c b/drivers/nvdimm/pfn_devs.c index a5c25cb87116..79fe02d6f657 100644 --- a/drivers/nvdimm/pfn_devs.c +++ b/drivers/nvdimm/pfn_devs.c @@ -561,14 +561,14 @@ int nd_pfn_validate(struct nd_pfn *nd_pfn, const char *sig) dev_dbg(&nd_pfn->dev, "align: %lx:%lx mode: %d:%d\n", nd_pfn->align, align, nd_pfn->mode, mode); - return -EINVAL; + return -EOPNOTSUPP; } } if (align > nvdimm_namespace_capacity(ndns)) { dev_err(&nd_pfn->dev, "alignment: %lx exceeds capacity %llx\n", align, nvdimm_namespace_capacity(ndns)); - return -EINVAL; + return -EOPNOTSUPP; } /* @@ -581,7 +581,7 @@ int nd_pfn_validate(struct nd_pfn *nd_pfn, const char *sig) if (offset >= resource_size(&nsio->res)) { dev_err(&nd_pfn->dev, "pfn array size exceeds capacity of %s\n", dev_name(&ndns->dev)); - return -EBUSY; + return -EOPNOTSUPP; } if ((align && !IS_ALIGNED(nsio->res.start + offset + start_pad, align)) @@ -589,7 +589,7 @@ int nd_pfn_validate(struct nd_pfn *nd_pfn, const char *sig) dev_err(&nd_pfn->dev, "bad offset: %#llx dax disabled align: %#lx\n", offset, align); - return -ENXIO; + return -EOPNOTSUPP; } return 0; From 6acd7d5ef264d8e9a8988cebf6eeb3567eaf60c6 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 27 Feb 2020 21:39:23 -0800 Subject: [PATCH 109/635] libnvdimm/namespace: Enforce memremap_compat_align() The pmem driver on PowerPC crashes with the following signature when instantiating misaligned namespaces that map their capacity via memremap_pages(). BUG: Unable to handle kernel data access at 0xc001000406000000 Faulting instruction address: 0xc000000000090790 NIP [c000000000090790] arch_add_memory+0xc0/0x130 LR [c000000000090744] arch_add_memory+0x74/0x130 Call Trace: arch_add_memory+0x74/0x130 (unreliable) memremap_pages+0x74c/0xa30 devm_memremap_pages+0x3c/0xa0 pmem_attach_disk+0x188/0x770 nvdimm_bus_probe+0xd8/0x470 With the assumption that only memremap_pages() has alignment constraints, enforce memremap_compat_align() for pmem_should_map_pages(), nd_pfn, and nd_dax cases. This includes preventing the creation of namespaces where the base address is misaligned and cases there infoblock padding parameters are invalid. Reported-by: Aneesh Kumar K.V Cc: Jeff Moyer Fixes: a3619190d62e ("libnvdimm/pfn: stop padding pmem namespaces to section alignment") Reviewed-by: Aneesh Kumar K.V Signed-off-by: Dan Williams --- drivers/nvdimm/namespace_devs.c | 17 +++++++++++++++++ drivers/nvdimm/pfn.h | 12 ++++++++++++ drivers/nvdimm/pfn_devs.c | 32 +++++++++++++++++++++++++++++--- 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/drivers/nvdimm/namespace_devs.c b/drivers/nvdimm/namespace_devs.c index 032dc61725ff..77e211c7d94d 100644 --- a/drivers/nvdimm/namespace_devs.c +++ b/drivers/nvdimm/namespace_devs.c @@ -10,6 +10,7 @@ #include #include "nd-core.h" #include "pmem.h" +#include "pfn.h" #include "nd.h" static void namespace_io_release(struct device *dev) @@ -1739,6 +1740,22 @@ struct nd_namespace_common *nvdimm_namespace_common_probe(struct device *dev) return ERR_PTR(-ENODEV); } + /* + * Note, alignment validation for fsdax and devdax mode + * namespaces happens in nd_pfn_validate() where infoblock + * padding parameters can be applied. + */ + if (pmem_should_map_pages(dev)) { + struct nd_namespace_io *nsio = to_nd_namespace_io(&ndns->dev); + struct resource *res = &nsio->res; + + if (!IS_ALIGNED(res->start | (res->end + 1), + memremap_compat_align())) { + dev_err(&ndns->dev, "%pr misaligned, unable to map\n", res); + return ERR_PTR(-EOPNOTSUPP); + } + } + if (is_namespace_pmem(&ndns->dev)) { struct nd_namespace_pmem *nspm; diff --git a/drivers/nvdimm/pfn.h b/drivers/nvdimm/pfn.h index acb19517f678..37cb1b8a2a39 100644 --- a/drivers/nvdimm/pfn.h +++ b/drivers/nvdimm/pfn.h @@ -24,6 +24,18 @@ struct nd_pfn_sb { __le64 npfns; __le32 mode; /* minor-version-1 additions for section alignment */ + /** + * @start_pad: Deprecated attribute to pad start-misaligned namespaces + * + * start_pad is deprecated because the original definition did + * not comprehend that dataoff is relative to the base address + * of the namespace not the start_pad adjusted base. The result + * is that the dax path is broken, but the block-I/O path is + * not. The kernel will no longer create namespaces using start + * padding, but it still supports block-I/O for legacy + * configurations mainly to allow a backup, reconfigure the + * namespace, and restore flow to repair dax operation. + */ __le32 start_pad; __le32 end_trunc; /* minor-version-2 record the base alignment of the mapping */ diff --git a/drivers/nvdimm/pfn_devs.c b/drivers/nvdimm/pfn_devs.c index 79fe02d6f657..34db557dbad1 100644 --- a/drivers/nvdimm/pfn_devs.c +++ b/drivers/nvdimm/pfn_devs.c @@ -446,6 +446,7 @@ static bool nd_supported_alignment(unsigned long align) int nd_pfn_validate(struct nd_pfn *nd_pfn, const char *sig) { u64 checksum, offset; + struct resource *res; enum nd_pfn_mode mode; struct nd_namespace_io *nsio; unsigned long align, start_pad; @@ -578,13 +579,14 @@ int nd_pfn_validate(struct nd_pfn *nd_pfn, const char *sig) * established. */ nsio = to_nd_namespace_io(&ndns->dev); - if (offset >= resource_size(&nsio->res)) { + res = &nsio->res; + if (offset >= resource_size(res)) { dev_err(&nd_pfn->dev, "pfn array size exceeds capacity of %s\n", dev_name(&ndns->dev)); return -EOPNOTSUPP; } - if ((align && !IS_ALIGNED(nsio->res.start + offset + start_pad, align)) + if ((align && !IS_ALIGNED(res->start + offset + start_pad, align)) || !IS_ALIGNED(offset, PAGE_SIZE)) { dev_err(&nd_pfn->dev, "bad offset: %#llx dax disabled align: %#lx\n", @@ -592,6 +594,18 @@ int nd_pfn_validate(struct nd_pfn *nd_pfn, const char *sig) return -EOPNOTSUPP; } + if (!IS_ALIGNED(res->start + le32_to_cpu(pfn_sb->start_pad), + memremap_compat_align())) { + dev_err(&nd_pfn->dev, "resource start misaligned\n"); + return -EOPNOTSUPP; + } + + if (!IS_ALIGNED(res->end + 1 - le32_to_cpu(pfn_sb->end_trunc), + memremap_compat_align())) { + dev_err(&nd_pfn->dev, "resource end misaligned\n"); + return -EOPNOTSUPP; + } + return 0; } EXPORT_SYMBOL(nd_pfn_validate); @@ -750,7 +764,19 @@ static int nd_pfn_init(struct nd_pfn *nd_pfn) start = nsio->res.start; size = resource_size(&nsio->res); npfns = PHYS_PFN(size - SZ_8K); - align = max(nd_pfn->align, SUBSECTION_SIZE); + align = max(nd_pfn->align, memremap_compat_align()); + + /* + * When @start is misaligned fail namespace creation. See + * the 'struct nd_pfn_sb' commentary on why ->start_pad is not + * an option. + */ + if (!IS_ALIGNED(start, memremap_compat_align())) { + dev_err(&nd_pfn->dev, "%s: start %pa misaligned to %#lx\n", + dev_name(&ndns->dev), &start, + memremap_compat_align()); + return -EINVAL; + } end_trunc = start + size - ALIGN_DOWN(start + size, align); if (nd_pfn->mode == PFN_MODE_PMEM) { /* From a0e374525def2ef18a078523e1faefb5ce2b05e5 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 30 Jan 2020 12:06:18 -0800 Subject: [PATCH 110/635] libnvdimm/region: Introduce NDD_LABELING The NDD_ALIASING flag is used to indicate where pmem capacity might alias with blk capacity and require labeling. It is also used to indicate whether the DIMM supports labeling. Separate this latter capability into its own flag so that the NDD_ALIASING flag is scoped to true aliased configurations. To my knowledge aliased configurations only exist in the ACPI spec, there are no known platforms that ship this support in production. This clarity allows namespace-capacity alignment constraints around interleave-ways to be relaxed. Cc: Vishal Verma Cc: Oliver O'Halloran Reviewed-by: Jeff Moyer Reviewed-by: Aneesh Kumar K.V Link: https://lore.kernel.org/r/158041477856.3889308.4212605617834097674.stgit@dwillia2-desk3.amr.corp.intel.com Signed-off-by: Dan Williams --- arch/powerpc/platforms/pseries/papr_scm.c | 2 +- drivers/acpi/nfit/core.c | 4 +++- drivers/nvdimm/dimm.c | 2 +- drivers/nvdimm/dimm_devs.c | 9 +++++---- drivers/nvdimm/namespace_devs.c | 2 +- drivers/nvdimm/nd.h | 2 +- drivers/nvdimm/region_devs.c | 10 +++++----- include/linux/libnvdimm.h | 2 ++ 8 files changed, 19 insertions(+), 14 deletions(-) diff --git a/arch/powerpc/platforms/pseries/papr_scm.c b/arch/powerpc/platforms/pseries/papr_scm.c index 0b4467e378e5..589858cb3203 100644 --- a/arch/powerpc/platforms/pseries/papr_scm.c +++ b/arch/powerpc/platforms/pseries/papr_scm.c @@ -328,7 +328,7 @@ static int papr_scm_nvdimm_init(struct papr_scm_priv *p) } dimm_flags = 0; - set_bit(NDD_ALIASING, &dimm_flags); + set_bit(NDD_LABELING, &dimm_flags); p->nvdimm = nvdimm_create(p->bus, p, NULL, dimm_flags, PAPR_SCM_DIMM_CMD_MASK, 0, NULL); diff --git a/drivers/acpi/nfit/core.c b/drivers/acpi/nfit/core.c index a3320f93616d..71d7f2aa1b12 100644 --- a/drivers/acpi/nfit/core.c +++ b/drivers/acpi/nfit/core.c @@ -2026,8 +2026,10 @@ static int acpi_nfit_register_dimms(struct acpi_nfit_desc *acpi_desc) continue; } - if (nfit_mem->bdw && nfit_mem->memdev_pmem) + if (nfit_mem->bdw && nfit_mem->memdev_pmem) { set_bit(NDD_ALIASING, &flags); + set_bit(NDD_LABELING, &flags); + } /* collate flags across all memdevs for this dimm */ list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { diff --git a/drivers/nvdimm/dimm.c b/drivers/nvdimm/dimm.c index 64776ed15bb3..7d4ddc4d9322 100644 --- a/drivers/nvdimm/dimm.c +++ b/drivers/nvdimm/dimm.c @@ -99,7 +99,7 @@ static int nvdimm_probe(struct device *dev) if (ndd->ns_current >= 0) { rc = nd_label_reserve_dpa(ndd); if (rc == 0) - nvdimm_set_aliasing(dev); + nvdimm_set_labeling(dev); } nvdimm_bus_unlock(dev); diff --git a/drivers/nvdimm/dimm_devs.c b/drivers/nvdimm/dimm_devs.c index 94ea6dba6b4f..39a61a514746 100644 --- a/drivers/nvdimm/dimm_devs.c +++ b/drivers/nvdimm/dimm_devs.c @@ -32,7 +32,7 @@ int nvdimm_check_config_data(struct device *dev) if (!nvdimm->cmd_mask || !test_bit(ND_CMD_GET_CONFIG_DATA, &nvdimm->cmd_mask)) { - if (test_bit(NDD_ALIASING, &nvdimm->flags)) + if (test_bit(NDD_LABELING, &nvdimm->flags)) return -ENXIO; else return -ENOTTY; @@ -173,11 +173,11 @@ int nvdimm_set_config_data(struct nvdimm_drvdata *ndd, size_t offset, return rc; } -void nvdimm_set_aliasing(struct device *dev) +void nvdimm_set_labeling(struct device *dev) { struct nvdimm *nvdimm = to_nvdimm(dev); - set_bit(NDD_ALIASING, &nvdimm->flags); + set_bit(NDD_LABELING, &nvdimm->flags); } void nvdimm_set_locked(struct device *dev) @@ -312,8 +312,9 @@ static ssize_t flags_show(struct device *dev, { struct nvdimm *nvdimm = to_nvdimm(dev); - return sprintf(buf, "%s%s\n", + return sprintf(buf, "%s%s%s\n", test_bit(NDD_ALIASING, &nvdimm->flags) ? "alias " : "", + test_bit(NDD_LABELING, &nvdimm->flags) ? "label " : "", test_bit(NDD_LOCKED, &nvdimm->flags) ? "lock " : ""); } static DEVICE_ATTR_RO(flags); diff --git a/drivers/nvdimm/namespace_devs.c b/drivers/nvdimm/namespace_devs.c index 77e211c7d94d..01f6c22f0d1a 100644 --- a/drivers/nvdimm/namespace_devs.c +++ b/drivers/nvdimm/namespace_devs.c @@ -2538,7 +2538,7 @@ static int init_active_labels(struct nd_region *nd_region) if (!ndd) { if (test_bit(NDD_LOCKED, &nvdimm->flags)) /* fail, label data may be unreadable */; - else if (test_bit(NDD_ALIASING, &nvdimm->flags)) + else if (test_bit(NDD_LABELING, &nvdimm->flags)) /* fail, labels needed to disambiguate dpa */; else return 0; diff --git a/drivers/nvdimm/nd.h b/drivers/nvdimm/nd.h index c9f6a5b5253a..ca39abe29c7c 100644 --- a/drivers/nvdimm/nd.h +++ b/drivers/nvdimm/nd.h @@ -252,7 +252,7 @@ int nvdimm_set_config_data(struct nvdimm_drvdata *ndd, size_t offset, void *buf, size_t len); long nvdimm_clear_poison(struct device *dev, phys_addr_t phys, unsigned int len); -void nvdimm_set_aliasing(struct device *dev); +void nvdimm_set_labeling(struct device *dev); void nvdimm_set_locked(struct device *dev); void nvdimm_clear_locked(struct device *dev); int nvdimm_security_setup_events(struct device *dev); diff --git a/drivers/nvdimm/region_devs.c b/drivers/nvdimm/region_devs.c index a19e535830d9..a5fc6e4c56ff 100644 --- a/drivers/nvdimm/region_devs.c +++ b/drivers/nvdimm/region_devs.c @@ -195,16 +195,16 @@ EXPORT_SYMBOL_GPL(nd_blk_region_set_provider_data); int nd_region_to_nstype(struct nd_region *nd_region) { if (is_memory(&nd_region->dev)) { - u16 i, alias; + u16 i, label; - for (i = 0, alias = 0; i < nd_region->ndr_mappings; i++) { + for (i = 0, label = 0; i < nd_region->ndr_mappings; i++) { struct nd_mapping *nd_mapping = &nd_region->mapping[i]; struct nvdimm *nvdimm = nd_mapping->nvdimm; - if (test_bit(NDD_ALIASING, &nvdimm->flags)) - alias++; + if (test_bit(NDD_LABELING, &nvdimm->flags)) + label++; } - if (alias) + if (label) return ND_DEVICE_NAMESPACE_PMEM; else return ND_DEVICE_NAMESPACE_IO; diff --git a/include/linux/libnvdimm.h b/include/linux/libnvdimm.h index 9df091bd30ba..18da4059be09 100644 --- a/include/linux/libnvdimm.h +++ b/include/linux/libnvdimm.h @@ -37,6 +37,8 @@ enum { NDD_WORK_PENDING = 4, /* ignore / filter NSLABEL_FLAG_LOCAL for this DIMM, i.e. no aliasing */ NDD_NOBLK = 5, + /* dimm supports namespace labels */ + NDD_LABELING = 6, /* need to set a limit somewhere, but yes, this is likely overkill */ ND_IOCTL_MAX_BUFLEN = SZ_4M, From 2522afb86a8cceba0f67dbf05772d21b76d79f06 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 30 Jan 2020 12:06:23 -0800 Subject: [PATCH 111/635] libnvdimm/region: Introduce an 'align' attribute The align attribute applies an alignment constraint for namespace creation in a region. Whereas the 'align' attribute of a namespace applied alignment padding via an info block, the 'align' attribute applies alignment constraints to the free space allocation. The default for 'align' is the maximum known memremap_compat_align() across all archs (16MiB from PowerPC at time of writing) multiplied by the number of interleave ways if there is blk-aliasing. The minimum is PAGE_SIZE and allows for the creation of cross-arch incompatible namespaces, just as previous kernels allowed, but the expectation is cross-arch and mode-independent compatibility by default. The regression risk with this change is limited to cases that were dependent on the ability to create unaligned namespaces, *and* for some reason are unable to opt-out of aligned namespaces by writing to 'regionX/align'. If such a scenario arises the default can be flipped from opt-out to opt-in of compat-aligned namespace creation, but that is a last resort. The kernel will otherwise continue to support existing defined misaligned namespaces. Unfortunately this change needs to touch several parts of the implementation at once: - region/available_size: expand busy extents to current align - region/max_available_extent: expand busy extents to current align - namespace/size: trim free space to current align ...to keep the free space accounting conforming to the dynamic align setting. Reported-by: Aneesh Kumar K.V Reported-by: Jeff Moyer Reviewed-by: Aneesh Kumar K.V Reviewed-by: Jeff Moyer Link: https://lore.kernel.org/r/158041478371.3889308.14542630147672668068.stgit@dwillia2-desk3.amr.corp.intel.com Signed-off-by: Dan Williams --- drivers/nvdimm/dimm_devs.c | 86 ++++++++++++++++++---- drivers/nvdimm/namespace_devs.c | 9 ++- drivers/nvdimm/nd.h | 1 + drivers/nvdimm/region_devs.c | 124 +++++++++++++++++++++++++++++--- 4 files changed, 193 insertions(+), 27 deletions(-) diff --git a/drivers/nvdimm/dimm_devs.c b/drivers/nvdimm/dimm_devs.c index 39a61a514746..b7b77e8d9027 100644 --- a/drivers/nvdimm/dimm_devs.c +++ b/drivers/nvdimm/dimm_devs.c @@ -563,6 +563,21 @@ int nvdimm_security_freeze(struct nvdimm *nvdimm) return rc; } +static unsigned long dpa_align(struct nd_region *nd_region) +{ + struct device *dev = &nd_region->dev; + + if (dev_WARN_ONCE(dev, !is_nvdimm_bus_locked(dev), + "bus lock required for capacity provision\n")) + return 0; + if (dev_WARN_ONCE(dev, !nd_region->ndr_mappings || nd_region->align + % nd_region->ndr_mappings, + "invalid region align %#lx mappings: %d\n", + nd_region->align, nd_region->ndr_mappings)) + return 0; + return nd_region->align / nd_region->ndr_mappings; +} + int alias_dpa_busy(struct device *dev, void *data) { resource_size_t map_end, blk_start, new; @@ -571,6 +586,7 @@ int alias_dpa_busy(struct device *dev, void *data) struct nd_region *nd_region; struct nvdimm_drvdata *ndd; struct resource *res; + unsigned long align; int i; if (!is_memory(dev)) @@ -608,13 +624,21 @@ int alias_dpa_busy(struct device *dev, void *data) * Find the free dpa from the end of the last pmem allocation to * the end of the interleave-set mapping. */ + align = dpa_align(nd_region); + if (!align) + return 0; + for_each_dpa_resource(ndd, res) { + resource_size_t start, end; + if (strncmp(res->name, "pmem", 4) != 0) continue; - if ((res->start >= blk_start && res->start < map_end) - || (res->end >= blk_start - && res->end <= map_end)) { - new = max(blk_start, min(map_end + 1, res->end + 1)); + + start = ALIGN_DOWN(res->start, align); + end = ALIGN(res->end + 1, align) - 1; + if ((start >= blk_start && start < map_end) + || (end >= blk_start && end <= map_end)) { + new = max(blk_start, min(map_end, end) + 1); if (new != blk_start) { blk_start = new; goto retry; @@ -654,6 +678,7 @@ resource_size_t nd_blk_available_dpa(struct nd_region *nd_region) .res = NULL, }; struct resource *res; + unsigned long align; if (!ndd) return 0; @@ -661,10 +686,20 @@ resource_size_t nd_blk_available_dpa(struct nd_region *nd_region) device_for_each_child(&nvdimm_bus->dev, &info, alias_dpa_busy); /* now account for busy blk allocations in unaliased dpa */ + align = dpa_align(nd_region); + if (!align) + return 0; for_each_dpa_resource(ndd, res) { + resource_size_t start, end, size; + if (strncmp(res->name, "blk", 3) != 0) continue; - info.available -= resource_size(res); + start = ALIGN_DOWN(res->start, align); + end = ALIGN(res->end + 1, align) - 1; + size = end - start + 1; + if (size >= info.available) + return 0; + info.available -= size; } return info.available; @@ -683,19 +718,31 @@ resource_size_t nd_pmem_max_contiguous_dpa(struct nd_region *nd_region, struct nvdimm_bus *nvdimm_bus; resource_size_t max = 0; struct resource *res; + unsigned long align; /* if a dimm is disabled the available capacity is zero */ if (!ndd) return 0; + align = dpa_align(nd_region); + if (!align) + return 0; + nvdimm_bus = walk_to_nvdimm_bus(ndd->dev); if (__reserve_free_pmem(&nd_region->dev, nd_mapping->nvdimm)) return 0; for_each_dpa_resource(ndd, res) { + resource_size_t start, end; + if (strcmp(res->name, "pmem-reserve") != 0) continue; - if (resource_size(res) > max) - max = resource_size(res); + /* trim free space relative to current alignment setting */ + start = ALIGN(res->start, align); + end = ALIGN_DOWN(res->end + 1, align) - 1; + if (end < start) + continue; + if (end - start + 1 > max) + max = end - start + 1; } release_free_pmem(nvdimm_bus, nd_mapping); return max; @@ -723,24 +770,33 @@ resource_size_t nd_pmem_available_dpa(struct nd_region *nd_region, struct nvdimm_drvdata *ndd = to_ndd(nd_mapping); struct resource *res; const char *reason; + unsigned long align; if (!ndd) return 0; + align = dpa_align(nd_region); + if (!align) + return 0; + map_start = nd_mapping->start; map_end = map_start + nd_mapping->size - 1; blk_start = max(map_start, map_end + 1 - *overlap); for_each_dpa_resource(ndd, res) { - if (res->start >= map_start && res->start < map_end) { + resource_size_t start, end; + + start = ALIGN_DOWN(res->start, align); + end = ALIGN(res->end + 1, align) - 1; + if (start >= map_start && start < map_end) { if (strncmp(res->name, "blk", 3) == 0) blk_start = min(blk_start, - max(map_start, res->start)); - else if (res->end > map_end) { + max(map_start, start)); + else if (end > map_end) { reason = "misaligned to iset"; goto err; } else - busy += resource_size(res); - } else if (res->end >= map_start && res->end <= map_end) { + busy += end - start + 1; + } else if (end >= map_start && end <= map_end) { if (strncmp(res->name, "blk", 3) == 0) { /* * If a BLK allocation overlaps the start of @@ -749,8 +805,8 @@ resource_size_t nd_pmem_available_dpa(struct nd_region *nd_region, */ blk_start = map_start; } else - busy += resource_size(res); - } else if (map_start > res->start && map_start < res->end) { + busy += end - start + 1; + } else if (map_start > start && map_start < end) { /* total eclipse of the mapping */ busy += nd_mapping->size; blk_start = map_start; @@ -760,7 +816,7 @@ resource_size_t nd_pmem_available_dpa(struct nd_region *nd_region, *overlap = map_end + 1 - blk_start; available = blk_start - map_start; if (busy < available) - return available - busy; + return ALIGN_DOWN(available - busy, align); return 0; err: diff --git a/drivers/nvdimm/namespace_devs.c b/drivers/nvdimm/namespace_devs.c index 01f6c22f0d1a..ae155e860fdc 100644 --- a/drivers/nvdimm/namespace_devs.c +++ b/drivers/nvdimm/namespace_devs.c @@ -542,6 +542,11 @@ static void space_valid(struct nd_region *nd_region, struct nvdimm_drvdata *ndd, { bool is_reserve = strcmp(label_id->id, "pmem-reserve") == 0; bool is_pmem = strncmp(label_id->id, "pmem", 4) == 0; + unsigned long align; + + align = nd_region->align / nd_region->ndr_mappings; + valid->start = ALIGN(valid->start, align); + valid->end = ALIGN_DOWN(valid->end + 1, align) - 1; if (valid->start >= valid->end) goto invalid; @@ -981,10 +986,10 @@ static ssize_t __size_store(struct device *dev, unsigned long long val) return -ENXIO; } - div_u64_rem(val, PAGE_SIZE * nd_region->ndr_mappings, &remainder); + div_u64_rem(val, nd_region->align, &remainder); if (remainder) { dev_dbg(dev, "%llu is not %ldK aligned\n", val, - (PAGE_SIZE * nd_region->ndr_mappings) / SZ_1K); + nd_region->align / SZ_1K); return -EINVAL; } diff --git a/drivers/nvdimm/nd.h b/drivers/nvdimm/nd.h index ca39abe29c7c..c4d69c1cce55 100644 --- a/drivers/nvdimm/nd.h +++ b/drivers/nvdimm/nd.h @@ -146,6 +146,7 @@ struct nd_region { struct device *btt_seed; struct device *pfn_seed; struct device *dax_seed; + unsigned long align; u16 ndr_mappings; u64 ndr_size; u64 ndr_start; diff --git a/drivers/nvdimm/region_devs.c b/drivers/nvdimm/region_devs.c index a5fc6e4c56ff..bf239e783940 100644 --- a/drivers/nvdimm/region_devs.c +++ b/drivers/nvdimm/region_devs.c @@ -216,21 +216,25 @@ int nd_region_to_nstype(struct nd_region *nd_region) } EXPORT_SYMBOL(nd_region_to_nstype); +static unsigned long long region_size(struct nd_region *nd_region) +{ + if (is_memory(&nd_region->dev)) { + return nd_region->ndr_size; + } else if (nd_region->ndr_mappings == 1) { + struct nd_mapping *nd_mapping = &nd_region->mapping[0]; + + return nd_mapping->size; + } + + return 0; +} + static ssize_t size_show(struct device *dev, struct device_attribute *attr, char *buf) { struct nd_region *nd_region = to_nd_region(dev); - unsigned long long size = 0; - if (is_memory(dev)) { - size = nd_region->ndr_size; - } else if (nd_region->ndr_mappings == 1) { - struct nd_mapping *nd_mapping = &nd_region->mapping[0]; - - size = nd_mapping->size; - } - - return sprintf(buf, "%llu\n", size); + return sprintf(buf, "%llu\n", region_size(nd_region)); } static DEVICE_ATTR_RO(size); @@ -529,6 +533,55 @@ static ssize_t read_only_store(struct device *dev, } static DEVICE_ATTR_RW(read_only); +static ssize_t align_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct nd_region *nd_region = to_nd_region(dev); + + return sprintf(buf, "%#lx\n", nd_region->align); +} + +static ssize_t align_store(struct device *dev, + struct device_attribute *attr, const char *buf, size_t len) +{ + struct nd_region *nd_region = to_nd_region(dev); + unsigned long val, dpa; + u32 remainder; + int rc; + + rc = kstrtoul(buf, 0, &val); + if (rc) + return rc; + + if (!nd_region->ndr_mappings) + return -ENXIO; + + /* + * Ensure space-align is evenly divisible by the region + * interleave-width because the kernel typically has no facility + * to determine which DIMM(s), dimm-physical-addresses, would + * contribute to the tail capacity in system-physical-address + * space for the namespace. + */ + dpa = val; + remainder = do_div(dpa, nd_region->ndr_mappings); + if (!is_power_of_2(dpa) || dpa < PAGE_SIZE + || val > region_size(nd_region) || remainder) + return -EINVAL; + + /* + * Given that space allocation consults this value multiple + * times ensure it does not change for the duration of the + * allocation. + */ + nvdimm_bus_lock(dev); + nd_region->align = val; + nvdimm_bus_unlock(dev); + + return len; +} +static DEVICE_ATTR_RW(align); + static ssize_t region_badblocks_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -571,6 +624,7 @@ static DEVICE_ATTR_RO(persistence_domain); static struct attribute *nd_region_attributes[] = { &dev_attr_size.attr, + &dev_attr_align.attr, &dev_attr_nstype.attr, &dev_attr_mappings.attr, &dev_attr_btt_seed.attr, @@ -626,6 +680,19 @@ static umode_t region_visible(struct kobject *kobj, struct attribute *a, int n) return a->mode; } + if (a == &dev_attr_align.attr) { + int i; + + for (i = 0; i < nd_region->ndr_mappings; i++) { + struct nd_mapping *nd_mapping = &nd_region->mapping[i]; + struct nvdimm *nvdimm = nd_mapping->nvdimm; + + if (test_bit(NDD_LABELING, &nvdimm->flags)) + return a->mode; + } + return 0; + } + if (a != &dev_attr_set_cookie.attr && a != &dev_attr_available_size.attr) return a->mode; @@ -935,6 +1002,42 @@ void nd_region_release_lane(struct nd_region *nd_region, unsigned int lane) } EXPORT_SYMBOL(nd_region_release_lane); +/* + * PowerPC requires this alignment for memremap_pages(). All other archs + * should be ok with SUBSECTION_SIZE (see memremap_compat_align()). + */ +#define MEMREMAP_COMPAT_ALIGN_MAX SZ_16M + +static unsigned long default_align(struct nd_region *nd_region) +{ + unsigned long align, per_mapping; + int i, mappings; + u32 remainder; + + if (is_nd_blk(&nd_region->dev)) + align = PAGE_SIZE; + else + align = MEMREMAP_COMPAT_ALIGN_MAX; + + for (i = 0; i < nd_region->ndr_mappings; i++) { + struct nd_mapping *nd_mapping = &nd_region->mapping[i]; + struct nvdimm *nvdimm = nd_mapping->nvdimm; + + if (test_bit(NDD_ALIASING, &nvdimm->flags)) { + align = MEMREMAP_COMPAT_ALIGN_MAX; + break; + } + } + + mappings = max_t(u16, 1, nd_region->ndr_mappings); + per_mapping = align; + remainder = do_div(per_mapping, mappings); + if (remainder) + align *= mappings; + + return align; +} + static struct nd_region *nd_region_create(struct nvdimm_bus *nvdimm_bus, struct nd_region_desc *ndr_desc, const struct device_type *dev_type, const char *caller) @@ -1039,6 +1142,7 @@ static struct nd_region *nd_region_create(struct nvdimm_bus *nvdimm_bus, dev->of_node = ndr_desc->of_node; nd_region->ndr_size = resource_size(ndr_desc->res); nd_region->ndr_start = ndr_desc->res->start; + nd_region->align = default_align(nd_region); if (ndr_desc->flush) nd_region->flush = ndr_desc->flush; else From 92e399c07b7e71805306e8ce82960f10400c4ffe Mon Sep 17 00:00:00 2001 From: Gwendal Grignou Date: Wed, 4 Mar 2020 15:21:08 -0800 Subject: [PATCH 112/635] platform/chrome: cros_usbpd_notify: Fix cros-usbpd-notify notifier cros-usbpd-notify notifier was returning NOTIFY_BAD when no host event was available in the MKBP message. But MKBP messages are used to transmit other information, so return NOTIFY_DONE instead, to allow other notifier to be called. Fixes: ec2daf6e33f9f ("platform: chrome: Add cros-usbpd-notify driver") Signed-off-by: Gwendal Grignou Signed-off-by: Benson Leung Signed-off-by: Enric Balletbo i Serra --- drivers/platform/chrome/cros_usbpd_notify.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/platform/chrome/cros_usbpd_notify.c b/drivers/platform/chrome/cros_usbpd_notify.c index 3851bbd6e9a3..ca2c0181a1db 100644 --- a/drivers/platform/chrome/cros_usbpd_notify.c +++ b/drivers/platform/chrome/cros_usbpd_notify.c @@ -84,7 +84,7 @@ static int cros_usbpd_notify_plat(struct notifier_block *nb, u32 host_event = cros_ec_get_host_event(ec_dev); if (!host_event) - return NOTIFY_BAD; + return NOTIFY_DONE; if (host_event & EC_HOST_EVENT_MASK(EC_HOST_EVENT_PD_MCU)) { blocking_notifier_call_chain(&cros_usbpd_notifier_list, From f5d84a21fab7d11ffdf740f6d44846f34f42c8e1 Mon Sep 17 00:00:00 2001 From: Prashant Malani Date: Mon, 16 Mar 2020 01:28:30 -0700 Subject: [PATCH 113/635] platform/chrome: cros_usbpd_notify: Add driver data struct Introduce a device driver data structure, cros_usbpd_notify_data, in which we can store the notifier block object and pointers to the struct cros_ec_device and struct device objects. This will make it more convenient to access these pointers when executing both platform and ACPI callbacks. Signed-off-by: Prashant Malani Signed-off-by: Enric Balletbo i Serra Signed-off-by: Benson Leung --- drivers/platform/chrome/cros_usbpd_notify.c | 28 ++++++++++++++------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/drivers/platform/chrome/cros_usbpd_notify.c b/drivers/platform/chrome/cros_usbpd_notify.c index ca2c0181a1db..d2af2130afec 100644 --- a/drivers/platform/chrome/cros_usbpd_notify.c +++ b/drivers/platform/chrome/cros_usbpd_notify.c @@ -16,6 +16,12 @@ static BLOCKING_NOTIFIER_HEAD(cros_usbpd_notifier_list); +struct cros_usbpd_notify_data { + struct device *dev; + struct cros_ec_device *ec; + struct notifier_block nb; +}; + /** * cros_usbpd_register_notify - Register a notifier callback for PD events. * @nb: Notifier block pointer to register @@ -98,18 +104,21 @@ static int cros_usbpd_notify_probe_plat(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct cros_ec_dev *ecdev = dev_get_drvdata(dev->parent); - struct notifier_block *nb; + struct cros_usbpd_notify_data *pdnotify; int ret; - nb = devm_kzalloc(dev, sizeof(*nb), GFP_KERNEL); - if (!nb) + pdnotify = devm_kzalloc(dev, sizeof(*pdnotify), GFP_KERNEL); + if (!pdnotify) return -ENOMEM; - nb->notifier_call = cros_usbpd_notify_plat; - dev_set_drvdata(dev, nb); + pdnotify->dev = dev; + pdnotify->ec = ecdev->ec_dev; + pdnotify->nb.notifier_call = cros_usbpd_notify_plat; + + dev_set_drvdata(dev, pdnotify); ret = blocking_notifier_chain_register(&ecdev->ec_dev->event_notifier, - nb); + &pdnotify->nb); if (ret < 0) { dev_err(dev, "Failed to register notifier\n"); return ret; @@ -122,10 +131,11 @@ static int cros_usbpd_notify_remove_plat(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct cros_ec_dev *ecdev = dev_get_drvdata(dev->parent); - struct notifier_block *nb = - (struct notifier_block *)dev_get_drvdata(dev); + struct cros_usbpd_notify_data *pdnotify = + (struct cros_usbpd_notify_data *)dev_get_drvdata(dev); - blocking_notifier_chain_unregister(&ecdev->ec_dev->event_notifier, nb); + blocking_notifier_chain_unregister(&ecdev->ec_dev->event_notifier, + &pdnotify->nb); return 0; } From 7e91e1ac60bb0f43e149fde7c5862d04ba27d7ad Mon Sep 17 00:00:00 2001 From: Prashant Malani Date: Mon, 16 Mar 2020 01:28:32 -0700 Subject: [PATCH 114/635] platform/chrome: cros_usbpd_notify: Amend ACPI driver to plat Convert the ACPI driver into the equivalent platform driver, with the same ACPI match table as before. This allows the device driver to access the parent platform EC device and its cros_ec_device struct, which will be required to communicate with the EC to pull PD Host event information from it. Also change the ACPI driver name to "cros-usbpd-notify-acpi" so that there is no confusion between it and the "regular" platform driver on platforms that have both CONFIG_ACPI and CONFIG_OF enabled. Signed-off-by: Prashant Malani Signed-off-by: Enric Balletbo i Serra Signed-off-by: Benson Leung --- drivers/platform/chrome/cros_usbpd_notify.c | 71 +++++++++++++++++---- 1 file changed, 59 insertions(+), 12 deletions(-) diff --git a/drivers/platform/chrome/cros_usbpd_notify.c b/drivers/platform/chrome/cros_usbpd_notify.c index d2af2130afec..76706a0d8f71 100644 --- a/drivers/platform/chrome/cros_usbpd_notify.c +++ b/drivers/platform/chrome/cros_usbpd_notify.c @@ -12,6 +12,7 @@ #include #define DRV_NAME "cros-usbpd-notify" +#define DRV_NAME_PLAT_ACPI "cros-usbpd-notify-acpi" #define ACPI_DRV_NAME "GOOG0003" static BLOCKING_NOTIFIER_HEAD(cros_usbpd_notifier_list); @@ -54,14 +55,61 @@ EXPORT_SYMBOL_GPL(cros_usbpd_unregister_notify); #ifdef CONFIG_ACPI -static int cros_usbpd_notify_add_acpi(struct acpi_device *adev) +static void cros_usbpd_notify_acpi(acpi_handle device, u32 event, void *data) { + blocking_notifier_call_chain(&cros_usbpd_notifier_list, event, NULL); +} + +static int cros_usbpd_notify_probe_acpi(struct platform_device *pdev) +{ + struct cros_usbpd_notify_data *pdnotify; + struct device *dev = &pdev->dev; + struct acpi_device *adev; + struct cros_ec_device *ec_dev; + acpi_status status; + + adev = ACPI_COMPANION(dev); + + pdnotify = devm_kzalloc(dev, sizeof(*pdnotify), GFP_KERNEL); + if (!pdnotify) + return -ENOMEM; + + /* Get the EC device pointer needed to talk to the EC. */ + ec_dev = dev_get_drvdata(dev->parent); + if (!ec_dev) { + /* + * We continue even for older devices which don't have the + * correct device heirarchy, namely, GOOG0003 is a child + * of GOOG0004. + */ + dev_warn(dev, "Couldn't get Chrome EC device pointer.\n"); + } + + pdnotify->dev = dev; + pdnotify->ec = ec_dev; + + status = acpi_install_notify_handler(adev->handle, + ACPI_ALL_NOTIFY, + cros_usbpd_notify_acpi, + pdnotify); + if (ACPI_FAILURE(status)) { + dev_warn(dev, "Failed to register notify handler %08x\n", + status); + return -EINVAL; + } + return 0; } -static void cros_usbpd_notify_acpi(struct acpi_device *adev, u32 event) +static int cros_usbpd_notify_remove_acpi(struct platform_device *pdev) { - blocking_notifier_call_chain(&cros_usbpd_notifier_list, event, NULL); + struct device *dev = &pdev->dev; + struct acpi_device *adev = ACPI_COMPANION(dev); + + acpi_remove_notify_handler(adev->handle, ACPI_ALL_NOTIFY, + cros_usbpd_notify_acpi); + + return 0; } static const struct acpi_device_id cros_usbpd_notify_acpi_device_ids[] = { @@ -70,14 +118,13 @@ static const struct acpi_device_id cros_usbpd_notify_acpi_device_ids[] = { }; MODULE_DEVICE_TABLE(acpi, cros_usbpd_notify_acpi_device_ids); -static struct acpi_driver cros_usbpd_notify_acpi_driver = { - .name = DRV_NAME, - .class = DRV_NAME, - .ids = cros_usbpd_notify_acpi_device_ids, - .ops = { - .add = cros_usbpd_notify_add_acpi, - .notify = cros_usbpd_notify_acpi, +static struct platform_driver cros_usbpd_notify_acpi_driver = { + .driver = { + .name = DRV_NAME_PLAT_ACPI, + .acpi_match_table = cros_usbpd_notify_acpi_device_ids, }, + .probe = cros_usbpd_notify_probe_acpi, + .remove = cros_usbpd_notify_remove_acpi, }; #endif /* CONFIG_ACPI */ @@ -157,7 +204,7 @@ static int __init cros_usbpd_notify_init(void) return ret; #ifdef CONFIG_ACPI - acpi_bus_register_driver(&cros_usbpd_notify_acpi_driver); + platform_driver_register(&cros_usbpd_notify_acpi_driver); #endif return 0; } @@ -165,7 +212,7 @@ static int __init cros_usbpd_notify_init(void) static void __exit cros_usbpd_notify_exit(void) { #ifdef CONFIG_ACPI - acpi_bus_unregister_driver(&cros_usbpd_notify_acpi_driver); + platform_driver_unregister(&cros_usbpd_notify_acpi_driver); #endif platform_driver_unregister(&cros_usbpd_notify_plat_driver); } From a88214089d67b0f246cf6ae4fb0a7e0735ff3595 Mon Sep 17 00:00:00 2001 From: Prashant Malani Date: Mon, 16 Mar 2020 01:28:34 -0700 Subject: [PATCH 115/635] platform/chrome: cros_usbpd_notify: Pull PD_HOST_EVENT status Read the PD host even status from the EC and send that to the notifier listeners, for more fine-grained event information. Signed-off-by: Prashant Malani Signed-off-by: Enric Balletbo i Serra Signed-off-by: Benson Leung --- drivers/platform/chrome/cros_usbpd_notify.c | 86 ++++++++++++++++++++- 1 file changed, 83 insertions(+), 3 deletions(-) diff --git a/drivers/platform/chrome/cros_usbpd_notify.c b/drivers/platform/chrome/cros_usbpd_notify.c index 76706a0d8f71..7f36142ab12a 100644 --- a/drivers/platform/chrome/cros_usbpd_notify.c +++ b/drivers/platform/chrome/cros_usbpd_notify.c @@ -53,11 +53,90 @@ void cros_usbpd_unregister_notify(struct notifier_block *nb) } EXPORT_SYMBOL_GPL(cros_usbpd_unregister_notify); +/** + * cros_ec_pd_command - Send a command to the EC. + * + * @ec_dev: EC device + * @command: EC command + * @outdata: EC command output data + * @outsize: Size of outdata + * @indata: EC command input data + * @insize: Size of indata + * + * Return: >= 0 on success, negative error number on failure. + */ +static int cros_ec_pd_command(struct cros_ec_device *ec_dev, + int command, + uint8_t *outdata, + int outsize, + uint8_t *indata, + int insize) +{ + struct cros_ec_command *msg; + int ret; + + msg = kzalloc(sizeof(*msg) + max(insize, outsize), GFP_KERNEL); + if (!msg) + return -ENOMEM; + + msg->command = command; + msg->outsize = outsize; + msg->insize = insize; + + if (outsize) + memcpy(msg->data, outdata, outsize); + + ret = cros_ec_cmd_xfer_status(ec_dev, msg); + if (ret < 0) + goto error; + + if (insize) + memcpy(indata, msg->data, insize); +error: + kfree(msg); + return ret; +} + +static void cros_usbpd_get_event_and_notify(struct device *dev, + struct cros_ec_device *ec_dev) +{ + struct ec_response_host_event_status host_event_status; + u32 event = 0; + int ret; + + /* + * We still send a 0 event out to older devices which don't + * have the updated device heirarchy. + */ + if (!ec_dev) { + dev_dbg(dev, + "EC device inaccessible; sending 0 event status.\n"); + goto send_notify; + } + + /* Check for PD host events on EC. */ + ret = cros_ec_pd_command(ec_dev, EC_CMD_PD_HOST_EVENT_STATUS, + NULL, 0, + (uint8_t *)&host_event_status, + sizeof(host_event_status)); + if (ret < 0) { + dev_warn(dev, "Can't get host event status (err: %d)\n", ret); + goto send_notify; + } + + event = host_event_status.status; + +send_notify: + blocking_notifier_call_chain(&cros_usbpd_notifier_list, event, NULL); +} + #ifdef CONFIG_ACPI static void cros_usbpd_notify_acpi(acpi_handle device, u32 event, void *data) { - blocking_notifier_call_chain(&cros_usbpd_notifier_list, event, NULL); + struct cros_usbpd_notify_data *pdnotify = data; + + cros_usbpd_get_event_and_notify(pdnotify->dev, pdnotify->ec); } static int cros_usbpd_notify_probe_acpi(struct platform_device *pdev) @@ -133,6 +212,8 @@ static int cros_usbpd_notify_plat(struct notifier_block *nb, unsigned long queued_during_suspend, void *data) { + struct cros_usbpd_notify_data *pdnotify = container_of(nb, + struct cros_usbpd_notify_data, nb); struct cros_ec_device *ec_dev = (struct cros_ec_device *)data; u32 host_event = cros_ec_get_host_event(ec_dev); @@ -140,8 +221,7 @@ static int cros_usbpd_notify_plat(struct notifier_block *nb, return NOTIFY_DONE; if (host_event & EC_HOST_EVENT_MASK(EC_HOST_EVENT_PD_MCU)) { - blocking_notifier_call_chain(&cros_usbpd_notifier_list, - host_event, NULL); + cros_usbpd_get_event_and_notify(pdnotify->dev, ec_dev); return NOTIFY_OK; } return NOTIFY_DONE; From 3cab1854b0c05c77f2d119d0ab70d3c33c9d5c61 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 16 Mar 2020 11:37:31 -0400 Subject: [PATCH 116/635] nfs: Fix up documentation in nfs_follow_referral() and nfs_do_submount() Fallout from the mount patches. Signed-off-by: Trond Myklebust --- fs/nfs/namespace.c | 5 +---- fs/nfs/nfs4namespace.c | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/fs/nfs/namespace.c b/fs/nfs/namespace.c index f3ece8ed3203..da67820462f2 100644 --- a/fs/nfs/namespace.c +++ b/fs/nfs/namespace.c @@ -247,10 +247,7 @@ void nfs_release_automount_timer(void) /** * nfs_do_submount - set up mountpoint when crossing a filesystem boundary - * @dentry: parent directory - * @fh: filehandle for new root dentry - * @fattr: attributes for new root inode - * @authflavor: security flavor to use when performing the mount + * @fc: pointer to struct nfs_fs_context * */ int nfs_do_submount(struct fs_context *fc) diff --git a/fs/nfs/nfs4namespace.c b/fs/nfs/nfs4namespace.c index 84026e7b8a5f..a3ab6e219061 100644 --- a/fs/nfs/nfs4namespace.c +++ b/fs/nfs/nfs4namespace.c @@ -354,7 +354,7 @@ static int try_location(struct fs_context *fc, /** * nfs_follow_referral - set up mountpoint when hitting a referral on moved error - * @dentry: parent directory + * @fc: pointer to struct nfs_fs_context * @locations: array of NFSv4 server location information * */ From 2484b17242896623b1603b371a6f981c4f197ece Mon Sep 17 00:00:00 2001 From: Yan Zhao Date: Thu, 5 Mar 2020 00:54:13 -0500 Subject: [PATCH 117/635] drm/i915/gvt: add support to command SWTESS_BASE_ADDRESS This cmd access is found on BDW. Signed-off-by: Yan Zhao Acked-by: Zhenyu Wang Signed-off-by: Zhenyu Wang Link: http://patchwork.freedesktop.org/patch/msgid/20200305055413.6833-1-yan.y.zhao@intel.com --- drivers/gpu/drm/i915/gvt/cmd_parser.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.c b/drivers/gpu/drm/i915/gvt/cmd_parser.c index 9e065ad0658f..bb204cfb9006 100644 --- a/drivers/gpu/drm/i915/gvt/cmd_parser.c +++ b/drivers/gpu/drm/i915/gvt/cmd_parser.c @@ -164,6 +164,7 @@ struct decode_info { #define OP_STATE_BASE_ADDRESS OP_3D_MEDIA(0x0, 0x1, 0x01) #define OP_STATE_SIP OP_3D_MEDIA(0x0, 0x1, 0x02) #define OP_3D_MEDIA_0_1_4 OP_3D_MEDIA(0x0, 0x1, 0x04) +#define OP_SWTESS_BASE_ADDRESS OP_3D_MEDIA(0x0, 0x1, 0x03) #define OP_3DSTATE_VF_STATISTICS_GM45 OP_3D_MEDIA(0x1, 0x0, 0x0B) @@ -2485,6 +2486,9 @@ static const struct cmd_info cmd_info[] = { {"OP_3D_MEDIA_0_1_4", OP_3D_MEDIA_0_1_4, F_LEN_VAR, R_RCS, D_ALL, ADDR_FIX_1(1), 8, NULL}, + {"OP_SWTESS_BASE_ADDRESS", OP_SWTESS_BASE_ADDRESS, + F_LEN_VAR, R_RCS, D_ALL, ADDR_FIX_2(1, 2), 3, NULL}, + {"3DSTATE_VS", OP_3DSTATE_VS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, {"3DSTATE_SF", OP_3DSTATE_SF, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, From dbafc67307ec06036b25b223a251af03fe07969a Mon Sep 17 00:00:00 2001 From: Yan Zhao Date: Wed, 18 Mar 2020 13:48:26 +0800 Subject: [PATCH 118/635] drm/i915/gvt: do not check len & max_len for lri lri ususally of variable len and far exceeding 127 dwords. Fixes: 00a33be40634 ("drm/i915/gvt: Add valid length check for MI variable commands") Signed-off-by: Yan Zhao Acked-by: Zhenyu Wang Signed-off-by: Zhenyu Wang Link: http://patchwork.freedesktop.org/patch/msgid/20200304095121.21609-1-yan.y.zhao@intel.com --- drivers/gpu/drm/i915/gvt/cmd_parser.c | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.c b/drivers/gpu/drm/i915/gvt/cmd_parser.c index bb204cfb9006..a3cc080a46c6 100644 --- a/drivers/gpu/drm/i915/gvt/cmd_parser.c +++ b/drivers/gpu/drm/i915/gvt/cmd_parser.c @@ -968,18 +968,6 @@ static int cmd_handler_lri(struct parser_exec_state *s) { int i, ret = 0; int cmd_len = cmd_length(s); - u32 valid_len = CMD_LEN(1); - - /* - * Official intel docs are somewhat sloppy , check the definition of - * MI_LOAD_REGISTER_IMM. - */ - #define MAX_VALID_LEN 127 - if ((cmd_len < valid_len) || (cmd_len > MAX_VALID_LEN)) { - gvt_err("len is not valid: len=%u valid_len=%u\n", - cmd_len, valid_len); - return -EFAULT; - } for (i = 1; i < cmd_len; i += 2) { if (IS_BROADWELL(s->engine->i915) && s->engine->id != RCS0) { From 000de5417107623925a4cf0310579f744ff43c28 Mon Sep 17 00:00:00 2001 From: Ansuel Smith Date: Tue, 4 Feb 2020 20:56:48 +0100 Subject: [PATCH 119/635] watchdog: qcom-wdt: disable pretimeout on timer platform Some platform like ipq806x doesn't support pretimeout and define some interrupts used by qcom,msm-timer. Change the driver to check and use pretimeout only on qcom,kpss-wdt as it's the only platform that actually supports it. Signed-off-by: Ansuel Smith Reviewed-by: Guenter Roeck Link: https://lore.kernel.org/r/20200204195648.23350-1-ansuelsmth@gmail.com [groeck: Conflict resolution] Signed-off-by: Guenter Roeck Signed-off-by: Wim Van Sebroeck --- drivers/watchdog/qcom-wdt.c | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/drivers/watchdog/qcom-wdt.c b/drivers/watchdog/qcom-wdt.c index eb47fe5ed280..460967a02da3 100644 --- a/drivers/watchdog/qcom-wdt.c +++ b/drivers/watchdog/qcom-wdt.c @@ -40,6 +40,11 @@ static const u32 reg_offset_data_kpss[] = { [WDT_BITE_TIME] = 0x14, }; +struct qcom_wdt_match_data { + const u32 *offset; + bool pretimeout; +}; + struct qcom_wdt { struct watchdog_device wdd; unsigned long rate; @@ -179,19 +184,29 @@ static void qcom_clk_disable_unprepare(void *data) clk_disable_unprepare(data); } +static const struct qcom_wdt_match_data match_data_apcs_tmr = { + .offset = reg_offset_data_apcs_tmr, + .pretimeout = false, +}; + +static const struct qcom_wdt_match_data match_data_kpss = { + .offset = reg_offset_data_kpss, + .pretimeout = true, +}; + static int qcom_wdt_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct qcom_wdt *wdt; struct resource *res; struct device_node *np = dev->of_node; - const u32 *regs; + const struct qcom_wdt_match_data *data; u32 percpu_offset; int irq, ret; struct clk *clk; - regs = of_device_get_match_data(dev); - if (!regs) { + data = of_device_get_match_data(dev); + if (!data) { dev_err(dev, "Unsupported QCOM WDT module\n"); return -ENODEV; } @@ -247,7 +262,7 @@ static int qcom_wdt_probe(struct platform_device *pdev) /* check if there is pretimeout support */ irq = platform_get_irq_optional(pdev, 0); - if (irq > 0) { + if (data->pretimeout && irq > 0) { ret = devm_request_irq(dev, irq, qcom_wdt_isr, IRQF_TRIGGER_RISING, "wdt_bark", &wdt->wdd); @@ -267,7 +282,7 @@ static int qcom_wdt_probe(struct platform_device *pdev) wdt->wdd.min_timeout = 1; wdt->wdd.max_timeout = 0x10000000U / wdt->rate; wdt->wdd.parent = dev; - wdt->layout = regs; + wdt->layout = data->offset; if (readl(wdt_addr(wdt, WDT_STS)) & 1) wdt->wdd.bootstatus = WDIOF_CARDRESET; @@ -311,9 +326,9 @@ static int __maybe_unused qcom_wdt_resume(struct device *dev) static SIMPLE_DEV_PM_OPS(qcom_wdt_pm_ops, qcom_wdt_suspend, qcom_wdt_resume); static const struct of_device_id qcom_wdt_of_table[] = { - { .compatible = "qcom,kpss-timer", .data = reg_offset_data_apcs_tmr }, - { .compatible = "qcom,scss-timer", .data = reg_offset_data_apcs_tmr }, - { .compatible = "qcom,kpss-wdt", .data = reg_offset_data_kpss }, + { .compatible = "qcom,kpss-timer", .data = &match_data_apcs_tmr }, + { .compatible = "qcom,scss-timer", .data = &match_data_apcs_tmr }, + { .compatible = "qcom,kpss-wdt", .data = &match_data_kpss }, { }, }; MODULE_DEVICE_TABLE(of, qcom_wdt_of_table); From b1413e6edc5ab55d77cfa2a0b45f4918c5e97e0f Mon Sep 17 00:00:00 2001 From: Loic Poulain Date: Tue, 11 Feb 2020 16:38:03 +0100 Subject: [PATCH 120/635] watchdog: pm8916_wdt: Add system sleep callbacks Add suspend and resume pm operations. Tested on dragonboard-410c. Signed-off-by: Loic Poulain Reviewed-by: Guenter Roeck Link: https://lore.kernel.org/r/1581435483-6796-1-git-send-email-loic.poulain@linaro.org Signed-off-by: Guenter Roeck Signed-off-by: Wim Van Sebroeck --- drivers/watchdog/pm8916_wdt.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/drivers/watchdog/pm8916_wdt.c b/drivers/watchdog/pm8916_wdt.c index 1213179f863c..0937b8d33104 100644 --- a/drivers/watchdog/pm8916_wdt.c +++ b/drivers/watchdog/pm8916_wdt.c @@ -192,6 +192,7 @@ static int pm8916_wdt_probe(struct platform_device *pdev) wdt->wdev.timeout = PM8916_WDT_DEFAULT_TIMEOUT; wdt->wdev.pretimeout = 0; watchdog_set_drvdata(&wdt->wdev, wdt); + platform_set_drvdata(pdev, wdt); watchdog_init_timeout(&wdt->wdev, 0, dev); pm8916_wdt_configure_timers(&wdt->wdev); @@ -199,6 +200,29 @@ static int pm8916_wdt_probe(struct platform_device *pdev) return devm_watchdog_register_device(dev, &wdt->wdev); } +static int __maybe_unused pm8916_wdt_suspend(struct device *dev) +{ + struct pm8916_wdt *wdt = dev_get_drvdata(dev); + + if (watchdog_active(&wdt->wdev)) + return pm8916_wdt_stop(&wdt->wdev); + + return 0; +} + +static int __maybe_unused pm8916_wdt_resume(struct device *dev) +{ + struct pm8916_wdt *wdt = dev_get_drvdata(dev); + + if (watchdog_active(&wdt->wdev)) + return pm8916_wdt_start(&wdt->wdev); + + return 0; +} + +static SIMPLE_DEV_PM_OPS(pm8916_wdt_pm_ops, pm8916_wdt_suspend, + pm8916_wdt_resume); + static const struct of_device_id pm8916_wdt_id_table[] = { { .compatible = "qcom,pm8916-wdt" }, { } @@ -210,6 +234,7 @@ static struct platform_driver pm8916_wdt_driver = { .driver = { .name = "pm8916-wdt", .of_match_table = of_match_ptr(pm8916_wdt_id_table), + .pm = &pm8916_wdt_pm_ops, }, }; module_platform_driver(pm8916_wdt_driver); From cc9cc794c329a9c4d0c1056b4b1f1632e9db0b8c Mon Sep 17 00:00:00 2001 From: Stephen Boyd Date: Wed, 19 Feb 2020 16:20:47 -0800 Subject: [PATCH 121/635] watchdog: qcom: Use irq flags from firmware The DT or ACPI tables should tell the driver what the irq flags are. Given that this driver probes only on DT based platforms and those DT platforms specify the irq flags we can safely drop the forced irq flag setting here. Cc: Andy Gross Cc: Bjorn Andersson Cc: Sai Prakash Ranjan Signed-off-by: Stephen Boyd Reviewed-by: Bjorn Andersson Reviewed-by: Guenter Roeck Reviewed-by: Sai Prakash Ranjan Link: https://lore.kernel.org/r/20200220002047.115000-1-swboyd@chromium.org [groeck: Context conflict resolution] Signed-off-by: Guenter Roeck Signed-off-by: Wim Van Sebroeck --- drivers/watchdog/qcom-wdt.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/watchdog/qcom-wdt.c b/drivers/watchdog/qcom-wdt.c index 460967a02da3..ab7465d186fd 100644 --- a/drivers/watchdog/qcom-wdt.c +++ b/drivers/watchdog/qcom-wdt.c @@ -263,8 +263,7 @@ static int qcom_wdt_probe(struct platform_device *pdev) /* check if there is pretimeout support */ irq = platform_get_irq_optional(pdev, 0); if (data->pretimeout && irq > 0) { - ret = devm_request_irq(dev, irq, qcom_wdt_isr, - IRQF_TRIGGER_RISING, + ret = devm_request_irq(dev, irq, qcom_wdt_isr, 0, "wdt_bark", &wdt->wdd); if (ret) return ret; From 15a2638ab46d5da1accc224e566d319d8d9a521e Mon Sep 17 00:00:00 2001 From: Anson Huang Date: Fri, 21 Feb 2020 10:00:28 +0800 Subject: [PATCH 122/635] watchdog: imx_sc_wdt: Remove unused includes There is nothing in use from init.h/reboot.h, remove them. Signed-off-by: Anson Huang Reviewed-by: Guenter Roeck Link: https://lore.kernel.org/r/1582250430-8872-1-git-send-email-Anson.Huang@nxp.com Signed-off-by: Guenter Roeck Signed-off-by: Wim Van Sebroeck --- drivers/watchdog/imx_sc_wdt.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/watchdog/imx_sc_wdt.c b/drivers/watchdog/imx_sc_wdt.c index 8ed89f032ebf..60a32469f7de 100644 --- a/drivers/watchdog/imx_sc_wdt.c +++ b/drivers/watchdog/imx_sc_wdt.c @@ -6,13 +6,11 @@ #include #include #include -#include #include #include #include #include #include -#include #include #define DEFAULT_TIMEOUT 60 From dca96e0117a040b2898899a32f9cbac3082dd3b2 Mon Sep 17 00:00:00 2001 From: Anson Huang Date: Fri, 21 Feb 2020 10:00:29 +0800 Subject: [PATCH 123/635] watchdog: imx7ulp: Remove unused include of init.h There is nothing in use from init.h, remove it. Signed-off-by: Anson Huang Link: https://lore.kernel.org/r/1582250430-8872-2-git-send-email-Anson.Huang@nxp.com Signed-off-by: Guenter Roeck Signed-off-by: Wim Van Sebroeck --- drivers/watchdog/imx7ulp_wdt.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/watchdog/imx7ulp_wdt.c b/drivers/watchdog/imx7ulp_wdt.c index 11b9e7c6b7f5..7993c8c41b3a 100644 --- a/drivers/watchdog/imx7ulp_wdt.c +++ b/drivers/watchdog/imx7ulp_wdt.c @@ -4,7 +4,6 @@ */ #include -#include #include #include #include From 3f9d51333129e16d77dcc9414bd548151d884c8a Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Sat, 29 Feb 2020 12:50:46 +0100 Subject: [PATCH 124/635] watchdog: wm831x_wdt: Remove GPIO handling An attempt to convert the driver to using GPIO descriptors (see Link tag) was discouraged in favor of deleting the handling of the update GPIO altogehter since there are no in-tree users. This patch deletes the GPIO handling instead. Cc: Richard Fitzgerald Cc: Charles Keepax Cc: Mark Brown Link: https://lore.kernel.org/linux-watchdog/20200210102209.289379-1-linus.walleij@linaro.org/ Signed-off-by: Linus Walleij Reviewed-by: Guenter Roeck Acked-by: Charles Keepax Link: https://lore.kernel.org/r/20200229115046.57781-1-linus.walleij@linaro.org Signed-off-by: Guenter Roeck Signed-off-by: Wim Van Sebroeck --- drivers/watchdog/wm831x_wdt.c | 27 --------------------------- include/linux/mfd/wm831x/pdata.h | 1 - 2 files changed, 28 deletions(-) diff --git a/drivers/watchdog/wm831x_wdt.c b/drivers/watchdog/wm831x_wdt.c index 030ce240620d..d96ad8f38bd2 100644 --- a/drivers/watchdog/wm831x_wdt.c +++ b/drivers/watchdog/wm831x_wdt.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include @@ -29,7 +28,6 @@ struct wm831x_wdt_drvdata { struct watchdog_device wdt; struct wm831x *wm831x; struct mutex lock; - int update_gpio; int update_state; }; @@ -103,14 +101,6 @@ static int wm831x_wdt_ping(struct watchdog_device *wdt_dev) mutex_lock(&driver_data->lock); - if (driver_data->update_gpio) { - gpio_set_value_cansleep(driver_data->update_gpio, - driver_data->update_state); - driver_data->update_state = !driver_data->update_state; - ret = 0; - goto out; - } - reg = wm831x_reg_read(wm831x, WM831X_WATCHDOG); if (!(reg & WM831X_WDOG_RST_SRC)) { @@ -239,23 +229,6 @@ static int wm831x_wdt_probe(struct platform_device *pdev) reg |= pdata->secondary << WM831X_WDOG_SECACT_SHIFT; reg |= pdata->software << WM831X_WDOG_RST_SRC_SHIFT; - if (pdata->update_gpio) { - ret = devm_gpio_request_one(dev, pdata->update_gpio, - GPIOF_OUT_INIT_LOW, - "Watchdog update"); - if (ret < 0) { - dev_err(wm831x->dev, - "Failed to request update GPIO: %d\n", - ret); - return ret; - } - - driver_data->update_gpio = pdata->update_gpio; - - /* Make sure the watchdog takes hardware updates */ - reg |= WM831X_WDOG_RST_SRC; - } - ret = wm831x_reg_unlock(wm831x); if (ret == 0) { ret = wm831x_reg_write(wm831x, WM831X_WATCHDOG, reg); diff --git a/include/linux/mfd/wm831x/pdata.h b/include/linux/mfd/wm831x/pdata.h index 986986fe4e4e..75aa94dadf1c 100644 --- a/include/linux/mfd/wm831x/pdata.h +++ b/include/linux/mfd/wm831x/pdata.h @@ -89,7 +89,6 @@ enum wm831x_watchdog_action { struct wm831x_watchdog_pdata { enum wm831x_watchdog_action primary, secondary; - int update_gpio; unsigned int software:1; }; From 9232c80659e941ffa6c7bfc96bd64c6172ef0452 Mon Sep 17 00:00:00 2001 From: Dmitry Safonov Date: Sun, 23 Feb 2020 11:49:39 +0000 Subject: [PATCH 125/635] watchdog: Add stop_on_reboot parameter to control reboot policy Many watchdog drivers use watchdog_stop_on_reboot() helper in order to stop the watchdog on system reboot. Unfortunately, this logic is coded in driver's probe function and doesn't allows user to decide what to do during shutdown/reboot. On the other side, Xen and Qemu watchdog drivers (xen_wdt and i6300esb) may be configured to either send NMI or turn off/reboot VM as the watchdog action. As the kernel may stuck at any state, sending NMIs can't reliably reboot the VM. At Arista, we benefited from the following set-up: the emulated watchdogs trigger VM reset and softdog is set to catch less severe conditions to generate vmcore. Just before reboot watchdog's timeout is increased to some good-enough value (3 mins). That keeps watchdog always running and guarantees that VM doesn't stuck. Provide new stop_on_reboot module parameter to let user control watchdog's reboot policy. Cc: Guenter Roeck Cc: Wim Van Sebroeck Cc: linux-watchdog@vger.kernel.org Signed-off-by: Dmitry Safonov Reviewed-by: Guenter Roeck Link: https://lore.kernel.org/r/20200223114939.194754-1-dima@arista.com Signed-off-by: Guenter Roeck Signed-off-by: Wim Van Sebroeck --- drivers/watchdog/watchdog_core.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/drivers/watchdog/watchdog_core.c b/drivers/watchdog/watchdog_core.c index 861daf4f37b2..423844757812 100644 --- a/drivers/watchdog/watchdog_core.c +++ b/drivers/watchdog/watchdog_core.c @@ -39,6 +39,10 @@ static DEFINE_IDA(watchdog_ida); +static int stop_on_reboot = -1; +module_param(stop_on_reboot, int, 0444); +MODULE_PARM_DESC(stop_on_reboot, "Stop watchdogs on reboot (0=keep watching, 1=stop)"); + /* * Deferred Registration infrastructure. * @@ -254,6 +258,14 @@ static int __watchdog_register_device(struct watchdog_device *wdd) } } + /* Module parameter to force watchdog policy on reboot. */ + if (stop_on_reboot != -1) { + if (stop_on_reboot) + set_bit(WDOG_STOP_ON_REBOOT, &wdd->status); + else + clear_bit(WDOG_STOP_ON_REBOOT, &wdd->status); + } + if (test_bit(WDOG_STOP_ON_REBOOT, &wdd->status)) { wdd->reboot_nb.notifier_call = watchdog_reboot_notifier; From 436867b6469a2ae38f6b5725aeeeb8211a2aca15 Mon Sep 17 00:00:00 2001 From: Anson Huang Date: Mon, 24 Feb 2020 10:51:27 +0800 Subject: [PATCH 126/635] watchdog: imx2_wdt: Drop .remove callback .remove callback implementation doesn' call clk_disable_unprepare() which is buggy, actually, we can just use devm_watchdog_register_device() and devm_add_action_or_reset() to handle all necessary operations for remove action, then .remove callback can be dropped. Signed-off-by: Anson Huang Reviewed-by: Guenter Roeck Link: https://lore.kernel.org/r/1582512687-13312-1-git-send-email-Anson.Huang@nxp.com Signed-off-by: Guenter Roeck Signed-off-by: Wim Van Sebroeck --- drivers/watchdog/imx2_wdt.c | 37 ++++++++++--------------------------- 1 file changed, 10 insertions(+), 27 deletions(-) diff --git a/drivers/watchdog/imx2_wdt.c b/drivers/watchdog/imx2_wdt.c index f8d58bf0bf66..1fe472f56cb3 100644 --- a/drivers/watchdog/imx2_wdt.c +++ b/drivers/watchdog/imx2_wdt.c @@ -244,6 +244,11 @@ static const struct regmap_config imx2_wdt_regmap_config = { .max_register = 0x8, }; +static void imx2_wdt_action(void *data) +{ + clk_disable_unprepare(data); +} + static int __init imx2_wdt_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -292,6 +297,10 @@ static int __init imx2_wdt_probe(struct platform_device *pdev) if (ret) return ret; + ret = devm_add_action_or_reset(dev, imx2_wdt_action, wdev->clk); + if (ret) + return ret; + regmap_read(wdev->regmap, IMX2_WDT_WRSR, &val); wdog->bootstatus = val & IMX2_WDT_WRSR_TOUT ? WDIOF_CARDRESET : 0; @@ -315,32 +324,7 @@ static int __init imx2_wdt_probe(struct platform_device *pdev) */ regmap_write(wdev->regmap, IMX2_WDT_WMCR, 0); - ret = watchdog_register_device(wdog); - if (ret) - goto disable_clk; - - dev_info(dev, "timeout %d sec (nowayout=%d)\n", - wdog->timeout, nowayout); - - return 0; - -disable_clk: - clk_disable_unprepare(wdev->clk); - return ret; -} - -static int __exit imx2_wdt_remove(struct platform_device *pdev) -{ - struct watchdog_device *wdog = platform_get_drvdata(pdev); - struct imx2_wdt_device *wdev = watchdog_get_drvdata(wdog); - - watchdog_unregister_device(wdog); - - if (imx2_wdt_is_running(wdev)) { - imx2_wdt_ping(wdog); - dev_crit(&pdev->dev, "Device removed: Expect reboot!\n"); - } - return 0; + return devm_watchdog_register_device(dev, wdog); } static void imx2_wdt_shutdown(struct platform_device *pdev) @@ -417,7 +401,6 @@ static const struct of_device_id imx2_wdt_dt_ids[] = { MODULE_DEVICE_TABLE(of, imx2_wdt_dt_ids); static struct platform_driver imx2_wdt_driver = { - .remove = __exit_p(imx2_wdt_remove), .shutdown = imx2_wdt_shutdown, .driver = { .name = DRIVER_NAME, From 982bb70517aef2225bad1d802887b733db492cc0 Mon Sep 17 00:00:00 2001 From: Tero Kristo Date: Thu, 12 Mar 2020 11:58:06 +0200 Subject: [PATCH 127/635] watchdog: reset last_hw_keepalive time at start Currently the watchdog core does not initialize the last_hw_keepalive time during watchdog startup. This will cause the watchdog to be pinged immediately if enough time has passed from the system boot-up time, and some types of watchdogs like K3 RTI does not like this. To avoid the issue, setup the last_hw_keepalive time during watchdog startup. Signed-off-by: Tero Kristo Reviewed-by: Guenter Roeck Link: https://lore.kernel.org/r/20200302200426.6492-3-t-kristo@ti.com Signed-off-by: Guenter Roeck Signed-off-by: Wim Van Sebroeck --- drivers/watchdog/watchdog_dev.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/watchdog/watchdog_dev.c b/drivers/watchdog/watchdog_dev.c index 8b5c742f24e8..7e4cd34a8c20 100644 --- a/drivers/watchdog/watchdog_dev.c +++ b/drivers/watchdog/watchdog_dev.c @@ -282,6 +282,7 @@ static int watchdog_start(struct watchdog_device *wdd) if (err == 0) { set_bit(WDOG_ACTIVE, &wdd->status); wd_data->last_keepalive = started_at; + wd_data->last_hw_keepalive = started_at; watchdog_update_worker(wdd); } From f667421324ed89a545b8d7efc69284059ea28d21 Mon Sep 17 00:00:00 2001 From: Tomer Maimon Date: Tue, 3 Mar 2020 12:01:14 +0200 Subject: [PATCH 128/635] watchdog: npcm: remove whitespaces Signed-off-by: Tomer Maimon Link: https://lore.kernel.org/r/20200303100114.87786-4-tmaimon77@gmail.com Reviewed-by: Guenter Roeck Signed-off-by: Guenter Roeck Signed-off-by: Wim Van Sebroeck --- drivers/watchdog/npcm_wdt.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/drivers/watchdog/npcm_wdt.c b/drivers/watchdog/npcm_wdt.c index 9c773c3d6d5d..765577f11c8d 100644 --- a/drivers/watchdog/npcm_wdt.c +++ b/drivers/watchdog/npcm_wdt.c @@ -103,30 +103,29 @@ static int npcm_wdt_stop(struct watchdog_device *wdd) return 0; } - static int npcm_wdt_set_timeout(struct watchdog_device *wdd, unsigned int timeout) { if (timeout < 2) wdd->timeout = 1; else if (timeout < 3) - wdd->timeout = 2; + wdd->timeout = 2; else if (timeout < 6) - wdd->timeout = 5; + wdd->timeout = 5; else if (timeout < 11) - wdd->timeout = 10; + wdd->timeout = 10; else if (timeout < 22) - wdd->timeout = 21; + wdd->timeout = 21; else if (timeout < 44) - wdd->timeout = 43; + wdd->timeout = 43; else if (timeout < 87) - wdd->timeout = 86; + wdd->timeout = 86; else if (timeout < 173) - wdd->timeout = 172; + wdd->timeout = 172; else if (timeout < 688) - wdd->timeout = 687; + wdd->timeout = 687; else - wdd->timeout = 2750; + wdd->timeout = 2750; if (watchdog_active(wdd)) npcm_wdt_start(wdd); From 2644f912b41012c1ce5ff9be99efeec721491b86 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Mon, 10 Feb 2020 11:15:46 +0100 Subject: [PATCH 129/635] backlight: pwm_bl: Switch to full GPIO descriptor The PWM backlight still supports passing a enable GPIO line as platform data using the legacy API. It turns out that ever board using this mechanism except one is pass .enable_gpio = -1. So we drop all these cargo-culted -1's from all instances of this platform data in the kernel. The remaning board, Palm TC, is converted to pass a machine descriptior table with the "enable" GPIO instead, and delete the platform data entry for enable_gpio and the code handling it and things should work smoothly with the new API. Signed-off-by: Linus Walleij Acked-by: Robert Jarzmik Acked-by: Krzysztof Kozlowski Reviewed-by: Daniel Thompson Signed-off-by: Lee Jones --- arch/arm/mach-pxa/cm-x300.c | 1 - arch/arm/mach-pxa/colibri-pxa270-income.c | 1 - arch/arm/mach-pxa/ezx.c | 1 - arch/arm/mach-pxa/hx4700.c | 1 - arch/arm/mach-pxa/lpd270.c | 1 - arch/arm/mach-pxa/magician.c | 1 - arch/arm/mach-pxa/mainstone.c | 1 - arch/arm/mach-pxa/mioa701.c | 1 - arch/arm/mach-pxa/palm27x.c | 1 - arch/arm/mach-pxa/palmtc.c | 11 ++++++++++- arch/arm/mach-pxa/palmte2.c | 1 - arch/arm/mach-pxa/pcm990-baseboard.c | 1 - arch/arm/mach-pxa/tavorevb.c | 2 -- arch/arm/mach-pxa/viper.c | 1 - arch/arm/mach-pxa/z2.c | 2 -- arch/arm/mach-pxa/zylonite.c | 1 - arch/arm/mach-s3c24xx/mach-h1940.c | 1 - arch/arm/mach-s3c24xx/mach-rx1950.c | 1 - arch/arm/mach-s3c64xx/dev-backlight.c | 3 --- arch/arm/mach-s3c64xx/mach-crag6410.c | 1 - arch/arm/mach-s3c64xx/mach-hmt.c | 1 - arch/arm/mach-s3c64xx/mach-smartq.c | 1 - arch/arm/mach-s3c64xx/mach-smdk6410.c | 2 +- arch/unicore32/kernel/puv3-nb0916.c | 1 - drivers/video/backlight/pwm_bl.c | 19 ------------------- include/linux/pwm_backlight.h | 2 -- 26 files changed, 11 insertions(+), 49 deletions(-) diff --git a/arch/arm/mach-pxa/cm-x300.c b/arch/arm/mach-pxa/cm-x300.c index 425855f456f2..2e35354b61f5 100644 --- a/arch/arm/mach-pxa/cm-x300.c +++ b/arch/arm/mach-pxa/cm-x300.c @@ -312,7 +312,6 @@ static struct pwm_lookup cm_x300_pwm_lookup[] = { static struct platform_pwm_backlight_data cm_x300_backlight_data = { .max_brightness = 100, .dft_brightness = 100, - .enable_gpio = -1, }; static struct platform_device cm_x300_backlight_device = { diff --git a/arch/arm/mach-pxa/colibri-pxa270-income.c b/arch/arm/mach-pxa/colibri-pxa270-income.c index dbad2f13706c..e5879e8b0682 100644 --- a/arch/arm/mach-pxa/colibri-pxa270-income.c +++ b/arch/arm/mach-pxa/colibri-pxa270-income.c @@ -202,7 +202,6 @@ static struct pwm_lookup income_pwm_lookup[] = { static struct platform_pwm_backlight_data income_backlight_data = { .max_brightness = 0x3ff, .dft_brightness = 0x1ff, - .enable_gpio = -1, }; static struct platform_device income_backlight = { diff --git a/arch/arm/mach-pxa/ezx.c b/arch/arm/mach-pxa/ezx.c index ec10851b63cf..eb85950e7c0e 100644 --- a/arch/arm/mach-pxa/ezx.c +++ b/arch/arm/mach-pxa/ezx.c @@ -55,7 +55,6 @@ static struct pwm_lookup ezx_pwm_lookup[] __maybe_unused = { static struct platform_pwm_backlight_data ezx_backlight_data = { .max_brightness = 1023, .dft_brightness = 1023, - .enable_gpio = -1, }; static struct platform_device ezx_backlight_device = { diff --git a/arch/arm/mach-pxa/hx4700.c b/arch/arm/mach-pxa/hx4700.c index 238a751a8797..1d4c5db54be2 100644 --- a/arch/arm/mach-pxa/hx4700.c +++ b/arch/arm/mach-pxa/hx4700.c @@ -556,7 +556,6 @@ static struct platform_device hx4700_lcd = { static struct platform_pwm_backlight_data backlight_data = { .max_brightness = 200, .dft_brightness = 100, - .enable_gpio = -1, }; static struct platform_device backlight = { diff --git a/arch/arm/mach-pxa/lpd270.c b/arch/arm/mach-pxa/lpd270.c index 20e00e970385..6fc40bc06910 100644 --- a/arch/arm/mach-pxa/lpd270.c +++ b/arch/arm/mach-pxa/lpd270.c @@ -277,7 +277,6 @@ static struct pwm_lookup lpd270_pwm_lookup[] = { static struct platform_pwm_backlight_data lpd270_backlight_data = { .max_brightness = 1, .dft_brightness = 1, - .enable_gpio = -1, }; static struct platform_device lpd270_backlight_device = { diff --git a/arch/arm/mach-pxa/magician.c b/arch/arm/mach-pxa/magician.c index 5d0591f93f4d..cd9fa465b9b2 100644 --- a/arch/arm/mach-pxa/magician.c +++ b/arch/arm/mach-pxa/magician.c @@ -401,7 +401,6 @@ static void magician_backlight_exit(struct device *dev) static struct platform_pwm_backlight_data backlight_data = { .max_brightness = 272, .dft_brightness = 100, - .enable_gpio = -1, .init = magician_backlight_init, .notify = magician_backlight_notify, .exit = magician_backlight_exit, diff --git a/arch/arm/mach-pxa/mainstone.c b/arch/arm/mach-pxa/mainstone.c index 1b7882920164..d1010ec26e9f 100644 --- a/arch/arm/mach-pxa/mainstone.c +++ b/arch/arm/mach-pxa/mainstone.c @@ -256,7 +256,6 @@ static struct pwm_lookup mainstone_pwm_lookup[] = { static struct platform_pwm_backlight_data mainstone_backlight_data = { .max_brightness = 1023, .dft_brightness = 1023, - .enable_gpio = -1, }; static struct platform_device mainstone_backlight_device = { diff --git a/arch/arm/mach-pxa/mioa701.c b/arch/arm/mach-pxa/mioa701.c index 0b8bae9610f1..d3af80317f2d 100644 --- a/arch/arm/mach-pxa/mioa701.c +++ b/arch/arm/mach-pxa/mioa701.c @@ -176,7 +176,6 @@ static struct pwm_lookup mioa701_pwm_lookup[] = { static struct platform_pwm_backlight_data mioa701_backlight_data = { .max_brightness = 100, .dft_brightness = 50, - .enable_gpio = -1, }; /* diff --git a/arch/arm/mach-pxa/palm27x.c b/arch/arm/mach-pxa/palm27x.c index b600b63af3a6..0d246a1aebbc 100644 --- a/arch/arm/mach-pxa/palm27x.c +++ b/arch/arm/mach-pxa/palm27x.c @@ -318,7 +318,6 @@ static void palm27x_backlight_exit(struct device *dev) static struct platform_pwm_backlight_data palm27x_backlight_data = { .max_brightness = 0xfe, .dft_brightness = 0x7e, - .enable_gpio = -1, .init = palm27x_backlight_init, .notify = palm27x_backlight_notify, .exit = palm27x_backlight_exit, diff --git a/arch/arm/mach-pxa/palmtc.c b/arch/arm/mach-pxa/palmtc.c index fda9deaaae02..455cb8ccaf26 100644 --- a/arch/arm/mach-pxa/palmtc.c +++ b/arch/arm/mach-pxa/palmtc.c @@ -174,6 +174,15 @@ static inline void palmtc_keys_init(void) {} * Backlight ******************************************************************************/ #if defined(CONFIG_BACKLIGHT_PWM) || defined(CONFIG_BACKLIGHT_PWM_MODULE) + +static struct gpiod_lookup_table palmtc_pwm_bl_gpio_table = { + .dev_id = "pwm-backlight.0", + .table = { + GPIO_LOOKUP("gpio-pxa", GPIO_NR_PALMTC_BL_POWER, + "enable", GPIO_ACTIVE_HIGH), + }, +}; + static struct pwm_lookup palmtc_pwm_lookup[] = { PWM_LOOKUP("pxa25x-pwm.1", 0, "pwm-backlight.0", NULL, PALMTC_PERIOD_NS, PWM_POLARITY_NORMAL), @@ -182,7 +191,6 @@ static struct pwm_lookup palmtc_pwm_lookup[] = { static struct platform_pwm_backlight_data palmtc_backlight_data = { .max_brightness = PALMTC_MAX_INTENSITY, .dft_brightness = PALMTC_MAX_INTENSITY, - .enable_gpio = GPIO_NR_PALMTC_BL_POWER, }; static struct platform_device palmtc_backlight = { @@ -195,6 +203,7 @@ static struct platform_device palmtc_backlight = { static void __init palmtc_pwm_init(void) { + gpiod_add_lookup_table(&palmtc_pwm_bl_gpio_table); pwm_add_table(palmtc_pwm_lookup, ARRAY_SIZE(palmtc_pwm_lookup)); platform_device_register(&palmtc_backlight); } diff --git a/arch/arm/mach-pxa/palmte2.c b/arch/arm/mach-pxa/palmte2.c index 7171014fd311..e3bcf58b4e63 100644 --- a/arch/arm/mach-pxa/palmte2.c +++ b/arch/arm/mach-pxa/palmte2.c @@ -175,7 +175,6 @@ static void palmte2_backlight_exit(struct device *dev) static struct platform_pwm_backlight_data palmte2_backlight_data = { .max_brightness = PALMTE2_MAX_INTENSITY, .dft_brightness = PALMTE2_MAX_INTENSITY, - .enable_gpio = -1, .init = palmte2_backlight_init, .notify = palmte2_backlight_notify, .exit = palmte2_backlight_exit, diff --git a/arch/arm/mach-pxa/pcm990-baseboard.c b/arch/arm/mach-pxa/pcm990-baseboard.c index cb1c56769fbc..bf613f88d70b 100644 --- a/arch/arm/mach-pxa/pcm990-baseboard.c +++ b/arch/arm/mach-pxa/pcm990-baseboard.c @@ -154,7 +154,6 @@ static struct pwm_lookup pcm990_pwm_lookup[] = { static struct platform_pwm_backlight_data pcm990_backlight_data = { .max_brightness = 1023, .dft_brightness = 1023, - .enable_gpio = -1, }; static struct platform_device pcm990_backlight_device = { diff --git a/arch/arm/mach-pxa/tavorevb.c b/arch/arm/mach-pxa/tavorevb.c index 93466fa3b0fe..a15eb3b9484d 100644 --- a/arch/arm/mach-pxa/tavorevb.c +++ b/arch/arm/mach-pxa/tavorevb.c @@ -178,13 +178,11 @@ static struct platform_pwm_backlight_data tavorevb_backlight_data[] = { /* primary backlight */ .max_brightness = 100, .dft_brightness = 100, - .enable_gpio = -1, }, [1] = { /* secondary backlight */ .max_brightness = 100, .dft_brightness = 100, - .enable_gpio = -1, }, }; diff --git a/arch/arm/mach-pxa/viper.c b/arch/arm/mach-pxa/viper.c index c06031da6676..3aa34e9a15d3 100644 --- a/arch/arm/mach-pxa/viper.c +++ b/arch/arm/mach-pxa/viper.c @@ -404,7 +404,6 @@ static void viper_backlight_exit(struct device *dev) static struct platform_pwm_backlight_data viper_backlight_data = { .max_brightness = 100, .dft_brightness = 100, - .enable_gpio = -1, .init = viper_backlight_init, .notify = viper_backlight_notify, .exit = viper_backlight_exit, diff --git a/arch/arm/mach-pxa/z2.c b/arch/arm/mach-pxa/z2.c index 900cefc4c5ea..21fd76bb09cd 100644 --- a/arch/arm/mach-pxa/z2.c +++ b/arch/arm/mach-pxa/z2.c @@ -210,13 +210,11 @@ static struct platform_pwm_backlight_data z2_backlight_data[] = { /* Keypad Backlight */ .max_brightness = 1023, .dft_brightness = 0, - .enable_gpio = -1, }, [1] = { /* LCD Backlight */ .max_brightness = 1023, .dft_brightness = 512, - .enable_gpio = -1, }, }; diff --git a/arch/arm/mach-pxa/zylonite.c b/arch/arm/mach-pxa/zylonite.c index bf2ab5bd49ec..79f0025fa17a 100644 --- a/arch/arm/mach-pxa/zylonite.c +++ b/arch/arm/mach-pxa/zylonite.c @@ -117,7 +117,6 @@ static struct pwm_lookup zylonite_pwm_lookup[] = { static struct platform_pwm_backlight_data zylonite_backlight_data = { .max_brightness = 100, .dft_brightness = 100, - .enable_gpio = -1, }; static struct platform_device zylonite_backlight_device = { diff --git a/arch/arm/mach-s3c24xx/mach-h1940.c b/arch/arm/mach-s3c24xx/mach-h1940.c index 74d6b68e91c7..e1c372e5447b 100644 --- a/arch/arm/mach-s3c24xx/mach-h1940.c +++ b/arch/arm/mach-s3c24xx/mach-h1940.c @@ -516,7 +516,6 @@ static void h1940_backlight_exit(struct device *dev) static struct platform_pwm_backlight_data backlight_data = { .max_brightness = 100, .dft_brightness = 50, - .enable_gpio = -1, .init = h1940_backlight_init, .notify = h1940_backlight_notify, .exit = h1940_backlight_exit, diff --git a/arch/arm/mach-s3c24xx/mach-rx1950.c b/arch/arm/mach-s3c24xx/mach-rx1950.c index 03d8f27cdc32..fde98b175c75 100644 --- a/arch/arm/mach-s3c24xx/mach-rx1950.c +++ b/arch/arm/mach-s3c24xx/mach-rx1950.c @@ -534,7 +534,6 @@ static int rx1950_backlight_notify(struct device *dev, int brightness) static struct platform_pwm_backlight_data rx1950_backlight_data = { .max_brightness = 24, .dft_brightness = 4, - .enable_gpio = -1, .init = rx1950_backlight_init, .notify = rx1950_backlight_notify, .exit = rx1950_backlight_exit, diff --git a/arch/arm/mach-s3c64xx/dev-backlight.c b/arch/arm/mach-s3c64xx/dev-backlight.c index 799cfdf0606b..09e6da305f60 100644 --- a/arch/arm/mach-s3c64xx/dev-backlight.c +++ b/arch/arm/mach-s3c64xx/dev-backlight.c @@ -65,7 +65,6 @@ static struct samsung_bl_drvdata samsung_dfl_bl_data __initdata = { .plat_data = { .max_brightness = 255, .dft_brightness = 255, - .enable_gpio = -1, .init = samsung_bl_init, .exit = samsung_bl_exit, }, @@ -111,8 +110,6 @@ void __init samsung_bl_set(struct samsung_bl_gpio_info *gpio_info, samsung_bl_data->dft_brightness = bl_data->dft_brightness; if (bl_data->lth_brightness) samsung_bl_data->lth_brightness = bl_data->lth_brightness; - if (bl_data->enable_gpio >= 0) - samsung_bl_data->enable_gpio = bl_data->enable_gpio; if (bl_data->init) samsung_bl_data->init = bl_data->init; if (bl_data->notify) diff --git a/arch/arm/mach-s3c64xx/mach-crag6410.c b/arch/arm/mach-s3c64xx/mach-crag6410.c index 8ec6a4f5eb05..da9654255e3f 100644 --- a/arch/arm/mach-s3c64xx/mach-crag6410.c +++ b/arch/arm/mach-s3c64xx/mach-crag6410.c @@ -114,7 +114,6 @@ static struct pwm_lookup crag6410_pwm_lookup[] = { static struct platform_pwm_backlight_data crag6410_backlight_data = { .max_brightness = 1000, .dft_brightness = 600, - .enable_gpio = -1, }; static struct platform_device crag6410_backlight_device = { diff --git a/arch/arm/mach-s3c64xx/mach-hmt.c b/arch/arm/mach-s3c64xx/mach-hmt.c index bfe9881d12cc..e7080215c624 100644 --- a/arch/arm/mach-s3c64xx/mach-hmt.c +++ b/arch/arm/mach-s3c64xx/mach-hmt.c @@ -115,7 +115,6 @@ static void hmt_bl_exit(struct device *dev) static struct platform_pwm_backlight_data hmt_backlight_data = { .max_brightness = 100 * 256, .dft_brightness = 40 * 256, - .enable_gpio = -1, .init = hmt_bl_init, .notify = hmt_bl_notify, .exit = hmt_bl_exit, diff --git a/arch/arm/mach-s3c64xx/mach-smartq.c b/arch/arm/mach-s3c64xx/mach-smartq.c index 829d5dbd69ee..5025db607c0f 100644 --- a/arch/arm/mach-s3c64xx/mach-smartq.c +++ b/arch/arm/mach-s3c64xx/mach-smartq.c @@ -150,7 +150,6 @@ static int smartq_bl_init(struct device *dev) static struct platform_pwm_backlight_data smartq_backlight_data = { .max_brightness = 1000, .dft_brightness = 600, - .enable_gpio = -1, .init = smartq_bl_init, }; diff --git a/arch/arm/mach-s3c64xx/mach-smdk6410.c b/arch/arm/mach-s3c64xx/mach-smdk6410.c index 908e5aa831c8..56f406c0c3dd 100644 --- a/arch/arm/mach-s3c64xx/mach-smdk6410.c +++ b/arch/arm/mach-s3c64xx/mach-smdk6410.c @@ -623,7 +623,7 @@ static struct pwm_lookup smdk6410_pwm_lookup[] = { }; static struct platform_pwm_backlight_data smdk6410_bl_data = { - .enable_gpio = -1, + /* Intentionally blank */ }; static struct dwc2_hsotg_plat smdk6410_hsotg_pdata; diff --git a/arch/unicore32/kernel/puv3-nb0916.c b/arch/unicore32/kernel/puv3-nb0916.c index a3bf2ffc54dd..e251f5028396 100644 --- a/arch/unicore32/kernel/puv3-nb0916.c +++ b/arch/unicore32/kernel/puv3-nb0916.c @@ -55,7 +55,6 @@ static struct pwm_lookup nb0916_pwm_lookup[] = { static struct platform_pwm_backlight_data nb0916_backlight_data = { .max_brightness = 100, .dft_brightness = 100, - .enable_gpio = -1, }; static struct gpio_keys_button nb0916_gpio_keys[] = { diff --git a/drivers/video/backlight/pwm_bl.c b/drivers/video/backlight/pwm_bl.c index efb4efc2a13d..82b8d7594701 100644 --- a/drivers/video/backlight/pwm_bl.c +++ b/drivers/video/backlight/pwm_bl.c @@ -7,7 +7,6 @@ #include #include -#include #include #include #include @@ -258,8 +257,6 @@ static int pwm_backlight_parse_dt(struct device *dev, &data->post_pwm_on_delay); of_property_read_u32(node, "pwm-off-delay-ms", &data->pwm_off_delay); - data->enable_gpio = -EINVAL; - /* * Determine the number of brightness levels, if this property is not * set a default table of brightness levels will be used. @@ -502,22 +499,6 @@ static int pwm_backlight_probe(struct platform_device *pdev) goto err_alloc; } - /* - * Compatibility fallback for drivers still using the integer GPIO - * platform data. Must go away soon. - */ - if (!pb->enable_gpio && gpio_is_valid(data->enable_gpio)) { - ret = devm_gpio_request_one(&pdev->dev, data->enable_gpio, - GPIOF_OUT_INIT_HIGH, "enable"); - if (ret < 0) { - dev_err(&pdev->dev, "failed to request GPIO#%d: %d\n", - data->enable_gpio, ret); - goto err_alloc; - } - - pb->enable_gpio = gpio_to_desc(data->enable_gpio); - } - /* * If the GPIO is not known to be already configured as output, that * is, if gpiod_get_direction returns either 1 or -EINVAL, change the diff --git a/include/linux/pwm_backlight.h b/include/linux/pwm_backlight.h index 8ea265a022fd..06086cb93b6f 100644 --- a/include/linux/pwm_backlight.h +++ b/include/linux/pwm_backlight.h @@ -16,8 +16,6 @@ struct platform_pwm_backlight_data { unsigned int *levels; unsigned int post_pwm_on_delay; unsigned int pwm_off_delay; - /* TODO remove once all users are switched to gpiod_* API */ - int enable_gpio; int (*init)(struct device *dev); int (*notify)(struct device *dev, int brightness); void (*notify_after)(struct device *dev, int brightness); From 7682ce2b12a0f9cf2a318c0d540a9f96780bfd7c Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Mon, 24 Feb 2020 17:58:41 +0100 Subject: [PATCH 130/635] PCI/ATS: Export symbols of PASID functions The Arm SMMUv3 driver uses pci_{enable,disable}_pasid() and related functions. Export them to allow the driver to be built as a module. Acked-by: Bjorn Helgaas Signed-off-by: Jean-Philippe Brucker Signed-off-by: Will Deacon --- drivers/pci/ats.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/pci/ats.c b/drivers/pci/ats.c index 3ef0bb281e7c..390e92f2d8d1 100644 --- a/drivers/pci/ats.c +++ b/drivers/pci/ats.c @@ -366,6 +366,7 @@ int pci_enable_pasid(struct pci_dev *pdev, int features) return 0; } +EXPORT_SYMBOL_GPL(pci_enable_pasid); /** * pci_disable_pasid - Disable the PASID capability @@ -390,6 +391,7 @@ void pci_disable_pasid(struct pci_dev *pdev) pdev->pasid_enabled = 0; } +EXPORT_SYMBOL_GPL(pci_disable_pasid); /** * pci_restore_pasid_state - Restore PASID capabilities @@ -441,6 +443,7 @@ int pci_pasid_features(struct pci_dev *pdev) return supported; } +EXPORT_SYMBOL_GPL(pci_pasid_features); #define PASID_NUMBER_SHIFT 8 #define PASID_NUMBER_MASK (0x1f << PASID_NUMBER_SHIFT) @@ -469,4 +472,5 @@ int pci_max_pasids(struct pci_dev *pdev) return (1 << supported); } +EXPORT_SYMBOL_GPL(pci_max_pasids); #endif /* CONFIG_PCI_PASID */ From 058c59a047d61601abf503563bbea818ee645c09 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Mon, 24 Feb 2020 17:58:42 +0100 Subject: [PATCH 131/635] iommu/arm-smmu-v3: Add support for PCI PASID Enable PASID for PCI devices that support it. Initialize PASID early in add_device() because it must be enabled before ATS. Tested-by: Zhangfei Gao Reviewed-by: Jonathan Cameron Signed-off-by: Jean-Philippe Brucker Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu-v3.c | 62 ++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index aa3ac2a03807..6b76df37025e 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -2628,6 +2628,53 @@ static void arm_smmu_disable_ats(struct arm_smmu_master *master) atomic_dec(&smmu_domain->nr_ats_masters); } +static int arm_smmu_enable_pasid(struct arm_smmu_master *master) +{ + int ret; + int features; + int num_pasids; + struct pci_dev *pdev; + + if (!dev_is_pci(master->dev)) + return -ENODEV; + + pdev = to_pci_dev(master->dev); + + features = pci_pasid_features(pdev); + if (features < 0) + return features; + + num_pasids = pci_max_pasids(pdev); + if (num_pasids <= 0) + return num_pasids; + + ret = pci_enable_pasid(pdev, features); + if (ret) { + dev_err(&pdev->dev, "Failed to enable PASID\n"); + return ret; + } + + master->ssid_bits = min_t(u8, ilog2(num_pasids), + master->smmu->ssid_bits); + return 0; +} + +static void arm_smmu_disable_pasid(struct arm_smmu_master *master) +{ + struct pci_dev *pdev; + + if (!dev_is_pci(master->dev)) + return; + + pdev = to_pci_dev(master->dev); + + if (!pdev->pasid_enabled) + return; + + master->ssid_bits = 0; + pci_disable_pasid(pdev); +} + static void arm_smmu_detach_dev(struct arm_smmu_master *master) { unsigned long flags; @@ -2831,13 +2878,23 @@ static int arm_smmu_add_device(struct device *dev) master->ssid_bits = min(smmu->ssid_bits, fwspec->num_pasid_bits); + /* + * Note that PASID must be enabled before, and disabled after ATS: + * PCI Express Base 4.0r1.0 - 10.5.1.3 ATS Control Register + * + * Behavior is undefined if this bit is Set and the value of the PASID + * Enable, Execute Requested Enable, or Privileged Mode Requested bits + * are changed. + */ + arm_smmu_enable_pasid(master); + if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB)) master->ssid_bits = min_t(u8, master->ssid_bits, CTXDESC_LINEAR_CDMAX); ret = iommu_device_link(&smmu->iommu, dev); if (ret) - goto err_free_master; + goto err_disable_pasid; group = iommu_group_get_for_dev(dev); if (IS_ERR(group)) { @@ -2850,6 +2907,8 @@ static int arm_smmu_add_device(struct device *dev) err_unlink: iommu_device_unlink(&smmu->iommu, dev); +err_disable_pasid: + arm_smmu_disable_pasid(master); err_free_master: kfree(master); fwspec->iommu_priv = NULL; @@ -2870,6 +2929,7 @@ static void arm_smmu_remove_device(struct device *dev) arm_smmu_detach_dev(master); iommu_group_remove_device(dev); iommu_device_unlink(&smmu->iommu, dev); + arm_smmu_disable_pasid(master); kfree(master); iommu_fwspec_free(dev); } From 87e5fe5b779a20fa02382aaf169015e68710b9ff Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Mon, 24 Feb 2020 17:58:43 +0100 Subject: [PATCH 132/635] iommu/arm-smmu-v3: Write level-1 descriptors atomically Use WRITE_ONCE() to make sure that the SMMU doesn't read incomplete stream table descriptors. Refer to the comment about 64-bit accesses, and add the comment to the equivalent context descriptor code. Signed-off-by: Jean-Philippe Brucker Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu-v3.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 6b76df37025e..068a16d0eabe 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -1531,6 +1531,7 @@ static void arm_smmu_write_cd_l1_desc(__le64 *dst, u64 val = (l1_desc->l2ptr_dma & CTXDESC_L1_DESC_L2PTR_MASK) | CTXDESC_L1_DESC_V; + /* See comment in arm_smmu_write_ctx_desc() */ WRITE_ONCE(*dst, cpu_to_le64(val)); } @@ -1726,7 +1727,8 @@ arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc) val |= FIELD_PREP(STRTAB_L1_DESC_SPAN, desc->span); val |= desc->l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK; - *dst = cpu_to_le64(val); + /* See comment in arm_smmu_write_ctx_desc() */ + WRITE_ONCE(*dst, cpu_to_le64(val)); } static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid) From 4ce8da453640147101bda418640394637c1a7cfc Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Mon, 24 Feb 2020 17:58:44 +0100 Subject: [PATCH 133/635] iommu/arm-smmu-v3: Add command queue batching helpers As more functions will implement command queue batching, add two helpers to simplify building a command list. Signed-off-by: Jean-Philippe Brucker Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu-v3.c | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 068a16d0eabe..beeec366bc41 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -548,6 +548,11 @@ struct arm_smmu_cmdq { atomic_t lock; }; +struct arm_smmu_cmdq_batch { + u64 cmds[CMDQ_BATCH_ENTRIES * CMDQ_ENT_DWORDS]; + int num; +}; + struct arm_smmu_evtq { struct arm_smmu_queue q; u32 max_stalls; @@ -1482,6 +1487,24 @@ static int arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu) return arm_smmu_cmdq_issue_cmdlist(smmu, NULL, 0, true); } +static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu, + struct arm_smmu_cmdq_batch *cmds, + struct arm_smmu_cmdq_ent *cmd) +{ + if (cmds->num == CMDQ_BATCH_ENTRIES) { + arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, false); + cmds->num = 0; + } + arm_smmu_cmdq_build_cmd(&cmds->cmds[cmds->num * CMDQ_ENT_DWORDS], cmd); + cmds->num++; +} + +static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu, + struct arm_smmu_cmdq_batch *cmds) +{ + return arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, true); +} + /* Context descriptor manipulation functions */ static void arm_smmu_sync_cd(struct arm_smmu_domain *smmu_domain, int ssid, bool leaf) @@ -2220,10 +2243,9 @@ static void arm_smmu_tlb_inv_range(unsigned long iova, size_t size, size_t granule, bool leaf, struct arm_smmu_domain *smmu_domain) { - u64 cmds[CMDQ_BATCH_ENTRIES * CMDQ_ENT_DWORDS]; struct arm_smmu_device *smmu = smmu_domain->smmu; unsigned long start = iova, end = iova + size; - int i = 0; + struct arm_smmu_cmdq_batch cmds = {}; struct arm_smmu_cmdq_ent cmd = { .tlbi = { .leaf = leaf, @@ -2242,18 +2264,11 @@ static void arm_smmu_tlb_inv_range(unsigned long iova, size_t size, } while (iova < end) { - if (i == CMDQ_BATCH_ENTRIES) { - arm_smmu_cmdq_issue_cmdlist(smmu, cmds, i, false); - i = 0; - } - cmd.tlbi.addr = iova; - arm_smmu_cmdq_build_cmd(&cmds[i * CMDQ_ENT_DWORDS], &cmd); + arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd); iova += granule; - i++; } - - arm_smmu_cmdq_issue_cmdlist(smmu, cmds, i, true); + arm_smmu_cmdq_batch_submit(smmu, &cmds); /* * Unfortunately, this can't be leaf-only since we may have From edd0351e7bc49555d8b5ad8438a65a7ca262c9f0 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Mon, 24 Feb 2020 17:58:45 +0100 Subject: [PATCH 134/635] iommu/arm-smmu-v3: Batch context descriptor invalidation Rather than publishing one command at a time when invalidating a context descriptor, batch the commands for all SIDs in the domain. Signed-off-by: Jean-Philippe Brucker Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu-v3.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index beeec366bc41..12b2a0fa747e 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -1512,6 +1512,7 @@ static void arm_smmu_sync_cd(struct arm_smmu_domain *smmu_domain, size_t i; unsigned long flags; struct arm_smmu_master *master; + struct arm_smmu_cmdq_batch cmds = {}; struct arm_smmu_device *smmu = smmu_domain->smmu; struct arm_smmu_cmdq_ent cmd = { .opcode = CMDQ_OP_CFGI_CD, @@ -1525,12 +1526,12 @@ static void arm_smmu_sync_cd(struct arm_smmu_domain *smmu_domain, list_for_each_entry(master, &smmu_domain->devices, domain_head) { for (i = 0; i < master->num_sids; i++) { cmd.cfgi.sid = master->sids[i]; - arm_smmu_cmdq_issue_cmd(smmu, &cmd); + arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd); } } spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); - arm_smmu_cmdq_issue_sync(smmu); + arm_smmu_cmdq_batch_submit(smmu, &cmds); } static int arm_smmu_alloc_cd_leaf_table(struct arm_smmu_device *smmu, From 9e773aee8c3e1b3ba019c5c7f8435aaa836c6130 Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Mon, 24 Feb 2020 17:58:46 +0100 Subject: [PATCH 135/635] iommu/arm-smmu-v3: Batch ATC invalidation commands Similar to commit 2af2e72b18b4 ("iommu/arm-smmu-v3: Defer TLB invalidation until ->iotlb_sync()"), build up a list of ATC invalidation commands and submit them all at once to the command queue instead of one-by-one. As there is only one caller of arm_smmu_atc_inv_master() left, we can simplify it and avoid passing in struct arm_smmu_cmdq_ent. Cc: Jean-Philippe Brucker Cc: Will Deacon Cc: Robin Murphy Cc: Joerg Roedel Signed-off-by: Rob Herring Signed-off-by: Jean-Philippe Brucker Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu-v3.c | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 12b2a0fa747e..4f0a38dae6db 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -2158,17 +2158,16 @@ arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size, cmd->atc.size = log2_span; } -static int arm_smmu_atc_inv_master(struct arm_smmu_master *master, - struct arm_smmu_cmdq_ent *cmd) +static int arm_smmu_atc_inv_master(struct arm_smmu_master *master) { int i; + struct arm_smmu_cmdq_ent cmd; - if (!master->ats_enabled) - return 0; + arm_smmu_atc_inv_to_cmd(0, 0, 0, &cmd); for (i = 0; i < master->num_sids; i++) { - cmd->atc.sid = master->sids[i]; - arm_smmu_cmdq_issue_cmd(master->smmu, cmd); + cmd.atc.sid = master->sids[i]; + arm_smmu_cmdq_issue_cmd(master->smmu, &cmd); } return arm_smmu_cmdq_issue_sync(master->smmu); @@ -2177,10 +2176,11 @@ static int arm_smmu_atc_inv_master(struct arm_smmu_master *master, static int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid, unsigned long iova, size_t size) { - int ret = 0; + int i; unsigned long flags; struct arm_smmu_cmdq_ent cmd; struct arm_smmu_master *master; + struct arm_smmu_cmdq_batch cmds = {}; if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS)) return 0; @@ -2205,11 +2205,18 @@ static int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, arm_smmu_atc_inv_to_cmd(ssid, iova, size, &cmd); spin_lock_irqsave(&smmu_domain->devices_lock, flags); - list_for_each_entry(master, &smmu_domain->devices, domain_head) - ret |= arm_smmu_atc_inv_master(master, &cmd); + list_for_each_entry(master, &smmu_domain->devices, domain_head) { + if (!master->ats_enabled) + continue; + + for (i = 0; i < master->num_sids; i++) { + cmd.atc.sid = master->sids[i]; + arm_smmu_cmdq_batch_add(smmu_domain->smmu, &cmds, &cmd); + } + } spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); - return ret ? -ETIMEDOUT : 0; + return arm_smmu_cmdq_batch_submit(smmu_domain->smmu, &cmds); } /* IO_PGTABLE API */ @@ -2629,7 +2636,6 @@ static void arm_smmu_enable_ats(struct arm_smmu_master *master) static void arm_smmu_disable_ats(struct arm_smmu_master *master) { - struct arm_smmu_cmdq_ent cmd; struct arm_smmu_domain *smmu_domain = master->domain; if (!master->ats_enabled) @@ -2641,8 +2647,7 @@ static void arm_smmu_disable_ats(struct arm_smmu_master *master) * ATC invalidation via the SMMU. */ wmb(); - arm_smmu_atc_inv_to_cmd(0, 0, 0, &cmd); - arm_smmu_atc_inv_master(master, &cmd); + arm_smmu_atc_inv_master(master); atomic_dec(&smmu_domain->nr_ats_masters); } From 6a481a95d4c198a2dd0a61f8877b92a375757db8 Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Mon, 24 Feb 2020 16:31:29 -0600 Subject: [PATCH 136/635] iommu/arm-smmu-v3: Add SMMUv3.2 range invalidation support Arm SMMUv3.2 adds support for TLB range invalidate operations. Support for range invalidate is determined by the RIL bit in the IDR3 register. The range invalidate is in units of the leaf page size and operates on 1-32 chunks of a power of 2 multiple pages. First, we determine from the size what power of 2 multiple we can use. Then we calculate how many chunks (1-31) of the power of 2 size for the range on the iteration. On each iteration, we move up in size by at least 5 bits. Cc: Jean-Philippe Brucker Cc: Will Deacon Cc: Robin Murphy Cc: Joerg Roedel Reviewed-by: Eric Auger Signed-off-by: Rob Herring Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu-v3.c | 69 +++++++++++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 4f0a38dae6db..a7222dd5b117 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -69,6 +69,9 @@ #define IDR1_SSIDSIZE GENMASK(10, 6) #define IDR1_SIDSIZE GENMASK(5, 0) +#define ARM_SMMU_IDR3 0xc +#define IDR3_RIL (1 << 10) + #define ARM_SMMU_IDR5 0x14 #define IDR5_STALL_MAX GENMASK(31, 16) #define IDR5_GRAN64K (1 << 6) @@ -346,9 +349,14 @@ #define CMDQ_CFGI_1_LEAF (1UL << 0) #define CMDQ_CFGI_1_RANGE GENMASK_ULL(4, 0) +#define CMDQ_TLBI_0_NUM GENMASK_ULL(16, 12) +#define CMDQ_TLBI_RANGE_NUM_MAX 31 +#define CMDQ_TLBI_0_SCALE GENMASK_ULL(24, 20) #define CMDQ_TLBI_0_VMID GENMASK_ULL(47, 32) #define CMDQ_TLBI_0_ASID GENMASK_ULL(63, 48) #define CMDQ_TLBI_1_LEAF (1UL << 0) +#define CMDQ_TLBI_1_TTL GENMASK_ULL(9, 8) +#define CMDQ_TLBI_1_TG GENMASK_ULL(11, 10) #define CMDQ_TLBI_1_VA_MASK GENMASK_ULL(63, 12) #define CMDQ_TLBI_1_IPA_MASK GENMASK_ULL(51, 12) @@ -473,9 +481,13 @@ struct arm_smmu_cmdq_ent { #define CMDQ_OP_TLBI_S2_IPA 0x2a #define CMDQ_OP_TLBI_NSNH_ALL 0x30 struct { + u8 num; + u8 scale; u16 asid; u16 vmid; bool leaf; + u8 ttl; + u8 tg; u64 addr; } tlbi; @@ -632,6 +644,7 @@ struct arm_smmu_device { #define ARM_SMMU_FEAT_HYP (1 << 12) #define ARM_SMMU_FEAT_STALL_FORCE (1 << 13) #define ARM_SMMU_FEAT_VAX (1 << 14) +#define ARM_SMMU_FEAT_RANGE_INV (1 << 15) u32 features; #define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0) @@ -900,14 +913,22 @@ static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent) cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31); break; case CMDQ_OP_TLBI_NH_VA: + cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num); + cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale); cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid); cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid); cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf); + cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl); + cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg); cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK; break; case CMDQ_OP_TLBI_S2_IPA: + cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num); + cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale); cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid); cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf); + cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl); + cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg); cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK; break; case CMDQ_OP_TLBI_NH_ASID: @@ -2252,7 +2273,8 @@ static void arm_smmu_tlb_inv_range(unsigned long iova, size_t size, struct arm_smmu_domain *smmu_domain) { struct arm_smmu_device *smmu = smmu_domain->smmu; - unsigned long start = iova, end = iova + size; + unsigned long start = iova, end = iova + size, num_pages = 0, tg = 0; + size_t inv_range = granule; struct arm_smmu_cmdq_batch cmds = {}; struct arm_smmu_cmdq_ent cmd = { .tlbi = { @@ -2271,10 +2293,48 @@ static void arm_smmu_tlb_inv_range(unsigned long iova, size_t size, cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid; } + if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) { + /* Get the leaf page size */ + tg = __ffs(smmu_domain->domain.pgsize_bitmap); + + /* Convert page size of 12,14,16 (log2) to 1,2,3 */ + cmd.tlbi.tg = (tg - 10) / 2; + + /* Determine what level the granule is at */ + cmd.tlbi.ttl = 4 - ((ilog2(granule) - 3) / (tg - 3)); + + num_pages = size >> tg; + } + while (iova < end) { + if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) { + /* + * On each iteration of the loop, the range is 5 bits + * worth of the aligned size remaining. + * The range in pages is: + * + * range = (num_pages & (0x1f << __ffs(num_pages))) + */ + unsigned long scale, num; + + /* Determine the power of 2 multiple number of pages */ + scale = __ffs(num_pages); + cmd.tlbi.scale = scale; + + /* Determine how many chunks of 2^scale size we have */ + num = (num_pages >> scale) & CMDQ_TLBI_RANGE_NUM_MAX; + cmd.tlbi.num = num - 1; + + /* range is num * 2^scale * pgsize */ + inv_range = num << (scale + tg); + + /* Clear out the lower order bits for the next iteration */ + num_pages -= num << scale; + } + cmd.tlbi.addr = iova; arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd); - iova += granule; + iova += inv_range; } arm_smmu_cmdq_batch_submit(smmu, &cmds); @@ -3783,6 +3843,11 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) if (smmu->sid_bits <= STRTAB_SPLIT) smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB; + /* IDR3 */ + reg = readl_relaxed(smmu->base + ARM_SMMU_IDR3); + if (FIELD_GET(IDR3_RIL, reg)) + smmu->features |= ARM_SMMU_FEAT_RANGE_INV; + /* IDR5 */ reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5); From 52355fb1919ef7ed9a38e0f3de6e928de1f57217 Mon Sep 17 00:00:00 2001 From: Jacob Pan Date: Tue, 17 Mar 2020 09:10:18 +0800 Subject: [PATCH 137/635] iommu/vt-d: Fix page request descriptor size Intel VT-d might support PRS (Page Reqest Support) when it's running in the scalable mode. Each page request descriptor occupies 32 bytes and is 32-bytes aligned. The page request descriptor offset mask should be 32-bytes aligned. Fixes: 5b438f4ba315d ("iommu/vt-d: Support page request in scalable mode") Signed-off-by: Lu Baolu Signed-off-by: Liu Yi L Signed-off-by: Jacob Pan Signed-off-by: Joerg Roedel --- drivers/iommu/intel-svm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c index d7f2a5358900..edac769fc03d 100644 --- a/drivers/iommu/intel-svm.c +++ b/drivers/iommu/intel-svm.c @@ -531,7 +531,7 @@ struct page_req_dsc { u64 priv_data[2]; }; -#define PRQ_RING_MASK ((0x1000 << PRQ_ORDER) - 0x10) +#define PRQ_RING_MASK ((0x1000 << PRQ_ORDER) - 0x20) static bool access_error(struct vm_area_struct *vma, struct page_req_dsc *req) { From c6f4ebdeba4cff590594df931ff1ee610c426431 Mon Sep 17 00:00:00 2001 From: Qian Cai Date: Tue, 17 Mar 2020 11:03:26 -0400 Subject: [PATCH 138/635] iommu/vt-d: Silence RCU-list debugging warning in dmar_find_atsr() dmar_find_atsr() calls list_for_each_entry_rcu() outside of an RCU read side critical section but with dmar_global_lock held. Silence this false positive. drivers/iommu/intel-iommu.c:4504 RCU-list traversed in non-reader section!! 1 lock held by swapper/0/1: #0: ffffffff9755bee8 (dmar_global_lock){+.+.}, at: intel_iommu_init+0x1a6/0xe19 Call Trace: dump_stack+0xa4/0xfe lockdep_rcu_suspicious+0xeb/0xf5 dmar_find_atsr+0x1ab/0x1c0 dmar_parse_one_atsr+0x64/0x220 dmar_walk_remapping_entries+0x130/0x380 dmar_table_init+0x166/0x243 intel_iommu_init+0x1ab/0xe19 pci_iommu_init+0x1a/0x44 do_one_initcall+0xae/0x4d0 kernel_init_freeable+0x412/0x4c5 kernel_init+0x19/0x193 Signed-off-by: Qian Cai Acked-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 4be549478691..ef0a5246700e 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -4501,7 +4501,8 @@ static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr) struct dmar_atsr_unit *atsru; struct acpi_dmar_atsr *tmp; - list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) { + list_for_each_entry_rcu(atsru, &dmar_atsr_units, list, + dmar_rcu_check()) { tmp = (struct acpi_dmar_atsr *)atsru->hdr; if (atsr->segment != tmp->segment) continue; From e1b0d18be384f8a8e0c333e3350a1d57ca788dee Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Fri, 6 Mar 2020 11:55:02 +0100 Subject: [PATCH 139/635] dt-bindings: thermal: rcar-gen3-thermal: Add r8a77961 support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document R-Car M3-W+ (R8A77961) SoC bindings. Signed-off-by: Geert Uytterhoeven Reviewed-by: Niklas Söderlund Acked-by: Rob Herring Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200306105503.24267-2-geert+renesas@glider.be --- Documentation/devicetree/bindings/thermal/rcar-gen3-thermal.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/thermal/rcar-gen3-thermal.txt b/Documentation/devicetree/bindings/thermal/rcar-gen3-thermal.txt index 12c740b975f7..2993fa720195 100644 --- a/Documentation/devicetree/bindings/thermal/rcar-gen3-thermal.txt +++ b/Documentation/devicetree/bindings/thermal/rcar-gen3-thermal.txt @@ -11,6 +11,7 @@ Required properties: - "renesas,r8a774b1-thermal" (RZ/G2N) - "renesas,r8a7795-thermal" (R-Car H3) - "renesas,r8a7796-thermal" (R-Car M3-W) + - "renesas,r8a77961-thermal" (R-Car M3-W+) - "renesas,r8a77965-thermal" (R-Car M3-N) - "renesas,r8a77980-thermal" (R-Car V3H) - reg : Address ranges of the thermal registers. Each sensor From 8d74bf79dfd51fa84a5b99ada868c09488e633e8 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Fri, 6 Mar 2020 11:55:03 +0100 Subject: [PATCH 140/635] thermal: rcar_gen3_thermal: Add r8a77961 support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add support for the Thermal Sensor/Chip Internal Voltage Monitor in the R-Car M3-W+ (R8A77961) SoC. According to the R-Car Gen3 Hardware Manual Errata for Revision 2.00 of Jan 31, 2020, the thermal parameters for R-Car M3-W+ are the same as for R-Car M3-W. Signed-off-by: Geert Uytterhoeven Reviewed-by: Niklas Söderlund Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200306105503.24267-3-geert+renesas@glider.be --- drivers/thermal/rcar_gen3_thermal.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/thermal/rcar_gen3_thermal.c b/drivers/thermal/rcar_gen3_thermal.c index a2bf9d8074cc..58fe7c1ef00b 100644 --- a/drivers/thermal/rcar_gen3_thermal.c +++ b/drivers/thermal/rcar_gen3_thermal.c @@ -322,6 +322,10 @@ static const struct of_device_id rcar_gen3_thermal_dt_ids[] = { .compatible = "renesas,r8a7796-thermal", .data = &rcar_gen3_ths_tj_1_m3_w, }, + { + .compatible = "renesas,r8a77961-thermal", + .data = &rcar_gen3_ths_tj_1_m3_w, + }, { .compatible = "renesas,r8a77965-thermal", .data = &rcar_gen3_ths_tj_1, From a9d8e61b936dd0cc23398a1e6a7a334d554e11b0 Mon Sep 17 00:00:00 2001 From: Anson Huang Date: Mon, 9 Mar 2020 22:04:28 +0800 Subject: [PATCH 141/635] thermal: imx: Remove unused includes Remove unused includes to simplify the code. Signed-off-by: Anson Huang Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/1583762668-12099-1-git-send-email-Anson.Huang@nxp.com --- drivers/thermal/imx_thermal.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/drivers/thermal/imx_thermal.c b/drivers/thermal/imx_thermal.c index e75dda53e71b..d2fa301e5033 100644 --- a/drivers/thermal/imx_thermal.c +++ b/drivers/thermal/imx_thermal.c @@ -3,24 +3,17 @@ // Copyright 2013 Freescale Semiconductor, Inc. #include -#include #include #include #include -#include -#include #include #include -#include #include #include #include #include -#include #include -#include #include -#include #include #define REG_SET 0x4 From 39056e8a989ef52486e063e34b4822b341e47b0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20S=C3=B6derlund?= Date: Tue, 10 Mar 2020 12:47:09 +0100 Subject: [PATCH 142/635] thermal: rcar_thermal: Handle probe error gracefully MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If the common register memory resource is not available the driver needs to fail gracefully to disable PM. Instead of returning the error directly store it in ret and use the already existing error path. Signed-off-by: Niklas Söderlund Reviewed-by: Geert Uytterhoeven Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200310114709.1483860-1-niklas.soderlund+renesas@ragnatech.se --- drivers/thermal/rcar_thermal.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/thermal/rcar_thermal.c b/drivers/thermal/rcar_thermal.c index 8f1aafa2044e..4a48d1d2a31c 100644 --- a/drivers/thermal/rcar_thermal.c +++ b/drivers/thermal/rcar_thermal.c @@ -521,8 +521,10 @@ static int rcar_thermal_probe(struct platform_device *pdev) res = platform_get_resource(pdev, IORESOURCE_MEM, mres++); common->base = devm_ioremap_resource(dev, res); - if (IS_ERR(common->base)) - return PTR_ERR(common->base); + if (IS_ERR(common->base)) { + ret = PTR_ERR(common->base); + goto error_unregister; + } idle = 0; /* polling delay is not needed */ } From 0aef1ee5af9ee9bf8c9267f7b8c10dafc057adf9 Mon Sep 17 00:00:00 2001 From: Amit Kucheria Date: Thu, 12 Mar 2020 18:06:58 +0530 Subject: [PATCH 143/635] drivers: thermal: tsens: De-constify struct tsens_features struct tsens_features is currently initialized as part of platform data at compile-time and not modifiable. We now have some usecases in feature detection across IP versions where it is more flexible to update the features after probing registers. Remove const qualifier from tsens_features and the encapsulating tsens_plat_data. Signed-off-by: Amit Kucheria Reviewed-by: Stephen Boyd Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/2919a72220470366ae11e0bb5330a4ea39838f71.1584015867.git.amit.kucheria@linaro.org --- drivers/thermal/qcom/tsens-8960.c | 2 +- drivers/thermal/qcom/tsens-v0_1.c | 6 +++--- drivers/thermal/qcom/tsens-v1.c | 6 +++--- drivers/thermal/qcom/tsens-v2.c | 6 +++--- drivers/thermal/qcom/tsens.h | 12 ++++++------ 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/drivers/thermal/qcom/tsens-8960.c b/drivers/thermal/qcom/tsens-8960.c index fb77acb8d13b..a383a57cfbbc 100644 --- a/drivers/thermal/qcom/tsens-8960.c +++ b/drivers/thermal/qcom/tsens-8960.c @@ -279,7 +279,7 @@ static const struct tsens_ops ops_8960 = { .resume = resume_8960, }; -const struct tsens_plat_data data_8960 = { +struct tsens_plat_data data_8960 = { .num_sensors = 11, .ops = &ops_8960, }; diff --git a/drivers/thermal/qcom/tsens-v0_1.c b/drivers/thermal/qcom/tsens-v0_1.c index 4b8dd6de02ce..959a9371d205 100644 --- a/drivers/thermal/qcom/tsens-v0_1.c +++ b/drivers/thermal/qcom/tsens-v0_1.c @@ -327,7 +327,7 @@ static int calibrate_8974(struct tsens_priv *priv) /* v0.1: 8916, 8974 */ -static const struct tsens_features tsens_v0_1_feat = { +static struct tsens_features tsens_v0_1_feat = { .ver_major = VER_0_1, .crit_int = 0, .adc = 1, @@ -377,7 +377,7 @@ static const struct tsens_ops ops_8916 = { .get_temp = get_temp_common, }; -const struct tsens_plat_data data_8916 = { +struct tsens_plat_data data_8916 = { .num_sensors = 5, .ops = &ops_8916, .hw_ids = (unsigned int []){0, 1, 2, 4, 5 }, @@ -392,7 +392,7 @@ static const struct tsens_ops ops_8974 = { .get_temp = get_temp_common, }; -const struct tsens_plat_data data_8974 = { +struct tsens_plat_data data_8974 = { .num_sensors = 11, .ops = &ops_8974, .feat = &tsens_v0_1_feat, diff --git a/drivers/thermal/qcom/tsens-v1.c b/drivers/thermal/qcom/tsens-v1.c index bd2ddb684a45..b682a4df0081 100644 --- a/drivers/thermal/qcom/tsens-v1.c +++ b/drivers/thermal/qcom/tsens-v1.c @@ -299,7 +299,7 @@ static int calibrate_8976(struct tsens_priv *priv) /* v1.x: msm8956,8976,qcs404,405 */ -static const struct tsens_features tsens_v1_feat = { +static struct tsens_features tsens_v1_feat = { .ver_major = VER_1_X, .crit_int = 0, .adc = 1, @@ -368,7 +368,7 @@ static const struct tsens_ops ops_generic_v1 = { .get_temp = get_temp_tsens_valid, }; -const struct tsens_plat_data data_tsens_v1 = { +struct tsens_plat_data data_tsens_v1 = { .ops = &ops_generic_v1, .feat = &tsens_v1_feat, .fields = tsens_v1_regfields, @@ -381,7 +381,7 @@ static const struct tsens_ops ops_8976 = { }; /* Valid for both MSM8956 and MSM8976. Sensor ID 3 is unused. */ -const struct tsens_plat_data data_8976 = { +struct tsens_plat_data data_8976 = { .num_sensors = 11, .ops = &ops_8976, .hw_ids = (unsigned int[]){0, 1, 2, 4, 5, 6, 7, 8, 9, 10}, diff --git a/drivers/thermal/qcom/tsens-v2.c b/drivers/thermal/qcom/tsens-v2.c index a4d15e1abfdd..f1c8ec62e69f 100644 --- a/drivers/thermal/qcom/tsens-v2.c +++ b/drivers/thermal/qcom/tsens-v2.c @@ -27,7 +27,7 @@ /* v2.x: 8996, 8998, sdm845 */ -static const struct tsens_features tsens_v2_feat = { +static struct tsens_features tsens_v2_feat = { .ver_major = VER_2_X, .crit_int = 1, .adc = 0, @@ -81,14 +81,14 @@ static const struct tsens_ops ops_generic_v2 = { .get_temp = get_temp_tsens_valid, }; -const struct tsens_plat_data data_tsens_v2 = { +struct tsens_plat_data data_tsens_v2 = { .ops = &ops_generic_v2, .feat = &tsens_v2_feat, .fields = tsens_v2_regfields, }; /* Kept around for backward compatibility with old msm8996.dtsi */ -const struct tsens_plat_data data_8996 = { +struct tsens_plat_data data_8996 = { .num_sensors = 13, .ops = &ops_generic_v2, .feat = &tsens_v2_feat, diff --git a/drivers/thermal/qcom/tsens.h b/drivers/thermal/qcom/tsens.h index e24a865fbc34..be364bf1d5a6 100644 --- a/drivers/thermal/qcom/tsens.h +++ b/drivers/thermal/qcom/tsens.h @@ -440,7 +440,7 @@ struct tsens_plat_data { const u32 num_sensors; const struct tsens_ops *ops; unsigned int *hw_ids; - const struct tsens_features *feat; + struct tsens_features *feat; const struct reg_field *fields; }; @@ -481,7 +481,7 @@ struct tsens_priv { struct regmap_field *rf[MAX_REGFIELDS]; struct tsens_context ctx; - const struct tsens_features *feat; + struct tsens_features *feat; const struct reg_field *fields; const struct tsens_ops *ops; @@ -502,15 +502,15 @@ int tsens_set_trips(void *_sensor, int low, int high); irqreturn_t tsens_irq_thread(int irq, void *data); /* TSENS target */ -extern const struct tsens_plat_data data_8960; +extern struct tsens_plat_data data_8960; /* TSENS v0.1 targets */ -extern const struct tsens_plat_data data_8916, data_8974; +extern struct tsens_plat_data data_8916, data_8974; /* TSENS v1 targets */ -extern const struct tsens_plat_data data_tsens_v1, data_8976; +extern struct tsens_plat_data data_tsens_v1, data_8976; /* TSENS v2 targets */ -extern const struct tsens_plat_data data_8996, data_tsens_v2; +extern struct tsens_plat_data data_8996, data_tsens_v2; #endif /* __QCOM_TSENS_H__ */ From e604bdd2a7e1087efaaa547644a57fbd8a273da1 Mon Sep 17 00:00:00 2001 From: Amit Kucheria Date: Thu, 12 Mar 2020 18:06:59 +0530 Subject: [PATCH 144/635] drivers: thermal: tsens: Pass around struct tsens_sensor as a constant All the sensor data is initialised at init time. Lock it down by passing it to functions as a constant. Signed-off-by: Amit Kucheria Reviewed-by: Bjorn Andersson Reviewed-by: Stephen Boyd Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/740f9254484c08d65869df578628eb523c0049ff.1584015867.git.amit.kucheria@linaro.org --- drivers/thermal/qcom/tsens-8960.c | 2 +- drivers/thermal/qcom/tsens-common.c | 15 ++++++++------- drivers/thermal/qcom/tsens.h | 6 +++--- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/drivers/thermal/qcom/tsens-8960.c b/drivers/thermal/qcom/tsens-8960.c index a383a57cfbbc..2a28a5af209e 100644 --- a/drivers/thermal/qcom/tsens-8960.c +++ b/drivers/thermal/qcom/tsens-8960.c @@ -245,7 +245,7 @@ static inline int code_to_mdegC(u32 adc_code, const struct tsens_sensor *s) return adc_code * slope + offset; } -static int get_temp_8960(struct tsens_sensor *s, int *temp) +static int get_temp_8960(const struct tsens_sensor *s, int *temp) { int ret; u32 code, trdy; diff --git a/drivers/thermal/qcom/tsens-common.c b/drivers/thermal/qcom/tsens-common.c index c8d57ee0a5bb..45e68f999bf6 100644 --- a/drivers/thermal/qcom/tsens-common.c +++ b/drivers/thermal/qcom/tsens-common.c @@ -128,7 +128,7 @@ static inline int code_to_degc(u32 adc_code, const struct tsens_sensor *s) * Return: Temperature in milliCelsius on success, a negative errno will * be returned in error cases */ -static int tsens_hw_to_mC(struct tsens_sensor *s, int field) +static int tsens_hw_to_mC(const struct tsens_sensor *s, int field) { struct tsens_priv *priv = s->priv; u32 resolution; @@ -160,7 +160,7 @@ static int tsens_hw_to_mC(struct tsens_sensor *s, int field) * * Return: ADC code or temperature in deciCelsius. */ -static int tsens_mC_to_hw(struct tsens_sensor *s, int temp) +static int tsens_mC_to_hw(const struct tsens_sensor *s, int temp) { struct tsens_priv *priv = s->priv; @@ -275,7 +275,8 @@ static int tsens_threshold_violated(struct tsens_priv *priv, u32 hw_id, } static int tsens_read_irq_state(struct tsens_priv *priv, u32 hw_id, - struct tsens_sensor *s, struct tsens_irq_data *d) + const struct tsens_sensor *s, + struct tsens_irq_data *d) { int ret; @@ -346,10 +347,10 @@ irqreturn_t tsens_irq_thread(int irq, void *data) for (i = 0; i < priv->num_sensors; i++) { bool trigger = false; - struct tsens_sensor *s = &priv->sensor[i]; + const struct tsens_sensor *s = &priv->sensor[i]; u32 hw_id = s->hw_id; - if (IS_ERR(priv->sensor[i].tzd)) + if (IS_ERR(s->tzd)) continue; if (!tsens_threshold_violated(priv, hw_id, &d)) continue; @@ -457,7 +458,7 @@ void tsens_disable_irq(struct tsens_priv *priv) regmap_field_write(priv->rf[INT_EN], 0); } -int get_temp_tsens_valid(struct tsens_sensor *s, int *temp) +int get_temp_tsens_valid(const struct tsens_sensor *s, int *temp) { struct tsens_priv *priv = s->priv; int hw_id = s->hw_id; @@ -486,7 +487,7 @@ int get_temp_tsens_valid(struct tsens_sensor *s, int *temp) return 0; } -int get_temp_common(struct tsens_sensor *s, int *temp) +int get_temp_common(const struct tsens_sensor *s, int *temp) { struct tsens_priv *priv = s->priv; int hw_id = s->hw_id; diff --git a/drivers/thermal/qcom/tsens.h b/drivers/thermal/qcom/tsens.h index be364bf1d5a6..70dc34c80537 100644 --- a/drivers/thermal/qcom/tsens.h +++ b/drivers/thermal/qcom/tsens.h @@ -67,7 +67,7 @@ struct tsens_ops { /* mandatory callbacks */ int (*init)(struct tsens_priv *priv); int (*calibrate)(struct tsens_priv *priv); - int (*get_temp)(struct tsens_sensor *s, int *temp); + int (*get_temp)(const struct tsens_sensor *s, int *temp); /* optional callbacks */ int (*enable)(struct tsens_priv *priv, int i); void (*disable)(struct tsens_priv *priv); @@ -494,8 +494,8 @@ struct tsens_priv { char *qfprom_read(struct device *dev, const char *cname); void compute_intercept_slope(struct tsens_priv *priv, u32 *pt1, u32 *pt2, u32 mode); int init_common(struct tsens_priv *priv); -int get_temp_tsens_valid(struct tsens_sensor *s, int *temp); -int get_temp_common(struct tsens_sensor *s, int *temp); +int get_temp_tsens_valid(const struct tsens_sensor *s, int *temp); +int get_temp_common(const struct tsens_sensor *s, int *temp); int tsens_enable_irq(struct tsens_priv *priv); void tsens_disable_irq(struct tsens_priv *priv); int tsens_set_trips(void *_sensor, int low, int high); From c1c6f3b39c043b8c7312ad691bae88fea201161d Mon Sep 17 00:00:00 2001 From: Amit Kucheria Date: Thu, 12 Mar 2020 18:07:00 +0530 Subject: [PATCH 145/635] drivers: thermal: tsens: use simpler variables We already dereference the sensor and save it into a variable. Use the variable directly to make the code easier to read. Signed-off-by: Amit Kucheria Reviewed-by: Bjorn Andersson Reviewed-by: Stephen Boyd Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/5dc4356edfb8dffa377fb561359bf41a6f1fdf17.1584015867.git.amit.kucheria@linaro.org --- drivers/thermal/qcom/tsens-common.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/thermal/qcom/tsens-common.c b/drivers/thermal/qcom/tsens-common.c index 45e68f999bf6..f044788681c4 100644 --- a/drivers/thermal/qcom/tsens-common.c +++ b/drivers/thermal/qcom/tsens-common.c @@ -369,7 +369,7 @@ irqreturn_t tsens_irq_thread(int irq, void *data) tsens_set_interrupt(priv, hw_id, UPPER, disable); if (d.up_thresh > temp) { dev_dbg(priv->dev, "[%u] %s: re-arm upper\n", - priv->sensor[i].hw_id, __func__); + hw_id, __func__); tsens_set_interrupt(priv, hw_id, UPPER, enable); } else { trigger = true; @@ -380,7 +380,7 @@ irqreturn_t tsens_irq_thread(int irq, void *data) tsens_set_interrupt(priv, hw_id, LOWER, disable); if (d.low_thresh < temp) { dev_dbg(priv->dev, "[%u] %s: re-arm low\n", - priv->sensor[i].hw_id, __func__); + hw_id, __func__); tsens_set_interrupt(priv, hw_id, LOWER, enable); } else { trigger = true; @@ -393,7 +393,7 @@ irqreturn_t tsens_irq_thread(int irq, void *data) if (trigger) { dev_dbg(priv->dev, "[%u] %s: TZ update trigger (%d mC)\n", hw_id, __func__, temp); - thermal_zone_device_update(priv->sensor[i].tzd, + thermal_zone_device_update(s->tzd, THERMAL_EVENT_UNSPECIFIED); } else { dev_dbg(priv->dev, "[%u] %s: no violation: %d\n", @@ -436,7 +436,7 @@ int tsens_set_trips(void *_sensor, int low, int high) spin_unlock_irqrestore(&priv->ul_lock, flags); dev_dbg(dev, "[%u] %s: (%d:%d)->(%d:%d)\n", - s->hw_id, __func__, d.low_thresh, d.up_thresh, cl_low, cl_high); + hw_id, __func__, d.low_thresh, d.up_thresh, cl_low, cl_high); return 0; } From f22a3bf0d2225fba438c46a25d3ab8823585a5e0 Mon Sep 17 00:00:00 2001 From: Amit Kucheria Date: Thu, 12 Mar 2020 18:07:01 +0530 Subject: [PATCH 146/635] drivers: thermal: tsens: Release device in success path We don't currently call put_device in case of successfully initialising the device. So we hold the reference and keep the device pinned forever. Allow control to fall through so we can use same code for success and error paths to put_device. As a part of this fixup, change devm_ioremap_resource to act on the same device pointer as that used to allocate regmap memory. That ensures that we are free to release op->dev after examining its resources. Signed-off-by: Amit Kucheria Reviewed-by: Bjorn Andersson Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/d3996667e9f976bb30e97e301585cb1023be422e.1584015867.git.amit.kucheria@linaro.org --- drivers/thermal/qcom/tsens-common.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/thermal/qcom/tsens-common.c b/drivers/thermal/qcom/tsens-common.c index f044788681c4..fd3b56d52341 100644 --- a/drivers/thermal/qcom/tsens-common.c +++ b/drivers/thermal/qcom/tsens-common.c @@ -603,7 +603,7 @@ int __init init_common(struct tsens_priv *priv) /* DT with separate SROT and TM address space */ priv->tm_offset = 0; res = platform_get_resource(op, IORESOURCE_MEM, 1); - srot_base = devm_ioremap_resource(&op->dev, res); + srot_base = devm_ioremap_resource(dev, res); if (IS_ERR(srot_base)) { ret = PTR_ERR(srot_base); goto err_put_device; @@ -621,7 +621,7 @@ int __init init_common(struct tsens_priv *priv) } res = platform_get_resource(op, IORESOURCE_MEM, 0); - tm_base = devm_ioremap_resource(&op->dev, res); + tm_base = devm_ioremap_resource(dev, res); if (IS_ERR(tm_base)) { ret = PTR_ERR(tm_base); goto err_put_device; @@ -688,8 +688,6 @@ int __init init_common(struct tsens_priv *priv) tsens_enable_irq(priv); tsens_debug_init(op); - return 0; - err_put_device: put_device(&op->dev); return ret; From 79125e03dbfc7148361787783c8a82a7ed60c718 Mon Sep 17 00:00:00 2001 From: Amit Kucheria Date: Thu, 12 Mar 2020 18:07:02 +0530 Subject: [PATCH 147/635] drivers: thermal: tsens: Add critical interrupt support TSENS IP v2.x adds critical threshold interrupt support for each sensor in addition to the upper/lower threshold interrupt. Add support in the driver. While the critical interrupts themselves aren't currently used by Linux, the HW line is also used by the TSENS watchdog. So this patch acts as infrastructure to enable watchdog functionality for the TSENS IP. Signed-off-by: Amit Kucheria Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/51b22461d4b5f85a817274568459db4579fd4298.1584015867.git.amit.kucheria@linaro.org --- drivers/thermal/qcom/tsens-common.c | 118 ++++++++++++++++++++++++++-- drivers/thermal/qcom/tsens-v2.c | 8 +- drivers/thermal/qcom/tsens.c | 65 +++++++++------ drivers/thermal/qcom/tsens.h | 68 ++++++++++++++++ 4 files changed, 223 insertions(+), 36 deletions(-) diff --git a/drivers/thermal/qcom/tsens-common.c b/drivers/thermal/qcom/tsens-common.c index fd3b56d52341..b7d599425605 100644 --- a/drivers/thermal/qcom/tsens-common.c +++ b/drivers/thermal/qcom/tsens-common.c @@ -23,6 +23,10 @@ * @low_thresh: lower threshold temperature value * @low_irq_mask: mask register for lower threshold irqs * @low_irq_clear: clear register for lower threshold irqs + * @crit_viol: critical threshold violated + * @crit_thresh: critical threshold temperature value + * @crit_irq_mask: mask register for critical threshold irqs + * @crit_irq_clear: clear register for critical threshold irqs * * Structure containing data about temperature threshold settings and * irq status if they were violated. @@ -36,6 +40,10 @@ struct tsens_irq_data { int low_thresh; u32 low_irq_mask; u32 low_irq_clear; + u32 crit_viol; + u32 crit_thresh; + u32 crit_irq_mask; + u32 crit_irq_clear; }; char *qfprom_read(struct device *dev, const char *cname) @@ -189,6 +197,9 @@ static void tsens_set_interrupt_v1(struct tsens_priv *priv, u32 hw_id, case LOWER: index = LOW_INT_CLEAR_0 + hw_id; break; + case CRITICAL: + /* No critical interrupts before v2 */ + return; } regmap_field_write(priv->rf[index], enable ? 0 : 1); } @@ -214,6 +225,10 @@ static void tsens_set_interrupt_v2(struct tsens_priv *priv, u32 hw_id, index_mask = LOW_INT_MASK_0 + hw_id; index_clear = LOW_INT_CLEAR_0 + hw_id; break; + case CRITICAL: + index_mask = CRIT_INT_MASK_0 + hw_id; + index_clear = CRIT_INT_CLEAR_0 + hw_id; + break; } if (enable) { @@ -268,7 +283,15 @@ static int tsens_threshold_violated(struct tsens_priv *priv, u32 hw_id, ret = regmap_field_read(priv->rf[LOWER_STATUS_0 + hw_id], &d->low_viol); if (ret) return ret; - if (d->up_viol || d->low_viol) + + if (priv->feat->crit_int) { + ret = regmap_field_read(priv->rf[CRITICAL_STATUS_0 + hw_id], + &d->crit_viol); + if (ret) + return ret; + } + + if (d->up_viol || d->low_viol || d->crit_viol) return 1; return 0; @@ -293,22 +316,37 @@ static int tsens_read_irq_state(struct tsens_priv *priv, u32 hw_id, ret = regmap_field_read(priv->rf[LOW_INT_MASK_0 + hw_id], &d->low_irq_mask); if (ret) return ret; + ret = regmap_field_read(priv->rf[CRIT_INT_CLEAR_0 + hw_id], + &d->crit_irq_clear); + if (ret) + return ret; + ret = regmap_field_read(priv->rf[CRIT_INT_MASK_0 + hw_id], + &d->crit_irq_mask); + if (ret) + return ret; + + d->crit_thresh = tsens_hw_to_mC(s, CRIT_THRESH_0 + hw_id); } else { /* No mask register on older TSENS */ d->up_irq_mask = 0; d->low_irq_mask = 0; + d->crit_irq_clear = 0; + d->crit_irq_mask = 0; + d->crit_thresh = 0; } d->up_thresh = tsens_hw_to_mC(s, UP_THRESH_0 + hw_id); d->low_thresh = tsens_hw_to_mC(s, LOW_THRESH_0 + hw_id); - dev_dbg(priv->dev, "[%u] %s%s: status(%u|%u) | clr(%u|%u) | mask(%u|%u)\n", - hw_id, __func__, (d->up_viol || d->low_viol) ? "(V)" : "", - d->low_viol, d->up_viol, d->low_irq_clear, d->up_irq_clear, - d->low_irq_mask, d->up_irq_mask); - dev_dbg(priv->dev, "[%u] %s%s: thresh: (%d:%d)\n", hw_id, __func__, - (d->up_viol || d->low_viol) ? "(violation)" : "", - d->low_thresh, d->up_thresh); + dev_dbg(priv->dev, "[%u] %s%s: status(%u|%u|%u) | clr(%u|%u|%u) | mask(%u|%u|%u)\n", + hw_id, __func__, + (d->up_viol || d->low_viol || d->crit_viol) ? "(V)" : "", + d->low_viol, d->up_viol, d->crit_viol, + d->low_irq_clear, d->up_irq_clear, d->crit_irq_clear, + d->low_irq_mask, d->up_irq_mask, d->crit_irq_mask); + dev_dbg(priv->dev, "[%u] %s%s: thresh: (%d:%d:%d)\n", hw_id, __func__, + (d->up_viol || d->low_viol || d->crit_viol) ? "(V)" : "", + d->low_thresh, d->up_thresh, d->crit_thresh); return 0; } @@ -322,6 +360,52 @@ static inline u32 masked_irq(u32 hw_id, u32 mask, enum tsens_ver ver) return 0; } +/** + * tsens_critical_irq_thread() - Threaded handler for critical interrupts + * @irq: irq number + * @data: tsens controller private data + * + * Check all sensors to find ones that violated their critical threshold limits. + * Clear and then re-enable the interrupt. + * + * The level-triggered interrupt might deassert if the temperature returned to + * within the threshold limits by the time the handler got scheduled. We + * consider the irq to have been handled in that case. + * + * Return: IRQ_HANDLED + */ +irqreturn_t tsens_critical_irq_thread(int irq, void *data) +{ + struct tsens_priv *priv = data; + struct tsens_irq_data d; + int temp, ret, i; + + for (i = 0; i < priv->num_sensors; i++) { + const struct tsens_sensor *s = &priv->sensor[i]; + u32 hw_id = s->hw_id; + + if (IS_ERR(s->tzd)) + continue; + if (!tsens_threshold_violated(priv, hw_id, &d)) + continue; + ret = get_temp_tsens_valid(s, &temp); + if (ret) { + dev_err(priv->dev, "[%u] %s: error reading sensor\n", + hw_id, __func__); + continue; + } + + tsens_read_irq_state(priv, hw_id, s, &d); + if (d.crit_viol && + !masked_irq(hw_id, d.crit_irq_mask, tsens_version(priv))) { + /* Mask critical interrupts, unused on Linux */ + tsens_set_interrupt(priv, hw_id, CRITICAL, false); + } + } + + return IRQ_HANDLED; +} + /** * tsens_irq_thread - Threaded interrupt handler for uplow interrupts * @irq: irq number @@ -684,6 +768,24 @@ int __init init_common(struct tsens_priv *priv) } } + if (priv->feat->crit_int) { + /* Loop might need changes if enum regfield_ids is reordered */ + for (j = CRITICAL_STATUS_0; j <= CRIT_THRESH_15; j += 16) { + for (i = 0; i < priv->feat->max_sensors; i++) { + int idx = j + i; + + priv->rf[idx] = + devm_regmap_field_alloc(dev, + priv->tm_map, + priv->fields[idx]); + if (IS_ERR(priv->rf[idx])) { + ret = PTR_ERR(priv->rf[idx]); + goto err_put_device; + } + } + } + } + spin_lock_init(&priv->ul_lock); tsens_enable_irq(priv); tsens_debug_init(op); diff --git a/drivers/thermal/qcom/tsens-v2.c b/drivers/thermal/qcom/tsens-v2.c index f1c8ec62e69f..ce5ef0055d13 100644 --- a/drivers/thermal/qcom/tsens-v2.c +++ b/drivers/thermal/qcom/tsens-v2.c @@ -51,8 +51,9 @@ static const struct reg_field tsens_v2_regfields[MAX_REGFIELDS] = { [INT_EN] = REG_FIELD(TM_INT_EN_OFF, 0, 2), /* TEMPERATURE THRESHOLDS */ - REG_FIELD_FOR_EACH_SENSOR16(LOW_THRESH, TM_Sn_UPPER_LOWER_THRESHOLD_OFF, 0, 11), - REG_FIELD_FOR_EACH_SENSOR16(UP_THRESH, TM_Sn_UPPER_LOWER_THRESHOLD_OFF, 12, 23), + REG_FIELD_FOR_EACH_SENSOR16(LOW_THRESH, TM_Sn_UPPER_LOWER_THRESHOLD_OFF, 0, 11), + REG_FIELD_FOR_EACH_SENSOR16(UP_THRESH, TM_Sn_UPPER_LOWER_THRESHOLD_OFF, 12, 23), + REG_FIELD_FOR_EACH_SENSOR16(CRIT_THRESH, TM_Sn_CRITICAL_THRESHOLD_OFF, 0, 11), /* INTERRUPTS [CLEAR/STATUS/MASK] */ REG_FIELD_SPLIT_BITS_0_15(LOW_INT_STATUS, TM_UPPER_LOWER_INT_STATUS_OFF), @@ -61,6 +62,9 @@ static const struct reg_field tsens_v2_regfields[MAX_REGFIELDS] = { REG_FIELD_SPLIT_BITS_16_31(UP_INT_STATUS, TM_UPPER_LOWER_INT_STATUS_OFF), REG_FIELD_SPLIT_BITS_16_31(UP_INT_CLEAR, TM_UPPER_LOWER_INT_CLEAR_OFF), REG_FIELD_SPLIT_BITS_16_31(UP_INT_MASK, TM_UPPER_LOWER_INT_MASK_OFF), + REG_FIELD_SPLIT_BITS_0_15(CRIT_INT_STATUS, TM_CRITICAL_INT_STATUS_OFF), + REG_FIELD_SPLIT_BITS_0_15(CRIT_INT_CLEAR, TM_CRITICAL_INT_CLEAR_OFF), + REG_FIELD_SPLIT_BITS_0_15(CRIT_INT_MASK, TM_CRITICAL_INT_MASK_OFF), /* Sn_STATUS */ REG_FIELD_FOR_EACH_SENSOR16(LAST_TEMP, TM_Sn_STATUS_OFF, 0, 11), diff --git a/drivers/thermal/qcom/tsens.c b/drivers/thermal/qcom/tsens.c index 0e7cf5236932..05d272132047 100644 --- a/drivers/thermal/qcom/tsens.c +++ b/drivers/thermal/qcom/tsens.c @@ -85,11 +85,42 @@ static const struct thermal_zone_of_device_ops tsens_of_ops = { .set_trips = tsens_set_trips, }; +static int tsens_register_irq(struct tsens_priv *priv, char *irqname, + irq_handler_t thread_fn) +{ + struct platform_device *pdev; + int ret, irq; + + pdev = of_find_device_by_node(priv->dev->of_node); + if (!pdev) + return -ENODEV; + + irq = platform_get_irq_byname(pdev, irqname); + if (irq < 0) { + ret = irq; + /* For old DTs with no IRQ defined */ + if (irq == -ENXIO) + ret = 0; + } else { + ret = devm_request_threaded_irq(&pdev->dev, irq, + NULL, thread_fn, + IRQF_TRIGGER_HIGH | IRQF_ONESHOT, + dev_name(&pdev->dev), priv); + if (ret) + dev_err(&pdev->dev, "%s: failed to get irq\n", + __func__); + else + enable_irq_wake(irq); + } + + put_device(&pdev->dev); + return ret; +} + static int tsens_register(struct tsens_priv *priv) { - int i, ret, irq; + int i, ret; struct thermal_zone_device *tzd; - struct platform_device *pdev; for (i = 0; i < priv->num_sensors; i++) { priv->sensor[i].priv = priv; @@ -103,32 +134,14 @@ static int tsens_register(struct tsens_priv *priv) priv->ops->enable(priv, i); } - pdev = of_find_device_by_node(priv->dev->of_node); - if (!pdev) - return -ENODEV; + ret = tsens_register_irq(priv, "uplow", tsens_irq_thread); + if (ret < 0) + return ret; - irq = platform_get_irq_byname(pdev, "uplow"); - if (irq < 0) { - ret = irq; - /* For old DTs with no IRQ defined */ - if (irq == -ENXIO) - ret = 0; - goto err_put_device; - } + if (priv->feat->crit_int) + ret = tsens_register_irq(priv, "critical", + tsens_critical_irq_thread); - ret = devm_request_threaded_irq(&pdev->dev, irq, - NULL, tsens_irq_thread, - IRQF_TRIGGER_HIGH | IRQF_ONESHOT, - dev_name(&pdev->dev), priv); - if (ret) { - dev_err(&pdev->dev, "%s: failed to get irq\n", __func__); - goto err_put_device; - } - - enable_irq_wake(irq); - -err_put_device: - put_device(&pdev->dev); return ret; } diff --git a/drivers/thermal/qcom/tsens.h b/drivers/thermal/qcom/tsens.h index 70dc34c80537..2466115a9881 100644 --- a/drivers/thermal/qcom/tsens.h +++ b/drivers/thermal/qcom/tsens.h @@ -23,6 +23,7 @@ struct tsens_priv; +/* IP version numbers in ascending order */ enum tsens_ver { VER_0_1 = 0, VER_1_X, @@ -32,6 +33,7 @@ enum tsens_ver { enum tsens_irq_type { LOWER, UPPER, + CRITICAL, }; /** @@ -374,6 +376,70 @@ enum regfield_ids { CRITICAL_STATUS_13, CRITICAL_STATUS_14, CRITICAL_STATUS_15, + CRIT_INT_STATUS_0, /* CRITICAL interrupt status */ + CRIT_INT_STATUS_1, + CRIT_INT_STATUS_2, + CRIT_INT_STATUS_3, + CRIT_INT_STATUS_4, + CRIT_INT_STATUS_5, + CRIT_INT_STATUS_6, + CRIT_INT_STATUS_7, + CRIT_INT_STATUS_8, + CRIT_INT_STATUS_9, + CRIT_INT_STATUS_10, + CRIT_INT_STATUS_11, + CRIT_INT_STATUS_12, + CRIT_INT_STATUS_13, + CRIT_INT_STATUS_14, + CRIT_INT_STATUS_15, + CRIT_INT_CLEAR_0, /* CRITICAL interrupt clear */ + CRIT_INT_CLEAR_1, + CRIT_INT_CLEAR_2, + CRIT_INT_CLEAR_3, + CRIT_INT_CLEAR_4, + CRIT_INT_CLEAR_5, + CRIT_INT_CLEAR_6, + CRIT_INT_CLEAR_7, + CRIT_INT_CLEAR_8, + CRIT_INT_CLEAR_9, + CRIT_INT_CLEAR_10, + CRIT_INT_CLEAR_11, + CRIT_INT_CLEAR_12, + CRIT_INT_CLEAR_13, + CRIT_INT_CLEAR_14, + CRIT_INT_CLEAR_15, + CRIT_INT_MASK_0, /* CRITICAL interrupt mask */ + CRIT_INT_MASK_1, + CRIT_INT_MASK_2, + CRIT_INT_MASK_3, + CRIT_INT_MASK_4, + CRIT_INT_MASK_5, + CRIT_INT_MASK_6, + CRIT_INT_MASK_7, + CRIT_INT_MASK_8, + CRIT_INT_MASK_9, + CRIT_INT_MASK_10, + CRIT_INT_MASK_11, + CRIT_INT_MASK_12, + CRIT_INT_MASK_13, + CRIT_INT_MASK_14, + CRIT_INT_MASK_15, + CRIT_THRESH_0, /* CRITICAL threshold values */ + CRIT_THRESH_1, + CRIT_THRESH_2, + CRIT_THRESH_3, + CRIT_THRESH_4, + CRIT_THRESH_5, + CRIT_THRESH_6, + CRIT_THRESH_7, + CRIT_THRESH_8, + CRIT_THRESH_9, + CRIT_THRESH_10, + CRIT_THRESH_11, + CRIT_THRESH_12, + CRIT_THRESH_13, + CRIT_THRESH_14, + CRIT_THRESH_15, MIN_STATUS_0, /* MIN threshold violated */ MIN_STATUS_1, MIN_STATUS_2, @@ -460,6 +526,7 @@ struct tsens_context { * @srot_map: pointer to SROT register address space * @tm_offset: deal with old device trees that don't address TM and SROT * address space separately + * @crit_lock: lock while processing critical threshold interrupts * @rf: array of regmap_fields used to store value of the field * @ctx: registers to be saved and restored during suspend/resume * @feat: features of the IP @@ -500,6 +567,7 @@ int tsens_enable_irq(struct tsens_priv *priv); void tsens_disable_irq(struct tsens_priv *priv); int tsens_set_trips(void *_sensor, int low, int high); irqreturn_t tsens_irq_thread(int irq, void *data); +irqreturn_t tsens_critical_irq_thread(int irq, void *data); /* TSENS target */ extern struct tsens_plat_data data_8960; From d22066c1afcda84306c96483357b8477b088ed30 Mon Sep 17 00:00:00 2001 From: Amit Kucheria Date: Thu, 12 Mar 2020 18:07:03 +0530 Subject: [PATCH 148/635] drivers: thermal: tsens: Add watchdog support TSENS IP v2.3 onwards adds support for a watchdog to detect if the TSENS HW FSM is stuck. Add support to detect and restart the FSM in the driver. The watchdog is configured by the bootloader, we just enable the watchdog bark as a debug feature in the kernel. Signed-off-by: Amit Kucheria Reviewed-by: Bjorn Andersson Reviewed-by: Stephen Boyd Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/a314747664a065db592ad77da7beae68128a5b6e.1584015867.git.amit.kucheria@linaro.org --- drivers/thermal/qcom/tsens-common.c | 47 +++++++++++++++++++++++++++++ drivers/thermal/qcom/tsens-v2.c | 10 ++++++ drivers/thermal/qcom/tsens.h | 14 +++++++++ 3 files changed, 71 insertions(+) diff --git a/drivers/thermal/qcom/tsens-common.c b/drivers/thermal/qcom/tsens-common.c index b7d599425605..172545366636 100644 --- a/drivers/thermal/qcom/tsens-common.c +++ b/drivers/thermal/qcom/tsens-common.c @@ -365,6 +365,7 @@ static inline u32 masked_irq(u32 hw_id, u32 mask, enum tsens_ver ver) * @irq: irq number * @data: tsens controller private data * + * Check FSM watchdog bark status and clear if needed. * Check all sensors to find ones that violated their critical threshold limits. * Clear and then re-enable the interrupt. * @@ -379,6 +380,29 @@ irqreturn_t tsens_critical_irq_thread(int irq, void *data) struct tsens_priv *priv = data; struct tsens_irq_data d; int temp, ret, i; + u32 wdog_status, wdog_count; + + if (priv->feat->has_watchdog) { + ret = regmap_field_read(priv->rf[WDOG_BARK_STATUS], + &wdog_status); + if (ret) + return ret; + + if (wdog_status) { + /* Clear WDOG interrupt */ + regmap_field_write(priv->rf[WDOG_BARK_CLEAR], 1); + regmap_field_write(priv->rf[WDOG_BARK_CLEAR], 0); + ret = regmap_field_read(priv->rf[WDOG_BARK_COUNT], + &wdog_count); + if (ret) + return ret; + if (wdog_count) + dev_dbg(priv->dev, "%s: watchdog count: %d\n", + __func__, wdog_count); + + /* Fall through to handle critical interrupts if any */ + } + } for (i = 0; i < priv->num_sensors; i++) { const struct tsens_sensor *s = &priv->sensor[i]; @@ -675,6 +699,7 @@ int __init init_common(struct tsens_priv *priv) { void __iomem *tm_base, *srot_base; struct device *dev = priv->dev; + u32 ver_minor; struct resource *res; u32 enabled; int ret, i, j; @@ -724,6 +749,9 @@ int __init init_common(struct tsens_priv *priv) if (IS_ERR(priv->rf[i])) return PTR_ERR(priv->rf[i]); } + ret = regmap_field_read(priv->rf[VER_MINOR], &ver_minor); + if (ret) + goto err_put_device; } priv->rf[TSENS_EN] = devm_regmap_field_alloc(dev, priv->srot_map, @@ -786,6 +814,25 @@ int __init init_common(struct tsens_priv *priv) } } + if (tsens_version(priv) > VER_1_X && ver_minor > 2) { + /* Watchdog is present only on v2.3+ */ + priv->feat->has_watchdog = 1; + for (i = WDOG_BARK_STATUS; i <= CC_MON_MASK; i++) { + priv->rf[i] = devm_regmap_field_alloc(dev, priv->tm_map, + priv->fields[i]); + if (IS_ERR(priv->rf[i])) { + ret = PTR_ERR(priv->rf[i]); + goto err_put_device; + } + } + /* + * Watchdog is already enabled, unmask the bark. + * Disable cycle completion monitoring + */ + regmap_field_write(priv->rf[WDOG_BARK_MASK], 0); + regmap_field_write(priv->rf[CC_MON_MASK], 1); + } + spin_lock_init(&priv->ul_lock); tsens_enable_irq(priv); tsens_debug_init(op); diff --git a/drivers/thermal/qcom/tsens-v2.c b/drivers/thermal/qcom/tsens-v2.c index ce5ef0055d13..b293ed32174b 100644 --- a/drivers/thermal/qcom/tsens-v2.c +++ b/drivers/thermal/qcom/tsens-v2.c @@ -24,6 +24,7 @@ #define TM_Sn_CRITICAL_THRESHOLD_OFF 0x0060 #define TM_Sn_STATUS_OFF 0x00a0 #define TM_TRDY_OFF 0x00e4 +#define TM_WDOG_LOG_OFF 0x013c /* v2.x: 8996, 8998, sdm845 */ @@ -66,6 +67,15 @@ static const struct reg_field tsens_v2_regfields[MAX_REGFIELDS] = { REG_FIELD_SPLIT_BITS_0_15(CRIT_INT_CLEAR, TM_CRITICAL_INT_CLEAR_OFF), REG_FIELD_SPLIT_BITS_0_15(CRIT_INT_MASK, TM_CRITICAL_INT_MASK_OFF), + /* WATCHDOG on v2.3 or later */ + [WDOG_BARK_STATUS] = REG_FIELD(TM_CRITICAL_INT_STATUS_OFF, 31, 31), + [WDOG_BARK_CLEAR] = REG_FIELD(TM_CRITICAL_INT_CLEAR_OFF, 31, 31), + [WDOG_BARK_MASK] = REG_FIELD(TM_CRITICAL_INT_MASK_OFF, 31, 31), + [CC_MON_STATUS] = REG_FIELD(TM_CRITICAL_INT_STATUS_OFF, 30, 30), + [CC_MON_CLEAR] = REG_FIELD(TM_CRITICAL_INT_CLEAR_OFF, 30, 30), + [CC_MON_MASK] = REG_FIELD(TM_CRITICAL_INT_MASK_OFF, 30, 30), + [WDOG_BARK_COUNT] = REG_FIELD(TM_WDOG_LOG_OFF, 0, 7), + /* Sn_STATUS */ REG_FIELD_FOR_EACH_SENSOR16(LAST_TEMP, TM_Sn_STATUS_OFF, 0, 11), REG_FIELD_FOR_EACH_SENSOR16(VALID, TM_Sn_STATUS_OFF, 21, 21), diff --git a/drivers/thermal/qcom/tsens.h b/drivers/thermal/qcom/tsens.h index 2466115a9881..ad7574fe0811 100644 --- a/drivers/thermal/qcom/tsens.h +++ b/drivers/thermal/qcom/tsens.h @@ -440,6 +440,18 @@ enum regfield_ids { CRIT_THRESH_13, CRIT_THRESH_14, CRIT_THRESH_15, + + /* WATCHDOG */ + WDOG_BARK_STATUS, + WDOG_BARK_CLEAR, + WDOG_BARK_MASK, + WDOG_BARK_COUNT, + + /* CYCLE COMPLETION MONITOR */ + CC_MON_STATUS, + CC_MON_CLEAR, + CC_MON_MASK, + MIN_STATUS_0, /* MIN threshold violated */ MIN_STATUS_1, MIN_STATUS_2, @@ -484,6 +496,7 @@ enum regfield_ids { * @adc: do the sensors only output adc code (instead of temperature)? * @srot_split: does the IP neatly splits the register space into SROT and TM, * with SROT only being available to secure boot firmware? + * @has_watchdog: does this IP support watchdog functionality? * @max_sensors: maximum sensors supported by this version of the IP */ struct tsens_features { @@ -491,6 +504,7 @@ struct tsens_features { unsigned int crit_int:1; unsigned int adc:1; unsigned int srot_split:1; + unsigned int has_watchdog:1; unsigned int max_sensors; }; From 348596969de1b637672bb5db25aa02cf5e6e018b Mon Sep 17 00:00:00 2001 From: Amit Kucheria Date: Thu, 12 Mar 2020 18:07:04 +0530 Subject: [PATCH 149/635] drivers: thermal: tsens: kernel-doc fixup Document ul_lock, threshold and control structure members and make the following kernel-doc invocation happy: $ scripts/kernel-doc -v -none drivers/thermal/qcom/* drivers/thermal/qcom/qcom-spmi-temp-alarm.c:105: info: Scanning doc for qpnp_tm_get_temp_stage drivers/thermal/qcom/tsens-common.c:18: info: Scanning doc for struct tsens_irq_data drivers/thermal/qcom/tsens-common.c:130: info: Scanning doc for tsens_hw_to_mC drivers/thermal/qcom/tsens-common.c:163: info: Scanning doc for tsens_mC_to_hw drivers/thermal/qcom/tsens-common.c:245: info: Scanning doc for tsens_set_interrupt drivers/thermal/qcom/tsens-common.c:268: info: Scanning doc for tsens_threshold_violated drivers/thermal/qcom/tsens-common.c:362: info: Scanning doc for tsens_critical_irq_thread drivers/thermal/qcom/tsens-common.c:438: info: Scanning doc for tsens_irq_thread drivers/thermal/qcom/tsens.h:41: info: Scanning doc for struct tsens_sensor drivers/thermal/qcom/tsens.h:59: info: Scanning doc for struct tsens_ops drivers/thermal/qcom/tsens.h:494: info: Scanning doc for struct tsens_features drivers/thermal/qcom/tsens.h:513: info: Scanning doc for struct tsens_plat_data drivers/thermal/qcom/tsens.h:529: info: Scanning doc for struct tsens_context Signed-off-by: Amit Kucheria Reviewed-by: Bjorn Andersson Reviewed-by: Stephen Boyd Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/7ea9c9ead90a91205a3f1717c0c86db9a51780ce.1584015867.git.amit.kucheria@linaro.org --- drivers/thermal/qcom/tsens.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/thermal/qcom/tsens.h b/drivers/thermal/qcom/tsens.h index ad7574fe0811..f49e516b3458 100644 --- a/drivers/thermal/qcom/tsens.h +++ b/drivers/thermal/qcom/tsens.h @@ -526,6 +526,8 @@ struct tsens_plat_data { /** * struct tsens_context - Registers to be saved/restored across a context loss + * @threshold: Threshold register value + * @control: Control register value */ struct tsens_context { int threshold; @@ -540,6 +542,7 @@ struct tsens_context { * @srot_map: pointer to SROT register address space * @tm_offset: deal with old device trees that don't address TM and SROT * address space separately + * @ul_lock: lock while processing upper/lower threshold interrupts * @crit_lock: lock while processing critical threshold interrupts * @rf: array of regmap_fields used to store value of the field * @ctx: registers to be saved and restored during suspend/resume From 8d3a6d4f43370109b0a0b38472fa046fad29db2d Mon Sep 17 00:00:00 2001 From: Amit Kucheria Date: Thu, 12 Mar 2020 18:07:05 +0530 Subject: [PATCH 150/635] drivers: thermal: tsens: Remove unnecessary irq flag IRQF_TRIGGER_HIGH is already specified through devicetree interrupts property. Remove it from code. Signed-off-by: Amit Kucheria Reviewed-by: Bjorn Andersson Reviewed-by: Stephen Boyd Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/8ac92e45b65fe411f4aaf70dcde4e7e7c3169b2d.1584015867.git.amit.kucheria@linaro.org --- drivers/thermal/qcom/tsens.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/thermal/qcom/tsens.c b/drivers/thermal/qcom/tsens.c index 05d272132047..2f77d235cf73 100644 --- a/drivers/thermal/qcom/tsens.c +++ b/drivers/thermal/qcom/tsens.c @@ -104,7 +104,7 @@ static int tsens_register_irq(struct tsens_priv *priv, char *irqname, } else { ret = devm_request_threaded_irq(&pdev->dev, irq, NULL, thread_fn, - IRQF_TRIGGER_HIGH | IRQF_ONESHOT, + IRQF_ONESHOT, dev_name(&pdev->dev), priv); if (ret) dev_err(&pdev->dev, "%s: failed to get irq\n", From 7617e771c16611b422c43b3b154dce2b3ee152d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20S=C3=B6derlund?= Date: Tue, 10 Mar 2020 18:00:27 +0100 Subject: [PATCH 151/635] thermal: rcar_thermal: Always update thermal zone on interrupt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since commit a1ade5653804b8eb ("thermal: rcar: check every rcar_thermal_update_temp() return value") the temperature is always read in rcar_thermal_get_current_temp() so comparing it before and after enabling interrupts have little effect. Remove the check and always update the thermal zone when we get an interrupt that the temperature have changed. Signed-off-by: Niklas Söderlund Reviewed-by: Geert Uytterhoeven Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200310170029.1648996-2-niklas.soderlund+renesas@ragnatech.se --- drivers/thermal/rcar_thermal.c | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/drivers/thermal/rcar_thermal.c b/drivers/thermal/rcar_thermal.c index 4a48d1d2a31c..fc54acdb914a 100644 --- a/drivers/thermal/rcar_thermal.c +++ b/drivers/thermal/rcar_thermal.c @@ -387,28 +387,17 @@ static void _rcar_thermal_irq_ctrl(struct rcar_thermal_priv *priv, int enable) static void rcar_thermal_work(struct work_struct *work) { struct rcar_thermal_priv *priv; - int cctemp, nctemp; int ret; priv = container_of(work, struct rcar_thermal_priv, work.work); - ret = rcar_thermal_get_current_temp(priv, &cctemp); - if (ret < 0) - return; - ret = rcar_thermal_update_temp(priv); if (ret < 0) return; rcar_thermal_irq_enable(priv); - ret = rcar_thermal_get_current_temp(priv, &nctemp); - if (ret < 0) - return; - - if (nctemp != cctemp) - thermal_zone_device_update(priv->zone, - THERMAL_EVENT_UNSPECIFIED); + thermal_zone_device_update(priv->zone, THERMAL_EVENT_UNSPECIFIED); } static u32 rcar_thermal_had_changed(struct rcar_thermal_priv *priv, u32 status) From 57ed737f1646579bf77070109c18ea78db690d18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20S=C3=B6derlund?= Date: Tue, 10 Mar 2020 18:00:28 +0100 Subject: [PATCH 152/635] thermal: rcar_thermal: Do not store ctemp in rcar_thermal_priv MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is no need to cache the ctemp value in the private data structure as it's always prefetched before it's used. Remove it from the structure and have rcar_thermal_update_temp return the value instead of storing it. Signed-off-by: Niklas Söderlund Reviewed-by: Geert Uytterhoeven Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200310170029.1648996-3-niklas.soderlund+renesas@ragnatech.se --- drivers/thermal/rcar_thermal.c | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/drivers/thermal/rcar_thermal.c b/drivers/thermal/rcar_thermal.c index fc54acdb914a..4a45b314ef30 100644 --- a/drivers/thermal/rcar_thermal.c +++ b/drivers/thermal/rcar_thermal.c @@ -95,7 +95,6 @@ struct rcar_thermal_priv { struct mutex lock; struct list_head list; int id; - u32 ctemp; }; #define rcar_thermal_for_each_priv(pos, common) \ @@ -201,7 +200,6 @@ static int rcar_thermal_update_temp(struct rcar_thermal_priv *priv) struct device *dev = rcar_priv_to_dev(priv); int i; u32 ctemp, old, new; - int ret = -EINVAL; mutex_lock(&priv->lock); @@ -247,32 +245,28 @@ static int rcar_thermal_update_temp(struct rcar_thermal_priv *priv) ((ctemp - 1) << 0))); } - dev_dbg(dev, "thermal%d %d -> %d\n", priv->id, priv->ctemp, ctemp); - - priv->ctemp = ctemp; - ret = 0; err_out_unlock: mutex_unlock(&priv->lock); - return ret; + + return ctemp ? ctemp : -EINVAL; } static int rcar_thermal_get_current_temp(struct rcar_thermal_priv *priv, int *temp) { - int tmp; - int ret; + int ctemp, tmp; - ret = rcar_thermal_update_temp(priv); - if (ret < 0) - return ret; + ctemp = rcar_thermal_update_temp(priv); + if (ctemp < 0) + return ctemp; mutex_lock(&priv->lock); if (priv->chip->ctemp_bands == 1) - tmp = MCELSIUS((priv->ctemp * 5) - 65); - else if (priv->ctemp < 24) - tmp = MCELSIUS(((priv->ctemp * 55) - 720) / 10); + tmp = MCELSIUS((ctemp * 5) - 65); + else if (ctemp < 24) + tmp = MCELSIUS(((ctemp * 55) - 720) / 10); else - tmp = MCELSIUS((priv->ctemp * 5) - 60); + tmp = MCELSIUS((ctemp * 5) - 60); mutex_unlock(&priv->lock); /* Guaranteed operating range is -45C to 125C. */ From 0fa04202078a7dae3a1199bd61951de3d915ede6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20S=C3=B6derlund?= Date: Tue, 10 Mar 2020 18:00:29 +0100 Subject: [PATCH 153/635] thermal: rcar_thermal: Remove lock in rcar_thermal_get_current_temp() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With the ctemp value returned instead of cached in the private data structure their is no need to take the lock when translating ctemp into a temperature. Signed-off-by: Niklas Söderlund Reviewed-by: Geert Uytterhoeven Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200310170029.1648996-4-niklas.soderlund+renesas@ragnatech.se --- drivers/thermal/rcar_thermal.c | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/drivers/thermal/rcar_thermal.c b/drivers/thermal/rcar_thermal.c index 4a45b314ef30..e0c1f2409035 100644 --- a/drivers/thermal/rcar_thermal.c +++ b/drivers/thermal/rcar_thermal.c @@ -254,24 +254,20 @@ err_out_unlock: static int rcar_thermal_get_current_temp(struct rcar_thermal_priv *priv, int *temp) { - int ctemp, tmp; + int ctemp; ctemp = rcar_thermal_update_temp(priv); if (ctemp < 0) return ctemp; - mutex_lock(&priv->lock); - if (priv->chip->ctemp_bands == 1) - tmp = MCELSIUS((ctemp * 5) - 65); - else if (ctemp < 24) - tmp = MCELSIUS(((ctemp * 55) - 720) / 10); - else - tmp = MCELSIUS((ctemp * 5) - 60); - mutex_unlock(&priv->lock); - /* Guaranteed operating range is -45C to 125C. */ - *temp = tmp; + if (priv->chip->ctemp_bands == 1) + *temp = MCELSIUS((ctemp * 5) - 65); + else if (ctemp < 24) + *temp = MCELSIUS(((ctemp * 55) - 720) / 10); + else + *temp = MCELSIUS((ctemp * 5) - 60); return 0; } From 85f0b61a6bede242a2d6b67f9f6e1c02a80808e7 Mon Sep 17 00:00:00 2001 From: Anson Huang Date: Wed, 11 Mar 2020 13:07:31 +0800 Subject: [PATCH 154/635] thermal: qoriq: Use devm_add_action_or_reset() to handle all cleanups Use devm_add_action_or_reset() to handle all cleanups of failure in .probe and .remove, then .remove callback can be dropped. Signed-off-by: Anson Huang Reviewed-by: Amit Kucheria Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/1583903252-2058-1-git-send-email-Anson.Huang@nxp.com --- drivers/thermal/qoriq_thermal.c | 35 +++++++++++++-------------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/drivers/thermal/qoriq_thermal.c b/drivers/thermal/qoriq_thermal.c index 874bc46e6c73..67a8d84c14fb 100644 --- a/drivers/thermal/qoriq_thermal.c +++ b/drivers/thermal/qoriq_thermal.c @@ -228,6 +228,14 @@ static const struct regmap_access_table qoriq_rd_table = { .n_yes_ranges = ARRAY_SIZE(qoriq_yes_ranges), }; +static void qoriq_tmu_action(void *p) +{ + struct qoriq_tmu_data *data = p; + + regmap_write(data->regmap, REGS_TMR, TMR_DISABLE); + clk_disable_unprepare(data->clk); +} + static int qoriq_tmu_probe(struct platform_device *pdev) { int ret; @@ -278,6 +286,10 @@ static int qoriq_tmu_probe(struct platform_device *pdev) return ret; } + ret = devm_add_action_or_reset(dev, qoriq_tmu_action, data); + if (ret) + return ret; + /* version register offset at: 0xbf8 on both v1 and v2 */ ret = regmap_read(data->regmap, REGS_IPBRR(0), &ver); if (ret) { @@ -290,34 +302,16 @@ static int qoriq_tmu_probe(struct platform_device *pdev) ret = qoriq_tmu_calibration(dev, data); /* TMU calibration */ if (ret < 0) - goto err; + return ret; ret = qoriq_tmu_register_tmu_zone(dev, data); if (ret < 0) { dev_err(dev, "Failed to register sensors\n"); - ret = -ENODEV; - goto err; + return ret; } platform_set_drvdata(pdev, data); - return 0; - -err: - clk_disable_unprepare(data->clk); - - return ret; -} - -static int qoriq_tmu_remove(struct platform_device *pdev) -{ - struct qoriq_tmu_data *data = platform_get_drvdata(pdev); - - /* Disable monitoring */ - regmap_write(data->regmap, REGS_TMR, TMR_DISABLE); - - clk_disable_unprepare(data->clk); - return 0; } @@ -365,7 +359,6 @@ static struct platform_driver qoriq_tmu = { .of_match_table = qoriq_tmu_match, }, .probe = qoriq_tmu_probe, - .remove = qoriq_tmu_remove, }; module_platform_driver(qoriq_tmu); From ce68eeca8f26c9691513d8f94ceaca0040585f52 Mon Sep 17 00:00:00 2001 From: Anson Huang Date: Wed, 11 Mar 2020 13:07:32 +0800 Subject: [PATCH 155/635] thermal: qoriq: Sort includes alphabetically Sort includes alphabetically for consistency, and take this chance to remove unused include of of_address.h. Signed-off-by: Anson Huang Reviewed-by: Amit Kucheria Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/1583903252-2058-2-git-send-email-Anson.Huang@nxp.com --- drivers/thermal/qoriq_thermal.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/thermal/qoriq_thermal.c b/drivers/thermal/qoriq_thermal.c index 67a8d84c14fb..028a6bbf75dc 100644 --- a/drivers/thermal/qoriq_thermal.c +++ b/drivers/thermal/qoriq_thermal.c @@ -3,12 +3,11 @@ // Copyright 2016 Freescale Semiconductor, Inc. #include -#include -#include #include #include +#include #include -#include +#include #include #include #include From 9db11010f27b9e96417a3a4d0b61115eed7fbb48 Mon Sep 17 00:00:00 2001 From: Anson Huang Date: Fri, 13 Mar 2020 16:28:14 +0800 Subject: [PATCH 156/635] thermal: imx: Calling imx_thermal_unregister_legacy_cooling() in .remove imx_thermal_unregister_legacy_cooling() should be used for handling legacy cpufreq cooling cleanups in .remove callback instead of calling cpufreq_cooling_unregister() and cpufreq_cpu_put() directly, especially for !CONFIG_CPU_FREQ scenario, no operation needed for handling legacy cpufreq cooling cleanups at all. Signed-off-by: Anson Huang Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/1584088094-24857-1-git-send-email-Anson.Huang@nxp.com --- drivers/thermal/imx_thermal.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/thermal/imx_thermal.c b/drivers/thermal/imx_thermal.c index d2fa301e5033..e761c9b42217 100644 --- a/drivers/thermal/imx_thermal.c +++ b/drivers/thermal/imx_thermal.c @@ -865,8 +865,7 @@ static int imx_thermal_remove(struct platform_device *pdev) clk_disable_unprepare(data->thermal_clk); thermal_zone_device_unregister(data->tz); - cpufreq_cooling_unregister(data->cdev); - cpufreq_cpu_put(data->policy); + imx_thermal_unregister_legacy_cooling(data); return 0; } From 8698977867313e25685710c364a7cb6b8c944d49 Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Fri, 13 Mar 2020 16:45:51 -0500 Subject: [PATCH 157/635] dt-bindings: thermal: sprd: Remove redundant 'maxItems' There's no need to specify 'maxItems' with the same value as the number of entries in 'items'. A meta-schema update will catch future cases. Cc: Orson Zhai Cc: Baolin Wang Cc: Chunyan Zhang Cc: Zhang Rui Cc: Daniel Lezcano Cc: Amit Kucheria Cc: linux-pm@vger.kernel.org Cc: devicetree@vger.kernel.org Signed-off-by: Rob Herring Reviewed-by: Baolin Wang Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200313214552.845-1-robh@kernel.org --- Documentation/devicetree/bindings/thermal/sprd-thermal.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/Documentation/devicetree/bindings/thermal/sprd-thermal.yaml b/Documentation/devicetree/bindings/thermal/sprd-thermal.yaml index 3bfe05504456..058c4cc06ba6 100644 --- a/Documentation/devicetree/bindings/thermal/sprd-thermal.yaml +++ b/Documentation/devicetree/bindings/thermal/sprd-thermal.yaml @@ -31,7 +31,6 @@ properties: Reference to nvmem nodes for the calibration data. nvmem-cell-names: - maxItems: 2 items: - const: thm_sign_cal - const: thm_ratio_cal From 01c354e2ec1059def2234f2aa393e07520dd9aee Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Fri, 13 Mar 2020 16:45:52 -0500 Subject: [PATCH 158/635] dt-bindings: thermal: qcom-tsens: Remove redundant 'maxItems' There's no need to specify 'maxItems' with the same value as the number of entries in 'items'. A meta-schema update will catch future cases. Cc: Andy Gross Cc: Bjorn Andersson Cc: Amit Kucheria Cc: Zhang Rui Cc: Daniel Lezcano Cc: linux-arm-msm@vger.kernel.org Cc: linux-pm@vger.kernel.org Cc: devicetree@vger.kernel.org Signed-off-by: Rob Herring Reviewed-by: Amit Kucheria Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200313214552.845-2-robh@kernel.org --- Documentation/devicetree/bindings/thermal/qcom-tsens.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/Documentation/devicetree/bindings/thermal/qcom-tsens.yaml b/Documentation/devicetree/bindings/thermal/qcom-tsens.yaml index c0ed030d0960..62b97a6d9b65 100644 --- a/Documentation/devicetree/bindings/thermal/qcom-tsens.yaml +++ b/Documentation/devicetree/bindings/thermal/qcom-tsens.yaml @@ -43,7 +43,6 @@ properties: - const: qcom,tsens-v2 reg: - maxItems: 2 items: - description: TM registers - description: SROT registers From fd96a316d2647c7566a41987c8ba32c7d55cfdf1 Mon Sep 17 00:00:00 2001 From: Yuantian Tang Date: Mon, 9 Mar 2020 12:54:11 +0800 Subject: [PATCH 159/635] dt-bindings: thermal: make cooling-maps property optional Cooling-maps doesn't have to be a required property because there may be no cooling device on system, or there are no enough cooling devices for each thermal zone in multiple thermal zone cases since cooling devices can't be shared. So make this property optional to remove such limitations. For thermal zones with no cooling-maps, there could be critic trips that can trigger CPU reset or shutdown. So they still can take actions. Signed-off-by: Yuantian Tang Reviewed-by: Amit Kucheria Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200309045411.21859-1-andy.tang@nxp.com --- Documentation/devicetree/bindings/thermal/thermal.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/thermal/thermal.txt b/Documentation/devicetree/bindings/thermal/thermal.txt index ca14ba959e0d..f78bec19ca35 100644 --- a/Documentation/devicetree/bindings/thermal/thermal.txt +++ b/Documentation/devicetree/bindings/thermal/thermal.txt @@ -142,11 +142,11 @@ Required properties: - trips: A sub-node which is a container of only trip point nodes Type: sub-node required to describe the thermal zone. +Optional property: - cooling-maps: A sub-node which is a container of only cooling device Type: sub-node map nodes, used to describe the relation between trips and cooling devices. -Optional property: - coefficients: An array of integers (one signed cell) containing Type: array coefficients to compose a linear relation between Elem size: one cell the sensors listed in the thermal-sensors property. From afa58b49ac52dff053e1bb363a425f09dbebc0a3 Mon Sep 17 00:00:00 2001 From: Clark Williams Date: Tue, 8 Oct 2019 13:00:21 +0200 Subject: [PATCH 160/635] thermal/x86_pkg_temp: Make pkg_temp_lock a raw_spinlock_t The spinlock pkg_temp_lock has the potential of being taken in atomic context because it can be acquired from the thermal IRQ vector. It's static and limited scope so go ahead and make it a raw spinlock. Signed-off-by: Clark Williams Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Zhang Rui Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20191008110021.2j44ayunal7fkb7i@linutronix.de --- drivers/thermal/intel/x86_pkg_temp_thermal.c | 24 ++++++++++---------- drivers/thermal/st/stm_thermal.c | 3 ++- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/drivers/thermal/intel/x86_pkg_temp_thermal.c b/drivers/thermal/intel/x86_pkg_temp_thermal.c index ddb4a973c698..e18758a8e911 100644 --- a/drivers/thermal/intel/x86_pkg_temp_thermal.c +++ b/drivers/thermal/intel/x86_pkg_temp_thermal.c @@ -63,7 +63,7 @@ static int max_id __read_mostly; /* Array of zone pointers */ static struct zone_device **zones; /* Serializes interrupt notification, work and hotplug */ -static DEFINE_SPINLOCK(pkg_temp_lock); +static DEFINE_RAW_SPINLOCK(pkg_temp_lock); /* Protects zone operation in the work function against hotplug removal */ static DEFINE_MUTEX(thermal_zone_mutex); @@ -266,12 +266,12 @@ static void pkg_temp_thermal_threshold_work_fn(struct work_struct *work) u64 msr_val, wr_val; mutex_lock(&thermal_zone_mutex); - spin_lock_irq(&pkg_temp_lock); + raw_spin_lock_irq(&pkg_temp_lock); ++pkg_work_cnt; zonedev = pkg_temp_thermal_get_dev(cpu); if (!zonedev) { - spin_unlock_irq(&pkg_temp_lock); + raw_spin_unlock_irq(&pkg_temp_lock); mutex_unlock(&thermal_zone_mutex); return; } @@ -285,7 +285,7 @@ static void pkg_temp_thermal_threshold_work_fn(struct work_struct *work) } enable_pkg_thres_interrupt(); - spin_unlock_irq(&pkg_temp_lock); + raw_spin_unlock_irq(&pkg_temp_lock); /* * If tzone is not NULL, then thermal_zone_mutex will prevent the @@ -310,7 +310,7 @@ static int pkg_thermal_notify(u64 msr_val) struct zone_device *zonedev; unsigned long flags; - spin_lock_irqsave(&pkg_temp_lock, flags); + raw_spin_lock_irqsave(&pkg_temp_lock, flags); ++pkg_interrupt_cnt; disable_pkg_thres_interrupt(); @@ -322,7 +322,7 @@ static int pkg_thermal_notify(u64 msr_val) pkg_thermal_schedule_work(zonedev->cpu, &zonedev->work); } - spin_unlock_irqrestore(&pkg_temp_lock, flags); + raw_spin_unlock_irqrestore(&pkg_temp_lock, flags); return 0; } @@ -368,9 +368,9 @@ static int pkg_temp_thermal_device_add(unsigned int cpu) zonedev->msr_pkg_therm_high); cpumask_set_cpu(cpu, &zonedev->cpumask); - spin_lock_irq(&pkg_temp_lock); + raw_spin_lock_irq(&pkg_temp_lock); zones[id] = zonedev; - spin_unlock_irq(&pkg_temp_lock); + raw_spin_unlock_irq(&pkg_temp_lock); return 0; } @@ -407,7 +407,7 @@ static int pkg_thermal_cpu_offline(unsigned int cpu) } /* Protect against work and interrupts */ - spin_lock_irq(&pkg_temp_lock); + raw_spin_lock_irq(&pkg_temp_lock); /* * Check whether this cpu was the current target and store the new @@ -439,9 +439,9 @@ static int pkg_thermal_cpu_offline(unsigned int cpu) * To cancel the work we need to drop the lock, otherwise * we might deadlock if the work needs to be flushed. */ - spin_unlock_irq(&pkg_temp_lock); + raw_spin_unlock_irq(&pkg_temp_lock); cancel_delayed_work_sync(&zonedev->work); - spin_lock_irq(&pkg_temp_lock); + raw_spin_lock_irq(&pkg_temp_lock); /* * If this is not the last cpu in the package and the work * did not run after we dropped the lock above, then we @@ -452,7 +452,7 @@ static int pkg_thermal_cpu_offline(unsigned int cpu) pkg_thermal_schedule_work(target, &zonedev->work); } - spin_unlock_irq(&pkg_temp_lock); + raw_spin_unlock_irq(&pkg_temp_lock); /* Final cleanup if this is the last cpu */ if (lastcpu) diff --git a/drivers/thermal/st/stm_thermal.c b/drivers/thermal/st/stm_thermal.c index ad9e3bf8fdf6..9314e3df6a42 100644 --- a/drivers/thermal/st/stm_thermal.c +++ b/drivers/thermal/st/stm_thermal.c @@ -478,7 +478,8 @@ static int stm_thermal_resume(struct device *dev) } #endif /* CONFIG_PM_SLEEP */ -SIMPLE_DEV_PM_OPS(stm_thermal_pm_ops, stm_thermal_suspend, stm_thermal_resume); +static SIMPLE_DEV_PM_OPS(stm_thermal_pm_ops, + stm_thermal_suspend, stm_thermal_resume); static const struct thermal_zone_of_device_ops stm_tz_ops = { .get_temp = stm_thermal_get_temp, From 141f15c66d9472c642f38aae010ed68289036d7c Mon Sep 17 00:00:00 2001 From: Denis Osterland-Heim Date: Sat, 21 Mar 2020 08:15:53 +0000 Subject: [PATCH 161/635] leds: pwm: remove header The header is only used by leds_pwm.c, so move contents to leds_pwm.c and remove it. Apply minor changes suggested by checkpatch. Remove deprecated and unused pwm_id member. Suggested-by: Pavel Machek Signed-off-by: Denis Osterland-Heim Signed-off-by: Pavel Machek --- drivers/leds/leds-pwm.c | 14 +++++++++++++- include/linux/leds_pwm.h | 22 ---------------------- 2 files changed, 13 insertions(+), 23 deletions(-) delete mode 100644 include/linux/leds_pwm.h diff --git a/drivers/leds/leds-pwm.c b/drivers/leds/leds-pwm.c index 9111cdede0ee..e1848e80aeb4 100644 --- a/drivers/leds/leds-pwm.c +++ b/drivers/leds/leds-pwm.c @@ -16,9 +16,21 @@ #include #include #include -#include #include +struct led_pwm { + const char *name; + const char *default_trigger; + u8 active_low; + unsigned int max_brightness; + unsigned int pwm_period_ns; +}; + +struct led_pwm_platform_data { + int num_leds; + struct led_pwm *leds; +}; + struct led_pwm_data { struct led_classdev cdev; struct pwm_device *pwm; diff --git a/include/linux/leds_pwm.h b/include/linux/leds_pwm.h deleted file mode 100644 index 93d101d28943..000000000000 --- a/include/linux/leds_pwm.h +++ /dev/null @@ -1,22 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * PWM LED driver data - see drivers/leds/leds-pwm.c - */ -#ifndef __LINUX_LEDS_PWM_H -#define __LINUX_LEDS_PWM_H - -struct led_pwm { - const char *name; - const char *default_trigger; - unsigned pwm_id __deprecated; - u8 active_low; - unsigned max_brightness; - unsigned pwm_period_ns; -}; - -struct led_pwm_platform_data { - int num_leds; - struct led_pwm *leds; -}; - -#endif From 4227685b4dda61a3c0d2f10d50d364cb9c1a669d Mon Sep 17 00:00:00 2001 From: Denis Osterland-Heim Date: Sat, 21 Mar 2020 08:15:53 +0000 Subject: [PATCH 162/635] leds: pwm: remove useless pwm_period_ns This member seems to was a way to pass PWM period to the LED. Since there is no header anymore, this is useless. Signed-off-by: Denis Osterland-Heim Signed-off-by: Pavel Machek --- drivers/leds/leds-pwm.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/leds/leds-pwm.c b/drivers/leds/leds-pwm.c index e1848e80aeb4..6caf8bea8cd5 100644 --- a/drivers/leds/leds-pwm.c +++ b/drivers/leds/leds-pwm.c @@ -23,7 +23,6 @@ struct led_pwm { const char *default_trigger; u8 active_low; unsigned int max_brightness; - unsigned int pwm_period_ns; }; struct led_pwm_platform_data { @@ -91,9 +90,6 @@ static int led_pwm_add(struct device *dev, struct led_pwm_priv *priv, pwm_init_state(led_data->pwm, &led_data->pwmstate); - if (!led_data->pwmstate.period) - led_data->pwmstate.period = led->pwm_period_ns; - ret = devm_led_classdev_register(dev, &led_data->cdev); if (ret == 0) { priv->num_leds++; From 7a630367759869612a3a7aec76cff4b436b7f9aa Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Thu, 19 Mar 2020 16:51:12 -0500 Subject: [PATCH 163/635] leds: leds-is31fl32xx: Replace zero-length array with flexible-array member The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertently introduced[3] to the codebase from now on. Also, notice that, dynamic memory allocations won't be affected by this change: "Flexible array members have incomplete type, and so the sizeof operator may not be applied. As a quirk of the original implementation of zero-length arrays, sizeof evaluates to zero."[1] This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Signed-off-by: Pavel Machek --- drivers/leds/leds-is31fl32xx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/leds/leds-is31fl32xx.c b/drivers/leds/leds-is31fl32xx.c index 6f29b8943913..cd768f991da1 100644 --- a/drivers/leds/leds-is31fl32xx.c +++ b/drivers/leds/leds-is31fl32xx.c @@ -44,7 +44,7 @@ struct is31fl32xx_priv { const struct is31fl32xx_chipdef *cdef; struct i2c_client *client; unsigned int num_leds; - struct is31fl32xx_led_data leds[0]; + struct is31fl32xx_led_data leds[]; }; /** From 7bbec6c4b51d2323459203510028e8344f79e126 Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Thu, 19 Mar 2020 16:51:46 -0500 Subject: [PATCH 164/635] leds: leds-pwm: Replace zero-length array with flexible-array member The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertently introduced[3] to the codebase from now on. Also, notice that, dynamic memory allocations won't be affected by this change: "Flexible array members have incomplete type, and so the sizeof operator may not be applied. As a quirk of the original implementation of zero-length arrays, sizeof evaluates to zero."[1] This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Signed-off-by: Pavel Machek --- drivers/leds/leds-pwm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/leds/leds-pwm.c b/drivers/leds/leds-pwm.c index 6caf8bea8cd5..6c8a724aac51 100644 --- a/drivers/leds/leds-pwm.c +++ b/drivers/leds/leds-pwm.c @@ -39,7 +39,7 @@ struct led_pwm_data { struct led_pwm_priv { int num_leds; - struct led_pwm_data leds[0]; + struct led_pwm_data leds[]; }; static int led_pwm_set(struct led_classdev *led_cdev, From fdc6b21e2444290b69ac9ffa936bf583830dc0de Mon Sep 17 00:00:00 2001 From: Prashant Malani Date: Mon, 16 Mar 2020 02:00:17 -0700 Subject: [PATCH 165/635] platform/chrome: Add Type C connector class driver Add a driver to implement the Type C connector class for Chrome OS devices with ECs (Embedded Controllers). The driver relies on firmware device specifications for various port attributes. On ACPI platforms, this is specified using the logical device with HID GOOG0014. On DT platforms, this is specified using the DT node with compatible string "google,cros-ec-typec". The driver reads the device FW node and uses the port attributes to register the typec ports with the Type C connector class framework, but doesn't do much else. Subsequent patches will add more functionality to the driver, including obtaining current port information (polarity, vconn role, current power role etc.) after querying the EC. Co-developed-by: Benson Leung Signed-off-by: Prashant Malani Reviewed-by: Heikki Krogerus Signed-off-by: Enric Balletbo i Serra --- drivers/platform/chrome/Kconfig | 11 ++ drivers/platform/chrome/Makefile | 1 + drivers/platform/chrome/cros_ec_typec.c | 238 ++++++++++++++++++++++++ 3 files changed, 250 insertions(+) create mode 100644 drivers/platform/chrome/cros_ec_typec.c diff --git a/drivers/platform/chrome/Kconfig b/drivers/platform/chrome/Kconfig index 5ae6c49f553d..03ea5129ed0c 100644 --- a/drivers/platform/chrome/Kconfig +++ b/drivers/platform/chrome/Kconfig @@ -214,6 +214,17 @@ config CROS_EC_SYSFS To compile this driver as a module, choose M here: the module will be called cros_ec_sysfs. +config CROS_EC_TYPEC + tristate "ChromeOS EC Type-C Connector Control" + depends on MFD_CROS_EC_DEV && TYPEC + default MFD_CROS_EC_DEV + help + If you say Y here, you get support for accessing Type C connector + information from the Chrome OS EC. + + To compile this driver as a module, choose M here: the module will be + called cros_ec_typec. + config CROS_USBPD_LOGGER tristate "Logging driver for USB PD charger" depends on CHARGER_CROS_USBPD diff --git a/drivers/platform/chrome/Makefile b/drivers/platform/chrome/Makefile index f6465f8ef0b5..198c155c7c4d 100644 --- a/drivers/platform/chrome/Makefile +++ b/drivers/platform/chrome/Makefile @@ -12,6 +12,7 @@ obj-$(CONFIG_CROS_EC_ISHTP) += cros_ec_ishtp.o obj-$(CONFIG_CROS_EC_RPMSG) += cros_ec_rpmsg.o obj-$(CONFIG_CROS_EC_SPI) += cros_ec_spi.o cros_ec_lpcs-objs := cros_ec_lpc.o cros_ec_lpc_mec.o +obj-$(CONFIG_CROS_EC_TYPEC) += cros_ec_typec.o obj-$(CONFIG_CROS_EC_LPC) += cros_ec_lpcs.o obj-$(CONFIG_CROS_EC_PROTO) += cros_ec_proto.o cros_ec_trace.o obj-$(CONFIG_CROS_KBD_LED_BACKLIGHT) += cros_kbd_led_backlight.o diff --git a/drivers/platform/chrome/cros_ec_typec.c b/drivers/platform/chrome/cros_ec_typec.c new file mode 100644 index 000000000000..02e6d5cbbbf7 --- /dev/null +++ b/drivers/platform/chrome/cros_ec_typec.c @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2020 Google LLC + * + * This driver provides the ability to view and manage Type C ports through the + * Chrome OS EC. + */ + +#include +#include +#include +#include +#include +#include +#include + +#define DRV_NAME "cros-ec-typec" + +/* Platform-specific data for the Chrome OS EC Type C controller. */ +struct cros_typec_data { + struct device *dev; + struct cros_ec_device *ec; + int num_ports; + /* Array of ports, indexed by port number. */ + struct typec_port *ports[EC_USB_PD_MAX_PORTS]; + /* Initial capabilities for each port. */ + struct typec_capability *caps[EC_USB_PD_MAX_PORTS]; +}; + +static int cros_typec_parse_port_props(struct typec_capability *cap, + struct fwnode_handle *fwnode, + struct device *dev) +{ + const char *buf; + int ret; + + memset(cap, 0, sizeof(*cap)); + ret = fwnode_property_read_string(fwnode, "power-role", &buf); + if (ret) { + dev_err(dev, "power-role not found: %d\n", ret); + return ret; + } + + ret = typec_find_port_power_role(buf); + if (ret < 0) + return ret; + cap->type = ret; + + ret = fwnode_property_read_string(fwnode, "data-role", &buf); + if (ret) { + dev_err(dev, "data-role not found: %d\n", ret); + return ret; + } + + ret = typec_find_port_data_role(buf); + if (ret < 0) + return ret; + cap->data = ret; + + ret = fwnode_property_read_string(fwnode, "try-power-role", &buf); + if (ret) { + dev_err(dev, "try-power-role not found: %d\n", ret); + return ret; + } + + ret = typec_find_power_role(buf); + if (ret < 0) + return ret; + cap->prefer_role = ret; + + cap->fwnode = fwnode; + + return 0; +} + +static int cros_typec_init_ports(struct cros_typec_data *typec) +{ + struct device *dev = typec->dev; + struct typec_capability *cap; + struct fwnode_handle *fwnode; + const char *port_prop; + int ret; + int i; + int nports; + u32 port_num = 0; + + nports = device_get_child_node_count(dev); + if (nports == 0) { + dev_err(dev, "No port entries found.\n"); + return -ENODEV; + } + + if (nports > typec->num_ports) { + dev_err(dev, "More ports listed than can be supported.\n"); + return -EINVAL; + } + + /* DT uses "reg" to specify port number. */ + port_prop = dev->of_node ? "reg" : "port-number"; + device_for_each_child_node(dev, fwnode) { + if (fwnode_property_read_u32(fwnode, port_prop, &port_num)) { + ret = -EINVAL; + dev_err(dev, "No port-number for port, aborting.\n"); + goto unregister_ports; + } + + if (port_num >= typec->num_ports) { + dev_err(dev, "Invalid port number.\n"); + ret = -EINVAL; + goto unregister_ports; + } + + dev_dbg(dev, "Registering port %d\n", port_num); + + cap = devm_kzalloc(dev, sizeof(*cap), GFP_KERNEL); + if (!cap) { + ret = -ENOMEM; + goto unregister_ports; + } + + typec->caps[port_num] = cap; + + ret = cros_typec_parse_port_props(cap, fwnode, dev); + if (ret < 0) + goto unregister_ports; + + typec->ports[port_num] = typec_register_port(dev, cap); + if (IS_ERR(typec->ports[port_num])) { + dev_err(dev, "Failed to register port %d\n", port_num); + ret = PTR_ERR(typec->ports[port_num]); + goto unregister_ports; + } + } + + return 0; + +unregister_ports: + for (i = 0; i < typec->num_ports; i++) + typec_unregister_port(typec->ports[i]); + return ret; +} + +static int cros_typec_ec_command(struct cros_typec_data *typec, + unsigned int version, + unsigned int command, + void *outdata, + unsigned int outsize, + void *indata, + unsigned int insize) +{ + struct cros_ec_command *msg; + int ret; + + msg = kzalloc(sizeof(*msg) + max(outsize, insize), GFP_KERNEL); + if (!msg) + return -ENOMEM; + + msg->version = version; + msg->command = command; + msg->outsize = outsize; + msg->insize = insize; + + if (outsize) + memcpy(msg->data, outdata, outsize); + + ret = cros_ec_cmd_xfer_status(typec->ec, msg); + if (ret >= 0 && insize) + memcpy(indata, msg->data, insize); + + kfree(msg); + return ret; +} + +#ifdef CONFIG_ACPI +static const struct acpi_device_id cros_typec_acpi_id[] = { + { "GOOG0014", 0 }, + {} +}; +MODULE_DEVICE_TABLE(acpi, cros_typec_acpi_id); +#endif + +#ifdef CONFIG_OF +static const struct of_device_id cros_typec_of_match[] = { + { .compatible = "google,cros-ec-typec", }, + {} +}; +MODULE_DEVICE_TABLE(of, cros_typec_of_match); +#endif + +static int cros_typec_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct cros_typec_data *typec; + struct ec_response_usb_pd_ports resp; + int ret; + + typec = devm_kzalloc(dev, sizeof(*typec), GFP_KERNEL); + if (!typec) + return -ENOMEM; + + typec->dev = dev; + typec->ec = dev_get_drvdata(pdev->dev.parent); + platform_set_drvdata(pdev, typec); + + ret = cros_typec_ec_command(typec, 0, EC_CMD_USB_PD_PORTS, NULL, 0, + &resp, sizeof(resp)); + if (ret < 0) + return ret; + + typec->num_ports = resp.num_ports; + if (typec->num_ports > EC_USB_PD_MAX_PORTS) { + dev_warn(typec->dev, + "Too many ports reported: %d, limiting to max: %d\n", + typec->num_ports, EC_USB_PD_MAX_PORTS); + typec->num_ports = EC_USB_PD_MAX_PORTS; + } + + ret = cros_typec_init_ports(typec); + if (ret < 0) + return ret; + + return 0; +} + +static struct platform_driver cros_typec_driver = { + .driver = { + .name = DRV_NAME, + .acpi_match_table = ACPI_PTR(cros_typec_acpi_id), + .of_match_table = of_match_ptr(cros_typec_of_match), + }, + .probe = cros_typec_probe, +}; + +module_platform_driver(cros_typec_driver); + +MODULE_AUTHOR("Prashant Malani "); +MODULE_DESCRIPTION("Chrome OS EC Type C control"); +MODULE_LICENSE("GPL"); From ad7c0510c99ef1568fcfd0ac7e2cefca3b30477a Mon Sep 17 00:00:00 2001 From: Prashant Malani Date: Mon, 16 Mar 2020 02:00:19 -0700 Subject: [PATCH 166/635] platform/chrome: cros_ec_typec: Update port info from EC After registering the ports at probe, get the current port information from EC and update the Type C connector class ports accordingly. Co-developed-by: Jon Flatley Signed-off-by: Prashant Malani Reviewed-by: Heikki Krogerus Signed-off-by: Enric Balletbo i Serra --- drivers/platform/chrome/cros_ec_typec.c | 121 +++++++++++++++++++++++- 1 file changed, 120 insertions(+), 1 deletion(-) diff --git a/drivers/platform/chrome/cros_ec_typec.c b/drivers/platform/chrome/cros_ec_typec.c index 02e6d5cbbbf7..874269c07073 100644 --- a/drivers/platform/chrome/cros_ec_typec.c +++ b/drivers/platform/chrome/cros_ec_typec.c @@ -21,6 +21,7 @@ struct cros_typec_data { struct device *dev; struct cros_ec_device *ec; int num_ports; + unsigned int cmd_ver; /* Array of ports, indexed by port number. */ struct typec_port *ports[EC_USB_PD_MAX_PORTS]; /* Initial capabilities for each port. */ @@ -171,6 +172,106 @@ static int cros_typec_ec_command(struct cros_typec_data *typec, return ret; } +static void cros_typec_set_port_params_v0(struct cros_typec_data *typec, + int port_num, struct ec_response_usb_pd_control *resp) +{ + struct typec_port *port = typec->ports[port_num]; + enum typec_orientation polarity; + + if (!resp->enabled) + polarity = TYPEC_ORIENTATION_NONE; + else if (!resp->polarity) + polarity = TYPEC_ORIENTATION_NORMAL; + else + polarity = TYPEC_ORIENTATION_REVERSE; + + typec_set_pwr_role(port, resp->role ? TYPEC_SOURCE : TYPEC_SINK); + typec_set_orientation(port, polarity); +} + +static void cros_typec_set_port_params_v1(struct cros_typec_data *typec, + int port_num, struct ec_response_usb_pd_control_v1 *resp) +{ + struct typec_port *port = typec->ports[port_num]; + enum typec_orientation polarity; + + if (!(resp->enabled & PD_CTRL_RESP_ENABLED_CONNECTED)) + polarity = TYPEC_ORIENTATION_NONE; + else if (!resp->polarity) + polarity = TYPEC_ORIENTATION_NORMAL; + else + polarity = TYPEC_ORIENTATION_REVERSE; + typec_set_orientation(port, polarity); + typec_set_data_role(port, resp->role & PD_CTRL_RESP_ROLE_DATA ? + TYPEC_HOST : TYPEC_DEVICE); + typec_set_pwr_role(port, resp->role & PD_CTRL_RESP_ROLE_POWER ? + TYPEC_SOURCE : TYPEC_SINK); + typec_set_vconn_role(port, resp->role & PD_CTRL_RESP_ROLE_VCONN ? + TYPEC_SOURCE : TYPEC_SINK); +} + +static int cros_typec_port_update(struct cros_typec_data *typec, int port_num) +{ + struct ec_params_usb_pd_control req; + struct ec_response_usb_pd_control_v1 resp; + int ret; + + if (port_num < 0 || port_num >= typec->num_ports) { + dev_err(typec->dev, "cannot get status for invalid port %d\n", + port_num); + return -EINVAL; + } + + req.port = port_num; + req.role = USB_PD_CTRL_ROLE_NO_CHANGE; + req.mux = USB_PD_CTRL_MUX_NO_CHANGE; + req.swap = USB_PD_CTRL_SWAP_NONE; + + ret = cros_typec_ec_command(typec, typec->cmd_ver, + EC_CMD_USB_PD_CONTROL, &req, sizeof(req), + &resp, sizeof(resp)); + if (ret < 0) + return ret; + + dev_dbg(typec->dev, "Enabled %d: 0x%hhx\n", port_num, resp.enabled); + dev_dbg(typec->dev, "Role %d: 0x%hhx\n", port_num, resp.role); + dev_dbg(typec->dev, "Polarity %d: 0x%hhx\n", port_num, resp.polarity); + dev_dbg(typec->dev, "State %d: %s\n", port_num, resp.state); + + if (typec->cmd_ver == 1) + cros_typec_set_port_params_v1(typec, port_num, &resp); + else + cros_typec_set_port_params_v0(typec, port_num, + (struct ec_response_usb_pd_control *) &resp); + + return 0; +} + +static int cros_typec_get_cmd_version(struct cros_typec_data *typec) +{ + struct ec_params_get_cmd_versions_v1 req_v1; + struct ec_response_get_cmd_versions resp; + int ret; + + /* We're interested in the PD control command version. */ + req_v1.cmd = EC_CMD_USB_PD_CONTROL; + ret = cros_typec_ec_command(typec, 1, EC_CMD_GET_CMD_VERSIONS, + &req_v1, sizeof(req_v1), &resp, + sizeof(resp)); + if (ret < 0) + return ret; + + if (resp.version_mask & EC_VER_MASK(1)) + typec->cmd_ver = 1; + else + typec->cmd_ver = 0; + + dev_dbg(typec->dev, "PD Control has version mask 0x%hhx\n", + typec->cmd_ver); + + return 0; +} + #ifdef CONFIG_ACPI static const struct acpi_device_id cros_typec_acpi_id[] = { { "GOOG0014", 0 }, @@ -192,7 +293,7 @@ static int cros_typec_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct cros_typec_data *typec; struct ec_response_usb_pd_ports resp; - int ret; + int ret, i; typec = devm_kzalloc(dev, sizeof(*typec), GFP_KERNEL); if (!typec) @@ -202,6 +303,12 @@ static int cros_typec_probe(struct platform_device *pdev) typec->ec = dev_get_drvdata(pdev->dev.parent); platform_set_drvdata(pdev, typec); + ret = cros_typec_get_cmd_version(typec); + if (ret < 0) { + dev_err(dev, "failed to get PD command version info\n"); + return ret; + } + ret = cros_typec_ec_command(typec, 0, EC_CMD_USB_PD_PORTS, NULL, 0, &resp, sizeof(resp)); if (ret < 0) @@ -219,7 +326,19 @@ static int cros_typec_probe(struct platform_device *pdev) if (ret < 0) return ret; + for (i = 0; i < typec->num_ports; i++) { + ret = cros_typec_port_update(typec, i); + if (ret < 0) + goto unregister_ports; + } + return 0; + +unregister_ports: + for (i = 0; i < typec->num_ports; i++) + if (typec->ports[i]) + typec_unregister_port(typec->ports[i]); + return ret; } static struct platform_driver cros_typec_driver = { From 45620da40f6fe627f6f98ba4f6edc2e43dd8d291 Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Fri, 20 Mar 2020 18:15:42 -0500 Subject: [PATCH 167/635] platform/chrome: cros_ec_chardev: Replace zero-length array with flexible-array member The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertently introduced[3] to the codebase from now on. Also, notice that, dynamic memory allocations won't be affected by this change: "Flexible array members have incomplete type, and so the sizeof operator may not be applied. As a quirk of the original implementation of zero-length arrays, sizeof evaluates to zero."[1] This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Signed-off-by: Enric Balletbo i Serra --- drivers/platform/chrome/cros_ec_chardev.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/platform/chrome/cros_ec_chardev.c b/drivers/platform/chrome/cros_ec_chardev.c index b51ab24055f3..e0bce869c49a 100644 --- a/drivers/platform/chrome/cros_ec_chardev.c +++ b/drivers/platform/chrome/cros_ec_chardev.c @@ -48,7 +48,7 @@ struct ec_event { struct list_head node; size_t size; u8 event_type; - u8 data[0]; + u8 data[]; }; static int ec_get_version(struct cros_ec_dev *ec, char *str, int maxlen) From e0db7c42b05f7b9d76174db56d683f0ec4f206c1 Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Fri, 20 Mar 2020 18:16:34 -0500 Subject: [PATCH 168/635] platform/chrome: wilco_ec: event: Replace zero-length array with flexible-array member The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertently introduced[3] to the codebase from now on. Also, notice that, dynamic memory allocations won't be affected by this change: "Flexible array members have incomplete type, and so the sizeof operator may not be applied. As a quirk of the original implementation of zero-length arrays, sizeof evaluates to zero."[1] This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Signed-off-by: Enric Balletbo i Serra --- drivers/platform/chrome/wilco_ec/event.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/platform/chrome/wilco_ec/event.c b/drivers/platform/chrome/wilco_ec/event.c index dba3d445623f..814518509739 100644 --- a/drivers/platform/chrome/wilco_ec/event.c +++ b/drivers/platform/chrome/wilco_ec/event.c @@ -79,7 +79,7 @@ static DEFINE_IDA(event_ida); struct ec_event { u16 size; u16 type; - u16 event[0]; + u16 event[]; } __packed; #define ec_event_num_words(ev) (ev->size - 1) @@ -96,7 +96,7 @@ struct ec_event_queue { int capacity; int head; int tail; - struct ec_event *entries[0]; + struct ec_event *entries[]; }; /* Maximum number of events to store in ec_event_queue */ From 8605cf0e852af3b2c771c18417499dc4ceed03d5 Mon Sep 17 00:00:00 2001 From: Misono Tomohiro Date: Wed, 28 Aug 2019 17:01:22 +0900 Subject: [PATCH 169/635] NFS: direct.c: Fix memory leak of dreq when nfs_get_lock_context fails When dreq is allocated by nfs_direct_req_alloc(), dreq->kref is initialized to 2. Therefore we need to call nfs_direct_req_release() twice to release the allocated dreq. Usually it is called in nfs_file_direct_{read, write}() and nfs_direct_complete(). However, current code only calls nfs_direct_req_relese() once if nfs_get_lock_context() fails in nfs_file_direct_{read, write}(). So, that case would result in memory leak. Fix this by adding the missing call. Signed-off-by: Misono Tomohiro Signed-off-by: Trond Myklebust --- fs/nfs/direct.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c index b768a0b42e82..ade2435551c8 100644 --- a/fs/nfs/direct.c +++ b/fs/nfs/direct.c @@ -571,6 +571,7 @@ ssize_t nfs_file_direct_read(struct kiocb *iocb, struct iov_iter *iter) l_ctx = nfs_get_lock_context(dreq->ctx); if (IS_ERR(l_ctx)) { result = PTR_ERR(l_ctx); + nfs_direct_req_release(dreq); goto out_release; } dreq->l_ctx = l_ctx; @@ -990,6 +991,7 @@ ssize_t nfs_file_direct_write(struct kiocb *iocb, struct iov_iter *iter) l_ctx = nfs_get_lock_context(dreq->ctx); if (IS_ERR(l_ctx)) { result = PTR_ERR(l_ctx); + nfs_direct_req_release(dreq); goto out_release; } dreq->l_ctx = l_ctx; From d5f5ee2a490547961c0b7ccdeef047c2018892bf Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Thu, 6 Feb 2020 02:54:32 -0500 Subject: [PATCH 170/635] tools/virtio: option to build an out of tree module Handy for testing with distro kernels. Warn that the resulting module is completely unsupported, and isn't intended for production use. Usage: make oot # builds vhost_test.ko, vhost.ko make oot-clean # cleans out files created Acked-by: Jason Wang Signed-off-by: Michael S. Tsirkin --- tools/virtio/Makefile | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/tools/virtio/Makefile b/tools/virtio/Makefile index 8e2a908115c2..f33f32f1d208 100644 --- a/tools/virtio/Makefile +++ b/tools/virtio/Makefile @@ -8,7 +8,32 @@ CFLAGS += -g -O2 -Werror -Wall -I. -I../include/ -I ../../usr/include/ -Wno-poin vpath %.c ../../drivers/virtio ../../drivers/vhost mod: ${MAKE} -C `pwd`/../.. M=`pwd`/vhost_test V=${V} -.PHONY: all test mod clean + +#oot: build vhost as an out of tree module for a distro kernel +#no effort is taken to make it actually build or work, but tends to mostly work +#if the distro kernel is very close to upstream +#unsupported! this is a development tool only, don't use the +#resulting modules in production! +OOT_KSRC=/lib/modules/$$(uname -r)/build +OOT_VHOST=`pwd`/../../drivers/vhost +#Everyone depends on vhost +#Tweak the below to enable more modules +OOT_CONFIGS=\ + CONFIG_VHOST=m \ + CONFIG_VHOST_NET=n \ + CONFIG_VHOST_SCSI=n \ + CONFIG_VHOST_VSOCK=n +OOT_BUILD=KCFLAGS="-I "${OOT_VHOST} ${MAKE} -C ${OOT_KSRC} V=${V} +oot-build: + echo "UNSUPPORTED! Don't use the resulting modules in production!" + ${OOT_BUILD} M=`pwd`/vhost_test + ${OOT_BUILD} M=${OOT_VHOST} ${OOT_CONFIGS} + +oot-clean: oot-build +oot: oot-build +oot-clean: OOT_BUILD+=clean + +.PHONY: all test mod clean vhost oot oot-clean oot-build clean: ${RM} *.o vringh_test virtio_test vhost_test/*.o vhost_test/.*.cmd \ vhost_test/Module.symvers vhost_test/modules.order *.d From 22b436c9b5682e877d34425d05576db74a8647e1 Mon Sep 17 00:00:00 2001 From: Yuri Benditovich Date: Mon, 2 Mar 2020 13:50:01 +0200 Subject: [PATCH 171/635] virtio-net: Introduce extended RSC feature VIRTIO_NET_F_RSC_EXT feature bit indicates that the device is able to provide extended RSC information. When the feature is negotiatede and 'gso_type' field in received packet is not GSO_NONE, the device reports number of coalesced packets in 'csum_start' field and number of duplicated acks in 'csum_offset' field and sets VIRTIO_NET_HDR_F_RSC_INFO in 'flags' field. Signed-off-by: Yuri Benditovich Link: https://lore.kernel.org/r/20200302115003.14877-2-yuri.benditovich@daynix.com Signed-off-by: Michael S. Tsirkin --- include/uapi/linux/virtio_net.h | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/include/uapi/linux/virtio_net.h b/include/uapi/linux/virtio_net.h index a3715a3224c1..6466c5979a93 100644 --- a/include/uapi/linux/virtio_net.h +++ b/include/uapi/linux/virtio_net.h @@ -57,6 +57,7 @@ * Steering */ #define VIRTIO_NET_F_CTRL_MAC_ADDR 23 /* Set MAC address */ +#define VIRTIO_NET_F_RSC_EXT 61 /* extended coalescing info */ #define VIRTIO_NET_F_STANDBY 62 /* Act as standby for another device * with the same MAC. */ @@ -104,6 +105,7 @@ struct virtio_net_config { struct virtio_net_hdr_v1 { #define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* Use csum_start, csum_offset */ #define VIRTIO_NET_HDR_F_DATA_VALID 2 /* Csum is valid */ +#define VIRTIO_NET_HDR_F_RSC_INFO 4 /* rsc info in csum_ fields */ __u8 flags; #define VIRTIO_NET_HDR_GSO_NONE 0 /* Not a GSO frame */ #define VIRTIO_NET_HDR_GSO_TCPV4 1 /* GSO frame, IPv4 TCP (TSO) */ @@ -113,8 +115,26 @@ struct virtio_net_hdr_v1 { __u8 gso_type; __virtio16 hdr_len; /* Ethernet + IP + tcp/udp hdrs */ __virtio16 gso_size; /* Bytes to append to hdr_len per frame */ - __virtio16 csum_start; /* Position to start checksumming from */ - __virtio16 csum_offset; /* Offset after that to place checksum */ + union { + struct { + __virtio16 csum_start; + __virtio16 csum_offset; + }; + /* Checksum calculation */ + struct { + /* Position to start checksumming from */ + __virtio16 start; + /* Offset after that to place checksum */ + __virtio16 offset; + } csum; + /* Receive Segment Coalescing */ + struct { + /* Number of coalesced segments */ + __le16 segments; + /* Number of duplicated acks */ + __le16 dup_acks; + } rsc; + }; __virtio16 num_buffers; /* Number of merged rx buffers */ }; From fd58bf674564f1731ca8a61a5150d40383f3df60 Mon Sep 17 00:00:00 2001 From: Yuri Benditovich Date: Mon, 2 Mar 2020 13:50:02 +0200 Subject: [PATCH 172/635] virtio-net: Introduce RSS receive steering feature RSS (Receive-side scaling) defines hash calculation rules and decision on receive virtqueue according to the calculated hash, provided mask to apply and provided indirection table containing indices of receive virqueues. The driver sends the control command to enable multiqueue and provide parameters for receive steering. Signed-off-by: Yuri Benditovich Link: https://lore.kernel.org/r/20200302115003.14877-3-yuri.benditovich@daynix.com Signed-off-by: Michael S. Tsirkin --- include/uapi/linux/virtio_net.h | 42 +++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/include/uapi/linux/virtio_net.h b/include/uapi/linux/virtio_net.h index 6466c5979a93..aec6fac3666a 100644 --- a/include/uapi/linux/virtio_net.h +++ b/include/uapi/linux/virtio_net.h @@ -57,6 +57,7 @@ * Steering */ #define VIRTIO_NET_F_CTRL_MAC_ADDR 23 /* Set MAC address */ +#define VIRTIO_NET_F_RSS 60 /* Supports RSS RX steering */ #define VIRTIO_NET_F_RSC_EXT 61 /* extended coalescing info */ #define VIRTIO_NET_F_STANDBY 62 /* Act as standby for another device * with the same MAC. @@ -70,6 +71,17 @@ #define VIRTIO_NET_S_LINK_UP 1 /* Link is up */ #define VIRTIO_NET_S_ANNOUNCE 2 /* Announcement is needed */ +/* supported/enabled hash types */ +#define VIRTIO_NET_RSS_HASH_TYPE_IPv4 (1 << 0) +#define VIRTIO_NET_RSS_HASH_TYPE_TCPv4 (1 << 1) +#define VIRTIO_NET_RSS_HASH_TYPE_UDPv4 (1 << 2) +#define VIRTIO_NET_RSS_HASH_TYPE_IPv6 (1 << 3) +#define VIRTIO_NET_RSS_HASH_TYPE_TCPv6 (1 << 4) +#define VIRTIO_NET_RSS_HASH_TYPE_UDPv6 (1 << 5) +#define VIRTIO_NET_RSS_HASH_TYPE_IP_EX (1 << 6) +#define VIRTIO_NET_RSS_HASH_TYPE_TCP_EX (1 << 7) +#define VIRTIO_NET_RSS_HASH_TYPE_UDP_EX (1 << 8) + struct virtio_net_config { /* The config defining mac address (if VIRTIO_NET_F_MAC) */ __u8 mac[ETH_ALEN]; @@ -93,6 +105,12 @@ struct virtio_net_config { * Any other value stands for unknown. */ __u8 duplex; + /* maximum size of RSS key */ + __u8 rss_max_key_size; + /* maximum number of indirection table entries */ + __le16 rss_max_indirection_table_length; + /* bitmask of supported VIRTIO_NET_RSS_HASH_ types */ + __le32 supported_hash_types; } __attribute__((packed)); /* @@ -248,7 +266,9 @@ struct virtio_net_ctrl_mac { /* * Control Receive Flow Steering - * + */ +#define VIRTIO_NET_CTRL_MQ 4 +/* * The command VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET * enables Receive Flow Steering, specifying the number of the transmit and * receive queues that will be used. After the command is consumed and acked by @@ -261,11 +281,29 @@ struct virtio_net_ctrl_mq { __virtio16 virtqueue_pairs; }; -#define VIRTIO_NET_CTRL_MQ 4 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000 +/* + * The command VIRTIO_NET_CTRL_MQ_RSS_CONFIG has the same effect as + * VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET does and additionally configures + * the receive steering to use a hash calculated for incoming packet + * to decide on receive virtqueue to place the packet. The command + * also provides parameters to calculate a hash and receive virtqueue. + */ +struct virtio_net_rss_config { + __le32 hash_types; + __le16 indirection_table_mask; + __le16 unclassified_queue; + __le16 indirection_table[1/* + indirection_table_mask */]; + __le16 max_tx_vq; + __u8 hash_key_length; + __u8 hash_key_data[/* hash_key_length */]; +}; + + #define VIRTIO_NET_CTRL_MQ_RSS_CONFIG 1 + /* * Control network offloads * From 3024e20958ee9e1554951df4d26aaf9f5cb7c210 Mon Sep 17 00:00:00 2001 From: Yuri Benditovich Date: Mon, 2 Mar 2020 13:50:03 +0200 Subject: [PATCH 173/635] virtio-net: Introduce hash report feature The feature VIRTIO_NET_F_HASH_REPORT extends the layout of the packet and requests the device to calculate hash on incoming packets and report it in the packet header. Signed-off-by: Yuri Benditovich Link: https://lore.kernel.org/r/20200302115003.14877-4-yuri.benditovich@daynix.com Signed-off-by: Michael S. Tsirkin --- include/uapi/linux/virtio_net.h | 36 +++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/include/uapi/linux/virtio_net.h b/include/uapi/linux/virtio_net.h index aec6fac3666a..19d23e5baa4e 100644 --- a/include/uapi/linux/virtio_net.h +++ b/include/uapi/linux/virtio_net.h @@ -57,6 +57,7 @@ * Steering */ #define VIRTIO_NET_F_CTRL_MAC_ADDR 23 /* Set MAC address */ +#define VIRTIO_NET_F_HASH_REPORT 57 /* Supports hash report */ #define VIRTIO_NET_F_RSS 60 /* Supports RSS RX steering */ #define VIRTIO_NET_F_RSC_EXT 61 /* extended coalescing info */ #define VIRTIO_NET_F_STANDBY 62 /* Act as standby for another device @@ -156,6 +157,23 @@ struct virtio_net_hdr_v1 { __virtio16 num_buffers; /* Number of merged rx buffers */ }; +struct virtio_net_hdr_v1_hash { + struct virtio_net_hdr_v1 hdr; + __le32 hash_value; +#define VIRTIO_NET_HASH_REPORT_NONE 0 +#define VIRTIO_NET_HASH_REPORT_IPv4 1 +#define VIRTIO_NET_HASH_REPORT_TCPv4 2 +#define VIRTIO_NET_HASH_REPORT_UDPv4 3 +#define VIRTIO_NET_HASH_REPORT_IPv6 4 +#define VIRTIO_NET_HASH_REPORT_TCPv6 5 +#define VIRTIO_NET_HASH_REPORT_UDPv6 6 +#define VIRTIO_NET_HASH_REPORT_IPv6_EX 7 +#define VIRTIO_NET_HASH_REPORT_TCPv6_EX 8 +#define VIRTIO_NET_HASH_REPORT_UDPv6_EX 9 + __le16 hash_report; + __le16 padding; +}; + #ifndef VIRTIO_NET_NO_LEGACY /* This header comes first in the scatter-gather list. * For legacy virtio, if VIRTIO_F_ANY_LAYOUT is not negotiated, it must @@ -304,6 +322,24 @@ struct virtio_net_rss_config { #define VIRTIO_NET_CTRL_MQ_RSS_CONFIG 1 +/* + * The command VIRTIO_NET_CTRL_MQ_HASH_CONFIG requests the device + * to include in the virtio header of the packet the value of the + * calculated hash and the report type of hash. It also provides + * parameters for hash calculation. The command requires feature + * VIRTIO_NET_F_HASH_REPORT to be negotiated to extend the + * layout of virtio header as defined in virtio_net_hdr_v1_hash. + */ +struct virtio_net_hash_config { + __le32 hash_types; + /* for compatibility with virtio_net_rss_config */ + __le16 reserved[4]; + __u8 hash_key_length; + __u8 hash_key_data[/* hash_key_length */]; +}; + + #define VIRTIO_NET_CTRL_MQ_HASH_CONFIG 2 + /* * Control network offloads * From 5a6b4cc5b7a1892a8d7f63d6cbac6e0ae2a9d031 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Wed, 5 Feb 2020 17:34:02 +0100 Subject: [PATCH 174/635] virtio-balloon: Switch back to OOM handler for VIRTIO_BALLOON_F_DEFLATE_ON_OOM Commit 71994620bb25 ("virtio_balloon: replace oom notifier with shrinker") changed the behavior when deflation happens automatically. Instead of deflating when called by the OOM handler, the shrinker is used. However, the balloon is not simply some slab cache that should be shrunk when under memory pressure. The shrinker does not have a concept of priorities, so this behavior cannot be configured. There was a report that this results in undesired side effects when inflating the balloon to shrink the page cache. [1] "When inflating the balloon against page cache (i.e. no free memory remains) vmscan.c will both shrink page cache, but also invoke the shrinkers -- including the balloon's shrinker. So the balloon driver allocates memory which requires reclaim, vmscan gets this memory by shrinking the balloon, and then the driver adds the memory back to the balloon. Basically a busy no-op." The name "deflate on OOM" makes it pretty clear when deflation should happen - after other approaches to reclaim memory failed, not while reclaiming. This allows to minimize the footprint of a guest - memory will only be taken out of the balloon when really needed. Especially, a drop_slab() will result in the whole balloon getting deflated - undesired. While handling it via the OOM handler might not be perfect, it keeps existing behavior. If we want a different behavior, then we need a new feature bit and document it properly (although, there should be a clear use case and the intended effects should be well described). Keep using the shrinker for VIRTIO_BALLOON_F_FREE_PAGE_HINT, because this has no such side effects. Always register the shrinker with VIRTIO_BALLOON_F_FREE_PAGE_HINT now. We are always allowed to reuse free pages that are still to be processed by the guest. The hypervisor takes care of identifying and resolving possible races between processing a hinting request and the guest reusing a page. In contrast to pre commit 71994620bb25 ("virtio_balloon: replace oom notifier with shrinker"), don't add a moodule parameter to configure the number of pages to deflate on OOM. Can be re-added if really needed. Also, pay attention that leak_balloon() returns the number of 4k pages - convert it properly in virtio_balloon_oom_notify(). Note1: using the OOM handler is frowned upon, but it really is what we need for this feature. Note2: without VIRTIO_BALLOON_F_MUST_TELL_HOST (iow, always with QEMU) we could actually skip sending deflation requests to our hypervisor, making the OOM path *very* simple. Besically freeing pages and updating the balloon. If the communication with the host ever becomes a problem on this call path. [1] https://www.spinics.net/lists/linux-virtualization/msg40863.html Test report by Tyler Sanderson: Test setup: VM with 16 CPU, 64GB RAM. Running Debian 10. We have a 42 GB file full of random bytes that we continually cat to /dev/null. This fills the page cache as the file is read. Meanwhile we trigger the balloon to inflate, with a target size of 53 GB. This setup causes the balloon inflation to pressure the page cache as the page cache is also trying to grow. Afterwards we shrink the balloon back to zero (so total deflate = total inflate). Without patch (kernel 4.19.0-5): Inflation never reaches the target until we stop the "cat file > /dev/null" process. Total inflation time was 542 seconds. The longest period that made no net forward progress was 315 seconds (see attached graph). Result of "grep balloon /proc/vmstat" after the test: balloon_inflate 154828377 balloon_deflate 154828377 With patch (kernel 5.6.0-rc4+): Total inflation duration was 63 seconds. No deflate-queue activity occurs when pressuring the page-cache. Result of "grep balloon /proc/vmstat" after the test: balloon_inflate 12968539 balloon_deflate 12968539 Conclusion: This patch fixes the issue. In the test it reduced inflate/deflate activity by 12x, and reduced inflation time by 8.6x. But more importantly, if we hadn't killed the "grep balloon /proc/vmstat" process then, without the patch, the inflation process would never reach the target. Attached [1] is a png of a graph showing the problematic behavior without this patch. It shows deflate-queue activity increasing linearly while balloon size stays constant over the course of more than 8 minutes of the test. [1] https://lore.kernel.org/linux-mm/CAJuQAmphPcfew1v_EOgAdSFiprzjiZjmOf3iJDmFX0gD6b9TYQ@mail.gmail.com/2-without_patch.png Full test report and discussion [2]: [2] https://lore.kernel.org/r/CAJuQAmphPcfew1v_EOgAdSFiprzjiZjmOf3iJDmFX0gD6b9TYQ@mail.gmail.com Tested-by: Tyler Sanderson Reported-by: Tyler Sanderson Cc: Michael S. Tsirkin Cc: Wei Wang Cc: Alexander Duyck Cc: David Rientjes Cc: Nadav Amit Cc: Michal Hocko Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20200205163402.42627-4-david@redhat.com Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_balloon.c | 107 +++++++++++++------------------- 1 file changed, 44 insertions(+), 63 deletions(-) diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c index 341458fd95ca..44375a22307b 100644 --- a/drivers/virtio/virtio_balloon.c +++ b/drivers/virtio/virtio_balloon.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -27,7 +28,9 @@ */ #define VIRTIO_BALLOON_PAGES_PER_PAGE (unsigned)(PAGE_SIZE >> VIRTIO_BALLOON_PFN_SHIFT) #define VIRTIO_BALLOON_ARRAY_PFNS_MAX 256 -#define VIRTBALLOON_OOM_NOTIFY_PRIORITY 80 +/* Maximum number of (4k) pages to deflate on OOM notifications. */ +#define VIRTIO_BALLOON_OOM_NR_PAGES 256 +#define VIRTIO_BALLOON_OOM_NOTIFY_PRIORITY 80 #define VIRTIO_BALLOON_FREE_PAGE_ALLOC_FLAG (__GFP_NORETRY | __GFP_NOWARN | \ __GFP_NOMEMALLOC) @@ -112,8 +115,11 @@ struct virtio_balloon { /* Memory statistics */ struct virtio_balloon_stat stats[VIRTIO_BALLOON_S_NR]; - /* To register a shrinker to shrink memory upon memory pressure */ + /* Shrinker to return free pages - VIRTIO_BALLOON_F_FREE_PAGE_HINT */ struct shrinker shrinker; + + /* OOM notifier to deflate on OOM - VIRTIO_BALLOON_F_DEFLATE_ON_OOM */ + struct notifier_block oom_nb; }; static struct virtio_device_id id_table[] = { @@ -788,50 +794,13 @@ static unsigned long shrink_free_pages(struct virtio_balloon *vb, return blocks_freed * VIRTIO_BALLOON_HINT_BLOCK_PAGES; } -static unsigned long leak_balloon_pages(struct virtio_balloon *vb, - unsigned long pages_to_free) -{ - return leak_balloon(vb, pages_to_free * VIRTIO_BALLOON_PAGES_PER_PAGE) / - VIRTIO_BALLOON_PAGES_PER_PAGE; -} - -static unsigned long shrink_balloon_pages(struct virtio_balloon *vb, - unsigned long pages_to_free) -{ - unsigned long pages_freed = 0; - - /* - * One invocation of leak_balloon can deflate at most - * VIRTIO_BALLOON_ARRAY_PFNS_MAX balloon pages, so we call it - * multiple times to deflate pages till reaching pages_to_free. - */ - while (vb->num_pages && pages_freed < pages_to_free) - pages_freed += leak_balloon_pages(vb, - pages_to_free - pages_freed); - - update_balloon_size(vb); - - return pages_freed; -} - static unsigned long virtio_balloon_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) { - unsigned long pages_to_free, pages_freed = 0; struct virtio_balloon *vb = container_of(shrinker, struct virtio_balloon, shrinker); - pages_to_free = sc->nr_to_scan; - - if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) - pages_freed = shrink_free_pages(vb, pages_to_free); - - if (pages_freed >= pages_to_free) - return pages_freed; - - pages_freed += shrink_balloon_pages(vb, pages_to_free - pages_freed); - - return pages_freed; + return shrink_free_pages(vb, sc->nr_to_scan); } static unsigned long virtio_balloon_shrinker_count(struct shrinker *shrinker, @@ -839,26 +808,22 @@ static unsigned long virtio_balloon_shrinker_count(struct shrinker *shrinker, { struct virtio_balloon *vb = container_of(shrinker, struct virtio_balloon, shrinker); - unsigned long count; - count = vb->num_pages / VIRTIO_BALLOON_PAGES_PER_PAGE; - count += vb->num_free_page_blocks * VIRTIO_BALLOON_HINT_BLOCK_PAGES; - - return count; + return vb->num_free_page_blocks * VIRTIO_BALLOON_HINT_BLOCK_PAGES; } -static void virtio_balloon_unregister_shrinker(struct virtio_balloon *vb) +static int virtio_balloon_oom_notify(struct notifier_block *nb, + unsigned long dummy, void *parm) { - unregister_shrinker(&vb->shrinker); -} + struct virtio_balloon *vb = container_of(nb, + struct virtio_balloon, oom_nb); + unsigned long *freed = parm; -static int virtio_balloon_register_shrinker(struct virtio_balloon *vb) -{ - vb->shrinker.scan_objects = virtio_balloon_shrinker_scan; - vb->shrinker.count_objects = virtio_balloon_shrinker_count; - vb->shrinker.seeks = DEFAULT_SEEKS; + *freed += leak_balloon(vb, VIRTIO_BALLOON_OOM_NR_PAGES) / + VIRTIO_BALLOON_PAGES_PER_PAGE; + update_balloon_size(vb); - return register_shrinker(&vb->shrinker); + return NOTIFY_OK; } static int virtballoon_probe(struct virtio_device *vdev) @@ -935,22 +900,35 @@ static int virtballoon_probe(struct virtio_device *vdev) virtio_cwrite(vb->vdev, struct virtio_balloon_config, poison_val, &poison_val); } - } - /* - * We continue to use VIRTIO_BALLOON_F_DEFLATE_ON_OOM to decide if a - * shrinker needs to be registered to relieve memory pressure. - */ - if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) { - err = virtio_balloon_register_shrinker(vb); + + /* + * We're allowed to reuse any free pages, even if they are + * still to be processed by the host. + */ + vb->shrinker.scan_objects = virtio_balloon_shrinker_scan; + vb->shrinker.count_objects = virtio_balloon_shrinker_count; + vb->shrinker.seeks = DEFAULT_SEEKS; + err = register_shrinker(&vb->shrinker); if (err) goto out_del_balloon_wq; } + if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) { + vb->oom_nb.notifier_call = virtio_balloon_oom_notify; + vb->oom_nb.priority = VIRTIO_BALLOON_OOM_NOTIFY_PRIORITY; + err = register_oom_notifier(&vb->oom_nb); + if (err < 0) + goto out_unregister_shrinker; + } + virtio_device_ready(vdev); if (towards_target(vb)) virtballoon_changed(vdev); return 0; +out_unregister_shrinker: + if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) + unregister_shrinker(&vb->shrinker); out_del_balloon_wq: if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) destroy_workqueue(vb->balloon_wq); @@ -989,8 +967,11 @@ static void virtballoon_remove(struct virtio_device *vdev) { struct virtio_balloon *vb = vdev->priv; - if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) - virtio_balloon_unregister_shrinker(vb); + if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) + unregister_oom_notifier(&vb->oom_nb); + if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) + unregister_shrinker(&vb->shrinker); + spin_lock_irq(&vb->stop_update_lock); vb->stop_update = true; spin_unlock_irq(&vb->stop_update_lock); From 671aa926a90aacf553857cd7eabfa04a3c95b091 Mon Sep 17 00:00:00 2001 From: Sumeet Pawnikar Date: Fri, 6 Mar 2020 15:49:12 +0530 Subject: [PATCH 175/635] thermal: int340x: processor_thermal: Add Tiger Lake support Added new PCI id for Tiger Lake processor thermal device along with MMIO RAPL support. Signed-off-by: Sumeet Pawnikar Signed-off-by: Zhang Rui Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/1583489952-29612-1-git-send-email-sumeet.r.pawnikar@intel.com --- .../processor_thermal_device.c | 5 ++++ tools/thermal/tmon/tmon.c | 26 +++++++------------ 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c index b1fd34516e28..297db1d2d960 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c @@ -45,6 +45,9 @@ /* JasperLake thermal reporting device */ #define PCI_DEVICE_ID_PROC_JSL_THERMAL 0x4503 +/* TigerLake thermal reporting device */ +#define PCI_DEVICE_ID_PROC_TGL_THERMAL 0x9A03 + #define DRV_NAME "proc_thermal" struct power_config { @@ -728,6 +731,8 @@ static const struct pci_device_id proc_thermal_pci_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_PROC_ICL_THERMAL), .driver_data = (kernel_ulong_t)&rapl_mmio_hsw, }, { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_PROC_JSL_THERMAL)}, + { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_PROC_TGL_THERMAL), + .driver_data = (kernel_ulong_t)&rapl_mmio_hsw, }, { 0, }, }; diff --git a/tools/thermal/tmon/tmon.c b/tools/thermal/tmon/tmon.c index 83ec6e482f12..7eb3216a27f4 100644 --- a/tools/thermal/tmon/tmon.c +++ b/tools/thermal/tmon/tmon.c @@ -46,7 +46,7 @@ static void start_daemon_mode(void); pthread_t event_tid; pthread_mutex_t input_lock; -void usage() +void usage(void) { printf("Usage: tmon [OPTION...]\n"); printf(" -c, --control cooling device in control\n"); @@ -62,7 +62,7 @@ void usage() exit(0); } -void version() +void version(void) { printf("TMON version %s\n", VERSION); exit(EXIT_SUCCESS); @@ -70,7 +70,6 @@ void version() static void tmon_cleanup(void) { - syslog(LOG_INFO, "TMON exit cleanup\n"); fflush(stdout); refresh(); @@ -96,7 +95,6 @@ static void tmon_cleanup(void) exit(1); } - static void tmon_sig_handler(int sig) { syslog(LOG_INFO, "TMON caught signal %d\n", sig); @@ -120,7 +118,6 @@ static void tmon_sig_handler(int sig) tmon_exit = true; } - static void start_syslog(void) { if (debug_on) @@ -167,7 +164,6 @@ static void prepare_logging(void) return; } - fprintf(tmon_log, "#----------- THERMAL SYSTEM CONFIG -------------\n"); for (i = 0; i < ptdata.nr_tz_sensor; i++) { char binding_str[33]; /* size of long + 1 */ @@ -175,7 +171,7 @@ static void prepare_logging(void) memset(binding_str, 0, sizeof(binding_str)); for (j = 0; j < 32; j++) - binding_str[j] = (ptdata.tzi[i].cdev_binding & 1< 0) + else if (pid > 0) /* kill parent */ exit(EXIT_SUCCESS); @@ -366,11 +360,9 @@ static void start_daemon_mode() if ((chdir("/")) < 0) exit(EXIT_FAILURE); - sleep(10); close(STDIN_FILENO); close(STDOUT_FILENO); close(STDERR_FILENO); - } From f21431f2de33b1bb99064e57d3ccaa772b92f333 Mon Sep 17 00:00:00 2001 From: Takashi Iwai Date: Wed, 11 Mar 2020 10:27:18 +0100 Subject: [PATCH 176/635] thermal: int340x_thermal: Use scnprintf() for avoiding potential buffer overflow Since snprintf() returns the would-be-output size instead of the actual output size, the succeeding calls may go beyond the given buffer limit. Fix it by replacing with scnprintf(). Signed-off-by: Takashi Iwai Reviewed-by: Pandruvada, Srinivas Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200311092718.24052-1-tiwai@suse.de --- drivers/thermal/intel/int340x_thermal/int3400_thermal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/thermal/intel/int340x_thermal/int3400_thermal.c b/drivers/thermal/intel/int340x_thermal/int3400_thermal.c index efae0c02d898..529df7174239 100644 --- a/drivers/thermal/intel/int340x_thermal/int3400_thermal.c +++ b/drivers/thermal/intel/int340x_thermal/int3400_thermal.c @@ -65,7 +65,7 @@ static ssize_t available_uuids_show(struct device *dev, for (i = 0; i < INT3400_THERMAL_MAXIMUM_UUID; i++) { if (priv->uuid_bitmap & (1 << i)) if (PAGE_SIZE - length > 0) - length += snprintf(&buf[length], + length += scnprintf(&buf[length], PAGE_SIZE - length, "%s\n", int3400_thermal_uuids[i]); From 968ea0dffa9396e9f8e00f12fd69be95f5a09191 Mon Sep 17 00:00:00 2001 From: Anson Huang Date: Thu, 19 Mar 2020 16:26:20 +0800 Subject: [PATCH 177/635] thermal: imx_sc_thermal: Fix incorrect data type The temperature value passed from SCU could be negative value, the data type should be signed instead of unsigned. Fixes: e20db70dba1c ("thermal: imx_sc: add i.MX system controller thermal support") Signed-off-by: Anson Huang Reviewed-by: Amit Kucheria Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/1584606380-9972-1-git-send-email-Anson.Huang@nxp.com --- drivers/thermal/imx_sc_thermal.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/thermal/imx_sc_thermal.c b/drivers/thermal/imx_sc_thermal.c index dbb277af232d..a8723b1eb8b0 100644 --- a/drivers/thermal/imx_sc_thermal.c +++ b/drivers/thermal/imx_sc_thermal.c @@ -30,8 +30,8 @@ struct req_get_temp { } __packed __aligned(4); struct resp_get_temp { - u16 celsius; - u8 tenths; + s16 celsius; + s8 tenths; } __packed __aligned(4); struct imx_sc_msg_misc_get_temp { From 0a8cdc8b14ef9f4474d38f0c364c82515798ade0 Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Thu, 19 Mar 2020 13:48:38 -0500 Subject: [PATCH 178/635] thermal: qcom: tsens.h: Replace zero-length array with flexible-array member The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertently introduced[3] to the codebase from now on. Also, notice that, dynamic memory allocations won't be affected by this change: "Flexible array members have incomplete type, and so the sizeof operator may not be applied. As a quirk of the original implementation of zero-length arrays, sizeof evaluates to zero."[1] This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Reviewed-by: Amit Kucheria Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200319184838.GA25767@embeddedor.com --- drivers/thermal/qcom/tsens.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/thermal/qcom/tsens.h b/drivers/thermal/qcom/tsens.h index f49e516b3458..502acf0e6828 100644 --- a/drivers/thermal/qcom/tsens.h +++ b/drivers/thermal/qcom/tsens.h @@ -572,7 +572,7 @@ struct tsens_priv { struct dentry *debug_root; struct dentry *debug; - struct tsens_sensor sensor[0]; + struct tsens_sensor sensor[]; }; char *qfprom_read(struct device *dev, const char *cname); From 526e6effe6f3350bd99740588a259d5f2def9fdc Mon Sep 17 00:00:00 2001 From: Anson Huang Date: Fri, 20 Mar 2020 11:26:29 +0800 Subject: [PATCH 179/635] dt-bindings: thermal: imx8mm-thermal: Add support for i.MX8MP Add thermal binding doc for Freescale's i.MX8MP Thermal Monitoring Unit. Signed-off-by: Anson Huang Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/1584674791-9717-1-git-send-email-Anson.Huang@nxp.com --- Documentation/devicetree/bindings/thermal/imx8mm-thermal.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/thermal/imx8mm-thermal.txt b/Documentation/devicetree/bindings/thermal/imx8mm-thermal.txt index d09ae82b49ee..3629d3c7e76a 100644 --- a/Documentation/devicetree/bindings/thermal/imx8mm-thermal.txt +++ b/Documentation/devicetree/bindings/thermal/imx8mm-thermal.txt @@ -1,10 +1,10 @@ * Thermal Monitoring Unit (TMU) on Freescale i.MX8MM SoC Required properties: -- compatible : Must be "fsl,imx8mm-tmu". +- compatible : Must be "fsl,imx8mm-tmu" or "fsl,imx8mp-tmu". - reg : Address range of TMU registers. - clocks : TMU's clock source. -- #thermal-sensor-cells : Should be 0. See ./thermal.txt for a description. +- #thermal-sensor-cells : Should be 0 or 1. See ./thermal.txt for a description. Example: tmu: tmu@30260000 { From 2b8f1f0337c55fb26347dda90112dfd8fe326ec5 Mon Sep 17 00:00:00 2001 From: Anson Huang Date: Fri, 20 Mar 2020 11:26:30 +0800 Subject: [PATCH 180/635] thermal: imx8mm: Add i.MX8MP support i.MX8MP shares same TMU with i.MX8MM, the only difference is i.MX8MP has two thermal sensors while i.MX8MM ONLY has one, add multiple sensors support for i.MX8MM TMU driver. Signed-off-by: Anson Huang Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/1584674791-9717-2-git-send-email-Anson.Huang@nxp.com --- drivers/thermal/imx8mm_thermal.c | 164 +++++++++++++++++++++++++------ 1 file changed, 135 insertions(+), 29 deletions(-) diff --git a/drivers/thermal/imx8mm_thermal.c b/drivers/thermal/imx8mm_thermal.c index d597cebb0545..c32308ba8d15 100644 --- a/drivers/thermal/imx8mm_thermal.c +++ b/drivers/thermal/imx8mm_thermal.c @@ -5,59 +5,143 @@ * Author: Anson Huang */ +#include #include #include #include #include #include -#include +#include #include #include #include "thermal_core.h" #define TER 0x0 /* TMU enable */ +#define TPS 0x4 #define TRITSR 0x20 /* TMU immediate temp */ #define TER_EN BIT(31) -#define TRITSR_VAL_MASK 0xff +#define TRITSR_TEMP0_VAL_MASK 0xff +#define TRITSR_TEMP1_VAL_MASK 0xff0000 -#define TEMP_LOW_LIMIT 10 +#define PROBE_SEL_ALL GENMASK(31, 30) + +#define probe_status_offset(x) (30 + x) +#define SIGN_BIT BIT(7) +#define TEMP_VAL_MASK GENMASK(6, 0) + +#define VER1_TEMP_LOW_LIMIT 10000 +#define VER2_TEMP_LOW_LIMIT -40000 +#define VER2_TEMP_HIGH_LIMIT 125000 + +#define TMU_VER1 0x1 +#define TMU_VER2 0x2 + +struct thermal_soc_data { + u32 num_sensors; + u32 version; + int (*get_temp)(void *, int *); +}; + +struct tmu_sensor { + struct imx8mm_tmu *priv; + u32 hw_id; + struct thermal_zone_device *tzd; +}; struct imx8mm_tmu { - struct thermal_zone_device *tzd; void __iomem *base; struct clk *clk; + const struct thermal_soc_data *socdata; + struct tmu_sensor sensors[0]; }; +static int imx8mm_tmu_get_temp(void *data, int *temp) +{ + struct tmu_sensor *sensor = data; + struct imx8mm_tmu *tmu = sensor->priv; + u32 val; + + val = readl_relaxed(tmu->base + TRITSR) & TRITSR_TEMP0_VAL_MASK; + *temp = val * 1000; + if (*temp < VER1_TEMP_LOW_LIMIT) + return -EAGAIN; + + return 0; +} + +static int imx8mp_tmu_get_temp(void *data, int *temp) +{ + struct tmu_sensor *sensor = data; + struct imx8mm_tmu *tmu = sensor->priv; + bool ready; + u32 val; + + ready = test_bit(probe_status_offset(sensor->hw_id), + tmu->base + TRITSR); + if (!ready) + return -EAGAIN; + + val = readl_relaxed(tmu->base + TRITSR); + val = sensor->hw_id ? FIELD_GET(TRITSR_TEMP1_VAL_MASK, val) : + FIELD_GET(TRITSR_TEMP0_VAL_MASK, val); + if (val & SIGN_BIT) /* negative */ + val = (~(val & TEMP_VAL_MASK) + 1); + + *temp = val * 1000; + if (*temp < VER2_TEMP_LOW_LIMIT || *temp > VER2_TEMP_HIGH_LIMIT) + return -EAGAIN; + + return 0; +} + static int tmu_get_temp(void *data, int *temp) { - struct imx8mm_tmu *tmu = data; - u32 val; + struct tmu_sensor *sensor = data; + struct imx8mm_tmu *tmu = sensor->priv; - val = readl_relaxed(tmu->base + TRITSR) & TRITSR_VAL_MASK; - if (val < TEMP_LOW_LIMIT) - return -EAGAIN; - - *temp = val * 1000; - - return 0; + return tmu->socdata->get_temp(data, temp); } static struct thermal_zone_of_device_ops tmu_tz_ops = { .get_temp = tmu_get_temp, }; +static void imx8mm_tmu_enable(struct imx8mm_tmu *tmu, bool enable) +{ + u32 val; + + val = readl_relaxed(tmu->base + TER); + val = enable ? (val | TER_EN) : (val & ~TER_EN); + writel_relaxed(val, tmu->base + TER); +} + +static void imx8mm_tmu_probe_sel_all(struct imx8mm_tmu *tmu) +{ + u32 val; + + val = readl_relaxed(tmu->base + TPS); + val |= PROBE_SEL_ALL; + writel_relaxed(val, tmu->base + TPS); +} + static int imx8mm_tmu_probe(struct platform_device *pdev) { + const struct thermal_soc_data *data; struct imx8mm_tmu *tmu; - u32 val; int ret; + int i; - tmu = devm_kzalloc(&pdev->dev, sizeof(struct imx8mm_tmu), GFP_KERNEL); + data = of_device_get_match_data(&pdev->dev); + + tmu = devm_kzalloc(&pdev->dev, struct_size(tmu, sensors, + data->num_sensors), GFP_KERNEL); if (!tmu) return -ENOMEM; + tmu->socdata = data; + tmu->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(tmu->base)) return PTR_ERR(tmu->base); @@ -77,20 +161,32 @@ static int imx8mm_tmu_probe(struct platform_device *pdev) return ret; } - tmu->tzd = devm_thermal_zone_of_sensor_register(&pdev->dev, 0, - tmu, &tmu_tz_ops); - if (IS_ERR(tmu->tzd)) { - dev_err(&pdev->dev, - "failed to register thermal zone sensor: %d\n", ret); - return PTR_ERR(tmu->tzd); + /* disable the monitor during initialization */ + imx8mm_tmu_enable(tmu, false); + + for (i = 0; i < data->num_sensors; i++) { + tmu->sensors[i].priv = tmu; + tmu->sensors[i].tzd = + devm_thermal_zone_of_sensor_register(&pdev->dev, i, + &tmu->sensors[i], + &tmu_tz_ops); + if (IS_ERR(tmu->sensors[i].tzd)) { + dev_err(&pdev->dev, + "failed to register thermal zone sensor[%d]: %d\n", + i, ret); + return PTR_ERR(tmu->sensors[i].tzd); + } + tmu->sensors[i].hw_id = i; } platform_set_drvdata(pdev, tmu); + /* enable all the probes for V2 TMU */ + if (tmu->socdata->version == TMU_VER2) + imx8mm_tmu_probe_sel_all(tmu); + /* enable the monitor */ - val = readl_relaxed(tmu->base + TER); - val |= TER_EN; - writel_relaxed(val, tmu->base + TER); + imx8mm_tmu_enable(tmu, true); return 0; } @@ -98,12 +194,9 @@ static int imx8mm_tmu_probe(struct platform_device *pdev) static int imx8mm_tmu_remove(struct platform_device *pdev) { struct imx8mm_tmu *tmu = platform_get_drvdata(pdev); - u32 val; /* disable TMU */ - val = readl_relaxed(tmu->base + TER); - val &= ~TER_EN; - writel_relaxed(val, tmu->base + TER); + imx8mm_tmu_enable(tmu, false); clk_disable_unprepare(tmu->clk); platform_set_drvdata(pdev, NULL); @@ -111,8 +204,21 @@ static int imx8mm_tmu_remove(struct platform_device *pdev) return 0; } +static struct thermal_soc_data imx8mm_tmu_data = { + .num_sensors = 1, + .version = TMU_VER1, + .get_temp = imx8mm_tmu_get_temp, +}; + +static struct thermal_soc_data imx8mp_tmu_data = { + .num_sensors = 2, + .version = TMU_VER2, + .get_temp = imx8mp_tmu_get_temp, +}; + static const struct of_device_id imx8mm_tmu_table[] = { - { .compatible = "fsl,imx8mm-tmu", }, + { .compatible = "fsl,imx8mm-tmu", .data = &imx8mm_tmu_data, }, + { .compatible = "fsl,imx8mp-tmu", .data = &imx8mp_tmu_data, }, { }, }; From ff44f672d74178b3be19d41a169b98b3e391d4ce Mon Sep 17 00:00:00 2001 From: Willy Wolff Date: Sat, 21 Mar 2020 09:27:40 +0000 Subject: [PATCH 181/635] thermal/drivers/cpufreq_cooling: Fix return of cpufreq_set_cur_state When setting the cooling device current state from userspace via sysfs, the operation fails by returning an -EINVAL. It appears the recent changes with the per-policy frequency QoS introduced a regression as reported by: https://lkml.org/lkml/2020/3/20/599 The function freq_qos_update_request returns 0 or 1 describing update effectiveness, and a negative error code on failure. However, cpufreq_set_cur_state returns 0 on success or an error code otherwise. Consider the QoS update as successful if the function does not return an error. Fixes: 3000ce3c52f8b ("cpufreq: Use per-policy frequency QoS") Signed-off-by: Willy Wolff Acked-by: Viresh Kumar Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200321092740.7vvwfxsebcrznydh@macmini.local --- drivers/thermal/cpufreq_cooling.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/thermal/cpufreq_cooling.c b/drivers/thermal/cpufreq_cooling.c index fe83d7a210d4..af55ac08e1bd 100644 --- a/drivers/thermal/cpufreq_cooling.c +++ b/drivers/thermal/cpufreq_cooling.c @@ -431,6 +431,7 @@ static int cpufreq_set_cur_state(struct thermal_cooling_device *cdev, unsigned long state) { struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata; + int ret; /* Request state should be less than max_level */ if (WARN_ON(state > cpufreq_cdev->max_level)) @@ -442,8 +443,9 @@ static int cpufreq_set_cur_state(struct thermal_cooling_device *cdev, cpufreq_cdev->cpufreq_state = state; - return freq_qos_update_request(&cpufreq_cdev->qos_req, - get_state_freq(cpufreq_cdev, state)); + ret = freq_qos_update_request(&cpufreq_cdev->qos_req, + get_state_freq(cpufreq_cdev, state)); + return ret < 0 ? ret : 0; } /* Bind cpufreq callbacks to thermal cooling device ops */ From 40ea568593f20647e7d58deb7fcfa69b11a37d8b Mon Sep 17 00:00:00 2001 From: Daniel Lezcano Date: Sat, 21 Mar 2020 20:31:07 +0100 Subject: [PATCH 182/635] thermal/drivers/cpufreq_cooling: Remove abusing WARN_ON The WARN_ON macros are used at the entry functions state2power() and set_cur_state(). state2power() is called with the max_state retrieved from get_max_state which returns cpufreq_cdev->max_level, then it check if max_state is > cpufreq_cdev->max_level. The test does not really makes sense but let's assume we want to make sure to catch an error if the code evolves. However the WARN_ON is overkill. set_cur_state() is also called from userspace if we write to the sysfs. It is easy to see a stack dumped by just writing to sysfs /sys/class/thermal/cooling_device0/cur_state a value greater than "max_level". A bit scary. Returing -EINVAL is enough. Remove these WARN_ON. Signed-off-by: Daniel Lezcano Acked-by: Viresh Kumar Link: https://lore.kernel.org/r/20200321193107.21590-1-daniel.lezcano@linaro.org --- drivers/thermal/cpufreq_cooling.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/thermal/cpufreq_cooling.c b/drivers/thermal/cpufreq_cooling.c index af55ac08e1bd..d66791a71320 100644 --- a/drivers/thermal/cpufreq_cooling.c +++ b/drivers/thermal/cpufreq_cooling.c @@ -273,7 +273,7 @@ static int cpufreq_state2power(struct thermal_cooling_device *cdev, struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata; /* Request state should be less than max_level */ - if (WARN_ON(state > cpufreq_cdev->max_level)) + if (state > cpufreq_cdev->max_level) return -EINVAL; num_cpus = cpumask_weight(cpufreq_cdev->policy->cpus); @@ -434,7 +434,7 @@ static int cpufreq_set_cur_state(struct thermal_cooling_device *cdev, int ret; /* Request state should be less than max_level */ - if (WARN_ON(state > cpufreq_cdev->max_level)) + if (state > cpufreq_cdev->max_level) return -EINVAL; /* Check if the old cooling action is same as new cooling action */ From 76a5c400aae52f541737de982e2b5a2e1f423bab Mon Sep 17 00:00:00 2001 From: Anson Huang Date: Mon, 23 Mar 2020 22:19:16 +0800 Subject: [PATCH 183/635] thermal: imx8mm: Fix build warning of incorrect argument type Fix below sparse warning: drivers/thermal/imx8mm_thermal.c:82:36: sparse: sparse: incorrect type in argument 2 (different address spaces), expected unsigned long const volatile *addr drivers/thermal/imx8mm_thermal.c:82:36: sparse: expected unsigned long const volatile *addr Signed-off-by: Anson Huang Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/1584973156-25734-1-git-send-email-Anson.Huang@nxp.com --- drivers/thermal/imx8mm_thermal.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/thermal/imx8mm_thermal.c b/drivers/thermal/imx8mm_thermal.c index c32308ba8d15..0d60f8d7894f 100644 --- a/drivers/thermal/imx8mm_thermal.c +++ b/drivers/thermal/imx8mm_thermal.c @@ -75,15 +75,14 @@ static int imx8mp_tmu_get_temp(void *data, int *temp) { struct tmu_sensor *sensor = data; struct imx8mm_tmu *tmu = sensor->priv; + unsigned long val; bool ready; - u32 val; - ready = test_bit(probe_status_offset(sensor->hw_id), - tmu->base + TRITSR); + val = readl_relaxed(tmu->base + TRITSR); + ready = test_bit(probe_status_offset(sensor->hw_id), &val); if (!ready) return -EAGAIN; - val = readl_relaxed(tmu->base + TRITSR); val = sensor->hw_id ? FIELD_GET(TRITSR_TEMP1_VAL_MASK, val) : FIELD_GET(TRITSR_TEMP0_VAL_MASK, val); if (val & SIGN_BIT) /* negative */ From 49db3997d82fe51655d75944fa68a08143782a5d Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Tue, 24 Mar 2020 11:12:10 -0700 Subject: [PATCH 184/635] Input: goodix - refactor IRQ pin GPIO accesses Suspending Goodix touchscreens requires changing the interrupt pin to output before sending them a power-down command. Followed by wiggling the interrupt pin to wake the device up, after which it is put back in input mode. So far we have only effectively supported this on devices which use devicetree. On X86 ACPI platforms both looking up the pins; and using a pin as both IRQ and GPIO is a bit more complicated. E.g. on some devices we cannot directly access the IRQ pin as GPIO and we need to call ACPI methods to control it instead. This commit adds a new irq_pin_access_method field to the goodix_chip_data struct and adds goodix_irq_direction_output and goodix_irq_direction_input helpers which together abstract the GPIO accesses to the IRQ pin. This is a preparation patch for adding support for properly suspending the touchscreen on X86 ACPI platforms. BugLink: https://bugzilla.redhat.com/show_bug.cgi?id=1786317 BugLink: https://github.com/nexus511/gpd-ubuntu-packages/issues/10 BugLink: https://bugzilla.kernel.org/show_bug.cgi?id=199207 Signed-off-by: Hans de Goede Reviewed-by: Bastien Nocera Link: https://lore.kernel.org/r/20200307121505.3707-1-hdegoede@redhat.com Signed-off-by: Dmitry Torokhov --- drivers/input/touchscreen/goodix.c | 62 ++++++++++++++++++++++++------ 1 file changed, 51 insertions(+), 11 deletions(-) diff --git a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c index 37b35ab97beb..f5e2e0956d74 100644 --- a/drivers/input/touchscreen/goodix.c +++ b/drivers/input/touchscreen/goodix.c @@ -31,6 +31,11 @@ struct goodix_ts_data; +enum goodix_irq_pin_access_method { + IRQ_PIN_ACCESS_NONE, + IRQ_PIN_ACCESS_GPIO, +}; + struct goodix_chip_data { u16 config_addr; int config_len; @@ -53,6 +58,7 @@ struct goodix_ts_data { const char *cfg_name; struct completion firmware_loading_complete; unsigned long irq_flags; + enum goodix_irq_pin_access_method irq_pin_access_method; unsigned int contact_size; }; @@ -518,17 +524,48 @@ static int goodix_send_cfg(struct goodix_ts_data *ts, return 0; } +static int goodix_irq_direction_output(struct goodix_ts_data *ts, + int value) +{ + switch (ts->irq_pin_access_method) { + case IRQ_PIN_ACCESS_NONE: + dev_err(&ts->client->dev, + "%s called without an irq_pin_access_method set\n", + __func__); + return -EINVAL; + case IRQ_PIN_ACCESS_GPIO: + return gpiod_direction_output(ts->gpiod_int, value); + } + + return -EINVAL; /* Never reached */ +} + +static int goodix_irq_direction_input(struct goodix_ts_data *ts) +{ + switch (ts->irq_pin_access_method) { + case IRQ_PIN_ACCESS_NONE: + dev_err(&ts->client->dev, + "%s called without an irq_pin_access_method set\n", + __func__); + return -EINVAL; + case IRQ_PIN_ACCESS_GPIO: + return gpiod_direction_input(ts->gpiod_int); + } + + return -EINVAL; /* Never reached */ +} + static int goodix_int_sync(struct goodix_ts_data *ts) { int error; - error = gpiod_direction_output(ts->gpiod_int, 0); + error = goodix_irq_direction_output(ts, 0); if (error) return error; msleep(50); /* T5: 50ms */ - error = gpiod_direction_input(ts->gpiod_int); + error = goodix_irq_direction_input(ts); if (error) return error; @@ -552,7 +589,7 @@ static int goodix_reset(struct goodix_ts_data *ts) msleep(20); /* T2: > 10ms */ /* HIGH: 0x28/0x29, LOW: 0xBA/0xBB */ - error = gpiod_direction_output(ts->gpiod_int, ts->client->addr == 0x14); + error = goodix_irq_direction_output(ts, ts->client->addr == 0x14); if (error) return error; @@ -633,6 +670,9 @@ static int goodix_get_gpio_config(struct goodix_ts_data *ts) ts->gpiod_rst = gpiod; + if (ts->gpiod_int && ts->gpiod_rst) + ts->irq_pin_access_method = IRQ_PIN_ACCESS_GPIO; + return 0; } @@ -911,7 +951,7 @@ static int goodix_ts_probe(struct i2c_client *client, if (error) return error; - if (ts->gpiod_int && ts->gpiod_rst) { + if (ts->irq_pin_access_method == IRQ_PIN_ACCESS_GPIO) { /* reset the controller */ error = goodix_reset(ts); if (error) { @@ -934,7 +974,7 @@ static int goodix_ts_probe(struct i2c_client *client, ts->chip = goodix_get_chip_data(ts->id); - if (ts->gpiod_int && ts->gpiod_rst) { + if (ts->irq_pin_access_method == IRQ_PIN_ACCESS_GPIO) { /* update device config */ ts->cfg_name = devm_kasprintf(&client->dev, GFP_KERNEL, "goodix_%d_cfg.bin", ts->id); @@ -965,7 +1005,7 @@ static int goodix_ts_remove(struct i2c_client *client) { struct goodix_ts_data *ts = i2c_get_clientdata(client); - if (ts->gpiod_int && ts->gpiod_rst) + if (ts->irq_pin_access_method == IRQ_PIN_ACCESS_GPIO) wait_for_completion(&ts->firmware_loading_complete); return 0; @@ -978,7 +1018,7 @@ static int __maybe_unused goodix_suspend(struct device *dev) int error; /* We need gpio pins to suspend/resume */ - if (!ts->gpiod_int || !ts->gpiod_rst) { + if (ts->irq_pin_access_method == IRQ_PIN_ACCESS_NONE) { disable_irq(client->irq); return 0; } @@ -989,7 +1029,7 @@ static int __maybe_unused goodix_suspend(struct device *dev) goodix_free_irq(ts); /* Output LOW on the INT pin for 5 ms */ - error = gpiod_direction_output(ts->gpiod_int, 0); + error = goodix_irq_direction_output(ts, 0); if (error) { goodix_request_irq(ts); return error; @@ -1001,7 +1041,7 @@ static int __maybe_unused goodix_suspend(struct device *dev) GOODIX_CMD_SCREEN_OFF); if (error) { dev_err(&ts->client->dev, "Screen off command failed\n"); - gpiod_direction_input(ts->gpiod_int); + goodix_irq_direction_input(ts); goodix_request_irq(ts); return -EAGAIN; } @@ -1021,7 +1061,7 @@ static int __maybe_unused goodix_resume(struct device *dev) struct goodix_ts_data *ts = i2c_get_clientdata(client); int error; - if (!ts->gpiod_int || !ts->gpiod_rst) { + if (ts->irq_pin_access_method == IRQ_PIN_ACCESS_NONE) { enable_irq(client->irq); return 0; } @@ -1030,7 +1070,7 @@ static int __maybe_unused goodix_resume(struct device *dev) * Exit sleep mode by outputting HIGH level to INT pin * for 2ms~5ms. */ - error = gpiod_direction_output(ts->gpiod_int, 1); + error = goodix_irq_direction_output(ts, 1); if (error) return error; From e070a97b929b58b60d54ae924fe0469a1c7dab73 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Tue, 24 Mar 2020 11:12:35 -0700 Subject: [PATCH 185/635] Input: goodix - make loading the config from disk independent from the GPIO setup At least on X86 ACPI platforms it is not necessary to load the touchscreen controller config from disk, if it needs to be loaded this has already been done by the BIOS / UEFI firmware. Even on other (e.g. devicetree) platforms the config-loading as currently done has the issue that the loaded cfg file is based on the controller model, but the actual cfg is device specific, so the cfg files are not part of linux-firmware and this can only work with a device specific OS image which includes the cfg file. And we do not need access to the GPIOs at all to load the config, if we do not have access we can still load the config. So all in all tying the decision to try to load the config from disk to being able to access the GPIOs is not desirable. This commit adds a new load_cfg_from_disk boolean to control the firmware loading instead. This commits sets the new bool to true when we set irq_pin_access_method to IRQ_PIN_ACCESS_GPIO, so there are no functional changes. BugLink: https://bugzilla.redhat.com/show_bug.cgi?id=1786317 BugLink: https://github.com/nexus511/gpd-ubuntu-packages/issues/10 BugLink: https://bugzilla.kernel.org/show_bug.cgi?id=199207 Reviewed-by: Bastien Nocera Signed-off-by: Hans de Goede Link: https://lore.kernel.org/r/20200307121505.3707-2-hdegoede@redhat.com Signed-off-by: Dmitry Torokhov --- drivers/input/touchscreen/goodix.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c index f5e2e0956d74..cb294615b060 100644 --- a/drivers/input/touchscreen/goodix.c +++ b/drivers/input/touchscreen/goodix.c @@ -56,6 +56,7 @@ struct goodix_ts_data { u16 id; u16 version; const char *cfg_name; + bool load_cfg_from_disk; struct completion firmware_loading_complete; unsigned long irq_flags; enum goodix_irq_pin_access_method irq_pin_access_method; @@ -670,8 +671,10 @@ static int goodix_get_gpio_config(struct goodix_ts_data *ts) ts->gpiod_rst = gpiod; - if (ts->gpiod_int && ts->gpiod_rst) + if (ts->gpiod_int && ts->gpiod_rst) { + ts->load_cfg_from_disk = true; ts->irq_pin_access_method = IRQ_PIN_ACCESS_GPIO; + } return 0; } @@ -974,7 +977,7 @@ static int goodix_ts_probe(struct i2c_client *client, ts->chip = goodix_get_chip_data(ts->id); - if (ts->irq_pin_access_method == IRQ_PIN_ACCESS_GPIO) { + if (ts->load_cfg_from_disk) { /* update device config */ ts->cfg_name = devm_kasprintf(&client->dev, GFP_KERNEL, "goodix_%d_cfg.bin", ts->id); @@ -1005,7 +1008,7 @@ static int goodix_ts_remove(struct i2c_client *client) { struct goodix_ts_data *ts = i2c_get_clientdata(client); - if (ts->irq_pin_access_method == IRQ_PIN_ACCESS_GPIO) + if (ts->load_cfg_from_disk) wait_for_completion(&ts->firmware_loading_complete); return 0; @@ -1017,14 +1020,15 @@ static int __maybe_unused goodix_suspend(struct device *dev) struct goodix_ts_data *ts = i2c_get_clientdata(client); int error; + if (ts->load_cfg_from_disk) + wait_for_completion(&ts->firmware_loading_complete); + /* We need gpio pins to suspend/resume */ if (ts->irq_pin_access_method == IRQ_PIN_ACCESS_NONE) { disable_irq(client->irq); return 0; } - wait_for_completion(&ts->firmware_loading_complete); - /* Free IRQ as IRQ pin is used as output in the suspend sequence */ goodix_free_irq(ts); From 1921dacef72d5c1fd6a63bfffba3997f34935e7f Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Tue, 24 Mar 2020 11:23:04 -0700 Subject: [PATCH 186/635] Input: goodix - make resetting the controller at probe independent from the GPIO setup Before this commit we would always reset the controller at probe when we have access to the GPIOs which are necessary to do a reset. Doing the reset requires access to the GPIOs, but just because we have access to the GPIOs does not mean that we should always reset the controller at probe. On X86 ACPI platforms the BIOS / UEFI firmware will already have reset the controller and it will have loaded the device specific config into the controller. Doing the reset sometimes causes the controller to lose its configuration, so on X86 ACPI platforms this is not a good idea. This commit adds a new reset_controller_at_probe boolean to control the reset at probe behavior. This commits sets the new bool to true when we set irq_pin_access_method to IRQ_PIN_ACCESS_GPIO, so there are no functional changes. BugLink: https://bugzilla.redhat.com/show_bug.cgi?id=1786317 BugLink: https://github.com/nexus511/gpd-ubuntu-packages/issues/10 BugLink: https://bugzilla.kernel.org/show_bug.cgi?id=199207 Reviewed-by: Bastien Nocera Signed-off-by: Hans de Goede Link: https://lore.kernel.org/r/20200307121505.3707-3-hdegoede@redhat.com Signed-off-by: Dmitry Torokhov --- drivers/input/touchscreen/goodix.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c index cb294615b060..70212ea7c631 100644 --- a/drivers/input/touchscreen/goodix.c +++ b/drivers/input/touchscreen/goodix.c @@ -56,6 +56,7 @@ struct goodix_ts_data { u16 id; u16 version; const char *cfg_name; + bool reset_controller_at_probe; bool load_cfg_from_disk; struct completion firmware_loading_complete; unsigned long irq_flags; @@ -672,6 +673,7 @@ static int goodix_get_gpio_config(struct goodix_ts_data *ts) ts->gpiod_rst = gpiod; if (ts->gpiod_int && ts->gpiod_rst) { + ts->reset_controller_at_probe = true; ts->load_cfg_from_disk = true; ts->irq_pin_access_method = IRQ_PIN_ACCESS_GPIO; } @@ -954,7 +956,7 @@ static int goodix_ts_probe(struct i2c_client *client, if (error) return error; - if (ts->irq_pin_access_method == IRQ_PIN_ACCESS_GPIO) { + if (ts->reset_controller_at_probe) { /* reset the controller */ error = goodix_reset(ts); if (error) { From a7d4b171660cadd0b410be132759267731da11bb Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Tue, 24 Mar 2020 11:23:21 -0700 Subject: [PATCH 187/635] Input: goodix - add support for getting IRQ + reset GPIOs on Cherry Trail devices On most Cherry Trail (x86, UEFI + ACPI) devices the ACPI tables do not have a _DSD with a "daffd814-6eba-4d8c-8a91-bc9bbf4aa301" UUID, adding "irq-gpios" and "reset-gpios" mappings, so we cannot get the GPIOS by name without first manually adding mappings ourselves. These devices contain 1 GpioInt and 1 GpioIo resource in their _CRS table: Method (_CRS, 0, NotSerialized) // _CRS: Current Resource Settings { Name (RBUF, ResourceTemplate () { I2cSerialBusV2 (0x0014, ControllerInitiated, 0x00061A80, AddressingMode7Bit, "\\_SB.PCI0.I2C2", 0x00, ResourceConsumer, , Exclusive, ) GpioInt (Edge, ActiveLow, Shared, PullDefault, 0x0000, "\\_SB.GPO1", 0x00, ResourceConsumer, , ) { // Pin list 0x0013 } GpioIo (Shared, PullDefault, 0x0000, 0x0000, IoRestrictionOutputOnly, "\\_SB.GPO1", 0x00, ResourceConsumer, , ) { // Pin list 0x0019 } }) Return (RBUF) /* \_SB_.PCI0.I2C2.TCS1._CRS.RBUF */ } There is no fixed order for these 2. This commit adds code to check that there is 1 of each as expected and then registers a mapping matching their order using devm_acpi_dev_add_driver_gpios(). This gives us access to both GPIOs allowing us to properly suspend the controller during suspend, and making it possible to reset the controller if necessary. BugLink: https://bugzilla.redhat.com/show_bug.cgi?id=1786317 BugLink: https://github.com/nexus511/gpd-ubuntu-packages/issues/10 BugLink: https://bugzilla.kernel.org/show_bug.cgi?id=199207 Signed-off-by: Hans de Goede Reviewed-by: Bastien Nocera Link: https://lore.kernel.org/r/20200307121505.3707-4-hdegoede@redhat.com Signed-off-by: Dmitry Torokhov --- drivers/input/touchscreen/goodix.c | 128 ++++++++++++++++++++++++++++- 1 file changed, 124 insertions(+), 4 deletions(-) diff --git a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c index 70212ea7c631..f7af5409f31c 100644 --- a/drivers/input/touchscreen/goodix.c +++ b/drivers/input/touchscreen/goodix.c @@ -34,6 +34,7 @@ struct goodix_ts_data; enum goodix_irq_pin_access_method { IRQ_PIN_ACCESS_NONE, IRQ_PIN_ACCESS_GPIO, + IRQ_PIN_ACCESS_ACPI_GPIO, }; struct goodix_chip_data { @@ -53,6 +54,8 @@ struct goodix_ts_data { struct regulator *vddio; struct gpio_desc *gpiod_int; struct gpio_desc *gpiod_rst; + int gpio_count; + int gpio_int_idx; u16 id; u16 version; const char *cfg_name; @@ -537,6 +540,12 @@ static int goodix_irq_direction_output(struct goodix_ts_data *ts, return -EINVAL; case IRQ_PIN_ACCESS_GPIO: return gpiod_direction_output(ts->gpiod_int, value); + case IRQ_PIN_ACCESS_ACPI_GPIO: + /* + * The IRQ pin triggers on a falling edge, so its gets marked + * as active-low, use output_raw to avoid the value inversion. + */ + return gpiod_direction_output_raw(ts->gpiod_int, value); } return -EINVAL; /* Never reached */ @@ -551,6 +560,7 @@ static int goodix_irq_direction_input(struct goodix_ts_data *ts) __func__); return -EINVAL; case IRQ_PIN_ACCESS_GPIO: + case IRQ_PIN_ACCESS_ACPI_GPIO: return gpiod_direction_input(ts->gpiod_int); } @@ -615,6 +625,94 @@ static int goodix_reset(struct goodix_ts_data *ts) return 0; } +#if defined CONFIG_X86 && defined CONFIG_ACPI +static const struct acpi_gpio_params first_gpio = { 0, 0, false }; +static const struct acpi_gpio_params second_gpio = { 1, 0, false }; + +static const struct acpi_gpio_mapping acpi_goodix_int_first_gpios[] = { + { GOODIX_GPIO_INT_NAME "-gpios", &first_gpio, 1 }, + { GOODIX_GPIO_RST_NAME "-gpios", &second_gpio, 1 }, + { }, +}; + +static const struct acpi_gpio_mapping acpi_goodix_int_last_gpios[] = { + { GOODIX_GPIO_RST_NAME "-gpios", &first_gpio, 1 }, + { GOODIX_GPIO_INT_NAME "-gpios", &second_gpio, 1 }, + { }, +}; + +static int goodix_resource(struct acpi_resource *ares, void *data) +{ + struct goodix_ts_data *ts = data; + struct device *dev = &ts->client->dev; + struct acpi_resource_gpio *gpio; + + switch (ares->type) { + case ACPI_RESOURCE_TYPE_GPIO: + gpio = &ares->data.gpio; + if (gpio->connection_type == ACPI_RESOURCE_GPIO_TYPE_INT) { + if (ts->gpio_int_idx == -1) { + ts->gpio_int_idx = ts->gpio_count; + } else { + dev_err(dev, "More then one GpioInt resource, ignoring ACPI GPIO resources\n"); + ts->gpio_int_idx = -2; + } + } + ts->gpio_count++; + break; + default: + break; + } + + return 0; +} + +/* + * This function gets called in case we fail to get the irq GPIO directly + * because the ACPI tables lack GPIO-name to APCI _CRS index mappings + * (no _DSD UUID daffd814-6eba-4d8c-8a91-bc9bbf4aa301 data). + * In that case we add our own mapping and then goodix_get_gpio_config() + * retries to get the GPIOs based on the added mapping. + */ +static int goodix_add_acpi_gpio_mappings(struct goodix_ts_data *ts) +{ + const struct acpi_gpio_mapping *gpio_mapping = NULL; + struct device *dev = &ts->client->dev; + LIST_HEAD(resources); + int ret; + + ts->gpio_count = 0; + ts->gpio_int_idx = -1; + ret = acpi_dev_get_resources(ACPI_COMPANION(dev), &resources, + goodix_resource, ts); + if (ret < 0) { + dev_err(dev, "Error getting ACPI resources: %d\n", ret); + return ret; + } + + acpi_dev_free_resource_list(&resources); + + if (ts->gpio_count == 2 && ts->gpio_int_idx == 0) { + ts->irq_pin_access_method = IRQ_PIN_ACCESS_ACPI_GPIO; + gpio_mapping = acpi_goodix_int_first_gpios; + } else if (ts->gpio_count == 2 && ts->gpio_int_idx == 1) { + ts->irq_pin_access_method = IRQ_PIN_ACCESS_ACPI_GPIO; + gpio_mapping = acpi_goodix_int_last_gpios; + } else { + dev_warn(dev, "Unexpected ACPI resources: gpio_count %d, gpio_int_idx %d\n", + ts->gpio_count, ts->gpio_int_idx); + return -EINVAL; + } + + return devm_acpi_dev_add_driver_gpios(dev, gpio_mapping); +} +#else +static int goodix_add_acpi_gpio_mappings(struct goodix_ts_data *ts) +{ + return -EINVAL; +} +#endif /* CONFIG_X86 && CONFIG_ACPI */ + /** * goodix_get_gpio_config - Get GPIO config from ACPI/DT * @@ -625,6 +723,7 @@ static int goodix_get_gpio_config(struct goodix_ts_data *ts) int error; struct device *dev; struct gpio_desc *gpiod; + bool added_acpi_mappings = false; if (!ts->client) return -EINVAL; @@ -648,6 +747,7 @@ static int goodix_get_gpio_config(struct goodix_ts_data *ts) return error; } +retry_get_irq_gpio: /* Get the interrupt GPIO pin number */ gpiod = devm_gpiod_get_optional(dev, GOODIX_GPIO_INT_NAME, GPIOD_IN); if (IS_ERR(gpiod)) { @@ -657,6 +757,11 @@ static int goodix_get_gpio_config(struct goodix_ts_data *ts) GOODIX_GPIO_INT_NAME, error); return error; } + if (!gpiod && has_acpi_companion(dev) && !added_acpi_mappings) { + added_acpi_mappings = true; + if (goodix_add_acpi_gpio_mappings(ts) == 0) + goto retry_get_irq_gpio; + } ts->gpiod_int = gpiod; @@ -672,10 +777,25 @@ static int goodix_get_gpio_config(struct goodix_ts_data *ts) ts->gpiod_rst = gpiod; - if (ts->gpiod_int && ts->gpiod_rst) { - ts->reset_controller_at_probe = true; - ts->load_cfg_from_disk = true; - ts->irq_pin_access_method = IRQ_PIN_ACCESS_GPIO; + switch (ts->irq_pin_access_method) { + case IRQ_PIN_ACCESS_ACPI_GPIO: + /* + * We end up here if goodix_add_acpi_gpio_mappings() has + * called devm_acpi_dev_add_driver_gpios() because the ACPI + * tables did not contain name to index mappings. + * Check that we successfully got both GPIOs after we've + * added our own acpi_gpio_mapping and if we did not get both + * GPIOs reset irq_pin_access_method to IRQ_PIN_ACCESS_NONE. + */ + if (!ts->gpiod_int || !ts->gpiod_rst) + ts->irq_pin_access_method = IRQ_PIN_ACCESS_NONE; + break; + default: + if (ts->gpiod_int && ts->gpiod_rst) { + ts->reset_controller_at_probe = true; + ts->load_cfg_from_disk = true; + ts->irq_pin_access_method = IRQ_PIN_ACCESS_GPIO; + } } return 0; From 67abd9eeb458f3d4f2bbc250e9d4d796fa18f26c Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Tue, 24 Mar 2020 11:23:35 -0700 Subject: [PATCH 188/635] Input: goodix - add support for getting IRQ + reset GPIOs on Bay Trail devices On most Bay Trail (x86, UEFI + ACPI) devices the ACPI tables do not have a _DSD with a "daffd814-6eba-4d8c-8a91-bc9bbf4aa301" UUID, adding "irq-gpios" and "reset-gpios" mappings, so we cannot get the GPIOS by name without first manually adding mappings ourselves. These devices contain 2 GpioIo resource in their _CRS table, on all 4 such devices which I have access to, the order of the 2 GPIOs is reset, int. Note that the GPIO to which the touchscreen controller irq pin is connected is configured in direct-irq mode on these Bay Trail devices, the pinctrl-baytrail.c driver still allows controlling the pin as a GPIO in this case, but this is not necessarily the case on other X86 ACPI platforms, nor do we have a guarantee that the GPIO order is the same elsewhere, so we limit the use of a _CRS table with 2 GpioIo resources to Bay Trail devices only. BugLink: https://bugzilla.redhat.com/show_bug.cgi?id=1786317 BugLink: https://github.com/nexus511/gpd-ubuntu-packages/issues/10 BugLink: https://bugzilla.kernel.org/show_bug.cgi?id=199207 Reviewed-by: Bastien Nocera Signed-off-by: Hans de Goede Link: https://lore.kernel.org/r/20200307121505.3707-5-hdegoede@redhat.com Signed-off-by: Dmitry Torokhov --- drivers/input/touchscreen/goodix.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c index f7af5409f31c..37862f7dfde3 100644 --- a/drivers/input/touchscreen/goodix.c +++ b/drivers/input/touchscreen/goodix.c @@ -626,6 +626,21 @@ static int goodix_reset(struct goodix_ts_data *ts) } #if defined CONFIG_X86 && defined CONFIG_ACPI +#include +#include + +static const struct x86_cpu_id baytrail_cpu_ids[] = { + { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_SILVERMONT, X86_FEATURE_ANY, }, + {} +}; + +static inline bool is_byt(void) +{ + const struct x86_cpu_id *id = x86_match_cpu(baytrail_cpu_ids); + + return !!id; +} + static const struct acpi_gpio_params first_gpio = { 0, 0, false }; static const struct acpi_gpio_params second_gpio = { 1, 0, false }; @@ -698,6 +713,10 @@ static int goodix_add_acpi_gpio_mappings(struct goodix_ts_data *ts) } else if (ts->gpio_count == 2 && ts->gpio_int_idx == 1) { ts->irq_pin_access_method = IRQ_PIN_ACCESS_ACPI_GPIO; gpio_mapping = acpi_goodix_int_last_gpios; + } else if (is_byt() && ts->gpio_count == 2 && ts->gpio_int_idx == -1) { + dev_info(dev, "No ACPI GpioInt resource, assuming that the GPIO order is reset, int\n"); + ts->irq_pin_access_method = IRQ_PIN_ACCESS_ACPI_GPIO; + gpio_mapping = acpi_goodix_int_last_gpios; } else { dev_warn(dev, "Unexpected ACPI resources: gpio_count %d, gpio_int_idx %d\n", ts->gpio_count, ts->gpio_int_idx); From c5fca485320e83b1bb964ad5559ec20f14c943be Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Tue, 24 Mar 2020 11:24:31 -0700 Subject: [PATCH 189/635] Input: goodix - add support for controlling the IRQ pin through ACPI methods Some Apollo Lake (x86, UEFI + ACPI) devices only list the reset GPIO in their _CRS table and the bit-banging of the IRQ line necessary to wake-up the controller from suspend can be done by calling 2 Goodix custom / specific ACPI methods. This commit adds support for controlling the IRQ line in this matter, allowing us to properly suspend the touchscreen controller on such devices. BugLink: https://bugzilla.redhat.com/show_bug.cgi?id=1786317 BugLink: https://github.com/nexus511/gpd-ubuntu-packages/issues/10 BugLink: https://bugzilla.kernel.org/show_bug.cgi?id=199207 Reviewed-by: Bastien Nocera Signed-off-by: Hans de Goede Link: https://lore.kernel.org/r/20200307121505.3707-6-hdegoede@redhat.com Signed-off-by: Dmitry Torokhov --- drivers/input/touchscreen/goodix.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c index 37862f7dfde3..8e31b4bc6275 100644 --- a/drivers/input/touchscreen/goodix.c +++ b/drivers/input/touchscreen/goodix.c @@ -35,6 +35,7 @@ enum goodix_irq_pin_access_method { IRQ_PIN_ACCESS_NONE, IRQ_PIN_ACCESS_GPIO, IRQ_PIN_ACCESS_ACPI_GPIO, + IRQ_PIN_ACCESS_ACPI_METHOD, }; struct goodix_chip_data { @@ -532,6 +533,9 @@ static int goodix_send_cfg(struct goodix_ts_data *ts, static int goodix_irq_direction_output(struct goodix_ts_data *ts, int value) { + struct device *dev = &ts->client->dev; + acpi_status status; + switch (ts->irq_pin_access_method) { case IRQ_PIN_ACCESS_NONE: dev_err(&ts->client->dev, @@ -546,6 +550,10 @@ static int goodix_irq_direction_output(struct goodix_ts_data *ts, * as active-low, use output_raw to avoid the value inversion. */ return gpiod_direction_output_raw(ts->gpiod_int, value); + case IRQ_PIN_ACCESS_ACPI_METHOD: + status = acpi_execute_simple_method(ACPI_HANDLE(dev), + "INTO", value); + return ACPI_SUCCESS(status) ? 0 : -EIO; } return -EINVAL; /* Never reached */ @@ -553,6 +561,9 @@ static int goodix_irq_direction_output(struct goodix_ts_data *ts, static int goodix_irq_direction_input(struct goodix_ts_data *ts) { + struct device *dev = &ts->client->dev; + acpi_status status; + switch (ts->irq_pin_access_method) { case IRQ_PIN_ACCESS_NONE: dev_err(&ts->client->dev, @@ -562,6 +573,10 @@ static int goodix_irq_direction_input(struct goodix_ts_data *ts) case IRQ_PIN_ACCESS_GPIO: case IRQ_PIN_ACCESS_ACPI_GPIO: return gpiod_direction_input(ts->gpiod_int); + case IRQ_PIN_ACCESS_ACPI_METHOD: + status = acpi_evaluate_object(ACPI_HANDLE(dev), "INTI", + NULL, NULL); + return ACPI_SUCCESS(status) ? 0 : -EIO; } return -EINVAL; /* Never reached */ @@ -656,6 +671,11 @@ static const struct acpi_gpio_mapping acpi_goodix_int_last_gpios[] = { { }, }; +static const struct acpi_gpio_mapping acpi_goodix_reset_only_gpios[] = { + { GOODIX_GPIO_RST_NAME "-gpios", &first_gpio, 1 }, + { }, +}; + static int goodix_resource(struct acpi_resource *ares, void *data) { struct goodix_ts_data *ts = data; @@ -713,6 +733,12 @@ static int goodix_add_acpi_gpio_mappings(struct goodix_ts_data *ts) } else if (ts->gpio_count == 2 && ts->gpio_int_idx == 1) { ts->irq_pin_access_method = IRQ_PIN_ACCESS_ACPI_GPIO; gpio_mapping = acpi_goodix_int_last_gpios; + } else if (ts->gpio_count == 1 && ts->gpio_int_idx == -1 && + acpi_has_method(ACPI_HANDLE(dev), "INTI") && + acpi_has_method(ACPI_HANDLE(dev), "INTO")) { + dev_info(dev, "Using ACPI INTI and INTO methods for IRQ pin access\n"); + ts->irq_pin_access_method = IRQ_PIN_ACCESS_ACPI_METHOD; + gpio_mapping = acpi_goodix_reset_only_gpios; } else if (is_byt() && ts->gpio_count == 2 && ts->gpio_int_idx == -1) { dev_info(dev, "No ACPI GpioInt resource, assuming that the GPIO order is reset, int\n"); ts->irq_pin_access_method = IRQ_PIN_ACCESS_ACPI_GPIO; @@ -809,6 +835,10 @@ retry_get_irq_gpio: if (!ts->gpiod_int || !ts->gpiod_rst) ts->irq_pin_access_method = IRQ_PIN_ACCESS_NONE; break; + case IRQ_PIN_ACCESS_ACPI_METHOD: + if (!ts->gpiod_rst) + ts->irq_pin_access_method = IRQ_PIN_ACCESS_NONE; + break; default: if (ts->gpiod_int && ts->gpiod_rst) { ts->reset_controller_at_probe = true; From aebfc52c09499eb3885aa352bb17b103109b73df Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Tue, 24 Mar 2020 11:29:10 -0700 Subject: [PATCH 190/635] Input: goodix - move defines to above struct goodix_ts_data declaration Move the defines to above the struct goodix_ts_data declaration, so that the MAX defines can be used inside the struct goodix_ts_data declaration. No functional changes, just moving a block of code. BugLink: https://bugzilla.redhat.com/show_bug.cgi?id=1786317 BugLink: https://github.com/nexus511/gpd-ubuntu-packages/issues/10 BugLink: https://bugzilla.kernel.org/show_bug.cgi?id=199207 Reviewed-by: Bastien Nocera Signed-off-by: Hans de Goede Link: https://lore.kernel.org/r/20200307121505.3707-7-hdegoede@redhat.com Signed-off-by: Dmitry Torokhov --- drivers/input/touchscreen/goodix.c | 60 +++++++++++++++--------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c index 8e31b4bc6275..e03ecaf7b822 100644 --- a/drivers/input/touchscreen/goodix.c +++ b/drivers/input/touchscreen/goodix.c @@ -29,6 +29,36 @@ #include #include +#define GOODIX_GPIO_INT_NAME "irq" +#define GOODIX_GPIO_RST_NAME "reset" + +#define GOODIX_MAX_HEIGHT 4096 +#define GOODIX_MAX_WIDTH 4096 +#define GOODIX_INT_TRIGGER 1 +#define GOODIX_CONTACT_SIZE 8 +#define GOODIX_MAX_CONTACT_SIZE 9 +#define GOODIX_MAX_CONTACTS 10 + +#define GOODIX_CONFIG_MAX_LENGTH 240 +#define GOODIX_CONFIG_911_LENGTH 186 +#define GOODIX_CONFIG_967_LENGTH 228 + +/* Register defines */ +#define GOODIX_REG_COMMAND 0x8040 +#define GOODIX_CMD_SCREEN_OFF 0x05 + +#define GOODIX_READ_COOR_ADDR 0x814E +#define GOODIX_GT1X_REG_CONFIG_DATA 0x8050 +#define GOODIX_GT9X_REG_CONFIG_DATA 0x8047 +#define GOODIX_REG_ID 0x8140 + +#define GOODIX_BUFFER_STATUS_READY BIT(7) +#define GOODIX_BUFFER_STATUS_TIMEOUT 20 + +#define RESOLUTION_LOC 1 +#define MAX_CONTACTS_LOC 5 +#define TRIGGER_LOC 6 + struct goodix_ts_data; enum goodix_irq_pin_access_method { @@ -68,36 +98,6 @@ struct goodix_ts_data { unsigned int contact_size; }; -#define GOODIX_GPIO_INT_NAME "irq" -#define GOODIX_GPIO_RST_NAME "reset" - -#define GOODIX_MAX_HEIGHT 4096 -#define GOODIX_MAX_WIDTH 4096 -#define GOODIX_INT_TRIGGER 1 -#define GOODIX_CONTACT_SIZE 8 -#define GOODIX_MAX_CONTACT_SIZE 9 -#define GOODIX_MAX_CONTACTS 10 - -#define GOODIX_CONFIG_MAX_LENGTH 240 -#define GOODIX_CONFIG_911_LENGTH 186 -#define GOODIX_CONFIG_967_LENGTH 228 - -/* Register defines */ -#define GOODIX_REG_COMMAND 0x8040 -#define GOODIX_CMD_SCREEN_OFF 0x05 - -#define GOODIX_READ_COOR_ADDR 0x814E -#define GOODIX_GT1X_REG_CONFIG_DATA 0x8050 -#define GOODIX_GT9X_REG_CONFIG_DATA 0x8047 -#define GOODIX_REG_ID 0x8140 - -#define GOODIX_BUFFER_STATUS_READY BIT(7) -#define GOODIX_BUFFER_STATUS_TIMEOUT 20 - -#define RESOLUTION_LOC 1 -#define MAX_CONTACTS_LOC 5 -#define TRIGGER_LOC 6 - static int goodix_check_cfg_8(struct goodix_ts_data *ts, const struct firmware *cfg); static int goodix_check_cfg_16(struct goodix_ts_data *ts, From de956caed952c7b07a5a7137037f735bd42050ca Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Tue, 24 Mar 2020 11:29:26 -0700 Subject: [PATCH 191/635] Input: goodix - save a copy of the config from goodix_read_config() Save a copy of the config in goodix_read_config(), this is a preparation patch for restoring the config if it was lost after a supend/resume cycle. BugLink: https://bugzilla.redhat.com/show_bug.cgi?id=1786317 BugLink: https://github.com/nexus511/gpd-ubuntu-packages/issues/10 BugLink: https://bugzilla.kernel.org/show_bug.cgi?id=199207 Reviewed-by: Bastien Nocera Signed-off-by: Hans de Goede Link: https://lore.kernel.org/r/20200307121505.3707-8-hdegoede@redhat.com Signed-off-by: Dmitry Torokhov --- drivers/input/touchscreen/goodix.c | 47 ++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c index e03ecaf7b822..7e318938b723 100644 --- a/drivers/input/touchscreen/goodix.c +++ b/drivers/input/touchscreen/goodix.c @@ -72,6 +72,7 @@ struct goodix_chip_data { u16 config_addr; int config_len; int (*check_config)(struct goodix_ts_data *, const struct firmware *); + void (*calc_config_checksum)(struct goodix_ts_data *ts); }; struct goodix_ts_data { @@ -96,35 +97,42 @@ struct goodix_ts_data { unsigned long irq_flags; enum goodix_irq_pin_access_method irq_pin_access_method; unsigned int contact_size; + u8 config[GOODIX_CONFIG_MAX_LENGTH]; }; static int goodix_check_cfg_8(struct goodix_ts_data *ts, const struct firmware *cfg); static int goodix_check_cfg_16(struct goodix_ts_data *ts, const struct firmware *cfg); +static void goodix_calc_cfg_checksum_8(struct goodix_ts_data *ts); +static void goodix_calc_cfg_checksum_16(struct goodix_ts_data *ts); static const struct goodix_chip_data gt1x_chip_data = { .config_addr = GOODIX_GT1X_REG_CONFIG_DATA, .config_len = GOODIX_CONFIG_MAX_LENGTH, .check_config = goodix_check_cfg_16, + .calc_config_checksum = goodix_calc_cfg_checksum_16, }; static const struct goodix_chip_data gt911_chip_data = { .config_addr = GOODIX_GT9X_REG_CONFIG_DATA, .config_len = GOODIX_CONFIG_911_LENGTH, .check_config = goodix_check_cfg_8, + .calc_config_checksum = goodix_calc_cfg_checksum_8, }; static const struct goodix_chip_data gt967_chip_data = { .config_addr = GOODIX_GT9X_REG_CONFIG_DATA, .config_len = GOODIX_CONFIG_967_LENGTH, .check_config = goodix_check_cfg_8, + .calc_config_checksum = goodix_calc_cfg_checksum_8, }; static const struct goodix_chip_data gt9x_chip_data = { .config_addr = GOODIX_GT9X_REG_CONFIG_DATA, .config_len = GOODIX_CONFIG_MAX_LENGTH, .check_config = goodix_check_cfg_8, + .calc_config_checksum = goodix_calc_cfg_checksum_8, }; static const unsigned long goodix_irq_flags[] = { @@ -458,6 +466,19 @@ static int goodix_check_cfg_8(struct goodix_ts_data *ts, return 0; } +static void goodix_calc_cfg_checksum_8(struct goodix_ts_data *ts) +{ + int i, raw_cfg_len = ts->chip->config_len - 2; + u8 check_sum = 0; + + for (i = 0; i < raw_cfg_len; i++) + check_sum += ts->config[i]; + check_sum = (~check_sum) + 1; + + ts->config[raw_cfg_len] = check_sum; + ts->config[raw_cfg_len + 1] = 1; /* Set "config_fresh" bit */ +} + static int goodix_check_cfg_16(struct goodix_ts_data *ts, const struct firmware *cfg) { @@ -482,6 +503,19 @@ static int goodix_check_cfg_16(struct goodix_ts_data *ts, return 0; } +static void goodix_calc_cfg_checksum_16(struct goodix_ts_data *ts) +{ + int i, raw_cfg_len = ts->chip->config_len - 3; + u16 check_sum = 0; + + for (i = 0; i < raw_cfg_len; i += 2) + check_sum += get_unaligned_be16(&ts->config[i]); + check_sum = (~check_sum) + 1; + + put_unaligned_be16(check_sum, &ts->config[raw_cfg_len]); + ts->config[raw_cfg_len + 2] = 1; /* Set "config_fresh" bit */ +} + /** * goodix_check_cfg - Checks if config fw is valid * @@ -859,12 +893,11 @@ retry_get_irq_gpio: */ static void goodix_read_config(struct goodix_ts_data *ts) { - u8 config[GOODIX_CONFIG_MAX_LENGTH]; int x_max, y_max; int error; error = goodix_i2c_read(ts->client, ts->chip->config_addr, - config, ts->chip->config_len); + ts->config, ts->chip->config_len); if (error) { dev_warn(&ts->client->dev, "Error reading config: %d\n", error); @@ -873,15 +906,17 @@ static void goodix_read_config(struct goodix_ts_data *ts) return; } - ts->int_trigger_type = config[TRIGGER_LOC] & 0x03; - ts->max_touch_num = config[MAX_CONTACTS_LOC] & 0x0f; + ts->int_trigger_type = ts->config[TRIGGER_LOC] & 0x03; + ts->max_touch_num = ts->config[MAX_CONTACTS_LOC] & 0x0f; - x_max = get_unaligned_le16(&config[RESOLUTION_LOC]); - y_max = get_unaligned_le16(&config[RESOLUTION_LOC + 2]); + x_max = get_unaligned_le16(&ts->config[RESOLUTION_LOC]); + y_max = get_unaligned_le16(&ts->config[RESOLUTION_LOC + 2]); if (x_max && y_max) { input_abs_set_max(ts->input_dev, ABS_MT_POSITION_X, x_max - 1); input_abs_set_max(ts->input_dev, ABS_MT_POSITION_Y, y_max - 1); } + + ts->chip->calc_config_checksum(ts); } /** From 686e8a2489baf6dcbd87de07f37fef07a706dd41 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Tue, 24 Mar 2020 11:33:16 -0700 Subject: [PATCH 192/635] Input: goodix - add minimum firmware size check Our goodix_check_cfg_* helpers do things like: int i, raw_cfg_len = cfg->size - 2; ... if (check_sum != cfg->data[raw_cfg_len]) { When cfg->size < 2, this will end up indexing the cfg->data array with a negative value, which will not end well. To fix this this commit adds a new GOODIX_CONFIG_MIN_LENGTH define and adds a minimum size check for firmware-config files using this new define. For consistency this commit also adds a new GOODIX_CONFIG_GT9X_LENGTH for the length used for recent gt9xx and gt1xxx chips, instead of using GOODIX_CONFIG_MAX_LENGTH for this, so that if other length defines get added in the future it will be clear that the MIN and MAX defines should contain the min and max values of all the other defines. Signed-off-by: Hans de Goede Reviewed-by: Bastien Nocera Link: https://lore.kernel.org/r/20200307121505.3707-9-hdegoede@redhat.com Signed-off-by: Dmitry Torokhov --- drivers/input/touchscreen/goodix.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c index 7e318938b723..7cca9998ab77 100644 --- a/drivers/input/touchscreen/goodix.c +++ b/drivers/input/touchscreen/goodix.c @@ -39,9 +39,11 @@ #define GOODIX_MAX_CONTACT_SIZE 9 #define GOODIX_MAX_CONTACTS 10 -#define GOODIX_CONFIG_MAX_LENGTH 240 +#define GOODIX_CONFIG_MIN_LENGTH 186 #define GOODIX_CONFIG_911_LENGTH 186 #define GOODIX_CONFIG_967_LENGTH 228 +#define GOODIX_CONFIG_GT9X_LENGTH 240 +#define GOODIX_CONFIG_MAX_LENGTH 240 /* Register defines */ #define GOODIX_REG_COMMAND 0x8040 @@ -109,7 +111,7 @@ static void goodix_calc_cfg_checksum_16(struct goodix_ts_data *ts); static const struct goodix_chip_data gt1x_chip_data = { .config_addr = GOODIX_GT1X_REG_CONFIG_DATA, - .config_len = GOODIX_CONFIG_MAX_LENGTH, + .config_len = GOODIX_CONFIG_GT9X_LENGTH, .check_config = goodix_check_cfg_16, .calc_config_checksum = goodix_calc_cfg_checksum_16, }; @@ -130,7 +132,7 @@ static const struct goodix_chip_data gt967_chip_data = { static const struct goodix_chip_data gt9x_chip_data = { .config_addr = GOODIX_GT9X_REG_CONFIG_DATA, - .config_len = GOODIX_CONFIG_MAX_LENGTH, + .config_len = GOODIX_CONFIG_GT9X_LENGTH, .check_config = goodix_check_cfg_8, .calc_config_checksum = goodix_calc_cfg_checksum_8, }; @@ -525,7 +527,8 @@ static void goodix_calc_cfg_checksum_16(struct goodix_ts_data *ts) static int goodix_check_cfg(struct goodix_ts_data *ts, const struct firmware *cfg) { - if (cfg->size > GOODIX_CONFIG_MAX_LENGTH) { + if (cfg->size < GOODIX_CONFIG_MIN_LENGTH || + cfg->size > GOODIX_CONFIG_MAX_LENGTH) { dev_err(&ts->client->dev, "The length of the config fw is not correct"); return -EINVAL; From 1a67448f1563c7fc5d3e19e6897e04a7b5424bf2 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Tue, 24 Mar 2020 11:33:50 -0700 Subject: [PATCH 193/635] Input: goodix - make goodix_send_cfg() take a raw buffer as argument Make goodix_send_cfg() take a raw buffer as argument instead of a struct firmware *cfg, so that it can also be used to restore the config on resume if necessary. BugLink: https://bugzilla.redhat.com/show_bug.cgi?id=1786317 BugLink: https://github.com/nexus511/gpd-ubuntu-packages/issues/10 BugLink: https://bugzilla.kernel.org/show_bug.cgi?id=199207 Reviewed-by: Bastien Nocera Signed-off-by: Hans de Goede Link: https://lore.kernel.org/r/20200307121505.3707-10-hdegoede@redhat.com Signed-off-by: Dmitry Torokhov --- drivers/input/touchscreen/goodix.c | 48 ++++++++++++++---------------- 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c index 7cca9998ab77..958a5d65e374 100644 --- a/drivers/input/touchscreen/goodix.c +++ b/drivers/input/touchscreen/goodix.c @@ -73,7 +73,7 @@ enum goodix_irq_pin_access_method { struct goodix_chip_data { u16 config_addr; int config_len; - int (*check_config)(struct goodix_ts_data *, const struct firmware *); + int (*check_config)(struct goodix_ts_data *ts, const u8 *cfg, int len); void (*calc_config_checksum)(struct goodix_ts_data *ts); }; @@ -103,9 +103,9 @@ struct goodix_ts_data { }; static int goodix_check_cfg_8(struct goodix_ts_data *ts, - const struct firmware *cfg); + const u8 *cfg, int len); static int goodix_check_cfg_16(struct goodix_ts_data *ts, - const struct firmware *cfg); + const u8 *cfg, int len); static void goodix_calc_cfg_checksum_8(struct goodix_ts_data *ts); static void goodix_calc_cfg_checksum_16(struct goodix_ts_data *ts); @@ -444,22 +444,21 @@ static int goodix_request_irq(struct goodix_ts_data *ts) ts->irq_flags, ts->client->name, ts); } -static int goodix_check_cfg_8(struct goodix_ts_data *ts, - const struct firmware *cfg) +static int goodix_check_cfg_8(struct goodix_ts_data *ts, const u8 *cfg, int len) { - int i, raw_cfg_len = cfg->size - 2; + int i, raw_cfg_len = len - 2; u8 check_sum = 0; for (i = 0; i < raw_cfg_len; i++) - check_sum += cfg->data[i]; + check_sum += cfg[i]; check_sum = (~check_sum) + 1; - if (check_sum != cfg->data[raw_cfg_len]) { + if (check_sum != cfg[raw_cfg_len]) { dev_err(&ts->client->dev, "The checksum of the config fw is not correct"); return -EINVAL; } - if (cfg->data[raw_cfg_len + 1] != 1) { + if (cfg[raw_cfg_len + 1] != 1) { dev_err(&ts->client->dev, "Config fw must have Config_Fresh register set"); return -EINVAL; @@ -481,22 +480,22 @@ static void goodix_calc_cfg_checksum_8(struct goodix_ts_data *ts) ts->config[raw_cfg_len + 1] = 1; /* Set "config_fresh" bit */ } -static int goodix_check_cfg_16(struct goodix_ts_data *ts, - const struct firmware *cfg) +static int goodix_check_cfg_16(struct goodix_ts_data *ts, const u8 *cfg, + int len) { - int i, raw_cfg_len = cfg->size - 3; + int i, raw_cfg_len = len - 3; u16 check_sum = 0; for (i = 0; i < raw_cfg_len; i += 2) - check_sum += get_unaligned_be16(&cfg->data[i]); + check_sum += get_unaligned_be16(&cfg[i]); check_sum = (~check_sum) + 1; - if (check_sum != get_unaligned_be16(&cfg->data[raw_cfg_len])) { + if (check_sum != get_unaligned_be16(&cfg[raw_cfg_len])) { dev_err(&ts->client->dev, "The checksum of the config fw is not correct"); return -EINVAL; } - if (cfg->data[raw_cfg_len + 2] != 1) { + if (cfg[raw_cfg_len + 2] != 1) { dev_err(&ts->client->dev, "Config fw must have Config_Fresh register set"); return -EINVAL; @@ -524,17 +523,16 @@ static void goodix_calc_cfg_checksum_16(struct goodix_ts_data *ts) * @ts: goodix_ts_data pointer * @cfg: firmware config data */ -static int goodix_check_cfg(struct goodix_ts_data *ts, - const struct firmware *cfg) +static int goodix_check_cfg(struct goodix_ts_data *ts, const u8 *cfg, int len) { - if (cfg->size < GOODIX_CONFIG_MIN_LENGTH || - cfg->size > GOODIX_CONFIG_MAX_LENGTH) { + if (len < GOODIX_CONFIG_MIN_LENGTH || + len > GOODIX_CONFIG_MAX_LENGTH) { dev_err(&ts->client->dev, "The length of the config fw is not correct"); return -EINVAL; } - return ts->chip->check_config(ts, cfg); + return ts->chip->check_config(ts, cfg, len); } /** @@ -543,17 +541,15 @@ static int goodix_check_cfg(struct goodix_ts_data *ts, * @ts: goodix_ts_data pointer * @cfg: config firmware to write to device */ -static int goodix_send_cfg(struct goodix_ts_data *ts, - const struct firmware *cfg) +static int goodix_send_cfg(struct goodix_ts_data *ts, const u8 *cfg, int len) { int error; - error = goodix_check_cfg(ts, cfg); + error = goodix_check_cfg(ts, cfg, len); if (error) return error; - error = goodix_i2c_write(ts->client, ts->chip->config_addr, cfg->data, - cfg->size); + error = goodix_i2c_write(ts->client, ts->chip->config_addr, cfg, len); if (error) { dev_err(&ts->client->dev, "Failed to write config data: %d", error); @@ -1094,7 +1090,7 @@ static void goodix_config_cb(const struct firmware *cfg, void *ctx) if (cfg) { /* send device configuration to the firmware */ - error = goodix_send_cfg(ts, cfg); + error = goodix_send_cfg(ts, cfg->data, cfg->size); if (error) goto err_release_cfg; } From 273ec6bd9af56feabdc72405fa546ef92dc50bc9 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Tue, 24 Mar 2020 11:34:11 -0700 Subject: [PATCH 194/635] Input: goodix - restore config on resume if necessary Some devices, e.g the Trekstor Primetab S11B, lose there config over a suspend/resume cycle (likely the controller loses power during suspend). This commit reads back the config version on resume and if matches the expected config version it resets the controller and resends the config we read back and saved at probe time. BugLink: https://bugzilla.redhat.com/show_bug.cgi?id=1786317 BugLink: https://github.com/nexus511/gpd-ubuntu-packages/issues/10 BugLink: https://bugzilla.kernel.org/show_bug.cgi?id=199207 Reviewed-by: Bastien Nocera Signed-off-by: Hans de Goede Link: https://lore.kernel.org/r/20200307121505.3707-11-hdegoede@redhat.com Signed-off-by: Dmitry Torokhov --- drivers/input/touchscreen/goodix.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c index 958a5d65e374..7811500ba3b5 100644 --- a/drivers/input/touchscreen/goodix.c +++ b/drivers/input/touchscreen/goodix.c @@ -1268,6 +1268,7 @@ static int __maybe_unused goodix_resume(struct device *dev) { struct i2c_client *client = to_i2c_client(dev); struct goodix_ts_data *ts = i2c_get_clientdata(client); + u8 config_ver; int error; if (ts->irq_pin_access_method == IRQ_PIN_ACCESS_NONE) { @@ -1289,6 +1290,27 @@ static int __maybe_unused goodix_resume(struct device *dev) if (error) return error; + error = goodix_i2c_read(ts->client, ts->chip->config_addr, + &config_ver, 1); + if (error) + dev_warn(dev, "Error reading config version: %d, resetting controller\n", + error); + else if (config_ver != ts->config[0]) + dev_info(dev, "Config version mismatch %d != %d, resetting controller\n", + config_ver, ts->config[0]); + + if (error != 0 || config_ver != ts->config[0]) { + error = goodix_reset(ts); + if (error) { + dev_err(dev, "Controller reset failed.\n"); + return error; + } + + error = goodix_send_cfg(ts, ts->config, ts->chip->config_len); + if (error) + return error; + } + error = goodix_request_irq(ts); if (error) return error; From 9e1f2d70ec3998e6d9948f8ac616db21fa4243d5 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Tue, 24 Mar 2020 11:37:23 -0700 Subject: [PATCH 195/635] Input: goodix - try to reset the controller if the i2c-test fails On some ACPI/x86 devices (where we use one of the ACPI IRQ pin access methods) the firmware is buggy, it does not properly reset the controller at boot, and we cannot communicate with it. Normally on ACPI/x86 devices we do not want to reset the controller at probe time since in some cases this causes the controller to loose its configuration and this is loaded into it by the system's firmware. So on these systems we leave the reset_controller_at_probe flag unset, even though we have a access to both the IRQ and reset pins and thus could reset it. In the case of the buggy firmware we have to reset the controller to actually be able to talk to it. This commit adds a special case for this, if the goodix_i2c_test() fails, and we have not reset the controller yet; and we do have a way to reset the controller then retry the i2c-test after resetting the controller. This fixes the driver failing at probe on ACPI/x86 systems with this firmware bug. Reported-and-tested-by: Dmitry Mastykin Signed-off-by: Hans de Goede Link: https://lore.kernel.org/r/20200311191013.10826-2-hdegoede@redhat.com Signed-off-by: Dmitry Torokhov --- drivers/input/touchscreen/goodix.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c index 7811500ba3b5..99a88bd3d212 100644 --- a/drivers/input/touchscreen/goodix.c +++ b/drivers/input/touchscreen/goodix.c @@ -1159,6 +1159,7 @@ static int goodix_ts_probe(struct i2c_client *client, if (error) return error; +reset: if (ts->reset_controller_at_probe) { /* reset the controller */ error = goodix_reset(ts); @@ -1170,6 +1171,12 @@ static int goodix_ts_probe(struct i2c_client *client, error = goodix_i2c_test(client); if (error) { + if (!ts->reset_controller_at_probe && + ts->irq_pin_access_method != IRQ_PIN_ACCESS_NONE) { + /* Retry after a controller reset */ + ts->reset_controller_at_probe = true; + goto reset; + } dev_err(&client->dev, "I2C communication failure: %d\n", error); return error; } From 24ef83f6e31d20fc121a7cd732b04b498475fca3 Mon Sep 17 00:00:00 2001 From: Dmitry Mastykin Date: Tue, 24 Mar 2020 11:38:28 -0700 Subject: [PATCH 196/635] Input: goodix - fix spurious key release events The goodix panel sends spurious interrupts after a 'finger up' event, which always cause a timeout. We were exiting the interrupt handler by reporting touch_num == 0, but this was still processed as valid and caused the code to use the uninitialised point_data, creating spurious key release events. Report an error from the interrupt handler so as to avoid processing invalid point_data further. Signed-off-by: Dmitry Mastykin Reviewed-by: Bastien Nocera Link: https://lore.kernel.org/r/20200316075302.3759-2-dmastykin@astralinux.ru Signed-off-by: Dmitry Torokhov --- drivers/input/touchscreen/goodix.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c index 99a88bd3d212..4cfebdde7d80 100644 --- a/drivers/input/touchscreen/goodix.c +++ b/drivers/input/touchscreen/goodix.c @@ -345,7 +345,7 @@ static int goodix_ts_read_input_report(struct goodix_ts_data *ts, u8 *data) * The Goodix panel will send spurious interrupts after a * 'finger up' event, which will always cause a timeout. */ - return 0; + return -ENOMSG; } static void goodix_ts_report_touch_8b(struct goodix_ts_data *ts, u8 *coor_data) From 492ee52c1cd4b6145bc39f73a20d2cf698eb33d2 Mon Sep 17 00:00:00 2001 From: Dmitry Mastykin Date: Tue, 24 Mar 2020 11:38:49 -0700 Subject: [PATCH 197/635] Input: goodix - add support for more then one touch-key Some devices with a goodix touchscreen have more then 1 capacitive touch-key. This commit replaces the current support for a single touch-key, which ignored the reported key-code. With support for up to 7 touch-keys, based upon checking the key-code which is post-fixed to any reported touch-data. KEY_LEFTMETA is assigned to the first touch-key (it will still be the default keycode for devices with a single touch-key). KEY_F1, KEY_F2... are assigned as default keycode for the other touch-keys. This commit also add supports for keycode remapping, so that systemd-udev's hwdb can be used to remap the codes to send keycodes to match the icons on the buttons for devices with more then 1 touch-key. Signed-off-by: Dmitry Mastykin Reviewed-by: Bastien Nocera Link: https://lore.kernel.org/r/20200316075302.3759-1-dmastykin@astralinux.ru Signed-off-by: Dmitry Torokhov --- drivers/input/touchscreen/goodix.c | 62 +++++++++++++++++++++++------- 1 file changed, 49 insertions(+), 13 deletions(-) diff --git a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c index 4cfebdde7d80..9805372aba5d 100644 --- a/drivers/input/touchscreen/goodix.c +++ b/drivers/input/touchscreen/goodix.c @@ -38,6 +38,7 @@ #define GOODIX_CONTACT_SIZE 8 #define GOODIX_MAX_CONTACT_SIZE 9 #define GOODIX_MAX_CONTACTS 10 +#define GOODIX_MAX_KEYS 7 #define GOODIX_CONFIG_MIN_LENGTH 186 #define GOODIX_CONFIG_911_LENGTH 186 @@ -55,6 +56,7 @@ #define GOODIX_REG_ID 0x8140 #define GOODIX_BUFFER_STATUS_READY BIT(7) +#define GOODIX_HAVE_KEY BIT(4) #define GOODIX_BUFFER_STATUS_TIMEOUT 20 #define RESOLUTION_LOC 1 @@ -100,6 +102,7 @@ struct goodix_ts_data { enum goodix_irq_pin_access_method irq_pin_access_method; unsigned int contact_size; u8 config[GOODIX_CONFIG_MAX_LENGTH]; + unsigned short keymap[GOODIX_MAX_KEYS]; }; static int goodix_check_cfg_8(struct goodix_ts_data *ts, @@ -302,6 +305,13 @@ static int goodix_ts_read_input_report(struct goodix_ts_data *ts, u8 *data) unsigned long max_timeout; int touch_num; int error; + u16 addr = GOODIX_READ_COOR_ADDR; + /* + * We are going to read 1-byte header, + * ts->contact_size * max(1, touch_num) bytes of coordinates + * and 1-byte footer which contains the touch-key code. + */ + const int header_contact_keycode_size = 1 + ts->contact_size + 1; /* * The 'buffer status' bit, which indicates that the data is valid, is @@ -310,8 +320,8 @@ static int goodix_ts_read_input_report(struct goodix_ts_data *ts, u8 *data) */ max_timeout = jiffies + msecs_to_jiffies(GOODIX_BUFFER_STATUS_TIMEOUT); do { - error = goodix_i2c_read(ts->client, GOODIX_READ_COOR_ADDR, - data, ts->contact_size + 1); + error = goodix_i2c_read(ts->client, addr, data, + header_contact_keycode_size); if (error) { dev_err(&ts->client->dev, "I2C transfer error: %d\n", error); @@ -324,11 +334,10 @@ static int goodix_ts_read_input_report(struct goodix_ts_data *ts, u8 *data) return -EPROTO; if (touch_num > 1) { - data += 1 + ts->contact_size; + addr += header_contact_keycode_size; + data += header_contact_keycode_size; error = goodix_i2c_read(ts->client, - GOODIX_READ_COOR_ADDR + - 1 + ts->contact_size, - data, + addr, data, ts->contact_size * (touch_num - 1)); if (error) @@ -378,6 +387,25 @@ static void goodix_ts_report_touch_9b(struct goodix_ts_data *ts, u8 *coor_data) input_report_abs(ts->input_dev, ABS_MT_WIDTH_MAJOR, input_w); } +static void goodix_ts_report_key(struct goodix_ts_data *ts, u8 *data) +{ + int touch_num; + u8 key_value; + int i; + + if (data[0] & GOODIX_HAVE_KEY) { + touch_num = data[0] & 0x0f; + key_value = data[1 + ts->contact_size * touch_num]; + for (i = 0; i < GOODIX_MAX_KEYS; i++) + if (key_value & BIT(i)) + input_report_key(ts->input_dev, + ts->keymap[i], 1); + } else { + for (i = 0; i < GOODIX_MAX_KEYS; i++) + input_report_key(ts->input_dev, ts->keymap[i], 0); + } +} + /** * goodix_process_events - Process incoming events * @@ -388,7 +416,7 @@ static void goodix_ts_report_touch_9b(struct goodix_ts_data *ts, u8 *coor_data) */ static void goodix_process_events(struct goodix_ts_data *ts) { - u8 point_data[1 + GOODIX_MAX_CONTACT_SIZE * GOODIX_MAX_CONTACTS]; + u8 point_data[2 + GOODIX_MAX_CONTACT_SIZE * GOODIX_MAX_CONTACTS]; int touch_num; int i; @@ -396,11 +424,7 @@ static void goodix_process_events(struct goodix_ts_data *ts) if (touch_num < 0) return; - /* - * Bit 4 of the first byte reports the status of the capacitive - * Windows/Home button. - */ - input_report_key(ts->input_dev, KEY_LEFTMETA, point_data[0] & BIT(4)); + goodix_ts_report_key(ts, point_data); for (i = 0; i < touch_num; i++) if (ts->contact_size == 9) @@ -986,6 +1010,7 @@ static int goodix_i2c_test(struct i2c_client *client) static int goodix_configure_dev(struct goodix_ts_data *ts) { int error; + int i; ts->int_trigger_type = GOODIX_INT_TRIGGER; ts->max_touch_num = GOODIX_MAX_CONTACTS; @@ -1003,8 +1028,19 @@ static int goodix_configure_dev(struct goodix_ts_data *ts) ts->input_dev->id.product = ts->id; ts->input_dev->id.version = ts->version; + ts->input_dev->keycode = ts->keymap; + ts->input_dev->keycodesize = sizeof(ts->keymap[0]); + ts->input_dev->keycodemax = GOODIX_MAX_KEYS; + /* Capacitive Windows/Home button on some devices */ - input_set_capability(ts->input_dev, EV_KEY, KEY_LEFTMETA); + for (i = 0; i < GOODIX_MAX_KEYS; ++i) { + if (i == 0) + ts->keymap[i] = KEY_LEFTMETA; + else + ts->keymap[i] = KEY_F1 + (i - 1); + + input_set_capability(ts->input_dev, EV_KEY, ts->keymap[i]); + } input_set_capability(ts->input_dev, EV_ABS, ABS_MT_POSITION_X); input_set_capability(ts->input_dev, EV_ABS, ABS_MT_POSITION_Y); From 8c2327ef7999c00d69258f03ab35c280e4dcfe46 Mon Sep 17 00:00:00 2001 From: Icenowy Zheng Date: Tue, 24 Mar 2020 11:57:46 -0700 Subject: [PATCH 198/635] dt-bindings: input: touchscreen: add compatible string for Goodix GT917S Goodix GT917S is a new touchscreen chip from Goodix. Add its compatible string to the device tree binding. Signed-off-by: Icenowy Zheng Acked-by: Rob Herring Reviewed-by: Bastien Nocera Link: https://lore.kernel.org/r/20200228010146.12215-2-icenowy@aosc.io Signed-off-by: Dmitry Torokhov --- Documentation/devicetree/bindings/input/touchscreen/goodix.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/input/touchscreen/goodix.yaml b/Documentation/devicetree/bindings/input/touchscreen/goodix.yaml index d7c3262b2494..109e5ca8d6e8 100644 --- a/Documentation/devicetree/bindings/input/touchscreen/goodix.yaml +++ b/Documentation/devicetree/bindings/input/touchscreen/goodix.yaml @@ -21,6 +21,7 @@ properties: - goodix,gt911 - goodix,gt9110 - goodix,gt912 + - goodix,gt917s - goodix,gt927 - goodix,gt9271 - goodix,gt928 From cae102e2fc0832b0633fcb4e051cc1338b37145c Mon Sep 17 00:00:00 2001 From: Icenowy Zheng Date: Tue, 24 Mar 2020 11:57:58 -0700 Subject: [PATCH 199/635] Input: goodix - use string-based chip ID For Goodix GT917S chip, the chip ID string is "917S", which contains not only numbers now. Use string-based chip ID in the driver to support this chip and further chips with alphanumber ID. Signed-off-by: Icenowy Zheng Reviewed-by: Bastien Nocera Link: https://lore.kernel.org/r/20200228010146.12215-3-icenowy@aosc.io Signed-off-by: Dmitry Torokhov --- drivers/input/touchscreen/goodix.c | 67 +++++++++++++++++------------- 1 file changed, 39 insertions(+), 28 deletions(-) diff --git a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c index 9805372aba5d..ca5e82eb4e07 100644 --- a/drivers/input/touchscreen/goodix.c +++ b/drivers/input/touchscreen/goodix.c @@ -79,6 +79,13 @@ struct goodix_chip_data { void (*calc_config_checksum)(struct goodix_ts_data *ts); }; +struct goodix_chip_id { + const char *id; + const struct goodix_chip_data *data; +}; + +#define GOODIX_ID_MAX_LEN 4 + struct goodix_ts_data { struct i2c_client *client; struct input_dev *input_dev; @@ -92,7 +99,7 @@ struct goodix_ts_data { struct gpio_desc *gpiod_rst; int gpio_count; int gpio_int_idx; - u16 id; + char id[GOODIX_ID_MAX_LEN + 1]; u16 version; const char *cfg_name; bool reset_controller_at_probe; @@ -140,6 +147,22 @@ static const struct goodix_chip_data gt9x_chip_data = { .calc_config_checksum = goodix_calc_cfg_checksum_8, }; +static const struct goodix_chip_id goodix_chip_ids[] = { + { .id = "1151", .data = >1x_chip_data }, + { .id = "5663", .data = >1x_chip_data }, + { .id = "5688", .data = >1x_chip_data }, + + { .id = "911", .data = >911_chip_data }, + { .id = "9271", .data = >911_chip_data }, + { .id = "9110", .data = >911_chip_data }, + { .id = "927", .data = >911_chip_data }, + { .id = "928", .data = >911_chip_data }, + + { .id = "912", .data = >967_chip_data }, + { .id = "967", .data = >967_chip_data }, + { } +}; + static const unsigned long goodix_irq_flags[] = { IRQ_TYPE_EDGE_RISING, IRQ_TYPE_EDGE_FALLING, @@ -276,28 +299,16 @@ static int goodix_i2c_write_u8(struct i2c_client *client, u16 reg, u8 value) return goodix_i2c_write(client, reg, &value, sizeof(value)); } -static const struct goodix_chip_data *goodix_get_chip_data(u16 id) +static const struct goodix_chip_data *goodix_get_chip_data(const char *id) { - switch (id) { - case 1151: - case 5663: - case 5688: - return >1x_chip_data; + unsigned int i; - case 911: - case 9271: - case 9110: - case 927: - case 928: - return >911_chip_data; - - case 912: - case 967: - return >967_chip_data; - - default: - return >9x_chip_data; + for (i = 0; goodix_chip_ids[i].id; i++) { + if (!strcmp(goodix_chip_ids[i].id, id)) + return goodix_chip_ids[i].data; } + + return >9x_chip_data; } static int goodix_ts_read_input_report(struct goodix_ts_data *ts, u8 *data) @@ -951,7 +962,7 @@ static int goodix_read_version(struct goodix_ts_data *ts) { int error; u8 buf[6]; - char id_str[5]; + char id_str[GOODIX_ID_MAX_LEN + 1]; error = goodix_i2c_read(ts->client, GOODIX_REG_ID, buf, sizeof(buf)); if (error) { @@ -959,14 +970,13 @@ static int goodix_read_version(struct goodix_ts_data *ts) return error; } - memcpy(id_str, buf, 4); - id_str[4] = 0; - if (kstrtou16(id_str, 10, &ts->id)) - ts->id = 0x1001; + memcpy(id_str, buf, GOODIX_ID_MAX_LEN); + id_str[GOODIX_ID_MAX_LEN] = 0; + strscpy(ts->id, id_str, GOODIX_ID_MAX_LEN + 1); ts->version = get_unaligned_le16(&buf[4]); - dev_info(&ts->client->dev, "ID %d, version: %04x\n", ts->id, + dev_info(&ts->client->dev, "ID %s, version: %04x\n", ts->id, ts->version); return 0; @@ -1025,7 +1035,8 @@ static int goodix_configure_dev(struct goodix_ts_data *ts) ts->input_dev->phys = "input/ts"; ts->input_dev->id.bustype = BUS_I2C; ts->input_dev->id.vendor = 0x0416; - ts->input_dev->id.product = ts->id; + if (kstrtou16(ts->id, 10, &ts->input_dev->id.product)) + ts->input_dev->id.product = 0x1001; ts->input_dev->id.version = ts->version; ts->input_dev->keycode = ts->keymap; @@ -1228,7 +1239,7 @@ reset: if (ts->load_cfg_from_disk) { /* update device config */ ts->cfg_name = devm_kasprintf(&client->dev, GFP_KERNEL, - "goodix_%d_cfg.bin", ts->id); + "goodix_%s_cfg.bin", ts->id); if (!ts->cfg_name) return -ENOMEM; From b5e458af8a1447db78818fc273364ef88c2aa1dc Mon Sep 17 00:00:00 2001 From: Icenowy Zheng Date: Tue, 24 Mar 2020 12:25:58 -0700 Subject: [PATCH 200/635] Input: goodix - add support for Goodix GT917S Goodix GT917S is a touchscreen chip from Goodix that is in the GT1x family. Add its support by assigning the gt1x config to it. Signed-off-by: Icenowy Zheng Reviewed-by: Bastien Nocera Link: https://lore.kernel.org/r/20200228010146.12215-4-icenowy@aosc.io Signed-off-by: Dmitry Torokhov --- drivers/input/touchscreen/goodix.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c index ca5e82eb4e07..973fb636cafc 100644 --- a/drivers/input/touchscreen/goodix.c +++ b/drivers/input/touchscreen/goodix.c @@ -151,6 +151,7 @@ static const struct goodix_chip_id goodix_chip_ids[] = { { .id = "1151", .data = >1x_chip_data }, { .id = "5663", .data = >1x_chip_data }, { .id = "5688", .data = >1x_chip_data }, + { .id = "917S", .data = >1x_chip_data }, { .id = "911", .data = >911_chip_data }, { .id = "9271", .data = >911_chip_data }, @@ -1397,6 +1398,7 @@ static const struct of_device_id goodix_of_match[] = { { .compatible = "goodix,gt911" }, { .compatible = "goodix,gt9110" }, { .compatible = "goodix,gt912" }, + { .compatible = "goodix,gt917s" }, { .compatible = "goodix,gt927" }, { .compatible = "goodix,gt9271" }, { .compatible = "goodix,gt928" }, From ec7f23e1d8adcd18dc3fc18c3b06daa681fe1e7e Mon Sep 17 00:00:00 2001 From: Yannick Fertre Date: Tue, 24 Mar 2020 12:30:19 -0700 Subject: [PATCH 201/635] dt-bindings: touchscreen: goodix: support of gt9147 Add support for it by adding compatible. The chip data on GT9147 is similar to GT912, like - config data register has 0x8047 address - config data register max len is 240 - config data checksum has 8-bit Signed-off-by: Yannick Fertre Acked-by: Rob Herring Link: https://lore.kernel.org/r/1583144308-3781-2-git-send-email-yannick.fertre@st.com Signed-off-by: Dmitry Torokhov --- Documentation/devicetree/bindings/input/touchscreen/goodix.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/input/touchscreen/goodix.yaml b/Documentation/devicetree/bindings/input/touchscreen/goodix.yaml index 109e5ca8d6e8..18cdaf0a90c1 100644 --- a/Documentation/devicetree/bindings/input/touchscreen/goodix.yaml +++ b/Documentation/devicetree/bindings/input/touchscreen/goodix.yaml @@ -21,6 +21,7 @@ properties: - goodix,gt911 - goodix,gt9110 - goodix,gt912 + - goodix,gt9147 - goodix,gt917s - goodix,gt927 - goodix,gt9271 From b9a1c116846e9ab9d4872d61604207cd5aef09b4 Mon Sep 17 00:00:00 2001 From: Yannick Fertre Date: Tue, 24 Mar 2020 12:32:27 -0700 Subject: [PATCH 202/635] Input: goodix - support gt9147 touchpanel Add support for it by adding compatible and supported chip data (default settings used). The chip data on GT9147 is similar to GT912, like - config data register has 0x8047 address - config data register max len is 240 - config data checksum has 8-bit Signed-off-by: Yannick Fertre Reviewed-by: Bastien Nocera Link: https://lore.kernel.org/r/1583144308-3781-3-git-send-email-yannick.fertre@st.com Signed-off-by: Dmitry Torokhov --- drivers/input/touchscreen/goodix.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c index 973fb636cafc..47f812b804c8 100644 --- a/drivers/input/touchscreen/goodix.c +++ b/drivers/input/touchscreen/goodix.c @@ -160,6 +160,7 @@ static const struct goodix_chip_id goodix_chip_ids[] = { { .id = "928", .data = >911_chip_data }, { .id = "912", .data = >967_chip_data }, + { .id = "9147", .data = >967_chip_data }, { .id = "967", .data = >967_chip_data }, { } }; @@ -1398,6 +1399,7 @@ static const struct of_device_id goodix_of_match[] = { { .compatible = "goodix,gt911" }, { .compatible = "goodix,gt9110" }, { .compatible = "goodix,gt912" }, + { .compatible = "goodix,gt9147" }, { .compatible = "goodix,gt917s" }, { .compatible = "goodix,gt927" }, { .compatible = "goodix,gt9271" }, From cc5117d6e4988e684f68ce5b5db312a3f0f943df Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 6 Mar 2020 17:11:46 -0800 Subject: [PATCH 203/635] Input: of_touchscreen - explicitly choose axis The 'axis + 1' calculation is implicit and potentially error prone. Moreover, few lines before the axis is set explicitly for both X and Y. Do the same when retrieving different properties for X and Y. Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20200303180917.12563-1-andriy.shevchenko@linux.intel.com Signed-off-by: Dmitry Torokhov --- drivers/input/touchscreen/of_touchscreen.c | 35 +++++++++++----------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/drivers/input/touchscreen/of_touchscreen.c b/drivers/input/touchscreen/of_touchscreen.c index e16ec4c7043a..97342e14b4f1 100644 --- a/drivers/input/touchscreen/of_touchscreen.c +++ b/drivers/input/touchscreen/of_touchscreen.c @@ -66,7 +66,7 @@ void touchscreen_parse_properties(struct input_dev *input, bool multitouch, { struct device *dev = input->dev.parent; struct input_absinfo *absinfo; - unsigned int axis; + unsigned int axis, axis_x, axis_y; unsigned int minimum, maximum, fuzz; bool data_present; @@ -74,33 +74,34 @@ void touchscreen_parse_properties(struct input_dev *input, bool multitouch, if (!input->absinfo) return; - axis = multitouch ? ABS_MT_POSITION_X : ABS_X; + axis_x = multitouch ? ABS_MT_POSITION_X : ABS_X; + axis_y = multitouch ? ABS_MT_POSITION_Y : ABS_Y; + data_present = touchscreen_get_prop_u32(dev, "touchscreen-min-x", - input_abs_get_min(input, axis), + input_abs_get_min(input, axis_x), &minimum) | touchscreen_get_prop_u32(dev, "touchscreen-size-x", input_abs_get_max(input, - axis) + 1, + axis_x) + 1, &maximum) | touchscreen_get_prop_u32(dev, "touchscreen-fuzz-x", - input_abs_get_fuzz(input, axis), + input_abs_get_fuzz(input, axis_x), &fuzz); if (data_present) - touchscreen_set_params(input, axis, minimum, maximum - 1, fuzz); + touchscreen_set_params(input, axis_x, minimum, maximum - 1, fuzz); - axis = multitouch ? ABS_MT_POSITION_Y : ABS_Y; data_present = touchscreen_get_prop_u32(dev, "touchscreen-min-y", - input_abs_get_min(input, axis), + input_abs_get_min(input, axis_y), &minimum) | touchscreen_get_prop_u32(dev, "touchscreen-size-y", input_abs_get_max(input, - axis) + 1, + axis_y) + 1, &maximum) | touchscreen_get_prop_u32(dev, "touchscreen-fuzz-y", - input_abs_get_fuzz(input, axis), + input_abs_get_fuzz(input, axis_y), &fuzz); if (data_present) - touchscreen_set_params(input, axis, minimum, maximum - 1, fuzz); + touchscreen_set_params(input, axis_y, minimum, maximum - 1, fuzz); axis = multitouch ? ABS_MT_PRESSURE : ABS_PRESSURE; data_present = touchscreen_get_prop_u32(dev, @@ -117,15 +118,13 @@ void touchscreen_parse_properties(struct input_dev *input, bool multitouch, if (!prop) return; - axis = multitouch ? ABS_MT_POSITION_X : ABS_X; - - prop->max_x = input_abs_get_max(input, axis); - prop->max_y = input_abs_get_max(input, axis + 1); + prop->max_x = input_abs_get_max(input, axis_x); + prop->max_y = input_abs_get_max(input, axis_y); prop->invert_x = device_property_read_bool(dev, "touchscreen-inverted-x"); if (prop->invert_x) { - absinfo = &input->absinfo[axis]; + absinfo = &input->absinfo[axis_x]; absinfo->maximum -= absinfo->minimum; absinfo->minimum = 0; } @@ -133,7 +132,7 @@ void touchscreen_parse_properties(struct input_dev *input, bool multitouch, prop->invert_y = device_property_read_bool(dev, "touchscreen-inverted-y"); if (prop->invert_y) { - absinfo = &input->absinfo[axis + 1]; + absinfo = &input->absinfo[axis_y]; absinfo->maximum -= absinfo->minimum; absinfo->minimum = 0; } @@ -141,7 +140,7 @@ void touchscreen_parse_properties(struct input_dev *input, bool multitouch, prop->swap_x_y = device_property_read_bool(dev, "touchscreen-swapped-x-y"); if (prop->swap_x_y) - swap(input->absinfo[axis], input->absinfo[axis + 1]); + swap(input->absinfo[axis_x], input->absinfo[axis_y]); } EXPORT_SYMBOL(touchscreen_parse_properties); From 10ad4845cd3d760c191ab4ecef446b99a6b2c779 Mon Sep 17 00:00:00 2001 From: Benjamin Gaignard Date: Sat, 15 Feb 2020 15:47:48 -0800 Subject: [PATCH 204/635] dt-bindings: touchscreen: Convert edt-ft5x06 to json-schema Convert the EDT-FT5x06 to DT schema using json-schema. Signed-off-by: Benjamin Gaignard Reviewed-by: Rob Herring Link: https://lore.kernel.org/r/20200207084657.31195-1-benjamin.gaignard@st.com Signed-off-by: Dmitry Torokhov --- .../bindings/input/touchscreen/edt-ft5x06.txt | 77 ----------- .../input/touchscreen/edt-ft5x06.yaml | 125 ++++++++++++++++++ 2 files changed, 125 insertions(+), 77 deletions(-) delete mode 100644 Documentation/devicetree/bindings/input/touchscreen/edt-ft5x06.txt create mode 100644 Documentation/devicetree/bindings/input/touchscreen/edt-ft5x06.yaml diff --git a/Documentation/devicetree/bindings/input/touchscreen/edt-ft5x06.txt b/Documentation/devicetree/bindings/input/touchscreen/edt-ft5x06.txt deleted file mode 100644 index 0e57315e9cbd..000000000000 --- a/Documentation/devicetree/bindings/input/touchscreen/edt-ft5x06.txt +++ /dev/null @@ -1,77 +0,0 @@ -FocalTech EDT-FT5x06 Polytouch driver -===================================== - -There are 5 variants of the chip for various touch panel sizes -FT5206GE1 2.8" .. 3.8" -FT5306DE4 4.3" .. 7" -FT5406EE8 7" .. 8.9" -FT5506EEG 7" .. 8.9" -FT5726NEI 5.7” .. 11.6" - -The software interface is identical for all those chips, so that -currently there is no need for the driver to distinguish between the -different chips. Nevertheless distinct compatible strings are used so -that a distinction can be added if necessary without changing the DT -bindings. - - -Required properties: - - compatible: "edt,edt-ft5206" - or: "edt,edt-ft5306" - or: "edt,edt-ft5406" - or: "edt,edt-ft5506" - or: "evervision,ev-ft5726" - or: "focaltech,ft6236" - - - reg: I2C slave address of the chip (0x38) - - interrupts: interrupt specification for the touchdetect - interrupt - -Optional properties: - - reset-gpios: GPIO specification for the RESET input - - wake-gpios: GPIO specification for the WAKE input - - vcc-supply: Regulator that supplies the touchscreen - - - pinctrl-names: should be "default" - - pinctrl-0: a phandle pointing to the pin settings for the - control gpios - - - wakeup-source: If present the device will act as wakeup-source - - - threshold: allows setting the "click"-threshold in the range - from 0 to 80. - - - gain: allows setting the sensitivity in the range from 0 to - 31. Note that lower values indicate higher - sensitivity. - - - offset: allows setting the edge compensation in the range from - 0 to 31. - - - offset-x: Same as offset, but applies only to the horizontal position. - Range from 0 to 80, only supported by evervision,ev-ft5726 - devices. - - - offset-y: Same as offset, but applies only to the vertical position. - Range from 0 to 80, only supported by evervision,ev-ft5726 - devices. - - - touchscreen-size-x : See touchscreen.txt - - touchscreen-size-y : See touchscreen.txt - - touchscreen-fuzz-x : See touchscreen.txt - - touchscreen-fuzz-y : See touchscreen.txt - - touchscreen-inverted-x : See touchscreen.txt - - touchscreen-inverted-y : See touchscreen.txt - - touchscreen-swapped-x-y : See touchscreen.txt - -Example: - polytouch: edt-ft5x06@38 { - compatible = "edt,edt-ft5406", "edt,edt-ft5x06"; - reg = <0x38>; - pinctrl-names = "default"; - pinctrl-0 = <&edt_ft5x06_pins>; - interrupt-parent = <&gpio2>; - interrupts = <5 IRQ_TYPE_EDGE_FALLING>; - reset-gpios = <&gpio2 6 GPIO_ACTIVE_LOW>; - wake-gpios = <&gpio4 9 GPIO_ACTIVE_HIGH>; - }; diff --git a/Documentation/devicetree/bindings/input/touchscreen/edt-ft5x06.yaml b/Documentation/devicetree/bindings/input/touchscreen/edt-ft5x06.yaml new file mode 100644 index 000000000000..8d58709d4b47 --- /dev/null +++ b/Documentation/devicetree/bindings/input/touchscreen/edt-ft5x06.yaml @@ -0,0 +1,125 @@ +# SPDX-License-Identifier: GPL-2.0 +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/input/touchscreen/edt-ft5x06.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: FocalTech EDT-FT5x06 Polytouch Bindings + +description: | + There are 5 variants of the chip for various touch panel sizes + FT5206GE1 2.8" .. 3.8" + FT5306DE4 4.3" .. 7" + FT5406EE8 7" .. 8.9" + FT5506EEG 7" .. 8.9" + FT5726NEI 5.7” .. 11.6" + +maintainers: + - Dmitry Torokhov + +allOf: + - $ref: touchscreen.yaml# + - if: + properties: + compatible: + contains: + enum: + - evervision,ev-ft5726 + + then: + properties: + offset-x: true + offset-y: true + +properties: + compatible: + enum: + - edt,edt-ft5206 + - edt,edt-ft5306 + - edt,edt-ft5406 + - edt,edt-ft5506 + - evervision,ev-ft5726 + - focaltech,ft6236 + + reg: + const: 0x38 + + interrupts: + maxItems: 1 + + reset-gpios: + maxItems: 1 + + wake-gpios: + maxItems: 1 + + wakeup-source: true + + vcc-supply: + maxItems: 1 + + gain: + description: Allows setting the sensitivity in the range from 0 to 31. + Note that lower values indicate higher sensitivity. + allOf: + - $ref: /schemas/types.yaml#/definitions/uint32 + - minimum: 0 + - maximum: 31 + + offset: + description: Allows setting the edge compensation in the range from 0 to 31. + allOf: + - $ref: /schemas/types.yaml#/definitions/uint32 + - minimum: 0 + - maximum: 31 + + offset-x: + description: Same as offset, but applies only to the horizontal position. + Range from 0 to 80, only supported by evervision,ev-ft5726 devices. + allOf: + - $ref: /schemas/types.yaml#/definitions/uint32 + - minimum: 0 + - maximum: 80 + + offset-y: + description: Same as offset, but applies only to the vertical position. + Range from 0 to 80, only supported by evervision,ev-ft5726 devices. + allOf: + - $ref: /schemas/types.yaml#/definitions/uint32 + - minimum: 0 + - maximum: 80 + + touchscreen-size-x: true + touchscreen-size-y: true + touchscreen-fuzz-x: true + touchscreen-fuzz-y: true + touchscreen-inverted-x: true + touchscreen-inverted-y: true + touchscreen-swapped-x-y: true + interrupt-controller: true + +additionalProperties: false + +required: + - compatible + - reg + - interrupts + +examples: + - | + #include + #include + i2c@00000000 { + #address-cells = <1>; + #size-cells = <0>; + edt-ft5x06@38 { + compatible = "edt,edt-ft5406"; + reg = <0x38>; + interrupt-parent = <&gpio2>; + interrupts = <5 IRQ_TYPE_EDGE_FALLING>; + reset-gpios = <&gpio2 6 GPIO_ACTIVE_LOW>; + wake-gpios = <&gpio4 9 GPIO_ACTIVE_HIGH>; + }; + }; + +... From ee0c8e494cc3c135350cd5c4752e82af3feae1ab Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Thu, 20 Feb 2020 10:00:32 +0100 Subject: [PATCH 205/635] backlight: corgi: Convert to use GPIO descriptors The code in the Corgi backlight driver can be considerably simplified by moving to GPIO descriptors and lookup tables from the board files instead of passing GPIO numbers using the old API. Make sure to encode inversion semantics for the Akita and Spitz platforms inside the GPIO lookup table and drop the custom inversion semantics from the driver. All in-tree users are converted in this patch. Signed-off-by: Linus Walleij Acked-by: Robert Jarzmik Reviewed-by: Daniel Thompson Signed-off-by: Lee Jones --- arch/arm/mach-pxa/corgi.c | 12 +++++- arch/arm/mach-pxa/spitz.c | 34 +++++++++++---- drivers/video/backlight/corgi_lcd.c | 64 ++++++++--------------------- include/linux/spi/corgi_lcd.h | 3 -- 4 files changed, 52 insertions(+), 61 deletions(-) diff --git a/arch/arm/mach-pxa/corgi.c b/arch/arm/mach-pxa/corgi.c index f2d73289230f..593c7f793da5 100644 --- a/arch/arm/mach-pxa/corgi.c +++ b/arch/arm/mach-pxa/corgi.c @@ -563,13 +563,20 @@ static void corgi_bl_kick_battery(void) } } +static struct gpiod_lookup_table corgi_lcdcon_gpio_table = { + .dev_id = "spi1.1", + .table = { + GPIO_LOOKUP("gpio-pxa", CORGI_GPIO_BACKLIGHT_CONT, + "BL_CONT", GPIO_ACTIVE_HIGH), + { }, + }, +}; + static struct corgi_lcd_platform_data corgi_lcdcon_info = { .init_mode = CORGI_LCD_MODE_VGA, .max_intensity = 0x2f, .default_intensity = 0x1f, .limit_mask = 0x0b, - .gpio_backlight_cont = CORGI_GPIO_BACKLIGHT_CONT, - .gpio_backlight_on = -1, .kick_battery = corgi_bl_kick_battery, }; @@ -609,6 +616,7 @@ static struct spi_board_info corgi_spi_devices[] = { static void __init corgi_init_spi(void) { pxa2xx_set_spi_info(1, &corgi_spi_info); + gpiod_add_lookup_table(&corgi_lcdcon_gpio_table); spi_register_board_info(ARRAY_AND_SIZE(corgi_spi_devices)); } #else diff --git a/arch/arm/mach-pxa/spitz.c b/arch/arm/mach-pxa/spitz.c index a4fdc399d152..371008e9bb02 100644 --- a/arch/arm/mach-pxa/spitz.c +++ b/arch/arm/mach-pxa/spitz.c @@ -525,13 +525,33 @@ static void spitz_bl_kick_battery(void) } } +static struct gpiod_lookup_table spitz_lcdcon_gpio_table = { + .dev_id = "spi2.1", + .table = { + GPIO_LOOKUP("gpio-pxa", SPITZ_GPIO_BACKLIGHT_CONT, + "BL_CONT", GPIO_ACTIVE_LOW), + GPIO_LOOKUP("gpio-pxa", SPITZ_GPIO_BACKLIGHT_ON, + "BL_ON", GPIO_ACTIVE_HIGH), + { }, + }, +}; + +static struct gpiod_lookup_table akita_lcdcon_gpio_table = { + .dev_id = "spi2.1", + .table = { + GPIO_LOOKUP("gpio-pxa", AKITA_GPIO_BACKLIGHT_CONT, + "BL_CONT", GPIO_ACTIVE_LOW), + GPIO_LOOKUP("gpio-pxa", AKITA_GPIO_BACKLIGHT_ON, + "BL_ON", GPIO_ACTIVE_HIGH), + { }, + }, +}; + static struct corgi_lcd_platform_data spitz_lcdcon_info = { .init_mode = CORGI_LCD_MODE_VGA, .max_intensity = 0x2f, .default_intensity = 0x1f, .limit_mask = 0x0b, - .gpio_backlight_cont = SPITZ_GPIO_BACKLIGHT_CONT, - .gpio_backlight_on = SPITZ_GPIO_BACKLIGHT_ON, .kick_battery = spitz_bl_kick_battery, }; @@ -574,12 +594,10 @@ static struct pxa2xx_spi_controller spitz_spi_info = { static void __init spitz_spi_init(void) { - struct corgi_lcd_platform_data *lcd_data = &spitz_lcdcon_info; - - if (machine_is_akita()) { - lcd_data->gpio_backlight_cont = AKITA_GPIO_BACKLIGHT_CONT; - lcd_data->gpio_backlight_on = AKITA_GPIO_BACKLIGHT_ON; - } + if (machine_is_akita()) + gpiod_add_lookup_table(&akita_lcdcon_gpio_table); + else + gpiod_add_lookup_table(&spitz_lcdcon_gpio_table); pxa2xx_set_spi_info(2, &spitz_spi_info); spi_register_board_info(ARRAY_AND_SIZE(spitz_spi_devices)); diff --git a/drivers/video/backlight/corgi_lcd.c b/drivers/video/backlight/corgi_lcd.c index 68f7592c5060..25ef0cbd7583 100644 --- a/drivers/video/backlight/corgi_lcd.c +++ b/drivers/video/backlight/corgi_lcd.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include #include @@ -90,9 +90,8 @@ struct corgi_lcd { int mode; char buf[2]; - int gpio_backlight_on; - int gpio_backlight_cont; - int gpio_backlight_cont_inverted; + struct gpio_desc *backlight_on; + struct gpio_desc *backlight_cont; void (*kick_battery)(void); }; @@ -403,13 +402,13 @@ static int corgi_bl_set_intensity(struct corgi_lcd *lcd, int intensity) corgi_ssp_lcdtg_send(lcd, DUTYCTRL_ADRS, intensity); /* Bit 5 via GPIO_BACKLIGHT_CONT */ - cont = !!(intensity & 0x20) ^ lcd->gpio_backlight_cont_inverted; + cont = !!(intensity & 0x20); - if (gpio_is_valid(lcd->gpio_backlight_cont)) - gpio_set_value_cansleep(lcd->gpio_backlight_cont, cont); + if (lcd->backlight_cont) + gpiod_set_value_cansleep(lcd->backlight_cont, cont); - if (gpio_is_valid(lcd->gpio_backlight_on)) - gpio_set_value_cansleep(lcd->gpio_backlight_on, intensity); + if (lcd->backlight_on) + gpiod_set_value_cansleep(lcd->backlight_on, intensity); if (lcd->kick_battery) lcd->kick_battery(); @@ -482,48 +481,17 @@ static int setup_gpio_backlight(struct corgi_lcd *lcd, struct corgi_lcd_platform_data *pdata) { struct spi_device *spi = lcd->spi_dev; - int err; - lcd->gpio_backlight_on = -1; - lcd->gpio_backlight_cont = -1; + lcd->backlight_on = devm_gpiod_get_optional(&spi->dev, + "BL_ON", GPIOD_OUT_LOW); + if (IS_ERR(lcd->backlight_on)) + return PTR_ERR(lcd->backlight_on); - if (gpio_is_valid(pdata->gpio_backlight_on)) { - err = devm_gpio_request(&spi->dev, pdata->gpio_backlight_on, - "BL_ON"); - if (err) { - dev_err(&spi->dev, - "failed to request GPIO%d for backlight_on\n", - pdata->gpio_backlight_on); - return err; - } + lcd->backlight_cont = devm_gpiod_get_optional(&spi->dev, "BL_CONT", + GPIOD_OUT_LOW); + if (IS_ERR(lcd->backlight_cont)) + return PTR_ERR(lcd->backlight_cont); - lcd->gpio_backlight_on = pdata->gpio_backlight_on; - gpio_direction_output(lcd->gpio_backlight_on, 0); - } - - if (gpio_is_valid(pdata->gpio_backlight_cont)) { - err = devm_gpio_request(&spi->dev, pdata->gpio_backlight_cont, - "BL_CONT"); - if (err) { - dev_err(&spi->dev, - "failed to request GPIO%d for backlight_cont\n", - pdata->gpio_backlight_cont); - return err; - } - - lcd->gpio_backlight_cont = pdata->gpio_backlight_cont; - - /* spitz and akita use both GPIOs for backlight, and - * have inverted polarity of GPIO_BACKLIGHT_CONT - */ - if (gpio_is_valid(lcd->gpio_backlight_on)) { - lcd->gpio_backlight_cont_inverted = 1; - gpio_direction_output(lcd->gpio_backlight_cont, 1); - } else { - lcd->gpio_backlight_cont_inverted = 0; - gpio_direction_output(lcd->gpio_backlight_cont, 0); - } - } return 0; } diff --git a/include/linux/spi/corgi_lcd.h b/include/linux/spi/corgi_lcd.h index edf4beccdadb..0b857616919c 100644 --- a/include/linux/spi/corgi_lcd.h +++ b/include/linux/spi/corgi_lcd.h @@ -11,9 +11,6 @@ struct corgi_lcd_platform_data { int default_intensity; int limit_mask; - int gpio_backlight_on; /* -1 if n/a */ - int gpio_backlight_cont; /* -1 if n/a */ - void (*notify)(int intensity); void (*kick_battery)(void); }; From 89c8023fd46167a41246a56b31d1b3c9a20b6970 Mon Sep 17 00:00:00 2001 From: Liwei Song Date: Wed, 25 Mar 2020 11:50:13 +0800 Subject: [PATCH 206/635] nfsroot: set tcp as the default transport protocol UDP is disabled by default in commit b24ee6c64ca7 ("NFS: allow deprecation of NFS UDP protocol"), but the default mount options is still udp, change it to tcp to avoid the "Unsupported transport protocol udp" error if no protocol is specified when mount nfs. Fixes: b24ee6c64ca7 ("NFS: allow deprecation of NFS UDP protocol") Signed-off-by: Liwei Song Signed-off-by: Trond Myklebust --- fs/nfs/nfsroot.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/nfs/nfsroot.c b/fs/nfs/nfsroot.c index effaa4247b91..8d3278805602 100644 --- a/fs/nfs/nfsroot.c +++ b/fs/nfs/nfsroot.c @@ -88,7 +88,7 @@ #define NFS_ROOT "/tftpboot/%s" /* Default NFSROOT mount options. */ -#define NFS_DEF_OPTIONS "vers=2,udp,rsize=4096,wsize=4096" +#define NFS_DEF_OPTIONS "vers=2,tcp,rsize=4096,wsize=4096" /* Parameters passed from the kernel command line */ static char nfs_root_parms[NFS_MAXPATHLEN + 1] __initdata = ""; From aa3367c91d360db4dc7cbd458c05a6a631aa7af1 Mon Sep 17 00:00:00 2001 From: Petr Vorel Date: Tue, 24 Mar 2020 21:08:49 +0100 Subject: [PATCH 207/635] NFS: Don't specify NFS version in "UDP not supported" error UDP was originally disabled in 6da1a034362f for NFSv4. Later in b24ee6c64ca7 UDP is by default disabled by NFS_DISABLE_UDP_SUPPORT=y for all NFS versions. Therefore remove v4 from error message. Fixes: b24ee6c64ca7 ("NFS: allow deprecation of NFS UDP protocol") Signed-off-by: Petr Vorel Signed-off-by: Trond Myklebust --- fs/nfs/fs_context.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/nfs/fs_context.c b/fs/nfs/fs_context.c index e113fcb4bb4c..566dd59570e6 100644 --- a/fs/nfs/fs_context.c +++ b/fs/nfs/fs_context.c @@ -1135,7 +1135,7 @@ out_no_address: return nfs_invalf(fc, "NFS4: mount program didn't pass remote address"); out_invalid_transport_udp: - return nfs_invalf(fc, "NFSv4: Unsupported transport protocol udp"); + return nfs_invalf(fc, "NFS: Unsupported transport protocol udp"); } #endif @@ -1257,7 +1257,7 @@ out_v4_not_compiled: nfs_errorf(fc, "NFS: NFSv4 is not compiled into kernel"); return -EPROTONOSUPPORT; out_invalid_transport_udp: - return nfs_invalf(fc, "NFSv4: Unsupported transport protocol udp"); + return nfs_invalf(fc, "NFS: Unsupported transport protocol udp"); out_no_address: return nfs_invalf(fc, "NFS: mount program didn't pass remote address"); out_mountproto_mismatch: From 518110cd0dedd7685ca365f464dd62c21c0a99e4 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Tue, 24 Mar 2020 13:54:42 +0100 Subject: [PATCH 208/635] dma-buf: Improve CONFIG_DMABUF_MOVE_NOTIFY help text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Improve the help text for the CONFIG_DMABUF_MOVE_NOTIFY symbol by: 1. Removing duplicated single quotes, 2. Adding a missing subject, 3. Fixing a misspelling of "yet", 4. Wrapping long lines. Fixes: bb42df4662a44765 ("dma-buf: add dynamic DMA-buf handling v15") Reviewed-by: Christian König Signed-off-by: Geert Uytterhoeven Signed-off-by: Alex Deucher Link: https://patchwork.freedesktop.org/patch/msgid/20200324125442.21983-1-geert+renesas@glider.be --- drivers/dma-buf/Kconfig | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/dma-buf/Kconfig b/drivers/dma-buf/Kconfig index ef73b678419c..9626673f1d83 100644 --- a/drivers/dma-buf/Kconfig +++ b/drivers/dma-buf/Kconfig @@ -43,11 +43,12 @@ config DMABUF_MOVE_NOTIFY bool "Move notify between drivers (EXPERIMENTAL)" default n help - Don''t pin buffers if the dynamic DMA-buf interface is available on both the - exporter as well as the importer. This fixes a security problem where - userspace is able to pin unrestricted amounts of memory through DMA-buf. - But marked experimental because we don''t jet have a consistent execution - context and memory management between drivers. + Don't pin buffers if the dynamic DMA-buf interface is available on + both the exporter as well as the importer. This fixes a security + problem where userspace is able to pin unrestricted amounts of memory + through DMA-buf. + This is marked experimental because we don't yet have a consistent + execution context and memory management between drivers. config DMABUF_SELFTESTS tristate "Selftests for the dma-buf interfaces" From 99f155d0776fb8838e326efce056aa08e25433d0 Mon Sep 17 00:00:00 2001 From: Sam Ravnborg Date: Sat, 14 Mar 2020 16:30:45 +0100 Subject: [PATCH 209/635] dt-bindings: display: drop data-mapping from panel-dpi data-mapping may not be the best way to describe the data format used between panels and display interface. Drop it from the panel-dpi binding so we do not start to rely on it. We can then work out how to best describe this mapping and when we know it, we can add it to this binding. Fixes: fa10224087f1 ("dt-bindings: display: add data-mapping to panel-dpi") Cc: Sam Ravnborg Cc: Rob Herring Cc: Maxime Ripard Cc: Thierry Reding Cc: Laurent Pinchart Cc: dri-devel@lists.freedesktop.org Signed-off-by: Sam Ravnborg Reviewed-by: Laurent Pinchart Link: https://patchwork.freedesktop.org/patch/msgid/20200314153047.2486-2-sam@ravnborg.org --- .../devicetree/bindings/display/panel/panel-dpi.yaml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/Documentation/devicetree/bindings/display/panel/panel-dpi.yaml b/Documentation/devicetree/bindings/display/panel/panel-dpi.yaml index f63870384c00..0cd74c8dab42 100644 --- a/Documentation/devicetree/bindings/display/panel/panel-dpi.yaml +++ b/Documentation/devicetree/bindings/display/panel/panel-dpi.yaml @@ -21,15 +21,6 @@ properties: - {} - const: panel-dpi - data-mapping: - enum: - - rgb24 - - rgb565 - - bgr666 - description: | - Describes the media format, how the display panel is connected - to the display interface. - backlight: true enable-gpios: true height-mm: true @@ -52,7 +43,6 @@ examples: compatible = "osddisplays,osd057T0559-34ts", "panel-dpi"; label = "osddisplay"; power-supply = <&vcc_supply>; - data-mapping = "rgb565"; backlight = <&backlight>; port { From d021d751c14752a0266865700f6f212fab40a18c Mon Sep 17 00:00:00 2001 From: Sam Ravnborg Date: Sat, 14 Mar 2020 16:30:46 +0100 Subject: [PATCH 210/635] drm/panel-simple: drop use of data-mapping property The "data-mapping" property may not be the best way to describe the interface between panels and display interfaces. Drop use of in the panel-simple driver, so we have time to find the right way to describe this interface. Fixes: 4a1d0dbc8332 ("drm/panel: simple: add panel-dpi support") Cc: Sam Ravnborg Cc: Oleksandr Suvorov Cc: Maxime Ripard Cc: Thierry Reding Cc: Laurent Pinchart Cc: dri-devel@lists.freedesktop.org Cc: Rob Herring Signed-off-by: Sam Ravnborg Reviewed-by: Laurent Pinchart Link: https://patchwork.freedesktop.org/patch/msgid/20200314153047.2486-3-sam@ravnborg.org --- drivers/gpu/drm/panel/panel-simple.c | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c index 0ce81b1f36af..3ad828eaefe1 100644 --- a/drivers/gpu/drm/panel/panel-simple.c +++ b/drivers/gpu/drm/panel/panel-simple.c @@ -361,7 +361,6 @@ static int panel_dpi_probe(struct device *dev, struct panel_desc *desc; unsigned int bus_flags; struct videomode vm; - const char *mapping; int ret; np = dev->of_node; @@ -386,16 +385,6 @@ static int panel_dpi_probe(struct device *dev, of_property_read_u32(np, "width-mm", &desc->size.width); of_property_read_u32(np, "height-mm", &desc->size.height); - of_property_read_string(np, "data-mapping", &mapping); - if (!strcmp(mapping, "rgb24")) - desc->bus_format = MEDIA_BUS_FMT_RGB888_1X24; - else if (!strcmp(mapping, "rgb565")) - desc->bus_format = MEDIA_BUS_FMT_RGB565_1X16; - else if (!strcmp(mapping, "bgr666")) - desc->bus_format = MEDIA_BUS_FMT_RGB666_1X18; - else if (!strcmp(mapping, "lvds666")) - desc->bus_format = MEDIA_BUS_FMT_RGB666_1X24_CPADHI; - /* Extract bus_flags from display_timing */ bus_flags = 0; vm.flags = timing->flags; From 4b566406085bc1fd5b6ba309fae70e6f6d6fa94e Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Thu, 19 Mar 2020 14:50:46 -0500 Subject: [PATCH 211/635] ACPI: NFIT: Replace zero-length array with flexible-array member The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertently introduced[3] to the codebase from now on. Also, notice that, dynamic memory allocations won't be affected by this change: "Flexible array members have incomplete type, and so the sizeof operator may not be applied. As a quirk of the original implementation of zero-length arrays, sizeof evaluates to zero."[1] This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Link: https://lore.kernel.org/r/20200319195046.GA452@embeddedor.com Signed-off-by: Dan Williams --- drivers/acpi/nfit/nfit.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/acpi/nfit/nfit.h b/drivers/acpi/nfit/nfit.h index 24241941181c..af09143ce403 100644 --- a/drivers/acpi/nfit/nfit.h +++ b/drivers/acpi/nfit/nfit.h @@ -144,32 +144,32 @@ struct nfit_spa { unsigned long ars_state; u32 clear_err_unit; u32 max_ars; - struct acpi_nfit_system_address spa[0]; + struct acpi_nfit_system_address spa[]; }; struct nfit_dcr { struct list_head list; - struct acpi_nfit_control_region dcr[0]; + struct acpi_nfit_control_region dcr[]; }; struct nfit_bdw { struct list_head list; - struct acpi_nfit_data_region bdw[0]; + struct acpi_nfit_data_region bdw[]; }; struct nfit_idt { struct list_head list; - struct acpi_nfit_interleave idt[0]; + struct acpi_nfit_interleave idt[]; }; struct nfit_flush { struct list_head list; - struct acpi_nfit_flush_address flush[0]; + struct acpi_nfit_flush_address flush[]; }; struct nfit_memdev { struct list_head list; - struct acpi_nfit_memory_map memdev[0]; + struct acpi_nfit_memory_map memdev[]; }; enum nfit_mem_flags { From df513a7711712758b9cb1a48d86712e7e1ee03f4 Mon Sep 17 00:00:00 2001 From: Olga Kornievskaia Date: Thu, 26 Mar 2020 10:24:51 -0400 Subject: [PATCH 212/635] SUNRPC: fix krb5p mount to provide large enough buffer in rq_rcvsize Ever since commit 2c94b8eca1a2 ("SUNRPC: Use au_rslack when computing reply buffer size"). It changed how "req->rq_rcvsize" is calculated. It used to use au_cslack value which was nice and large and changed it to au_rslack value which turns out to be too small. Since 5.1, v3 mount with sec=krb5p fails against an Ontap server because client's receive buffer it too small. For gss krb5p, we need to account for the mic token in the verifier, and the wrap token in the wrap token. RFC 4121 defines: mic token Octet no Name Description -------------------------------------------------------------- 0..1 TOK_ID Identification field. Tokens emitted by GSS_GetMIC() contain the hex value 04 04 expressed in big-endian order in this field. 2 Flags Attributes field, as described in section 4.2.2. 3..7 Filler Contains five octets of hex value FF. 8..15 SND_SEQ Sequence number field in clear text, expressed in big-endian order. 16..last SGN_CKSUM Checksum of the "to-be-signed" data and octet 0..15, as described in section 4.2.4. that's 16bytes (GSS_KRB5_TOK_HDR_LEN) + chksum wrap token Octet no Name Description -------------------------------------------------------------- 0..1 TOK_ID Identification field. Tokens emitted by GSS_Wrap() contain the hex value 05 04 expressed in big-endian order in this field. 2 Flags Attributes field, as described in section 4.2.2. 3 Filler Contains the hex value FF. 4..5 EC Contains the "extra count" field, in big- endian order as described in section 4.2.3. 6..7 RRC Contains the "right rotation count" in big- endian order, as described in section 4.2.5. 8..15 SND_SEQ Sequence number field in clear text, expressed in big-endian order. 16..last Data Encrypted data for Wrap tokens with confidentiality, or plaintext data followed by the checksum for Wrap tokens without confidentiality, as described in section 4.2.4. Also 16bytes of header (GSS_KRB5_TOK_HDR_LEN), encrypted data, and cksum (other things like padding) RFC 3961 defines known cksum sizes: Checksum type sumtype checksum section or value size reference --------------------------------------------------------------------- CRC32 1 4 6.1.3 rsa-md4 2 16 6.1.2 rsa-md4-des 3 24 6.2.5 des-mac 4 16 6.2.7 des-mac-k 5 8 6.2.8 rsa-md4-des-k 6 16 6.2.6 rsa-md5 7 16 6.1.1 rsa-md5-des 8 24 6.2.4 rsa-md5-des3 9 24 ?? sha1 (unkeyed) 10 20 ?? hmac-sha1-des3-kd 12 20 6.3 hmac-sha1-des3 13 20 ?? sha1 (unkeyed) 14 20 ?? hmac-sha1-96-aes128 15 20 [KRB5-AES] hmac-sha1-96-aes256 16 20 [KRB5-AES] [reserved] 0x8003 ? [GSS-KRB5] Linux kernel now mainly supports type 15,16 so max cksum size is 20bytes. (GSS_KRB5_MAX_CKSUM_LEN) Re-use already existing define of GSS_KRB5_MAX_SLACK_NEEDED that's used for encoding the gss_wrap tokens (same tokens are used in reply). Fixes: 2c94b8eca1a2 ("SUNRPC: Use au_rslack when computing reply buffer size") Signed-off-by: Olga Kornievskaia Reviewed-by: Chuck Lever Signed-off-by: Trond Myklebust --- net/sunrpc/auth_gss/auth_gss.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/net/sunrpc/auth_gss/auth_gss.c b/net/sunrpc/auth_gss/auth_gss.c index 45707a306f20..cfee4954f821 100644 --- a/net/sunrpc/auth_gss/auth_gss.c +++ b/net/sunrpc/auth_gss/auth_gss.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -1050,7 +1051,7 @@ gss_create_new(const struct rpc_auth_create_args *args, struct rpc_clnt *clnt) goto err_put_mech; auth = &gss_auth->rpc_auth; auth->au_cslack = GSS_CRED_SLACK >> 2; - auth->au_rslack = GSS_VERF_SLACK >> 2; + auth->au_rslack = GSS_KRB5_MAX_SLACK_NEEDED >> 2; auth->au_verfsize = GSS_VERF_SLACK >> 2; auth->au_ralign = GSS_VERF_SLACK >> 2; auth->au_flags = 0; From 329651b1f1965cdc0a66df5717d3ac2fccddc740 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Thu, 19 Mar 2020 10:13:05 -0400 Subject: [PATCH 213/635] pNFS/flexfiles: Simplify allocation of the mirror array Just allocate the array at the end of the layout segment structure, instead of allocating it as a separate array of pointers. Signed-off-by: Trond Myklebust --- fs/nfs/flexfilelayout/flexfilelayout.c | 21 +++++---------------- fs/nfs/flexfilelayout/flexfilelayout.h | 2 +- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c index e7d8ae4d0cc5..19728206e9c6 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.c +++ b/fs/nfs/flexfilelayout/flexfilelayout.c @@ -248,18 +248,10 @@ static void ff_layout_put_mirror(struct nfs4_ff_layout_mirror *mirror) static void ff_layout_free_mirror_array(struct nfs4_ff_layout_segment *fls) { - int i; + u32 i; - if (fls->mirror_array) { - for (i = 0; i < fls->mirror_array_cnt; i++) { - /* normally mirror_ds is freed in - * .free_deviceid_node but we still do it here - * for .alloc_lseg error path */ - ff_layout_put_mirror(fls->mirror_array[i]); - } - kfree(fls->mirror_array); - fls->mirror_array = NULL; - } + for (i = 0; i < fls->mirror_array_cnt; i++) + ff_layout_put_mirror(fls->mirror_array[i]); } static int ff_layout_check_layout(struct nfs4_layoutget_res *lgr) @@ -400,16 +392,13 @@ ff_layout_alloc_lseg(struct pnfs_layout_hdr *lh, goto out_err_free; rc = -ENOMEM; - fls = kzalloc(sizeof(*fls), gfp_flags); + fls = kzalloc(struct_size(fls, mirror_array, mirror_array_cnt), + gfp_flags); if (!fls) goto out_err_free; fls->mirror_array_cnt = mirror_array_cnt; fls->stripe_unit = stripe_unit; - fls->mirror_array = kcalloc(fls->mirror_array_cnt, - sizeof(fls->mirror_array[0]), gfp_flags); - if (fls->mirror_array == NULL) - goto out_err_free; for (i = 0; i < fls->mirror_array_cnt; i++) { struct nfs4_ff_layout_mirror *mirror; diff --git a/fs/nfs/flexfilelayout/flexfilelayout.h b/fs/nfs/flexfilelayout/flexfilelayout.h index 2f369966abf7..354a031c69b1 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.h +++ b/fs/nfs/flexfilelayout/flexfilelayout.h @@ -99,7 +99,7 @@ struct nfs4_ff_layout_segment { u64 stripe_unit; u32 flags; u32 mirror_array_cnt; - struct nfs4_ff_layout_mirror **mirror_array; + struct nfs4_ff_layout_mirror *mirror_array[]; }; struct nfs4_flexfile_layout { From 19573c939a854483c7c44e939821776db251596b Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Thu, 19 Mar 2020 17:29:12 -0400 Subject: [PATCH 214/635] NFS/pNFS: Refactor pnfs_generic_commit_pagelist() Refactor pnfs_generic_commit_pagelist() to simplify the conversion to layout segment based commit lists. Signed-off-by: Trond Myklebust --- fs/nfs/pnfs_nfs.c | 168 +++++++++++++++++++--------------------------- fs/nfs/write.c | 13 ++-- 2 files changed, 78 insertions(+), 103 deletions(-) diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c index 8b37e7f8e789..3d0942541618 100644 --- a/fs/nfs/pnfs_nfs.c +++ b/fs/nfs/pnfs_nfs.c @@ -156,103 +156,86 @@ restart: } EXPORT_SYMBOL_GPL(pnfs_generic_recover_commit_reqs); -static void pnfs_generic_retry_commit(struct nfs_commit_info *cinfo, int idx) +static struct pnfs_layout_segment * +pnfs_bucket_get_committing(struct list_head *head, + struct pnfs_commit_bucket *bucket, + struct nfs_commit_info *cinfo) { - struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds; - struct pnfs_commit_bucket *bucket; struct pnfs_layout_segment *freeme; struct list_head *pos; - LIST_HEAD(pages); - int i; - mutex_lock(&NFS_I(cinfo->inode)->commit_mutex); - for (i = idx; i < fl_cinfo->nbuckets; i++) { - bucket = &fl_cinfo->buckets[i]; + list_for_each(pos, &bucket->committing) + cinfo->ds->ncommitting--; + list_splice_init(&bucket->committing, head); + freeme = bucket->clseg; + bucket->clseg = NULL; + return freeme; +} + +static struct nfs_commit_data * +pnfs_bucket_fetch_commitdata(struct pnfs_commit_bucket *bucket, + struct nfs_commit_info *cinfo) +{ + struct nfs_commit_data *data = nfs_commitdata_alloc(false); + + if (!data) + return NULL; + data->lseg = pnfs_bucket_get_committing(&data->pages, bucket, cinfo); + return data; +} + +static void pnfs_generic_retry_commit(struct pnfs_commit_bucket *buckets, + unsigned int nbuckets, + struct nfs_commit_info *cinfo, + unsigned int idx) +{ + struct pnfs_commit_bucket *bucket; + struct pnfs_layout_segment *freeme; + LIST_HEAD(pages); + + for (bucket = buckets; idx < nbuckets; bucket++, idx++) { if (list_empty(&bucket->committing)) continue; - freeme = bucket->clseg; - bucket->clseg = NULL; - list_for_each(pos, &bucket->committing) - cinfo->ds->ncommitting--; - list_splice_init(&bucket->committing, &pages); - mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex); - nfs_retry_commit(&pages, freeme, cinfo, i); - pnfs_put_lseg(freeme); mutex_lock(&NFS_I(cinfo->inode)->commit_mutex); + freeme = pnfs_bucket_get_committing(&pages, bucket, cinfo); + mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex); + nfs_retry_commit(&pages, freeme, cinfo, idx); + pnfs_put_lseg(freeme); } - mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex); } static unsigned int -pnfs_generic_alloc_ds_commits(struct nfs_commit_info *cinfo, - struct list_head *list) +pnfs_bucket_alloc_ds_commits(struct list_head *list, + struct pnfs_commit_bucket *buckets, + unsigned int nbuckets, + struct nfs_commit_info *cinfo) { - struct pnfs_ds_commit_info *fl_cinfo; struct pnfs_commit_bucket *bucket; struct nfs_commit_data *data; - int i; + unsigned int i; unsigned int nreq = 0; - fl_cinfo = cinfo->ds; - bucket = fl_cinfo->buckets; - for (i = 0; i < fl_cinfo->nbuckets; i++, bucket++) { + for (i = 0, bucket = buckets; i < nbuckets; i++, bucket++) { if (list_empty(&bucket->committing)) continue; - data = nfs_commitdata_alloc(false); - if (!data) - break; - data->ds_commit_index = i; - list_add(&data->pages, list); - nreq++; + mutex_lock(&NFS_I(cinfo->inode)->commit_mutex); + if (!list_empty(&bucket->committing)) { + data = pnfs_bucket_fetch_commitdata(bucket, cinfo); + if (!data) + goto out_error; + data->ds_commit_index = i; + list_add_tail(&data->list, list); + atomic_inc(&cinfo->mds->rpcs_out); + nreq++; + } + mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex); } - - /* Clean up on error */ - pnfs_generic_retry_commit(cinfo, i); return nreq; -} - -static inline -void pnfs_fetch_commit_bucket_list(struct list_head *pages, - struct nfs_commit_data *data, - struct nfs_commit_info *cinfo) -{ - struct pnfs_commit_bucket *bucket; - struct list_head *pos; - - bucket = &cinfo->ds->buckets[data->ds_commit_index]; - mutex_lock(&NFS_I(cinfo->inode)->commit_mutex); - list_for_each(pos, &bucket->committing) - cinfo->ds->ncommitting--; - list_splice_init(&bucket->committing, pages); - data->lseg = bucket->clseg; - bucket->clseg = NULL; +out_error: mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex); - -} - -/* Helper function for pnfs_generic_commit_pagelist to catch an empty - * page list. This can happen when two commits race. - * - * This must be called instead of nfs_init_commit - call one or the other, but - * not both! - */ -static bool -pnfs_generic_commit_cancel_empty_pagelist(struct list_head *pages, - struct nfs_commit_data *data, - struct nfs_commit_info *cinfo) -{ - if (list_empty(pages)) { - if (atomic_dec_and_test(&cinfo->mds->rpcs_out)) - wake_up_var(&cinfo->mds->rpcs_out); - /* don't call nfs_commitdata_release - it tries to put - * the open_context which is not acquired until nfs_init_commit - * which has not been called on @data */ - WARN_ON_ONCE(data->context); - nfs_commit_free(data); - return true; - } - - return false; + /* Clean up on error */ + pnfs_generic_retry_commit(buckets, nbuckets, cinfo, i); + return nreq; } /* This follows nfs_commit_list pretty closely */ @@ -262,6 +245,7 @@ pnfs_generic_commit_pagelist(struct inode *inode, struct list_head *mds_pages, int (*initiate_commit)(struct nfs_commit_data *data, int how)) { + struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds; struct nfs_commit_data *data, *tmp; LIST_HEAD(list); unsigned int nreq = 0; @@ -269,40 +253,26 @@ pnfs_generic_commit_pagelist(struct inode *inode, struct list_head *mds_pages, if (!list_empty(mds_pages)) { data = nfs_commitdata_alloc(true); data->ds_commit_index = -1; - list_add(&data->pages, &list); + list_splice_init(mds_pages, &data->pages); + list_add_tail(&data->list, &list); + atomic_inc(&cinfo->mds->rpcs_out); nreq++; } - nreq += pnfs_generic_alloc_ds_commits(cinfo, &list); - + nreq += pnfs_bucket_alloc_ds_commits(&list, fl_cinfo->buckets, + fl_cinfo->nbuckets, cinfo); if (nreq == 0) goto out; - atomic_add(nreq, &cinfo->mds->rpcs_out); - - list_for_each_entry_safe(data, tmp, &list, pages) { - list_del_init(&data->pages); + list_for_each_entry_safe(data, tmp, &list, list) { + list_del(&data->list); if (data->ds_commit_index < 0) { - /* another commit raced with us */ - if (pnfs_generic_commit_cancel_empty_pagelist(mds_pages, - data, cinfo)) - continue; - - nfs_init_commit(data, mds_pages, NULL, cinfo); + nfs_init_commit(data, NULL, NULL, cinfo); nfs_initiate_commit(NFS_CLIENT(inode), data, NFS_PROTO(data->inode), data->mds_ops, how, 0); } else { - LIST_HEAD(pages); - - pnfs_fetch_commit_bucket_list(&pages, data, cinfo); - - /* another commit raced with us */ - if (pnfs_generic_commit_cancel_empty_pagelist(&pages, - data, cinfo)) - continue; - - nfs_init_commit(data, &pages, data->lseg, cinfo); + nfs_init_commit(data, NULL, data->lseg, cinfo); initiate_commit(data, how); } } diff --git a/fs/nfs/write.c b/fs/nfs/write.c index 5544ee6cfda8..1f8108f5a041 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -1746,14 +1746,19 @@ void nfs_init_commit(struct nfs_commit_data *data, struct pnfs_layout_segment *lseg, struct nfs_commit_info *cinfo) { - struct nfs_page *first = nfs_list_entry(head->next); - struct nfs_open_context *ctx = nfs_req_openctx(first); - struct inode *inode = d_inode(ctx->dentry); + struct nfs_page *first; + struct nfs_open_context *ctx; + struct inode *inode; /* Set up the RPC argument and reply structs * NB: take care not to mess about with data->commit et al. */ - list_splice_init(head, &data->pages); + if (head) + list_splice_init(head, &data->pages); + + first = nfs_list_entry(data->pages.next); + ctx = nfs_req_openctx(first); + inode = d_inode(ctx->dentry); data->inode = inode; data->cred = ctx->cred; From d7242c4641fba521a1ea9dbccb11a40cf38cd912 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Wed, 18 Mar 2020 17:22:47 -0400 Subject: [PATCH 215/635] pNFS: Add a helper to allocate the array of buckets Signed-off-by: Trond Myklebust --- fs/nfs/pnfs.h | 3 +++ fs/nfs/pnfs_nfs.c | 31 +++++++++++++++++++++++++++++++ include/linux/nfs_xdr.h | 15 ++++++++++++--- 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h index 7bfb6970134a..f6b1099aa151 100644 --- a/fs/nfs/pnfs.h +++ b/fs/nfs/pnfs.h @@ -366,6 +366,9 @@ bool nfs4_test_deviceid_unavailable(struct nfs4_deviceid_node *node); void nfs4_deviceid_purge_client(const struct nfs_client *); /* pnfs_nfs.c */ +struct pnfs_commit_array *pnfs_alloc_commit_array(size_t n, gfp_t gfp_flags); +void pnfs_free_commit_array(struct pnfs_commit_array *p); + void pnfs_generic_clear_request_commit(struct nfs_page *req, struct nfs_commit_info *cinfo); void pnfs_generic_commit_release(void *calldata); diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c index 3d0942541618..c8518ce3a4ef 100644 --- a/fs/nfs/pnfs_nfs.c +++ b/fs/nfs/pnfs_nfs.c @@ -87,6 +87,37 @@ out: } EXPORT_SYMBOL_GPL(pnfs_generic_clear_request_commit); +struct pnfs_commit_array * +pnfs_alloc_commit_array(size_t n, gfp_t gfp_flags) +{ + struct pnfs_commit_array *p; + struct pnfs_commit_bucket *b; + + p = kmalloc(struct_size(p, buckets, n), gfp_flags); + if (!p) + return NULL; + p->nbuckets = n; + INIT_LIST_HEAD(&p->cinfo_list); + INIT_LIST_HEAD(&p->lseg_list); + p->lseg = NULL; + for (b = &p->buckets[0]; n != 0; b++, n--) { + INIT_LIST_HEAD(&b->written); + INIT_LIST_HEAD(&b->committing); + b->wlseg = NULL; + b->clseg = NULL; + b->direct_verf.committed = NFS_INVALID_STABLE_HOW; + } + return p; +} +EXPORT_SYMBOL_GPL(pnfs_alloc_commit_array); + +void +pnfs_free_commit_array(struct pnfs_commit_array *p) +{ + kfree_rcu(p, rcu); +} +EXPORT_SYMBOL_GPL(pnfs_free_commit_array); + static int pnfs_generic_scan_ds_commit_list(struct pnfs_commit_bucket *bucket, struct nfs_commit_info *cinfo, diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index 94c77ed55ce1..e91c917c9c1c 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -1270,10 +1270,19 @@ struct pnfs_commit_bucket { struct nfs_writeverf direct_verf; }; +struct pnfs_commit_array { + struct list_head cinfo_list; + struct list_head lseg_list; + struct pnfs_layout_segment *lseg; + struct rcu_head rcu; + unsigned int nbuckets; + struct pnfs_commit_bucket buckets[]; +}; + struct pnfs_ds_commit_info { - int nwritten; - int ncommitting; - int nbuckets; + unsigned int nwritten; + unsigned int ncommitting; + unsigned int nbuckets; struct pnfs_commit_bucket *buckets; }; From a65a97b48694d34248195eb89bf3687403261056 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Wed, 25 Mar 2020 15:43:10 +0100 Subject: [PATCH 216/635] drm/vboxvideo: Add missing remove_conflicting_pci_framebuffers call, v2 The vboxvideo driver is missing a call to remove conflicting framebuffers. Surprisingly, when using legacy BIOS booting this does not really cause any issues. But when using UEFI to boot the VM then plymouth will draw on both the efifb /dev/fb0 and /dev/drm/card0 (which has registered /dev/fb1 as fbdev emulation). VirtualBox will actual display the output of both devices (I guess it is showing whatever was drawn last), this causes weird artifacts because of pitch issues in the efifb when the VM window is not sized at 1024x768 (the window will resize to its last size once the vboxvideo driver loads, changing the pitch). Adding the missing drm_fb_helper_remove_conflicting_pci_framebuffers() call fixes this. Changes in v2: -Make the drm_fb_helper_remove_conflicting_pci_framebuffers() call one of the first things we do in our probe() method Cc: stable@vger.kernel.org Fixes: 2695eae1f6d3 ("drm/vboxvideo: Switch to generic fbdev emulation") Signed-off-by: Hans de Goede Reviewed-by: Daniel Vetter Link: https://patchwork.freedesktop.org/patch/msgid/20200325144310.36779-1-hdegoede@redhat.com --- drivers/gpu/drm/vboxvideo/vbox_drv.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/gpu/drm/vboxvideo/vbox_drv.c b/drivers/gpu/drm/vboxvideo/vbox_drv.c index 8512d970a09f..ac8f75db2ecd 100644 --- a/drivers/gpu/drm/vboxvideo/vbox_drv.c +++ b/drivers/gpu/drm/vboxvideo/vbox_drv.c @@ -41,6 +41,10 @@ static int vbox_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (!vbox_check_supported(VBE_DISPI_ID_HGSMI)) return -ENODEV; + ret = drm_fb_helper_remove_conflicting_pci_framebuffers(pdev, "vboxvideodrmfb"); + if (ret) + return ret; + vbox = kzalloc(sizeof(*vbox), GFP_KERNEL); if (!vbox) return -ENOMEM; From eb0ff8074e0baecba2cd0c7813f6cfa99bafc430 Mon Sep 17 00:00:00 2001 From: Tina Zhang Date: Tue, 24 Mar 2020 20:30:21 +0800 Subject: [PATCH 217/635] drm/i915/gvt: Fix klocwork issues about data size Add llu suffix and cast operator to fix the klocwork warning about "Operands in a bitwise operation have different size" Signed-off-by: Tina Zhang Acked-by: Zhenyu Wang Signed-off-by: Zhenyu Wang Link: http://patchwork.freedesktop.org/patch/msgid/20200324123021.15831-1-tina.zhang@intel.com --- drivers/gpu/drm/i915/gvt/scheduler.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c index e1e6345700cc..9d67f33f37a0 100644 --- a/drivers/gpu/drm/i915/gvt/scheduler.c +++ b/drivers/gpu/drm/i915/gvt/scheduler.c @@ -296,8 +296,8 @@ shadow_context_descriptor_update(struct intel_context *ce, * Update bits 0-11 of the context descriptor which includes flags * like GEN8_CTX_* cached in desc_template */ - desc &= ~(0x3 << GEN8_CTX_ADDRESSING_MODE_SHIFT); - desc |= workload->ctx_desc.addressing_mode << + desc &= ~(0x3ull << GEN8_CTX_ADDRESSING_MODE_SHIFT); + desc |= (u64)workload->ctx_desc.addressing_mode << GEN8_CTX_ADDRESSING_MODE_SHIFT; ce->lrc_desc = desc; From d8a3c488c6d393eaac4418b5fc4d57c36310a92b Mon Sep 17 00:00:00 2001 From: Jeff LaBundy Date: Sun, 16 Feb 2020 17:32:05 -0600 Subject: [PATCH 218/635] dt-bindings: Add bindings for Azoteq IQS620A/621/622/624/625 This patch adds device tree bindings for the Azoteq IQS620A, IQS621, IQS622, IQS624 and IQS625 multi-function sensors. A total of three bindings are presented (one MFD and two child nodes); they are submitted as a single patch because the child node bindings have no meaning in the absence of the MFD binding. Signed-off-by: Jeff LaBundy Reviewed-by: Rob Herring Signed-off-by: Lee Jones --- .../bindings/input/iqs62x-keys.yaml | 132 +++++++++++++ .../devicetree/bindings/mfd/iqs62x.yaml | 179 ++++++++++++++++++ .../devicetree/bindings/pwm/iqs620a-pwm.yaml | 32 ++++ 3 files changed, 343 insertions(+) create mode 100644 Documentation/devicetree/bindings/input/iqs62x-keys.yaml create mode 100644 Documentation/devicetree/bindings/mfd/iqs62x.yaml create mode 100644 Documentation/devicetree/bindings/pwm/iqs620a-pwm.yaml diff --git a/Documentation/devicetree/bindings/input/iqs62x-keys.yaml b/Documentation/devicetree/bindings/input/iqs62x-keys.yaml new file mode 100644 index 000000000000..5625c222903a --- /dev/null +++ b/Documentation/devicetree/bindings/input/iqs62x-keys.yaml @@ -0,0 +1,132 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/input/iqs62x-keys.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Azoteq IQS620A/621/622/624/625 Keys and Switches + +maintainers: + - Jeff LaBundy + +description: | + The Azoteq IQS620A, IQS621, IQS622, IQS624 and IQS625 multi-function sensors + feature a variety of self-capacitive, mutual-inductive and Hall-effect sens- + ing capabilities that can facilitate a variety of contactless key and switch + applications. + + These functions are collectively represented by a "keys" child node from the + parent MFD driver. See Documentation/devicetree/bindings/mfd/iqs62x.yaml for + further details and examples. Sensor hardware configuration (self-capacitive + vs. mutual-inductive, etc.) is selected based on the device's firmware. + +properties: + compatible: + enum: + - azoteq,iqs620a-keys + - azoteq,iqs621-keys + - azoteq,iqs622-keys + - azoteq,iqs624-keys + - azoteq,iqs625-keys + + linux,keycodes: + allOf: + - $ref: /schemas/types.yaml#/definitions/uint32-array + - minItems: 1 + maxItems: 16 + description: | + Specifies the numeric keycodes associated with each available touch or + proximity event according to the following table. An 'x' indicates the + event is supported for a given device. Specify 0 for unused events. + + ------------------------------------------------------------------------- + | # | Event | IQS620A | IQS621 | IQS622 | IQS624 | IQS625 | + ------------------------------------------------------------------------- + | 0 | CH0 Touch | x | x | x | x | x | + | | Antenna 1 Touch* | x | | | | | + ------------------------------------------------------------------------- + | 1 | CH0 Proximity | x | x | x | x | x | + | | Antenna 1 Prox.* | x | | | | | + ------------------------------------------------------------------------- + | 2 | CH1 Touch | x | x | x | x | x | + | | Ant. 1 Deep Touch* | x | | | | | + ------------------------------------------------------------------------- + | 3 | CH1 Proximity | x | x | x | x | x | + ------------------------------------------------------------------------- + | 4 | CH2 Touch | x | | | | | + ------------------------------------------------------------------------- + | 5 | CH2 Proximity | x | | | | | + | | Antenna 2 Prox.* | x | | | | | + ------------------------------------------------------------------------- + | 6 | Metal (+) Touch** | x | x | | | | + | | Ant. 2 Deep Touch* | x | | | | | + ------------------------------------------------------------------------- + | 7 | Metal (+) Prox.** | x | x | | | | + | | Antenna 2 Touch* | x | | | | | + ------------------------------------------------------------------------- + | 8 | Metal (-) Touch** | x | x | | | | + ------------------------------------------------------------------------- + | 9 | Metal (-) Prox.** | x | x | | | | + ------------------------------------------------------------------------- + | 10 | SAR Active*** | x | | x | | | + ------------------------------------------------------------------------- + | 11 | SAR Quick Rel.*** | x | | x | | | + ------------------------------------------------------------------------- + | 12 | SAR Movement*** | x | | x | | | + ------------------------------------------------------------------------- + | 13 | SAR Filter Halt*** | x | | x | | | + ------------------------------------------------------------------------- + | 14 | Wheel Up | | | | x | | + ------------------------------------------------------------------------- + | 15 | Wheel Down | | | | x | | + ------------------------------------------------------------------------- + * Two-channel SAR. Replaces CH0-2 plus metal touch and proximity events + if enabled via firmware. + ** "+" and "-" refer to the polarity of a channel's delta (LTA - counts), + where "LTA" is defined as the channel's long-term average. + *** One-channel SAR. Replaces CH0-2 touch and proximity events if enabled + via firmware. + +patternProperties: + "^hall-switch-(north|south)$": + type: object + description: + Represents north/south-field Hall-effect sensor touch or proximity + events. Note that north/south-field orientation is reversed on the + IQS620AXzCSR device due to its flip-chip package. + + properties: + linux,code: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Numeric switch code associated with the event. + + azoteq,use-prox: + $ref: /schemas/types.yaml#/definitions/flag + description: + If present, specifies that Hall-effect sensor reporting should + use the device's wide-range proximity threshold instead of its + close-range touch threshold (default). + + required: + - linux,code + + additionalProperties: false + +if: + properties: + compatible: + contains: + enum: + - azoteq,iqs624-keys + - azoteq,iqs625-keys +then: + patternProperties: + "^hall-switch-(north|south)$": false + +required: + - compatible + - linux,keycodes + +additionalProperties: false + +... diff --git a/Documentation/devicetree/bindings/mfd/iqs62x.yaml b/Documentation/devicetree/bindings/mfd/iqs62x.yaml new file mode 100644 index 000000000000..541b06d80e73 --- /dev/null +++ b/Documentation/devicetree/bindings/mfd/iqs62x.yaml @@ -0,0 +1,179 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mfd/iqs62x.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Azoteq IQS620A/621/622/624/625 Multi-Function Sensors + +maintainers: + - Jeff LaBundy + +description: | + The Azoteq IQS620A, IQS621, IQS622, IQS624 and IQS625 multi-function sensors + integrate multiple sensing technologies in a single package. + + Link to datasheets: https://www.azoteq.com/ + +properties: + compatible: + enum: + - azoteq,iqs620a + - azoteq,iqs621 + - azoteq,iqs622 + - azoteq,iqs624 + - azoteq,iqs625 + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + firmware-name: + $ref: /schemas/types.yaml#/definitions/string + description: + Specifies the name of the calibration and configuration file selected by + the driver. If this property is omitted, the name is chosen based on the + device name with ".bin" as the extension (e.g. iqs620a.bin for IQS620A). + + keys: + $ref: ../input/iqs62x-keys.yaml + + pwm: + $ref: ../pwm/iqs620a-pwm.yaml + +required: + - compatible + - reg + - interrupts + +additionalProperties: false + +examples: + - | + /* + * Dual capacitive buttons with proximity-activated function, unipolar lid + * switch and panel-mounted LED. + */ + #include + #include + + i2c { + #address-cells = <1>; + #size-cells = <0>; + + iqs620a@44 { + compatible = "azoteq,iqs620a"; + reg = <0x44>; + interrupt-parent = <&gpio>; + interrupts = <17 IRQ_TYPE_LEVEL_LOW>; + + keys { + compatible = "azoteq,iqs620a-keys"; + + linux,keycodes = , + , + , + ; + + hall-switch-south { + linux,code = ; + azoteq,use-prox; + }; + }; + + iqs620a_pwm: pwm { + compatible = "azoteq,iqs620a-pwm"; + #pwm-cells = <2>; + }; + }; + }; + + pwmleds { + compatible = "pwm-leds"; + + panel { + pwms = <&iqs620a_pwm 0 1000000>; + max-brightness = <255>; + }; + }; + + - | + /* Single inductive button with bipolar dock/tablet-mode switch. */ + #include + #include + + i2c { + #address-cells = <1>; + #size-cells = <0>; + + iqs620a@44 { + compatible = "azoteq,iqs620a"; + reg = <0x44>; + interrupt-parent = <&gpio>; + interrupts = <17 IRQ_TYPE_LEVEL_LOW>; + + firmware-name = "iqs620a_coil.bin"; + + keys { + compatible = "azoteq,iqs620a-keys"; + + linux,keycodes = <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + ; + + hall-switch-north { + linux,code = ; + }; + + hall-switch-south { + linux,code = ; + }; + }; + }; + }; + + - | + /* Dual capacitive buttons with volume knob. */ + #include + #include + + i2c { + #address-cells = <1>; + #size-cells = <0>; + + iqs624@44 { + compatible = "azoteq,iqs624"; + reg = <0x44>; + interrupt-parent = <&gpio>; + interrupts = <17 IRQ_TYPE_LEVEL_LOW>; + + keys { + compatible = "azoteq,iqs624-keys"; + + linux,keycodes = , + <0>, + , + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + , + ; + }; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/pwm/iqs620a-pwm.yaml b/Documentation/devicetree/bindings/pwm/iqs620a-pwm.yaml new file mode 100644 index 000000000000..1d7c27be50da --- /dev/null +++ b/Documentation/devicetree/bindings/pwm/iqs620a-pwm.yaml @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pwm/iqs620a-pwm.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Azoteq IQS620A PWM Generator + +maintainers: + - Jeff LaBundy + +description: | + The Azoteq IQS620A multi-function sensor generates a fixed-frequency PWM + output represented by a "pwm" child node from the parent MFD driver. See + Documentation/devicetree/bindings/mfd/iqs62x.yaml for further details as + well as an example. + +properties: + compatible: + enum: + - azoteq,iqs620a-pwm + + "#pwm-cells": + const: 2 + +required: + - compatible + - "#pwm-cells" + +additionalProperties: false + +... From 4d9cf7df8d355e519adb8b2f8759c84e1e633070 Mon Sep 17 00:00:00 2001 From: Jeff LaBundy Date: Sun, 16 Feb 2020 17:32:06 -0600 Subject: [PATCH 219/635] mfd: Add support for Azoteq IQS620A/621/622/624/625 This patch adds core support for the Azoteq IQS620A, IQS621, IQS622, IQS624 and IQS625 multi-function sensors. Signed-off-by: Jeff LaBundy Signed-off-by: Lee Jones --- drivers/mfd/Kconfig | 13 + drivers/mfd/Makefile | 1 + drivers/mfd/iqs62x.c | 1063 ++++++++++++++++++++++++++++++++++++ include/linux/mfd/iqs62x.h | 139 +++++ 4 files changed, 1216 insertions(+) create mode 100644 drivers/mfd/iqs62x.c create mode 100644 include/linux/mfd/iqs62x.h diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig index 2b203290e7b9..daefcb6310f9 100644 --- a/drivers/mfd/Kconfig +++ b/drivers/mfd/Kconfig @@ -642,6 +642,19 @@ config MFD_IPAQ_MICRO AT90LS8535 microcontroller flashed with a special iPAQ firmware using the custom protocol implemented in this driver. +config MFD_IQS62X + tristate "Azoteq IQS620A/621/622/624/625 core support" + depends on I2C + select MFD_CORE + select REGMAP_I2C + help + Say Y here if you want to build core support for the Azoteq IQS620A, + IQS621, IQS622, IQS624 and IQS625 multi-function sensors. Additional + options must be selected to enable device-specific functions. + + To compile this driver as a module, choose M here: the module will + be called iqs62x. + config MFD_JANZ_CMODIO tristate "Janz CMOD-IO PCI MODULbus Carrier Board" select MFD_CORE diff --git a/drivers/mfd/Makefile b/drivers/mfd/Makefile index b83f172545e1..f935d10cbf0f 100644 --- a/drivers/mfd/Makefile +++ b/drivers/mfd/Makefile @@ -226,6 +226,7 @@ obj-$(CONFIG_MFD_AS3711) += as3711.o obj-$(CONFIG_MFD_AS3722) += as3722.o obj-$(CONFIG_MFD_STW481X) += stw481x.o obj-$(CONFIG_MFD_IPAQ_MICRO) += ipaq-micro.o +obj-$(CONFIG_MFD_IQS62X) += iqs62x.o obj-$(CONFIG_MFD_MENF21BMC) += menf21bmc.o obj-$(CONFIG_MFD_HI6421_PMIC) += hi6421-pmic-core.o obj-$(CONFIG_MFD_HI655X_PMIC) += hi655x-pmic.o diff --git a/drivers/mfd/iqs62x.c b/drivers/mfd/iqs62x.c new file mode 100644 index 000000000000..af764bc87d7c --- /dev/null +++ b/drivers/mfd/iqs62x.c @@ -0,0 +1,1063 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Azoteq IQS620A/621/622/624/625 Multi-Function Sensors + * + * Copyright (C) 2019 Jeff LaBundy + * + * These devices rely on application-specific register settings and calibration + * data developed in and exported from a suite of GUIs offered by the vendor. A + * separate tool converts the GUIs' ASCII-based output into a standard firmware + * file parsed by the driver. + * + * Link to datasheets and GUIs: https://www.azoteq.com/ + * + * Link to conversion tool: https://github.com/jlabundy/iqs62x-h2bin.git + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define IQS62X_PROD_NUM 0x00 + +#define IQS62X_SYS_FLAGS 0x10 +#define IQS62X_SYS_FLAGS_IN_ATI BIT(2) + +#define IQS620_HALL_FLAGS 0x16 +#define IQS621_HALL_FLAGS 0x19 +#define IQS622_HALL_FLAGS IQS621_HALL_FLAGS + +#define IQS624_INTERVAL_NUM 0x18 +#define IQS625_INTERVAL_NUM 0x12 + +#define IQS622_PROX_SETTINGS_4 0x48 +#define IQS620_PROX_SETTINGS_4 0x50 +#define IQS620_PROX_SETTINGS_4_SAR_EN BIT(7) + +#define IQS621_ALS_CAL_DIV_LUX 0x82 +#define IQS621_ALS_CAL_DIV_IR 0x83 + +#define IQS620_TEMP_CAL_MULT 0xC2 +#define IQS620_TEMP_CAL_DIV 0xC3 +#define IQS620_TEMP_CAL_OFFS 0xC4 + +#define IQS62X_SYS_SETTINGS 0xD0 +#define IQS62X_SYS_SETTINGS_SOFT_RESET BIT(7) +#define IQS62X_SYS_SETTINGS_ACK_RESET BIT(6) +#define IQS62X_SYS_SETTINGS_EVENT_MODE BIT(5) +#define IQS62X_SYS_SETTINGS_CLK_DIV BIT(4) +#define IQS62X_SYS_SETTINGS_REDO_ATI BIT(1) + +#define IQS62X_PWR_SETTINGS 0xD2 +#define IQS62X_PWR_SETTINGS_DIS_AUTO BIT(5) +#define IQS62X_PWR_SETTINGS_PWR_MODE_MASK (BIT(4) | BIT(3)) +#define IQS62X_PWR_SETTINGS_PWR_MODE_HALT (BIT(4) | BIT(3)) +#define IQS62X_PWR_SETTINGS_PWR_MODE_NORM 0 + +#define IQS62X_OTP_CMD 0xF0 +#define IQS62X_OTP_CMD_FG3 0x13 +#define IQS62X_OTP_DATA 0xF1 +#define IQS62X_MAX_REG 0xFF + +#define IQS62X_HALL_CAL_MASK GENMASK(3, 0) + +#define IQS62X_FW_REC_TYPE_INFO 0 +#define IQS62X_FW_REC_TYPE_PROD 1 +#define IQS62X_FW_REC_TYPE_HALL 2 +#define IQS62X_FW_REC_TYPE_MASK 3 +#define IQS62X_FW_REC_TYPE_DATA 4 + +#define IQS62X_ATI_POLL_SLEEP_US 10000 +#define IQS62X_ATI_POLL_TIMEOUT_US 500000 +#define IQS62X_ATI_STABLE_DELAY_MS 150 + +struct iqs62x_fw_rec { + u8 type; + u8 addr; + u8 len; + u8 data; +} __packed; + +struct iqs62x_fw_blk { + struct list_head list; + u8 addr; + u8 mask; + u8 len; + u8 data[]; +}; + +struct iqs62x_info { + u8 prod_num; + u8 sw_num; + u8 hw_num; +} __packed; + +static int iqs62x_dev_init(struct iqs62x_core *iqs62x) +{ + struct iqs62x_fw_blk *fw_blk; + unsigned int val; + int ret; + u8 clk_div = 1; + + list_for_each_entry(fw_blk, &iqs62x->fw_blk_head, list) { + if (fw_blk->mask) + ret = regmap_update_bits(iqs62x->regmap, fw_blk->addr, + fw_blk->mask, *fw_blk->data); + else + ret = regmap_raw_write(iqs62x->regmap, fw_blk->addr, + fw_blk->data, fw_blk->len); + if (ret) + return ret; + } + + switch (iqs62x->dev_desc->prod_num) { + case IQS620_PROD_NUM: + case IQS622_PROD_NUM: + ret = regmap_read(iqs62x->regmap, + iqs62x->dev_desc->prox_settings, &val); + if (ret) + return ret; + + if (val & IQS620_PROX_SETTINGS_4_SAR_EN) + iqs62x->ui_sel = IQS62X_UI_SAR1; + + /* fall through */ + + case IQS621_PROD_NUM: + ret = regmap_write(iqs62x->regmap, IQS620_GLBL_EVENT_MASK, + IQS620_GLBL_EVENT_MASK_PMU | + iqs62x->dev_desc->prox_mask | + iqs62x->dev_desc->sar_mask | + iqs62x->dev_desc->hall_mask | + iqs62x->dev_desc->hyst_mask | + iqs62x->dev_desc->temp_mask | + iqs62x->dev_desc->als_mask | + iqs62x->dev_desc->ir_mask); + if (ret) + return ret; + break; + + default: + ret = regmap_write(iqs62x->regmap, IQS624_HALL_UI, + IQS624_HALL_UI_WHL_EVENT | + IQS624_HALL_UI_INT_EVENT | + IQS624_HALL_UI_AUTO_CAL); + if (ret) + return ret; + + /* + * The IQS625 default interval divider is below the minimum + * permissible value, and the datasheet mandates that it is + * corrected during initialization (unless an updated value + * has already been provided by firmware). + * + * To protect against an unacceptably low user-entered value + * stored in the firmware, the same check is extended to the + * IQS624 as well. + */ + ret = regmap_read(iqs62x->regmap, IQS624_INTERVAL_DIV, &val); + if (ret) + return ret; + + if (val >= iqs62x->dev_desc->interval_div) + break; + + ret = regmap_write(iqs62x->regmap, IQS624_INTERVAL_DIV, + iqs62x->dev_desc->interval_div); + if (ret) + return ret; + } + + ret = regmap_read(iqs62x->regmap, IQS62X_SYS_SETTINGS, &val); + if (ret) + return ret; + + if (val & IQS62X_SYS_SETTINGS_CLK_DIV) + clk_div = iqs62x->dev_desc->clk_div; + + ret = regmap_write(iqs62x->regmap, IQS62X_SYS_SETTINGS, val | + IQS62X_SYS_SETTINGS_ACK_RESET | + IQS62X_SYS_SETTINGS_EVENT_MODE | + IQS62X_SYS_SETTINGS_REDO_ATI); + if (ret) + return ret; + + ret = regmap_read_poll_timeout(iqs62x->regmap, IQS62X_SYS_FLAGS, val, + !(val & IQS62X_SYS_FLAGS_IN_ATI), + IQS62X_ATI_POLL_SLEEP_US, + IQS62X_ATI_POLL_TIMEOUT_US * clk_div); + if (ret) + return ret; + + msleep(IQS62X_ATI_STABLE_DELAY_MS * clk_div); + + return 0; +} + +static int iqs62x_firmware_parse(struct iqs62x_core *iqs62x, + const struct firmware *fw) +{ + struct i2c_client *client = iqs62x->client; + struct iqs62x_fw_rec *fw_rec; + struct iqs62x_fw_blk *fw_blk; + unsigned int val; + size_t pos = 0; + int ret = 0; + u8 mask, len, *data; + u8 hall_cal_index = 0; + + while (pos < fw->size) { + if (pos + sizeof(*fw_rec) > fw->size) { + ret = -EINVAL; + break; + } + fw_rec = (struct iqs62x_fw_rec *)(fw->data + pos); + pos += sizeof(*fw_rec); + + if (pos + fw_rec->len - 1 > fw->size) { + ret = -EINVAL; + break; + } + pos += fw_rec->len - 1; + + switch (fw_rec->type) { + case IQS62X_FW_REC_TYPE_INFO: + continue; + + case IQS62X_FW_REC_TYPE_PROD: + if (fw_rec->data == iqs62x->dev_desc->prod_num) + continue; + + dev_err(&client->dev, + "Incompatible product number: 0x%02X\n", + fw_rec->data); + ret = -EINVAL; + break; + + case IQS62X_FW_REC_TYPE_HALL: + if (!hall_cal_index) { + ret = regmap_write(iqs62x->regmap, + IQS62X_OTP_CMD, + IQS62X_OTP_CMD_FG3); + if (ret) + break; + + ret = regmap_read(iqs62x->regmap, + IQS62X_OTP_DATA, &val); + if (ret) + break; + + hall_cal_index = val & IQS62X_HALL_CAL_MASK; + if (!hall_cal_index) { + dev_err(&client->dev, + "Uncalibrated device\n"); + ret = -ENODATA; + break; + } + } + + if (hall_cal_index > fw_rec->len) { + ret = -EINVAL; + break; + } + + mask = 0; + data = &fw_rec->data + hall_cal_index - 1; + len = sizeof(*data); + break; + + case IQS62X_FW_REC_TYPE_MASK: + if (fw_rec->len < (sizeof(mask) + sizeof(*data))) { + ret = -EINVAL; + break; + } + + mask = fw_rec->data; + data = &fw_rec->data + sizeof(mask); + len = sizeof(*data); + break; + + case IQS62X_FW_REC_TYPE_DATA: + mask = 0; + data = &fw_rec->data; + len = fw_rec->len; + break; + + default: + dev_err(&client->dev, + "Unrecognized record type: 0x%02X\n", + fw_rec->type); + ret = -EINVAL; + } + + if (ret) + break; + + fw_blk = devm_kzalloc(&client->dev, + struct_size(fw_blk, data, len), + GFP_KERNEL); + if (!fw_blk) { + ret = -ENOMEM; + break; + } + + fw_blk->addr = fw_rec->addr; + fw_blk->mask = mask; + fw_blk->len = len; + memcpy(fw_blk->data, data, len); + + list_add(&fw_blk->list, &iqs62x->fw_blk_head); + } + + release_firmware(fw); + + return ret; +} + +const struct iqs62x_event_desc iqs62x_events[IQS62X_NUM_EVENTS] = { + [IQS62X_EVENT_PROX_CH0_T] = { + .reg = IQS62X_EVENT_PROX, + .mask = BIT(4), + .val = BIT(4), + }, + [IQS62X_EVENT_PROX_CH0_P] = { + .reg = IQS62X_EVENT_PROX, + .mask = BIT(0), + .val = BIT(0), + }, + [IQS62X_EVENT_PROX_CH1_T] = { + .reg = IQS62X_EVENT_PROX, + .mask = BIT(5), + .val = BIT(5), + }, + [IQS62X_EVENT_PROX_CH1_P] = { + .reg = IQS62X_EVENT_PROX, + .mask = BIT(1), + .val = BIT(1), + }, + [IQS62X_EVENT_PROX_CH2_T] = { + .reg = IQS62X_EVENT_PROX, + .mask = BIT(6), + .val = BIT(6), + }, + [IQS62X_EVENT_PROX_CH2_P] = { + .reg = IQS62X_EVENT_PROX, + .mask = BIT(2), + .val = BIT(2), + }, + [IQS62X_EVENT_HYST_POS_T] = { + .reg = IQS62X_EVENT_HYST, + .mask = BIT(6) | BIT(7), + .val = BIT(6), + }, + [IQS62X_EVENT_HYST_POS_P] = { + .reg = IQS62X_EVENT_HYST, + .mask = BIT(5) | BIT(7), + .val = BIT(5), + }, + [IQS62X_EVENT_HYST_NEG_T] = { + .reg = IQS62X_EVENT_HYST, + .mask = BIT(6) | BIT(7), + .val = BIT(6) | BIT(7), + }, + [IQS62X_EVENT_HYST_NEG_P] = { + .reg = IQS62X_EVENT_HYST, + .mask = BIT(5) | BIT(7), + .val = BIT(5) | BIT(7), + }, + [IQS62X_EVENT_SAR1_ACT] = { + .reg = IQS62X_EVENT_HYST, + .mask = BIT(4), + .val = BIT(4), + }, + [IQS62X_EVENT_SAR1_QRD] = { + .reg = IQS62X_EVENT_HYST, + .mask = BIT(2), + .val = BIT(2), + }, + [IQS62X_EVENT_SAR1_MOVE] = { + .reg = IQS62X_EVENT_HYST, + .mask = BIT(1), + .val = BIT(1), + }, + [IQS62X_EVENT_SAR1_HALT] = { + .reg = IQS62X_EVENT_HYST, + .mask = BIT(0), + .val = BIT(0), + }, + [IQS62X_EVENT_WHEEL_UP] = { + .reg = IQS62X_EVENT_WHEEL, + .mask = BIT(7) | BIT(6), + .val = BIT(7), + }, + [IQS62X_EVENT_WHEEL_DN] = { + .reg = IQS62X_EVENT_WHEEL, + .mask = BIT(7) | BIT(6), + .val = BIT(7) | BIT(6), + }, + [IQS62X_EVENT_HALL_N_T] = { + .reg = IQS62X_EVENT_HALL, + .mask = BIT(2) | BIT(0), + .val = BIT(2), + }, + [IQS62X_EVENT_HALL_N_P] = { + .reg = IQS62X_EVENT_HALL, + .mask = BIT(1) | BIT(0), + .val = BIT(1), + }, + [IQS62X_EVENT_HALL_S_T] = { + .reg = IQS62X_EVENT_HALL, + .mask = BIT(2) | BIT(0), + .val = BIT(2) | BIT(0), + }, + [IQS62X_EVENT_HALL_S_P] = { + .reg = IQS62X_EVENT_HALL, + .mask = BIT(1) | BIT(0), + .val = BIT(1) | BIT(0), + }, + [IQS62X_EVENT_SYS_RESET] = { + .reg = IQS62X_EVENT_SYS, + .mask = BIT(7), + .val = BIT(7), + }, +}; +EXPORT_SYMBOL_GPL(iqs62x_events); + +static irqreturn_t iqs62x_irq(int irq, void *context) +{ + struct iqs62x_core *iqs62x = context; + struct i2c_client *client = iqs62x->client; + struct iqs62x_event_data event_data; + struct iqs62x_event_desc event_desc; + enum iqs62x_event_reg event_reg; + unsigned long event_flags = 0; + int ret, i, j; + u8 event_map[IQS62X_EVENT_SIZE]; + + /* + * The device asserts the RDY output to signal the beginning of a + * communication window, which is closed by an I2C stop condition. + * As such, all interrupt status is captured in a single read and + * broadcast to any interested sub-device drivers. + */ + ret = regmap_raw_read(iqs62x->regmap, IQS62X_SYS_FLAGS, event_map, + sizeof(event_map)); + if (ret) { + dev_err(&client->dev, "Failed to read device status: %d\n", + ret); + return IRQ_NONE; + } + + for (i = 0; i < sizeof(event_map); i++) { + event_reg = iqs62x->dev_desc->event_regs[iqs62x->ui_sel][i]; + + switch (event_reg) { + case IQS62X_EVENT_UI_LO: + event_data.ui_data = get_unaligned_le16(&event_map[i]); + + /* fall through */ + + case IQS62X_EVENT_UI_HI: + case IQS62X_EVENT_NONE: + continue; + + case IQS62X_EVENT_ALS: + event_data.als_flags = event_map[i]; + continue; + + case IQS62X_EVENT_IR: + event_data.ir_flags = event_map[i]; + continue; + + case IQS62X_EVENT_INTER: + event_data.interval = event_map[i]; + continue; + + case IQS62X_EVENT_HYST: + event_map[i] <<= iqs62x->dev_desc->hyst_shift; + + /* fall through */ + + case IQS62X_EVENT_WHEEL: + case IQS62X_EVENT_HALL: + case IQS62X_EVENT_PROX: + case IQS62X_EVENT_SYS: + break; + } + + for (j = 0; j < IQS62X_NUM_EVENTS; j++) { + event_desc = iqs62x_events[j]; + + if (event_desc.reg != event_reg) + continue; + + if ((event_map[i] & event_desc.mask) == event_desc.val) + event_flags |= BIT(j); + } + } + + /* + * The device resets itself in response to the I2C master stalling + * communication past a fixed timeout. In this case, all registers + * are restored and any interested sub-device drivers are notified. + */ + if (event_flags & BIT(IQS62X_EVENT_SYS_RESET)) { + dev_err(&client->dev, "Unexpected device reset\n"); + + ret = iqs62x_dev_init(iqs62x); + if (ret) { + dev_err(&client->dev, + "Failed to re-initialize device: %d\n", ret); + return IRQ_NONE; + } + } + + ret = blocking_notifier_call_chain(&iqs62x->nh, event_flags, + &event_data); + if (ret & NOTIFY_STOP_MASK) + return IRQ_NONE; + + /* + * Once the communication window is closed, a small delay is added to + * ensure the device's RDY output has been deasserted by the time the + * interrupt handler returns. + */ + usleep_range(50, 100); + + return IRQ_HANDLED; +} + +static void iqs62x_firmware_load(const struct firmware *fw, void *context) +{ + struct iqs62x_core *iqs62x = context; + struct i2c_client *client = iqs62x->client; + int ret; + + if (fw) { + ret = iqs62x_firmware_parse(iqs62x, fw); + if (ret) { + dev_err(&client->dev, "Failed to parse firmware: %d\n", + ret); + goto err_out; + } + } + + ret = iqs62x_dev_init(iqs62x); + if (ret) { + dev_err(&client->dev, "Failed to initialize device: %d\n", ret); + goto err_out; + } + + ret = devm_request_threaded_irq(&client->dev, client->irq, + NULL, iqs62x_irq, IRQF_ONESHOT, + client->name, iqs62x); + if (ret) { + dev_err(&client->dev, "Failed to request IRQ: %d\n", ret); + goto err_out; + } + + ret = devm_mfd_add_devices(&client->dev, PLATFORM_DEVID_NONE, + iqs62x->dev_desc->sub_devs, + iqs62x->dev_desc->num_sub_devs, + NULL, 0, NULL); + if (ret) + dev_err(&client->dev, "Failed to add sub-devices: %d\n", ret); + +err_out: + complete_all(&iqs62x->fw_done); +} + +static const struct mfd_cell iqs620at_sub_devs[] = { + { + .name = "iqs62x-keys", + .of_compatible = "azoteq,iqs620a-keys", + }, + { + .name = "iqs620a-pwm", + .of_compatible = "azoteq,iqs620a-pwm", + }, + { .name = "iqs620at-temp", }, +}; + +static const struct mfd_cell iqs620a_sub_devs[] = { + { + .name = "iqs62x-keys", + .of_compatible = "azoteq,iqs620a-keys", + }, + { + .name = "iqs620a-pwm", + .of_compatible = "azoteq,iqs620a-pwm", + }, +}; + +static const struct mfd_cell iqs621_sub_devs[] = { + { + .name = "iqs62x-keys", + .of_compatible = "azoteq,iqs621-keys", + }, + { .name = "iqs621-als", }, +}; + +static const struct mfd_cell iqs622_sub_devs[] = { + { + .name = "iqs62x-keys", + .of_compatible = "azoteq,iqs622-keys", + }, + { .name = "iqs621-als", }, +}; + +static const struct mfd_cell iqs624_sub_devs[] = { + { + .name = "iqs62x-keys", + .of_compatible = "azoteq,iqs624-keys", + }, + { .name = "iqs624-pos", }, +}; + +static const struct mfd_cell iqs625_sub_devs[] = { + { + .name = "iqs62x-keys", + .of_compatible = "azoteq,iqs625-keys", + }, + { .name = "iqs624-pos", }, +}; + +static const u8 iqs620at_cal_regs[] = { + IQS620_TEMP_CAL_MULT, + IQS620_TEMP_CAL_DIV, + IQS620_TEMP_CAL_OFFS, +}; + +static const u8 iqs621_cal_regs[] = { + IQS621_ALS_CAL_DIV_LUX, + IQS621_ALS_CAL_DIV_IR, +}; + +static const enum iqs62x_event_reg iqs620a_event_regs[][IQS62X_EVENT_SIZE] = { + [IQS62X_UI_PROX] = { + IQS62X_EVENT_SYS, /* 0x10 */ + IQS62X_EVENT_NONE, + IQS62X_EVENT_PROX, /* 0x12 */ + IQS62X_EVENT_HYST, /* 0x13 */ + IQS62X_EVENT_NONE, + IQS62X_EVENT_NONE, + IQS62X_EVENT_HALL, /* 0x16 */ + IQS62X_EVENT_NONE, + IQS62X_EVENT_NONE, + IQS62X_EVENT_NONE, + }, + [IQS62X_UI_SAR1] = { + IQS62X_EVENT_SYS, /* 0x10 */ + IQS62X_EVENT_NONE, + IQS62X_EVENT_NONE, + IQS62X_EVENT_HYST, /* 0x13 */ + IQS62X_EVENT_NONE, + IQS62X_EVENT_NONE, + IQS62X_EVENT_HALL, /* 0x16 */ + IQS62X_EVENT_NONE, + IQS62X_EVENT_NONE, + IQS62X_EVENT_NONE, + }, +}; + +static const enum iqs62x_event_reg iqs621_event_regs[][IQS62X_EVENT_SIZE] = { + [IQS62X_UI_PROX] = { + IQS62X_EVENT_SYS, /* 0x10 */ + IQS62X_EVENT_NONE, + IQS62X_EVENT_PROX, /* 0x12 */ + IQS62X_EVENT_HYST, /* 0x13 */ + IQS62X_EVENT_NONE, + IQS62X_EVENT_NONE, + IQS62X_EVENT_ALS, /* 0x16 */ + IQS62X_EVENT_UI_LO, /* 0x17 */ + IQS62X_EVENT_UI_HI, /* 0x18 */ + IQS62X_EVENT_HALL, /* 0x19 */ + }, +}; + +static const enum iqs62x_event_reg iqs622_event_regs[][IQS62X_EVENT_SIZE] = { + [IQS62X_UI_PROX] = { + IQS62X_EVENT_SYS, /* 0x10 */ + IQS62X_EVENT_NONE, + IQS62X_EVENT_PROX, /* 0x12 */ + IQS62X_EVENT_NONE, + IQS62X_EVENT_ALS, /* 0x14 */ + IQS62X_EVENT_NONE, + IQS62X_EVENT_IR, /* 0x16 */ + IQS62X_EVENT_UI_LO, /* 0x17 */ + IQS62X_EVENT_UI_HI, /* 0x18 */ + IQS62X_EVENT_HALL, /* 0x19 */ + }, + [IQS62X_UI_SAR1] = { + IQS62X_EVENT_SYS, /* 0x10 */ + IQS62X_EVENT_NONE, + IQS62X_EVENT_NONE, + IQS62X_EVENT_HYST, /* 0x13 */ + IQS62X_EVENT_ALS, /* 0x14 */ + IQS62X_EVENT_NONE, + IQS62X_EVENT_IR, /* 0x16 */ + IQS62X_EVENT_UI_LO, /* 0x17 */ + IQS62X_EVENT_UI_HI, /* 0x18 */ + IQS62X_EVENT_HALL, /* 0x19 */ + }, +}; + +static const enum iqs62x_event_reg iqs624_event_regs[][IQS62X_EVENT_SIZE] = { + [IQS62X_UI_PROX] = { + IQS62X_EVENT_SYS, /* 0x10 */ + IQS62X_EVENT_NONE, + IQS62X_EVENT_PROX, /* 0x12 */ + IQS62X_EVENT_NONE, + IQS62X_EVENT_WHEEL, /* 0x14 */ + IQS62X_EVENT_NONE, + IQS62X_EVENT_UI_LO, /* 0x16 */ + IQS62X_EVENT_UI_HI, /* 0x17 */ + IQS62X_EVENT_INTER, /* 0x18 */ + IQS62X_EVENT_NONE, + }, +}; + +static const enum iqs62x_event_reg iqs625_event_regs[][IQS62X_EVENT_SIZE] = { + [IQS62X_UI_PROX] = { + IQS62X_EVENT_SYS, /* 0x10 */ + IQS62X_EVENT_PROX, /* 0x11 */ + IQS62X_EVENT_INTER, /* 0x12 */ + IQS62X_EVENT_NONE, + IQS62X_EVENT_NONE, + IQS62X_EVENT_NONE, + IQS62X_EVENT_NONE, + IQS62X_EVENT_NONE, + IQS62X_EVENT_NONE, + IQS62X_EVENT_NONE, + }, +}; + +static const struct iqs62x_dev_desc iqs62x_devs[] = { + { + .dev_name = "iqs620at", + .sub_devs = iqs620at_sub_devs, + .num_sub_devs = ARRAY_SIZE(iqs620at_sub_devs), + + .prod_num = IQS620_PROD_NUM, + .sw_num = 0x08, + .cal_regs = iqs620at_cal_regs, + .num_cal_regs = ARRAY_SIZE(iqs620at_cal_regs), + + .prox_mask = BIT(0), + .sar_mask = BIT(1) | BIT(7), + .hall_mask = BIT(2), + .hyst_mask = BIT(3), + .temp_mask = BIT(4), + + .prox_settings = IQS620_PROX_SETTINGS_4, + .hall_flags = IQS620_HALL_FLAGS, + + .clk_div = 4, + .fw_name = "iqs620a.bin", + .event_regs = &iqs620a_event_regs[IQS62X_UI_PROX], + }, + { + .dev_name = "iqs620a", + .sub_devs = iqs620a_sub_devs, + .num_sub_devs = ARRAY_SIZE(iqs620a_sub_devs), + + .prod_num = IQS620_PROD_NUM, + .sw_num = 0x08, + + .prox_mask = BIT(0), + .sar_mask = BIT(1) | BIT(7), + .hall_mask = BIT(2), + .hyst_mask = BIT(3), + .temp_mask = BIT(4), + + .prox_settings = IQS620_PROX_SETTINGS_4, + .hall_flags = IQS620_HALL_FLAGS, + + .clk_div = 4, + .fw_name = "iqs620a.bin", + .event_regs = &iqs620a_event_regs[IQS62X_UI_PROX], + }, + { + .dev_name = "iqs621", + .sub_devs = iqs621_sub_devs, + .num_sub_devs = ARRAY_SIZE(iqs621_sub_devs), + + .prod_num = IQS621_PROD_NUM, + .sw_num = 0x09, + .cal_regs = iqs621_cal_regs, + .num_cal_regs = ARRAY_SIZE(iqs621_cal_regs), + + .prox_mask = BIT(0), + .hall_mask = BIT(1), + .als_mask = BIT(2), + .hyst_mask = BIT(3), + .temp_mask = BIT(4), + + .als_flags = IQS621_ALS_FLAGS, + .hall_flags = IQS621_HALL_FLAGS, + .hyst_shift = 5, + + .clk_div = 2, + .fw_name = "iqs621.bin", + .event_regs = &iqs621_event_regs[IQS62X_UI_PROX], + }, + { + .dev_name = "iqs622", + .sub_devs = iqs622_sub_devs, + .num_sub_devs = ARRAY_SIZE(iqs622_sub_devs), + + .prod_num = IQS622_PROD_NUM, + .sw_num = 0x06, + + .prox_mask = BIT(0), + .sar_mask = BIT(1), + .hall_mask = BIT(2), + .als_mask = BIT(3), + .ir_mask = BIT(4), + + .prox_settings = IQS622_PROX_SETTINGS_4, + .als_flags = IQS622_ALS_FLAGS, + .hall_flags = IQS622_HALL_FLAGS, + + .clk_div = 2, + .fw_name = "iqs622.bin", + .event_regs = &iqs622_event_regs[IQS62X_UI_PROX], + }, + { + .dev_name = "iqs624", + .sub_devs = iqs624_sub_devs, + .num_sub_devs = ARRAY_SIZE(iqs624_sub_devs), + + .prod_num = IQS624_PROD_NUM, + .sw_num = 0x0B, + + .interval = IQS624_INTERVAL_NUM, + .interval_div = 3, + + .clk_div = 2, + .fw_name = "iqs624.bin", + .event_regs = &iqs624_event_regs[IQS62X_UI_PROX], + }, + { + .dev_name = "iqs625", + .sub_devs = iqs625_sub_devs, + .num_sub_devs = ARRAY_SIZE(iqs625_sub_devs), + + .prod_num = IQS625_PROD_NUM, + .sw_num = 0x0B, + + .interval = IQS625_INTERVAL_NUM, + .interval_div = 10, + + .clk_div = 2, + .fw_name = "iqs625.bin", + .event_regs = &iqs625_event_regs[IQS62X_UI_PROX], + }, +}; + +static const struct regmap_config iqs62x_map_config = { + .reg_bits = 8, + .val_bits = 8, + .max_register = IQS62X_MAX_REG, +}; + +static int iqs62x_probe(struct i2c_client *client) +{ + struct iqs62x_core *iqs62x; + struct iqs62x_info info; + unsigned int val; + int ret, i, j; + u8 sw_num = 0; + const char *fw_name = NULL; + + iqs62x = devm_kzalloc(&client->dev, sizeof(*iqs62x), GFP_KERNEL); + if (!iqs62x) + return -ENOMEM; + + i2c_set_clientdata(client, iqs62x); + iqs62x->client = client; + + BLOCKING_INIT_NOTIFIER_HEAD(&iqs62x->nh); + INIT_LIST_HEAD(&iqs62x->fw_blk_head); + init_completion(&iqs62x->fw_done); + + iqs62x->regmap = devm_regmap_init_i2c(client, &iqs62x_map_config); + if (IS_ERR(iqs62x->regmap)) { + ret = PTR_ERR(iqs62x->regmap); + dev_err(&client->dev, "Failed to initialize register map: %d\n", + ret); + return ret; + } + + ret = regmap_raw_read(iqs62x->regmap, IQS62X_PROD_NUM, &info, + sizeof(info)); + if (ret) + return ret; + + /* + * The following sequence validates the device's product and software + * numbers. It then determines if the device is factory-calibrated by + * checking for nonzero values in the device's designated calibration + * registers (if applicable). Depending on the device, the absence of + * calibration data indicates a reduced feature set or invalid device. + * + * For devices given in both calibrated and uncalibrated versions, the + * calibrated version (e.g. IQS620AT) appears first in the iqs62x_devs + * array. The uncalibrated version (e.g. IQS620A) appears next and has + * the same product and software numbers, but no calibration registers + * are specified. + */ + for (i = 0; i < ARRAY_SIZE(iqs62x_devs); i++) { + if (info.prod_num != iqs62x_devs[i].prod_num) + continue; + + iqs62x->dev_desc = &iqs62x_devs[i]; + + if (info.sw_num < iqs62x->dev_desc->sw_num) + continue; + + sw_num = info.sw_num; + + /* + * Read each of the device's designated calibration registers, + * if any, and exit from the inner loop early if any are equal + * to zero (indicating the device is uncalibrated). This could + * be acceptable depending on the device (e.g. IQS620A instead + * of IQS620AT). + */ + for (j = 0; j < iqs62x->dev_desc->num_cal_regs; j++) { + ret = regmap_read(iqs62x->regmap, + iqs62x->dev_desc->cal_regs[j], &val); + if (ret) + return ret; + + if (!val) + break; + } + + /* + * If the number of nonzero values read from the device equals + * the number of designated calibration registers (which could + * be zero), exit from the outer loop early to signal that the + * device's product and software numbers match a known device, + * and the device is calibrated (if applicable). + */ + if (j == iqs62x->dev_desc->num_cal_regs) + break; + } + + if (!iqs62x->dev_desc) { + dev_err(&client->dev, "Unrecognized product number: 0x%02X\n", + info.prod_num); + return -EINVAL; + } + + if (!sw_num) { + dev_err(&client->dev, "Unrecognized software number: 0x%02X\n", + info.sw_num); + return -EINVAL; + } + + if (i == ARRAY_SIZE(iqs62x_devs)) { + dev_err(&client->dev, "Uncalibrated device\n"); + return -ENODATA; + } + + device_property_read_string(&client->dev, "firmware-name", &fw_name); + + ret = request_firmware_nowait(THIS_MODULE, FW_ACTION_HOTPLUG, + fw_name ? : iqs62x->dev_desc->fw_name, + &client->dev, GFP_KERNEL, iqs62x, + iqs62x_firmware_load); + if (ret) + dev_err(&client->dev, "Failed to request firmware: %d\n", ret); + + return ret; +} + +static int iqs62x_remove(struct i2c_client *client) +{ + struct iqs62x_core *iqs62x = i2c_get_clientdata(client); + + wait_for_completion(&iqs62x->fw_done); + + return 0; +} + +static int __maybe_unused iqs62x_suspend(struct device *dev) +{ + struct iqs62x_core *iqs62x = dev_get_drvdata(dev); + int ret; + + wait_for_completion(&iqs62x->fw_done); + + /* + * As per the datasheet, automatic mode switching must be disabled + * before the device is placed in or taken out of halt mode. + */ + ret = regmap_update_bits(iqs62x->regmap, IQS62X_PWR_SETTINGS, + IQS62X_PWR_SETTINGS_DIS_AUTO, 0xFF); + if (ret) + return ret; + + return regmap_update_bits(iqs62x->regmap, IQS62X_PWR_SETTINGS, + IQS62X_PWR_SETTINGS_PWR_MODE_MASK, + IQS62X_PWR_SETTINGS_PWR_MODE_HALT); +} + +static int __maybe_unused iqs62x_resume(struct device *dev) +{ + struct iqs62x_core *iqs62x = dev_get_drvdata(dev); + int ret; + + ret = regmap_update_bits(iqs62x->regmap, IQS62X_PWR_SETTINGS, + IQS62X_PWR_SETTINGS_PWR_MODE_MASK, + IQS62X_PWR_SETTINGS_PWR_MODE_NORM); + if (ret) + return ret; + + return regmap_update_bits(iqs62x->regmap, IQS62X_PWR_SETTINGS, + IQS62X_PWR_SETTINGS_DIS_AUTO, 0); +} + +static SIMPLE_DEV_PM_OPS(iqs62x_pm, iqs62x_suspend, iqs62x_resume); + +static const struct of_device_id iqs62x_of_match[] = { + { .compatible = "azoteq,iqs620a" }, + { .compatible = "azoteq,iqs621" }, + { .compatible = "azoteq,iqs622" }, + { .compatible = "azoteq,iqs624" }, + { .compatible = "azoteq,iqs625" }, + { } +}; +MODULE_DEVICE_TABLE(of, iqs62x_of_match); + +static struct i2c_driver iqs62x_i2c_driver = { + .driver = { + .name = "iqs62x", + .of_match_table = iqs62x_of_match, + .pm = &iqs62x_pm, + }, + .probe_new = iqs62x_probe, + .remove = iqs62x_remove, +}; +module_i2c_driver(iqs62x_i2c_driver); + +MODULE_AUTHOR("Jeff LaBundy "); +MODULE_DESCRIPTION("Azoteq IQS620A/621/622/624/625 Multi-Function Sensors"); +MODULE_LICENSE("GPL"); diff --git a/include/linux/mfd/iqs62x.h b/include/linux/mfd/iqs62x.h new file mode 100644 index 000000000000..043d3b6de9ec --- /dev/null +++ b/include/linux/mfd/iqs62x.h @@ -0,0 +1,139 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Azoteq IQS620A/621/622/624/625 Multi-Function Sensors + * + * Copyright (C) 2019 Jeff LaBundy + */ + +#ifndef __LINUX_MFD_IQS62X_H +#define __LINUX_MFD_IQS62X_H + +#define IQS620_PROD_NUM 0x41 +#define IQS621_PROD_NUM 0x46 +#define IQS622_PROD_NUM 0x42 +#define IQS624_PROD_NUM 0x43 +#define IQS625_PROD_NUM 0x4E + +#define IQS621_ALS_FLAGS 0x16 +#define IQS622_ALS_FLAGS 0x14 + +#define IQS624_HALL_UI 0x70 +#define IQS624_HALL_UI_WHL_EVENT BIT(4) +#define IQS624_HALL_UI_INT_EVENT BIT(3) +#define IQS624_HALL_UI_AUTO_CAL BIT(2) + +#define IQS624_INTERVAL_DIV 0x7D + +#define IQS620_GLBL_EVENT_MASK 0xD7 +#define IQS620_GLBL_EVENT_MASK_PMU BIT(6) + +#define IQS62X_NUM_KEYS 16 +#define IQS62X_NUM_EVENTS (IQS62X_NUM_KEYS + 5) + +#define IQS62X_EVENT_SIZE 10 + +enum iqs62x_ui_sel { + IQS62X_UI_PROX, + IQS62X_UI_SAR1, +}; + +enum iqs62x_event_reg { + IQS62X_EVENT_NONE, + IQS62X_EVENT_SYS, + IQS62X_EVENT_PROX, + IQS62X_EVENT_HYST, + IQS62X_EVENT_HALL, + IQS62X_EVENT_ALS, + IQS62X_EVENT_IR, + IQS62X_EVENT_WHEEL, + IQS62X_EVENT_INTER, + IQS62X_EVENT_UI_LO, + IQS62X_EVENT_UI_HI, +}; + +enum iqs62x_event_flag { + /* keys */ + IQS62X_EVENT_PROX_CH0_T, + IQS62X_EVENT_PROX_CH0_P, + IQS62X_EVENT_PROX_CH1_T, + IQS62X_EVENT_PROX_CH1_P, + IQS62X_EVENT_PROX_CH2_T, + IQS62X_EVENT_PROX_CH2_P, + IQS62X_EVENT_HYST_POS_T, + IQS62X_EVENT_HYST_POS_P, + IQS62X_EVENT_HYST_NEG_T, + IQS62X_EVENT_HYST_NEG_P, + IQS62X_EVENT_SAR1_ACT, + IQS62X_EVENT_SAR1_QRD, + IQS62X_EVENT_SAR1_MOVE, + IQS62X_EVENT_SAR1_HALT, + IQS62X_EVENT_WHEEL_UP, + IQS62X_EVENT_WHEEL_DN, + + /* switches */ + IQS62X_EVENT_HALL_N_T, + IQS62X_EVENT_HALL_N_P, + IQS62X_EVENT_HALL_S_T, + IQS62X_EVENT_HALL_S_P, + + /* everything else */ + IQS62X_EVENT_SYS_RESET, +}; + +struct iqs62x_event_data { + u16 ui_data; + u8 als_flags; + u8 ir_flags; + u8 interval; +}; + +struct iqs62x_event_desc { + enum iqs62x_event_reg reg; + u8 mask; + u8 val; +}; + +struct iqs62x_dev_desc { + const char *dev_name; + const struct mfd_cell *sub_devs; + int num_sub_devs; + + u8 prod_num; + u8 sw_num; + const u8 *cal_regs; + int num_cal_regs; + + u8 prox_mask; + u8 sar_mask; + u8 hall_mask; + u8 hyst_mask; + u8 temp_mask; + u8 als_mask; + u8 ir_mask; + + u8 prox_settings; + u8 als_flags; + u8 hall_flags; + u8 hyst_shift; + + u8 interval; + u8 interval_div; + + u8 clk_div; + const char *fw_name; + const enum iqs62x_event_reg (*event_regs)[IQS62X_EVENT_SIZE]; +}; + +struct iqs62x_core { + const struct iqs62x_dev_desc *dev_desc; + struct i2c_client *client; + struct regmap *regmap; + struct blocking_notifier_head nh; + struct list_head fw_blk_head; + struct completion fw_done; + enum iqs62x_ui_sel ui_sel; +}; + +extern const struct iqs62x_event_desc iqs62x_events[IQS62X_NUM_EVENTS]; + +#endif /* __LINUX_MFD_IQS62X_H */ From ce1cb0eec85bdd865a788a99274609bcce4ff086 Mon Sep 17 00:00:00 2001 From: Jeff LaBundy Date: Sun, 16 Feb 2020 17:32:07 -0600 Subject: [PATCH 220/635] input: keyboard: Add support for Azoteq IQS620A/621/622/624/625 This patch adds key and switch support for the Azoteq IQS620A, IQS621, IQS622, IQS624 and IQS625 multi-function sensors. Signed-off-by: Jeff LaBundy Acked-by: Dmitry Torokhov Signed-off-by: Lee Jones --- drivers/input/keyboard/Kconfig | 10 + drivers/input/keyboard/Makefile | 1 + drivers/input/keyboard/iqs62x-keys.c | 335 +++++++++++++++++++++++++++ 3 files changed, 346 insertions(+) create mode 100644 drivers/input/keyboard/iqs62x-keys.c diff --git a/drivers/input/keyboard/Kconfig b/drivers/input/keyboard/Kconfig index 4706ff09f0e8..28de965a08d5 100644 --- a/drivers/input/keyboard/Kconfig +++ b/drivers/input/keyboard/Kconfig @@ -663,6 +663,16 @@ config KEYBOARD_IPAQ_MICRO To compile this driver as a module, choose M here: the module will be called ipaq-micro-keys. +config KEYBOARD_IQS62X + tristate "Azoteq IQS620A/621/622/624/625 keys and switches" + depends on MFD_IQS62X + help + Say Y here to enable key and switch support for the Azoteq IQS620A, + IQS621, IQS622, IQS624 and IQS625 multi-function sensors. + + To compile this driver as a module, choose M here: the module will + be called iqs62x-keys. + config KEYBOARD_OMAP tristate "TI OMAP keypad support" depends on ARCH_OMAP1 diff --git a/drivers/input/keyboard/Makefile b/drivers/input/keyboard/Makefile index f5b17524adf2..1d689fdd5c00 100644 --- a/drivers/input/keyboard/Makefile +++ b/drivers/input/keyboard/Makefile @@ -28,6 +28,7 @@ obj-$(CONFIG_KEYBOARD_TCA8418) += tca8418_keypad.o obj-$(CONFIG_KEYBOARD_HIL) += hil_kbd.o obj-$(CONFIG_KEYBOARD_HIL_OLD) += hilkbd.o obj-$(CONFIG_KEYBOARD_IPAQ_MICRO) += ipaq-micro-keys.o +obj-$(CONFIG_KEYBOARD_IQS62X) += iqs62x-keys.o obj-$(CONFIG_KEYBOARD_IMX) += imx_keypad.o obj-$(CONFIG_KEYBOARD_IMX_SC_KEY) += imx_sc_key.o obj-$(CONFIG_KEYBOARD_HP6XX) += jornada680_kbd.o diff --git a/drivers/input/keyboard/iqs62x-keys.c b/drivers/input/keyboard/iqs62x-keys.c new file mode 100644 index 000000000000..93446b21f98f --- /dev/null +++ b/drivers/input/keyboard/iqs62x-keys.c @@ -0,0 +1,335 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Azoteq IQS620A/621/622/624/625 Keys and Switches + * + * Copyright (C) 2019 Jeff LaBundy + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum { + IQS62X_SW_HALL_N, + IQS62X_SW_HALL_S, +}; + +static const char * const iqs62x_switch_names[] = { + [IQS62X_SW_HALL_N] = "hall-switch-north", + [IQS62X_SW_HALL_S] = "hall-switch-south", +}; + +struct iqs62x_switch_desc { + enum iqs62x_event_flag flag; + unsigned int code; + bool enabled; +}; + +struct iqs62x_keys_private { + struct iqs62x_core *iqs62x; + struct input_dev *input; + struct notifier_block notifier; + struct iqs62x_switch_desc switches[ARRAY_SIZE(iqs62x_switch_names)]; + unsigned int keycode[IQS62X_NUM_KEYS]; + unsigned int keycodemax; + u8 interval; +}; + +static int iqs62x_keys_parse_prop(struct platform_device *pdev, + struct iqs62x_keys_private *iqs62x_keys) +{ + struct fwnode_handle *child; + unsigned int val; + int ret, i; + + ret = device_property_count_u32(&pdev->dev, "linux,keycodes"); + if (ret > IQS62X_NUM_KEYS) { + dev_err(&pdev->dev, "Too many keycodes present\n"); + return -EINVAL; + } else if (ret < 0) { + dev_err(&pdev->dev, "Failed to count keycodes: %d\n", ret); + return ret; + } + iqs62x_keys->keycodemax = ret; + + ret = device_property_read_u32_array(&pdev->dev, "linux,keycodes", + iqs62x_keys->keycode, + iqs62x_keys->keycodemax); + if (ret) { + dev_err(&pdev->dev, "Failed to read keycodes: %d\n", ret); + return ret; + } + + for (i = 0; i < ARRAY_SIZE(iqs62x_keys->switches); i++) { + child = device_get_named_child_node(&pdev->dev, + iqs62x_switch_names[i]); + if (!child) + continue; + + ret = fwnode_property_read_u32(child, "linux,code", &val); + if (ret) { + dev_err(&pdev->dev, "Failed to read switch code: %d\n", + ret); + return ret; + } + iqs62x_keys->switches[i].code = val; + iqs62x_keys->switches[i].enabled = true; + + if (fwnode_property_present(child, "azoteq,use-prox")) + iqs62x_keys->switches[i].flag = (i == IQS62X_SW_HALL_N ? + IQS62X_EVENT_HALL_N_P : + IQS62X_EVENT_HALL_S_P); + else + iqs62x_keys->switches[i].flag = (i == IQS62X_SW_HALL_N ? + IQS62X_EVENT_HALL_N_T : + IQS62X_EVENT_HALL_S_T); + } + + return 0; +} + +static int iqs62x_keys_init(struct iqs62x_keys_private *iqs62x_keys) +{ + struct iqs62x_core *iqs62x = iqs62x_keys->iqs62x; + enum iqs62x_event_flag flag; + unsigned int event_reg, val; + unsigned int event_mask = 0; + int ret, i; + + switch (iqs62x->dev_desc->prod_num) { + case IQS620_PROD_NUM: + case IQS621_PROD_NUM: + case IQS622_PROD_NUM: + event_reg = IQS620_GLBL_EVENT_MASK; + + /* + * Discreet button, hysteresis and SAR UI flags represent keys + * and are unmasked if mapped to a valid keycode. + */ + for (i = 0; i < iqs62x_keys->keycodemax; i++) { + if (iqs62x_keys->keycode[i] == KEY_RESERVED) + continue; + + if (iqs62x_events[i].reg == IQS62X_EVENT_PROX) + event_mask |= iqs62x->dev_desc->prox_mask; + else if (iqs62x_events[i].reg == IQS62X_EVENT_HYST) + event_mask |= (iqs62x->dev_desc->hyst_mask | + iqs62x->dev_desc->sar_mask); + } + + ret = regmap_read(iqs62x->regmap, iqs62x->dev_desc->hall_flags, + &val); + if (ret) + return ret; + + /* + * Hall UI flags represent switches and are unmasked if their + * corresponding child nodes are present. + */ + for (i = 0; i < ARRAY_SIZE(iqs62x_keys->switches); i++) { + if (!(iqs62x_keys->switches[i].enabled)) + continue; + + flag = iqs62x_keys->switches[i].flag; + + if (iqs62x_events[flag].reg != IQS62X_EVENT_HALL) + continue; + + event_mask |= iqs62x->dev_desc->hall_mask; + + input_report_switch(iqs62x_keys->input, + iqs62x_keys->switches[i].code, + (val & iqs62x_events[flag].mask) == + iqs62x_events[flag].val); + } + + input_sync(iqs62x_keys->input); + break; + + case IQS624_PROD_NUM: + event_reg = IQS624_HALL_UI; + + /* + * Interval change events represent keys and are unmasked if + * either wheel movement flag is mapped to a valid keycode. + */ + if (iqs62x_keys->keycode[IQS62X_EVENT_WHEEL_UP] != KEY_RESERVED) + event_mask |= IQS624_HALL_UI_INT_EVENT; + + if (iqs62x_keys->keycode[IQS62X_EVENT_WHEEL_DN] != KEY_RESERVED) + event_mask |= IQS624_HALL_UI_INT_EVENT; + + ret = regmap_read(iqs62x->regmap, iqs62x->dev_desc->interval, + &val); + if (ret) + return ret; + + iqs62x_keys->interval = val; + break; + + default: + return 0; + } + + return regmap_update_bits(iqs62x->regmap, event_reg, event_mask, 0); +} + +static int iqs62x_keys_notifier(struct notifier_block *notifier, + unsigned long event_flags, void *context) +{ + struct iqs62x_event_data *event_data = context; + struct iqs62x_keys_private *iqs62x_keys; + int ret, i; + + iqs62x_keys = container_of(notifier, struct iqs62x_keys_private, + notifier); + + if (event_flags & BIT(IQS62X_EVENT_SYS_RESET)) { + ret = iqs62x_keys_init(iqs62x_keys); + if (ret) { + dev_err(iqs62x_keys->input->dev.parent, + "Failed to re-initialize device: %d\n", ret); + return NOTIFY_BAD; + } + + return NOTIFY_OK; + } + + for (i = 0; i < iqs62x_keys->keycodemax; i++) { + if (iqs62x_events[i].reg == IQS62X_EVENT_WHEEL && + event_data->interval == iqs62x_keys->interval) + continue; + + input_report_key(iqs62x_keys->input, iqs62x_keys->keycode[i], + event_flags & BIT(i)); + } + + for (i = 0; i < ARRAY_SIZE(iqs62x_keys->switches); i++) + if (iqs62x_keys->switches[i].enabled) + input_report_switch(iqs62x_keys->input, + iqs62x_keys->switches[i].code, + event_flags & + BIT(iqs62x_keys->switches[i].flag)); + + input_sync(iqs62x_keys->input); + + if (event_data->interval == iqs62x_keys->interval) + return NOTIFY_OK; + + /* + * Each frame contains at most one wheel event (up or down), in which + * case a complementary release cycle is emulated. + */ + if (event_flags & BIT(IQS62X_EVENT_WHEEL_UP)) { + input_report_key(iqs62x_keys->input, + iqs62x_keys->keycode[IQS62X_EVENT_WHEEL_UP], + 0); + input_sync(iqs62x_keys->input); + } else if (event_flags & BIT(IQS62X_EVENT_WHEEL_DN)) { + input_report_key(iqs62x_keys->input, + iqs62x_keys->keycode[IQS62X_EVENT_WHEEL_DN], + 0); + input_sync(iqs62x_keys->input); + } + + iqs62x_keys->interval = event_data->interval; + + return NOTIFY_OK; +} + +static int iqs62x_keys_probe(struct platform_device *pdev) +{ + struct iqs62x_core *iqs62x = dev_get_drvdata(pdev->dev.parent); + struct iqs62x_keys_private *iqs62x_keys; + struct input_dev *input; + int ret, i; + + iqs62x_keys = devm_kzalloc(&pdev->dev, sizeof(*iqs62x_keys), + GFP_KERNEL); + if (!iqs62x_keys) + return -ENOMEM; + + platform_set_drvdata(pdev, iqs62x_keys); + + ret = iqs62x_keys_parse_prop(pdev, iqs62x_keys); + if (ret) + return ret; + + input = devm_input_allocate_device(&pdev->dev); + if (!input) + return -ENOMEM; + + input->keycodemax = iqs62x_keys->keycodemax; + input->keycode = iqs62x_keys->keycode; + input->keycodesize = sizeof(*iqs62x_keys->keycode); + + input->name = iqs62x->dev_desc->dev_name; + input->id.bustype = BUS_I2C; + + for (i = 0; i < iqs62x_keys->keycodemax; i++) + if (iqs62x_keys->keycode[i] != KEY_RESERVED) + input_set_capability(input, EV_KEY, + iqs62x_keys->keycode[i]); + + for (i = 0; i < ARRAY_SIZE(iqs62x_keys->switches); i++) + if (iqs62x_keys->switches[i].enabled) + input_set_capability(input, EV_SW, + iqs62x_keys->switches[i].code); + + iqs62x_keys->iqs62x = iqs62x; + iqs62x_keys->input = input; + + ret = iqs62x_keys_init(iqs62x_keys); + if (ret) { + dev_err(&pdev->dev, "Failed to initialize device: %d\n", ret); + return ret; + } + + ret = input_register_device(iqs62x_keys->input); + if (ret) { + dev_err(&pdev->dev, "Failed to register device: %d\n", ret); + return ret; + } + + iqs62x_keys->notifier.notifier_call = iqs62x_keys_notifier; + ret = blocking_notifier_chain_register(&iqs62x_keys->iqs62x->nh, + &iqs62x_keys->notifier); + if (ret) + dev_err(&pdev->dev, "Failed to register notifier: %d\n", ret); + + return ret; +} + +static int iqs62x_keys_remove(struct platform_device *pdev) +{ + struct iqs62x_keys_private *iqs62x_keys = platform_get_drvdata(pdev); + int ret; + + ret = blocking_notifier_chain_unregister(&iqs62x_keys->iqs62x->nh, + &iqs62x_keys->notifier); + if (ret) + dev_err(&pdev->dev, "Failed to unregister notifier: %d\n", ret); + + return ret; +} + +static struct platform_driver iqs62x_keys_platform_driver = { + .driver = { + .name = "iqs62x-keys", + }, + .probe = iqs62x_keys_probe, + .remove = iqs62x_keys_remove, +}; +module_platform_driver(iqs62x_keys_platform_driver); + +MODULE_AUTHOR("Jeff LaBundy "); +MODULE_DESCRIPTION("Azoteq IQS620A/621/622/624/625 Keys and Switches"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:iqs62x-keys"); From 8ba447109af4e0bcbc38fac1b134f207c66ab39e Mon Sep 17 00:00:00 2001 From: Jeff LaBundy Date: Sun, 16 Feb 2020 17:32:09 -0600 Subject: [PATCH 221/635] iio: temperature: Add support for Azoteq IQS620AT temperature sensor This patch adds support for the Azoteq IQS620AT temperature sensor, capable of reporting its absolute die temperature. Signed-off-by: Jeff LaBundy Reviewed-by: Jonathan Cameron Signed-off-by: Lee Jones --- drivers/iio/temperature/Kconfig | 10 +++ drivers/iio/temperature/Makefile | 1 + drivers/iio/temperature/iqs620at-temp.c | 97 +++++++++++++++++++++++++ 3 files changed, 108 insertions(+) create mode 100644 drivers/iio/temperature/iqs620at-temp.c diff --git a/drivers/iio/temperature/Kconfig b/drivers/iio/temperature/Kconfig index e1ccb4003015..f1f2a1499c9e 100644 --- a/drivers/iio/temperature/Kconfig +++ b/drivers/iio/temperature/Kconfig @@ -4,6 +4,16 @@ # menu "Temperature sensors" +config IQS620AT_TEMP + tristate "Azoteq IQS620AT temperature sensor" + depends on MFD_IQS62X || COMPILE_TEST + help + Say Y here if you want to build support for the Azoteq IQS620AT + temperature sensor. + + To compile this driver as a module, choose M here: the module + will be called iqs620at-temp. + config LTC2983 tristate "Analog Devices Multi-Sensor Digital Temperature Measurement System" depends on SPI diff --git a/drivers/iio/temperature/Makefile b/drivers/iio/temperature/Makefile index d6b850b0cf63..90c113115422 100644 --- a/drivers/iio/temperature/Makefile +++ b/drivers/iio/temperature/Makefile @@ -3,6 +3,7 @@ # Makefile for industrial I/O temperature drivers # +obj-$(CONFIG_IQS620AT_TEMP) += iqs620at-temp.o obj-$(CONFIG_LTC2983) += ltc2983.o obj-$(CONFIG_HID_SENSOR_TEMP) += hid-sensor-temperature.o obj-$(CONFIG_MAXIM_THERMOCOUPLE) += maxim_thermocouple.o diff --git a/drivers/iio/temperature/iqs620at-temp.c b/drivers/iio/temperature/iqs620at-temp.c new file mode 100644 index 000000000000..3fd52b3eb030 --- /dev/null +++ b/drivers/iio/temperature/iqs620at-temp.c @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Azoteq IQS620AT Temperature Sensor + * + * Copyright (C) 2019 Jeff LaBundy + */ + +#include +#include +#include +#include +#include +#include +#include + +#define IQS620_TEMP_UI_OUT 0x1A + +#define IQS620_TEMP_SCALE 1000 +#define IQS620_TEMP_OFFSET (-100) + +static int iqs620_temp_read_raw(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + int *val, int *val2, long mask) +{ + struct iqs62x_core *iqs62x = iio_device_get_drvdata(indio_dev); + int ret; + __le16 val_buf; + + switch (mask) { + case IIO_CHAN_INFO_RAW: + ret = regmap_raw_read(iqs62x->regmap, IQS620_TEMP_UI_OUT, + &val_buf, sizeof(val_buf)); + if (ret) + return ret; + + *val = le16_to_cpu(val_buf); + return IIO_VAL_INT; + + case IIO_CHAN_INFO_SCALE: + *val = IQS620_TEMP_SCALE; + return IIO_VAL_INT; + + case IIO_CHAN_INFO_OFFSET: + *val = IQS620_TEMP_OFFSET; + return IIO_VAL_INT; + + default: + return -EINVAL; + } +} + +static const struct iio_info iqs620_temp_info = { + .read_raw = &iqs620_temp_read_raw, +}; + +static const struct iio_chan_spec iqs620_temp_channels[] = { + { + .type = IIO_TEMP, + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | + BIT(IIO_CHAN_INFO_SCALE) | + BIT(IIO_CHAN_INFO_OFFSET), + }, +}; + +static int iqs620_temp_probe(struct platform_device *pdev) +{ + struct iqs62x_core *iqs62x = dev_get_drvdata(pdev->dev.parent); + struct iio_dev *indio_dev; + + indio_dev = devm_iio_device_alloc(&pdev->dev, 0); + if (!indio_dev) + return -ENOMEM; + + iio_device_set_drvdata(indio_dev, iqs62x); + + indio_dev->modes = INDIO_DIRECT_MODE; + indio_dev->dev.parent = &pdev->dev; + indio_dev->channels = iqs620_temp_channels; + indio_dev->num_channels = ARRAY_SIZE(iqs620_temp_channels); + indio_dev->name = iqs62x->dev_desc->dev_name; + indio_dev->info = &iqs620_temp_info; + + return devm_iio_device_register(&pdev->dev, indio_dev); +} + +static struct platform_driver iqs620_temp_platform_driver = { + .driver = { + .name = "iqs620at-temp", + }, + .probe = iqs620_temp_probe, +}; +module_platform_driver(iqs620_temp_platform_driver); + +MODULE_AUTHOR("Jeff LaBundy "); +MODULE_DESCRIPTION("Azoteq IQS620AT Temperature Sensor"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:iqs620at-temp"); From b081b73820945decb2fd39bdc0ccf46a1ddc6d53 Mon Sep 17 00:00:00 2001 From: Jeff LaBundy Date: Sun, 16 Feb 2020 17:32:10 -0600 Subject: [PATCH 222/635] iio: light: Add support for Azoteq IQS621/622 ambient light sensors This patch adds support for the Azoteq IQS621 and IQS622 ambient light sensors, both of which can report a four-bit representation of ambient light intensity. The IQS621 can additionally report illuminace directly in units of lux, while the IQS622 can report a four-bit representation of infrared light intensity. Furthermore, the IQS622 can report a unitless measurement of a target's proximity to the device. Signed-off-by: Jeff LaBundy Reviewed-by: Jonathan Cameron Signed-off-by: Lee Jones --- drivers/iio/light/Kconfig | 10 + drivers/iio/light/Makefile | 1 + drivers/iio/light/iqs621-als.c | 617 +++++++++++++++++++++++++++++++++ 3 files changed, 628 insertions(+) create mode 100644 drivers/iio/light/iqs621-als.c diff --git a/drivers/iio/light/Kconfig b/drivers/iio/light/Kconfig index 9968f982fbc7..baf7958b7ddd 100644 --- a/drivers/iio/light/Kconfig +++ b/drivers/iio/light/Kconfig @@ -173,6 +173,16 @@ config GP2AP020A00F To compile this driver as a module, choose M here: the module will be called gp2ap020a00f. +config IQS621_ALS + tristate "Azoteq IQS621/622 ambient light sensors" + depends on MFD_IQS62X || COMPILE_TEST + help + Say Y here if you want to build support for the Azoteq IQS621 + and IQS622 ambient light sensors. + + To compile this driver as a module, choose M here: the module + will be called iqs621-als. + config SENSORS_ISL29018 tristate "Intersil 29018 light and proximity sensor" depends on I2C diff --git a/drivers/iio/light/Makefile b/drivers/iio/light/Makefile index c98d1cefb861..988e8f4128c3 100644 --- a/drivers/iio/light/Makefile +++ b/drivers/iio/light/Makefile @@ -21,6 +21,7 @@ obj-$(CONFIG_IIO_CROS_EC_LIGHT_PROX) += cros_ec_light_prox.o obj-$(CONFIG_GP2AP020A00F) += gp2ap020a00f.o obj-$(CONFIG_HID_SENSOR_ALS) += hid-sensor-als.o obj-$(CONFIG_HID_SENSOR_PROX) += hid-sensor-prox.o +obj-$(CONFIG_IQS621_ALS) += iqs621-als.o obj-$(CONFIG_SENSORS_ISL29018) += isl29018.o obj-$(CONFIG_SENSORS_ISL29028) += isl29028.o obj-$(CONFIG_ISL29125) += isl29125.o diff --git a/drivers/iio/light/iqs621-als.c b/drivers/iio/light/iqs621-als.c new file mode 100644 index 000000000000..b2988a782bd0 --- /dev/null +++ b/drivers/iio/light/iqs621-als.c @@ -0,0 +1,617 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Azoteq IQS621/622 Ambient Light Sensors + * + * Copyright (C) 2019 Jeff LaBundy + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define IQS621_ALS_FLAGS_LIGHT BIT(7) +#define IQS621_ALS_FLAGS_RANGE GENMASK(3, 0) + +#define IQS621_ALS_UI_OUT 0x17 + +#define IQS621_ALS_THRESH_DARK 0x80 +#define IQS621_ALS_THRESH_LIGHT 0x81 + +#define IQS622_IR_RANGE 0x15 +#define IQS622_IR_FLAGS 0x16 +#define IQS622_IR_FLAGS_TOUCH BIT(1) +#define IQS622_IR_FLAGS_PROX BIT(0) + +#define IQS622_IR_UI_OUT 0x17 + +#define IQS622_IR_THRESH_PROX 0x91 +#define IQS622_IR_THRESH_TOUCH 0x92 + +struct iqs621_als_private { + struct iqs62x_core *iqs62x; + struct notifier_block notifier; + struct mutex lock; + bool light_en; + bool range_en; + bool prox_en; + u8 als_flags; + u8 ir_flags_mask; + u8 ir_flags; + u8 thresh_light; + u8 thresh_dark; + u8 thresh_prox; +}; + +static int iqs621_als_init(struct iqs621_als_private *iqs621_als) +{ + struct iqs62x_core *iqs62x = iqs621_als->iqs62x; + unsigned int event_mask = 0; + int ret; + + switch (iqs621_als->ir_flags_mask) { + case IQS622_IR_FLAGS_TOUCH: + ret = regmap_write(iqs62x->regmap, IQS622_IR_THRESH_TOUCH, + iqs621_als->thresh_prox); + break; + + case IQS622_IR_FLAGS_PROX: + ret = regmap_write(iqs62x->regmap, IQS622_IR_THRESH_PROX, + iqs621_als->thresh_prox); + break; + + default: + ret = regmap_write(iqs62x->regmap, IQS621_ALS_THRESH_LIGHT, + iqs621_als->thresh_light); + if (ret) + return ret; + + ret = regmap_write(iqs62x->regmap, IQS621_ALS_THRESH_DARK, + iqs621_als->thresh_dark); + } + + if (ret) + return ret; + + if (iqs621_als->light_en || iqs621_als->range_en) + event_mask |= iqs62x->dev_desc->als_mask; + + if (iqs621_als->prox_en) + event_mask |= iqs62x->dev_desc->ir_mask; + + return regmap_update_bits(iqs62x->regmap, IQS620_GLBL_EVENT_MASK, + event_mask, 0); +} + +static int iqs621_als_notifier(struct notifier_block *notifier, + unsigned long event_flags, void *context) +{ + struct iqs62x_event_data *event_data = context; + struct iqs621_als_private *iqs621_als; + struct iio_dev *indio_dev; + bool light_new, light_old; + bool prox_new, prox_old; + u8 range_new, range_old; + s64 timestamp; + int ret; + + iqs621_als = container_of(notifier, struct iqs621_als_private, + notifier); + indio_dev = iio_priv_to_dev(iqs621_als); + timestamp = iio_get_time_ns(indio_dev); + + mutex_lock(&iqs621_als->lock); + + if (event_flags & BIT(IQS62X_EVENT_SYS_RESET)) { + ret = iqs621_als_init(iqs621_als); + if (ret) { + dev_err(indio_dev->dev.parent, + "Failed to re-initialize device: %d\n", ret); + ret = NOTIFY_BAD; + } else { + ret = NOTIFY_OK; + } + + goto err_mutex; + } + + if (!iqs621_als->light_en && !iqs621_als->range_en && + !iqs621_als->prox_en) { + ret = NOTIFY_DONE; + goto err_mutex; + } + + /* IQS621 only */ + light_new = event_data->als_flags & IQS621_ALS_FLAGS_LIGHT; + light_old = iqs621_als->als_flags & IQS621_ALS_FLAGS_LIGHT; + + if (iqs621_als->light_en && light_new && !light_old) + iio_push_event(indio_dev, + IIO_UNMOD_EVENT_CODE(IIO_LIGHT, 0, + IIO_EV_TYPE_THRESH, + IIO_EV_DIR_RISING), + timestamp); + else if (iqs621_als->light_en && !light_new && light_old) + iio_push_event(indio_dev, + IIO_UNMOD_EVENT_CODE(IIO_LIGHT, 0, + IIO_EV_TYPE_THRESH, + IIO_EV_DIR_FALLING), + timestamp); + + /* IQS621 and IQS622 */ + range_new = event_data->als_flags & IQS621_ALS_FLAGS_RANGE; + range_old = iqs621_als->als_flags & IQS621_ALS_FLAGS_RANGE; + + if (iqs621_als->range_en && (range_new > range_old)) + iio_push_event(indio_dev, + IIO_UNMOD_EVENT_CODE(IIO_INTENSITY, 0, + IIO_EV_TYPE_CHANGE, + IIO_EV_DIR_RISING), + timestamp); + else if (iqs621_als->range_en && (range_new < range_old)) + iio_push_event(indio_dev, + IIO_UNMOD_EVENT_CODE(IIO_INTENSITY, 0, + IIO_EV_TYPE_CHANGE, + IIO_EV_DIR_FALLING), + timestamp); + + /* IQS622 only */ + prox_new = event_data->ir_flags & iqs621_als->ir_flags_mask; + prox_old = iqs621_als->ir_flags & iqs621_als->ir_flags_mask; + + if (iqs621_als->prox_en && prox_new && !prox_old) + iio_push_event(indio_dev, + IIO_UNMOD_EVENT_CODE(IIO_PROXIMITY, 0, + IIO_EV_TYPE_THRESH, + IIO_EV_DIR_RISING), + timestamp); + else if (iqs621_als->prox_en && !prox_new && prox_old) + iio_push_event(indio_dev, + IIO_UNMOD_EVENT_CODE(IIO_PROXIMITY, 0, + IIO_EV_TYPE_THRESH, + IIO_EV_DIR_FALLING), + timestamp); + + iqs621_als->als_flags = event_data->als_flags; + iqs621_als->ir_flags = event_data->ir_flags; + ret = NOTIFY_OK; + +err_mutex: + mutex_unlock(&iqs621_als->lock); + + return ret; +} + +static void iqs621_als_notifier_unregister(void *context) +{ + struct iqs621_als_private *iqs621_als = context; + struct iio_dev *indio_dev = iio_priv_to_dev(iqs621_als); + int ret; + + ret = blocking_notifier_chain_unregister(&iqs621_als->iqs62x->nh, + &iqs621_als->notifier); + if (ret) + dev_err(indio_dev->dev.parent, + "Failed to unregister notifier: %d\n", ret); +} + +static int iqs621_als_read_raw(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + int *val, int *val2, long mask) +{ + struct iqs621_als_private *iqs621_als = iio_priv(indio_dev); + struct iqs62x_core *iqs62x = iqs621_als->iqs62x; + int ret; + __le16 val_buf; + + switch (chan->type) { + case IIO_INTENSITY: + ret = regmap_read(iqs62x->regmap, chan->address, val); + if (ret) + return ret; + + *val &= IQS621_ALS_FLAGS_RANGE; + return IIO_VAL_INT; + + case IIO_PROXIMITY: + case IIO_LIGHT: + ret = regmap_raw_read(iqs62x->regmap, chan->address, &val_buf, + sizeof(val_buf)); + if (ret) + return ret; + + *val = le16_to_cpu(val_buf); + return IIO_VAL_INT; + + default: + return -EINVAL; + } +} + +static int iqs621_als_read_event_config(struct iio_dev *indio_dev, + const struct iio_chan_spec *chan, + enum iio_event_type type, + enum iio_event_direction dir) +{ + struct iqs621_als_private *iqs621_als = iio_priv(indio_dev); + int ret; + + mutex_lock(&iqs621_als->lock); + + switch (chan->type) { + case IIO_LIGHT: + ret = iqs621_als->light_en; + break; + + case IIO_INTENSITY: + ret = iqs621_als->range_en; + break; + + case IIO_PROXIMITY: + ret = iqs621_als->prox_en; + break; + + default: + ret = -EINVAL; + } + + mutex_unlock(&iqs621_als->lock); + + return ret; +} + +static int iqs621_als_write_event_config(struct iio_dev *indio_dev, + const struct iio_chan_spec *chan, + enum iio_event_type type, + enum iio_event_direction dir, + int state) +{ + struct iqs621_als_private *iqs621_als = iio_priv(indio_dev); + struct iqs62x_core *iqs62x = iqs621_als->iqs62x; + unsigned int val; + int ret; + + mutex_lock(&iqs621_als->lock); + + ret = regmap_read(iqs62x->regmap, iqs62x->dev_desc->als_flags, &val); + if (ret) + goto err_mutex; + iqs621_als->als_flags = val; + + switch (chan->type) { + case IIO_LIGHT: + ret = regmap_update_bits(iqs62x->regmap, IQS620_GLBL_EVENT_MASK, + iqs62x->dev_desc->als_mask, + iqs621_als->range_en || state ? 0 : + 0xFF); + if (!ret) + iqs621_als->light_en = state; + break; + + case IIO_INTENSITY: + ret = regmap_update_bits(iqs62x->regmap, IQS620_GLBL_EVENT_MASK, + iqs62x->dev_desc->als_mask, + iqs621_als->light_en || state ? 0 : + 0xFF); + if (!ret) + iqs621_als->range_en = state; + break; + + case IIO_PROXIMITY: + ret = regmap_read(iqs62x->regmap, IQS622_IR_FLAGS, &val); + if (ret) + goto err_mutex; + iqs621_als->ir_flags = val; + + ret = regmap_update_bits(iqs62x->regmap, IQS620_GLBL_EVENT_MASK, + iqs62x->dev_desc->ir_mask, + state ? 0 : 0xFF); + if (!ret) + iqs621_als->prox_en = state; + break; + + default: + ret = -EINVAL; + } + +err_mutex: + mutex_unlock(&iqs621_als->lock); + + return ret; +} + +static int iqs621_als_read_event_value(struct iio_dev *indio_dev, + const struct iio_chan_spec *chan, + enum iio_event_type type, + enum iio_event_direction dir, + enum iio_event_info info, + int *val, int *val2) +{ + struct iqs621_als_private *iqs621_als = iio_priv(indio_dev); + int ret = IIO_VAL_INT; + + mutex_lock(&iqs621_als->lock); + + switch (dir) { + case IIO_EV_DIR_RISING: + *val = iqs621_als->thresh_light * 16; + break; + + case IIO_EV_DIR_FALLING: + *val = iqs621_als->thresh_dark * 4; + break; + + case IIO_EV_DIR_EITHER: + if (iqs621_als->ir_flags_mask == IQS622_IR_FLAGS_TOUCH) + *val = iqs621_als->thresh_prox * 4; + else + *val = iqs621_als->thresh_prox; + break; + + default: + ret = -EINVAL; + } + + mutex_unlock(&iqs621_als->lock); + + return ret; +} + +static int iqs621_als_write_event_value(struct iio_dev *indio_dev, + const struct iio_chan_spec *chan, + enum iio_event_type type, + enum iio_event_direction dir, + enum iio_event_info info, + int val, int val2) +{ + struct iqs621_als_private *iqs621_als = iio_priv(indio_dev); + struct iqs62x_core *iqs62x = iqs621_als->iqs62x; + unsigned int thresh_reg, thresh_val; + u8 ir_flags_mask, *thresh_cache; + int ret = -EINVAL; + + mutex_lock(&iqs621_als->lock); + + switch (dir) { + case IIO_EV_DIR_RISING: + thresh_reg = IQS621_ALS_THRESH_LIGHT; + thresh_val = val / 16; + + thresh_cache = &iqs621_als->thresh_light; + ir_flags_mask = 0; + break; + + case IIO_EV_DIR_FALLING: + thresh_reg = IQS621_ALS_THRESH_DARK; + thresh_val = val / 4; + + thresh_cache = &iqs621_als->thresh_dark; + ir_flags_mask = 0; + break; + + case IIO_EV_DIR_EITHER: + /* + * The IQS622 supports two detection thresholds, both measured + * in the same arbitrary units reported by read_raw: proximity + * (0 through 255 in steps of 1), and touch (0 through 1020 in + * steps of 4). + * + * Based on the single detection threshold chosen by the user, + * select the hardware threshold that gives the best trade-off + * between range and resolution. + * + * By default, the close-range (but coarse) touch threshold is + * chosen during probe. + */ + switch (val) { + case 0 ... 255: + thresh_reg = IQS622_IR_THRESH_PROX; + thresh_val = val; + + ir_flags_mask = IQS622_IR_FLAGS_PROX; + break; + + case 256 ... 1020: + thresh_reg = IQS622_IR_THRESH_TOUCH; + thresh_val = val / 4; + + ir_flags_mask = IQS622_IR_FLAGS_TOUCH; + break; + + default: + goto err_mutex; + } + + thresh_cache = &iqs621_als->thresh_prox; + break; + + default: + goto err_mutex; + } + + if (thresh_val > 0xFF) + goto err_mutex; + + ret = regmap_write(iqs62x->regmap, thresh_reg, thresh_val); + if (ret) + goto err_mutex; + + *thresh_cache = thresh_val; + iqs621_als->ir_flags_mask = ir_flags_mask; + +err_mutex: + mutex_unlock(&iqs621_als->lock); + + return ret; +} + +static const struct iio_info iqs621_als_info = { + .read_raw = &iqs621_als_read_raw, + .read_event_config = iqs621_als_read_event_config, + .write_event_config = iqs621_als_write_event_config, + .read_event_value = iqs621_als_read_event_value, + .write_event_value = iqs621_als_write_event_value, +}; + +static const struct iio_event_spec iqs621_als_range_events[] = { + { + .type = IIO_EV_TYPE_CHANGE, + .dir = IIO_EV_DIR_EITHER, + .mask_separate = BIT(IIO_EV_INFO_ENABLE), + }, +}; + +static const struct iio_event_spec iqs621_als_light_events[] = { + { + .type = IIO_EV_TYPE_THRESH, + .dir = IIO_EV_DIR_EITHER, + .mask_separate = BIT(IIO_EV_INFO_ENABLE), + }, + { + .type = IIO_EV_TYPE_THRESH, + .dir = IIO_EV_DIR_RISING, + .mask_separate = BIT(IIO_EV_INFO_VALUE), + }, + { + .type = IIO_EV_TYPE_THRESH, + .dir = IIO_EV_DIR_FALLING, + .mask_separate = BIT(IIO_EV_INFO_VALUE), + }, +}; + +static const struct iio_chan_spec iqs621_als_channels[] = { + { + .type = IIO_INTENSITY, + .address = IQS621_ALS_FLAGS, + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW), + .event_spec = iqs621_als_range_events, + .num_event_specs = ARRAY_SIZE(iqs621_als_range_events), + }, + { + .type = IIO_LIGHT, + .address = IQS621_ALS_UI_OUT, + .info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED), + .event_spec = iqs621_als_light_events, + .num_event_specs = ARRAY_SIZE(iqs621_als_light_events), + }, +}; + +static const struct iio_event_spec iqs622_als_prox_events[] = { + { + .type = IIO_EV_TYPE_THRESH, + .dir = IIO_EV_DIR_EITHER, + .mask_separate = BIT(IIO_EV_INFO_ENABLE) | + BIT(IIO_EV_INFO_VALUE), + }, +}; + +static const struct iio_chan_spec iqs622_als_channels[] = { + { + .type = IIO_INTENSITY, + .channel2 = IIO_MOD_LIGHT_BOTH, + .address = IQS622_ALS_FLAGS, + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW), + .event_spec = iqs621_als_range_events, + .num_event_specs = ARRAY_SIZE(iqs621_als_range_events), + .modified = true, + }, + { + .type = IIO_INTENSITY, + .channel2 = IIO_MOD_LIGHT_IR, + .address = IQS622_IR_RANGE, + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW), + .modified = true, + }, + { + .type = IIO_PROXIMITY, + .address = IQS622_IR_UI_OUT, + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW), + .event_spec = iqs622_als_prox_events, + .num_event_specs = ARRAY_SIZE(iqs622_als_prox_events), + }, +}; + +static int iqs621_als_probe(struct platform_device *pdev) +{ + struct iqs62x_core *iqs62x = dev_get_drvdata(pdev->dev.parent); + struct iqs621_als_private *iqs621_als; + struct iio_dev *indio_dev; + unsigned int val; + int ret; + + indio_dev = devm_iio_device_alloc(&pdev->dev, sizeof(*iqs621_als)); + if (!indio_dev) + return -ENOMEM; + + iqs621_als = iio_priv(indio_dev); + iqs621_als->iqs62x = iqs62x; + + if (iqs62x->dev_desc->prod_num == IQS622_PROD_NUM) { + ret = regmap_read(iqs62x->regmap, IQS622_IR_THRESH_TOUCH, + &val); + if (ret) + return ret; + iqs621_als->thresh_prox = val; + iqs621_als->ir_flags_mask = IQS622_IR_FLAGS_TOUCH; + + indio_dev->channels = iqs622_als_channels; + indio_dev->num_channels = ARRAY_SIZE(iqs622_als_channels); + } else { + ret = regmap_read(iqs62x->regmap, IQS621_ALS_THRESH_LIGHT, + &val); + if (ret) + return ret; + iqs621_als->thresh_light = val; + + ret = regmap_read(iqs62x->regmap, IQS621_ALS_THRESH_DARK, + &val); + if (ret) + return ret; + iqs621_als->thresh_dark = val; + + indio_dev->channels = iqs621_als_channels; + indio_dev->num_channels = ARRAY_SIZE(iqs621_als_channels); + } + + indio_dev->modes = INDIO_DIRECT_MODE; + indio_dev->dev.parent = &pdev->dev; + indio_dev->name = iqs62x->dev_desc->dev_name; + indio_dev->info = &iqs621_als_info; + + mutex_init(&iqs621_als->lock); + + iqs621_als->notifier.notifier_call = iqs621_als_notifier; + ret = blocking_notifier_chain_register(&iqs621_als->iqs62x->nh, + &iqs621_als->notifier); + if (ret) { + dev_err(&pdev->dev, "Failed to register notifier: %d\n", ret); + return ret; + } + + ret = devm_add_action_or_reset(&pdev->dev, + iqs621_als_notifier_unregister, + iqs621_als); + if (ret) + return ret; + + return devm_iio_device_register(&pdev->dev, indio_dev); +} + +static struct platform_driver iqs621_als_platform_driver = { + .driver = { + .name = "iqs621-als", + }, + .probe = iqs621_als_probe, +}; +module_platform_driver(iqs621_als_platform_driver); + +MODULE_AUTHOR("Jeff LaBundy "); +MODULE_DESCRIPTION("Azoteq IQS621/622 Ambient Light Sensors"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:iqs621-als"); From 189c3c495ad7382099a641664171d8b047d9e9b5 Mon Sep 17 00:00:00 2001 From: Jeff LaBundy Date: Sun, 16 Feb 2020 17:32:11 -0600 Subject: [PATCH 223/635] iio: position: Add support for Azoteq IQS624/625 angle sensors This patch adds support for the Azoteq IQS624 and IQS625 angular position sensors, capable of reporting the angle of a rotating shaft down to 1 and 10 degrees of accuracy, respectively. This patch also introduces a home for linear and angular position sensors. Unlike resolvers, they are typically contactless and use the Hall effect. Signed-off-by: Jeff LaBundy Reviewed-by: Jonathan Cameron Signed-off-by: Lee Jones --- drivers/iio/Kconfig | 1 + drivers/iio/Makefile | 1 + drivers/iio/position/Kconfig | 19 ++ drivers/iio/position/Makefile | 7 + drivers/iio/position/iqs624-pos.c | 284 ++++++++++++++++++++++++++++++ 5 files changed, 312 insertions(+) create mode 100644 drivers/iio/position/Kconfig create mode 100644 drivers/iio/position/Makefile create mode 100644 drivers/iio/position/iqs624-pos.c diff --git a/drivers/iio/Kconfig b/drivers/iio/Kconfig index 5bd51853b15e..d5c073a8aa3e 100644 --- a/drivers/iio/Kconfig +++ b/drivers/iio/Kconfig @@ -88,6 +88,7 @@ source "drivers/iio/orientation/Kconfig" if IIO_TRIGGER source "drivers/iio/trigger/Kconfig" endif #IIO_TRIGGER +source "drivers/iio/position/Kconfig" source "drivers/iio/potentiometer/Kconfig" source "drivers/iio/potentiostat/Kconfig" source "drivers/iio/pressure/Kconfig" diff --git a/drivers/iio/Makefile b/drivers/iio/Makefile index bff682ad1cfb..1712011c0f4a 100644 --- a/drivers/iio/Makefile +++ b/drivers/iio/Makefile @@ -31,6 +31,7 @@ obj-y += light/ obj-y += magnetometer/ obj-y += multiplexer/ obj-y += orientation/ +obj-y += position/ obj-y += potentiometer/ obj-y += potentiostat/ obj-y += pressure/ diff --git a/drivers/iio/position/Kconfig b/drivers/iio/position/Kconfig new file mode 100644 index 000000000000..eda67f008c5b --- /dev/null +++ b/drivers/iio/position/Kconfig @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Linear and angular position sensors +# +# When adding new entries keep the list in alphabetical order + +menu "Linear and angular position sensors" + +config IQS624_POS + tristate "Azoteq IQS624/625 angular position sensors" + depends on MFD_IQS62X || COMPILE_TEST + help + Say Y here if you want to build support for the Azoteq IQS624 + and IQS625 angular position sensors. + + To compile this driver as a module, choose M here: the module + will be called iqs624-pos. + +endmenu diff --git a/drivers/iio/position/Makefile b/drivers/iio/position/Makefile new file mode 100644 index 000000000000..3cbe7a734352 --- /dev/null +++ b/drivers/iio/position/Makefile @@ -0,0 +1,7 @@ +# +# Makefile for IIO linear and angular position sensors +# + +# When adding new entries keep the list in alphabetical order + +obj-$(CONFIG_IQS624_POS) += iqs624-pos.o diff --git a/drivers/iio/position/iqs624-pos.c b/drivers/iio/position/iqs624-pos.c new file mode 100644 index 000000000000..77096c31c2ba --- /dev/null +++ b/drivers/iio/position/iqs624-pos.c @@ -0,0 +1,284 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Azoteq IQS624/625 Angular Position Sensors + * + * Copyright (C) 2019 Jeff LaBundy + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define IQS624_POS_DEG_OUT 0x16 + +#define IQS624_POS_SCALE1 (314159 / 180) +#define IQS624_POS_SCALE2 100000 + +struct iqs624_pos_private { + struct iqs62x_core *iqs62x; + struct notifier_block notifier; + struct mutex lock; + bool angle_en; + u16 angle; +}; + +static int iqs624_pos_angle_en(struct iqs62x_core *iqs62x, bool angle_en) +{ + unsigned int event_mask = IQS624_HALL_UI_WHL_EVENT; + + /* + * The IQS625 reports angular position in the form of coarse intervals, + * so only interval change events are unmasked. Conversely, the IQS624 + * reports angular position down to one degree of resolution, so wheel + * movement events are unmasked instead. + */ + if (iqs62x->dev_desc->prod_num == IQS625_PROD_NUM) + event_mask = IQS624_HALL_UI_INT_EVENT; + + return regmap_update_bits(iqs62x->regmap, IQS624_HALL_UI, event_mask, + angle_en ? 0 : 0xFF); +} + +static int iqs624_pos_notifier(struct notifier_block *notifier, + unsigned long event_flags, void *context) +{ + struct iqs62x_event_data *event_data = context; + struct iqs624_pos_private *iqs624_pos; + struct iqs62x_core *iqs62x; + struct iio_dev *indio_dev; + u16 angle = event_data->ui_data; + s64 timestamp; + int ret; + + iqs624_pos = container_of(notifier, struct iqs624_pos_private, + notifier); + indio_dev = iio_priv_to_dev(iqs624_pos); + timestamp = iio_get_time_ns(indio_dev); + + iqs62x = iqs624_pos->iqs62x; + if (iqs62x->dev_desc->prod_num == IQS625_PROD_NUM) + angle = event_data->interval; + + mutex_lock(&iqs624_pos->lock); + + if (event_flags & BIT(IQS62X_EVENT_SYS_RESET)) { + ret = iqs624_pos_angle_en(iqs62x, iqs624_pos->angle_en); + if (ret) { + dev_err(indio_dev->dev.parent, + "Failed to re-initialize device: %d\n", ret); + ret = NOTIFY_BAD; + } else { + ret = NOTIFY_OK; + } + } else if (iqs624_pos->angle_en && (angle != iqs624_pos->angle)) { + iio_push_event(indio_dev, + IIO_UNMOD_EVENT_CODE(IIO_ANGL, 0, + IIO_EV_TYPE_CHANGE, + IIO_EV_DIR_NONE), + timestamp); + + iqs624_pos->angle = angle; + ret = NOTIFY_OK; + } else { + ret = NOTIFY_DONE; + } + + mutex_unlock(&iqs624_pos->lock); + + return ret; +} + +static void iqs624_pos_notifier_unregister(void *context) +{ + struct iqs624_pos_private *iqs624_pos = context; + struct iio_dev *indio_dev = iio_priv_to_dev(iqs624_pos); + int ret; + + ret = blocking_notifier_chain_unregister(&iqs624_pos->iqs62x->nh, + &iqs624_pos->notifier); + if (ret) + dev_err(indio_dev->dev.parent, + "Failed to unregister notifier: %d\n", ret); +} + +static int iqs624_pos_angle_get(struct iqs62x_core *iqs62x, unsigned int *val) +{ + int ret; + __le16 val_buf; + + if (iqs62x->dev_desc->prod_num == IQS625_PROD_NUM) + return regmap_read(iqs62x->regmap, iqs62x->dev_desc->interval, + val); + + ret = regmap_raw_read(iqs62x->regmap, IQS624_POS_DEG_OUT, &val_buf, + sizeof(val_buf)); + if (ret) + return ret; + + *val = le16_to_cpu(val_buf); + + return 0; +} + +static int iqs624_pos_read_raw(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + int *val, int *val2, long mask) +{ + struct iqs624_pos_private *iqs624_pos = iio_priv(indio_dev); + struct iqs62x_core *iqs62x = iqs624_pos->iqs62x; + unsigned int scale = 1; + int ret; + + switch (mask) { + case IIO_CHAN_INFO_RAW: + ret = iqs624_pos_angle_get(iqs62x, val); + if (ret) + return ret; + + return IIO_VAL_INT; + + case IIO_CHAN_INFO_SCALE: + if (iqs62x->dev_desc->prod_num == IQS625_PROD_NUM) { + ret = regmap_read(iqs62x->regmap, IQS624_INTERVAL_DIV, + &scale); + if (ret) + return ret; + } + + *val = scale * IQS624_POS_SCALE1; + *val2 = IQS624_POS_SCALE2; + return IIO_VAL_FRACTIONAL; + + default: + return -EINVAL; + } +} + +static int iqs624_pos_read_event_config(struct iio_dev *indio_dev, + const struct iio_chan_spec *chan, + enum iio_event_type type, + enum iio_event_direction dir) +{ + struct iqs624_pos_private *iqs624_pos = iio_priv(indio_dev); + int ret; + + mutex_lock(&iqs624_pos->lock); + ret = iqs624_pos->angle_en; + mutex_unlock(&iqs624_pos->lock); + + return ret; +} + +static int iqs624_pos_write_event_config(struct iio_dev *indio_dev, + const struct iio_chan_spec *chan, + enum iio_event_type type, + enum iio_event_direction dir, + int state) +{ + struct iqs624_pos_private *iqs624_pos = iio_priv(indio_dev); + struct iqs62x_core *iqs62x = iqs624_pos->iqs62x; + unsigned int val; + int ret; + + mutex_lock(&iqs624_pos->lock); + + ret = iqs624_pos_angle_get(iqs62x, &val); + if (ret) + goto err_mutex; + + ret = iqs624_pos_angle_en(iqs62x, state); + if (ret) + goto err_mutex; + + iqs624_pos->angle = val; + iqs624_pos->angle_en = state; + +err_mutex: + mutex_unlock(&iqs624_pos->lock); + + return ret; +} + +static const struct iio_info iqs624_pos_info = { + .read_raw = &iqs624_pos_read_raw, + .read_event_config = iqs624_pos_read_event_config, + .write_event_config = iqs624_pos_write_event_config, +}; + +static const struct iio_event_spec iqs624_pos_events[] = { + { + .type = IIO_EV_TYPE_CHANGE, + .dir = IIO_EV_DIR_NONE, + .mask_separate = BIT(IIO_EV_INFO_ENABLE), + }, +}; + +static const struct iio_chan_spec iqs624_pos_channels[] = { + { + .type = IIO_ANGL, + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | + BIT(IIO_CHAN_INFO_SCALE), + .event_spec = iqs624_pos_events, + .num_event_specs = ARRAY_SIZE(iqs624_pos_events), + }, +}; + +static int iqs624_pos_probe(struct platform_device *pdev) +{ + struct iqs62x_core *iqs62x = dev_get_drvdata(pdev->dev.parent); + struct iqs624_pos_private *iqs624_pos; + struct iio_dev *indio_dev; + int ret; + + indio_dev = devm_iio_device_alloc(&pdev->dev, sizeof(*iqs624_pos)); + if (!indio_dev) + return -ENOMEM; + + iqs624_pos = iio_priv(indio_dev); + iqs624_pos->iqs62x = iqs62x; + + indio_dev->modes = INDIO_DIRECT_MODE; + indio_dev->dev.parent = &pdev->dev; + indio_dev->channels = iqs624_pos_channels; + indio_dev->num_channels = ARRAY_SIZE(iqs624_pos_channels); + indio_dev->name = iqs62x->dev_desc->dev_name; + indio_dev->info = &iqs624_pos_info; + + mutex_init(&iqs624_pos->lock); + + iqs624_pos->notifier.notifier_call = iqs624_pos_notifier; + ret = blocking_notifier_chain_register(&iqs624_pos->iqs62x->nh, + &iqs624_pos->notifier); + if (ret) { + dev_err(&pdev->dev, "Failed to register notifier: %d\n", ret); + return ret; + } + + ret = devm_add_action_or_reset(&pdev->dev, + iqs624_pos_notifier_unregister, + iqs624_pos); + if (ret) + return ret; + + return devm_iio_device_register(&pdev->dev, indio_dev); +} + +static struct platform_driver iqs624_pos_platform_driver = { + .driver = { + .name = "iqs624-pos", + }, + .probe = iqs624_pos_probe, +}; +module_platform_driver(iqs624_pos_platform_driver); + +MODULE_AUTHOR("Jeff LaBundy "); +MODULE_DESCRIPTION("Azoteq IQS624/625 Angular Position Sensors"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:iqs624-pos"); From 361dca7d97f5ef8d0f3f8036bb33626a67817867 Mon Sep 17 00:00:00 2001 From: Andreas Kemnade Date: Fri, 20 Mar 2020 09:10:59 +0100 Subject: [PATCH 224/635] dt-bindings: mfd: rn5t618: Document optional property interrupts These chips use interrupts for various things like RTC alarm. Signed-off-by: Andreas Kemnade Acked-by: Rob Herring Signed-off-by: Lee Jones --- Documentation/devicetree/bindings/mfd/rn5t618.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Documentation/devicetree/bindings/mfd/rn5t618.txt b/Documentation/devicetree/bindings/mfd/rn5t618.txt index b74e5e94d1cb..16778ea00dbc 100644 --- a/Documentation/devicetree/bindings/mfd/rn5t618.txt +++ b/Documentation/devicetree/bindings/mfd/rn5t618.txt @@ -15,6 +15,8 @@ Required properties: - reg: the I2C slave address of the device Optional properties: + - interrupts: interrupt mapping for IRQ + See Documentation/devicetree/bindings/interrupt-controller/interrupts.txt - system-power-controller: See Documentation/devicetree/bindings/power/power-controller.txt @@ -32,6 +34,8 @@ Example: pmic@32 { compatible = "ricoh,rn5t618"; reg = <0x32>; + interrupt-parent = <&gpio5>; + interrupts = <11 IRQ_TYPE_EDGE_FALLING>; system-power-controller; regulators { From 0c81604516afc0f3aedbb4dcf8215df7e5c7ef32 Mon Sep 17 00:00:00 2001 From: Andreas Kemnade Date: Fri, 20 Mar 2020 09:11:00 +0100 Subject: [PATCH 225/635] mfd: rn5t618: Add IRQ support This adds support for IRQ handling in the RC5T619 which is required for properly implementing subdevices like RTC. For now only definitions for the variant RC5T619 are included. Signed-off-by: Andreas Kemnade Signed-off-by: Lee Jones --- drivers/mfd/Kconfig | 1 + drivers/mfd/rn5t618.c | 78 ++++++++++++++++++++++++++++++++++++- include/linux/mfd/rn5t618.h | 15 +++++++ 3 files changed, 93 insertions(+), 1 deletion(-) diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig index 2b203290e7b9..a7067888a41e 100644 --- a/drivers/mfd/Kconfig +++ b/drivers/mfd/Kconfig @@ -1058,6 +1058,7 @@ config MFD_RN5T618 depends on OF select MFD_CORE select REGMAP_I2C + select REGMAP_IRQ help Say yes here to add support for the Ricoh RN5T567, RN5T618, RC5T619 PMIC. diff --git a/drivers/mfd/rn5t618.c b/drivers/mfd/rn5t618.c index ead2e79036a9..b66dc4605d56 100644 --- a/drivers/mfd/rn5t618.c +++ b/drivers/mfd/rn5t618.c @@ -8,6 +8,8 @@ #include #include +#include +#include #include #include #include @@ -46,9 +48,56 @@ static const struct regmap_config rn5t618_regmap_config = { .cache_type = REGCACHE_RBTREE, }; +static const struct regmap_irq rc5t619_irqs[] = { + REGMAP_IRQ_REG(RN5T618_IRQ_SYS, 0, BIT(0)), + REGMAP_IRQ_REG(RN5T618_IRQ_DCDC, 0, BIT(1)), + REGMAP_IRQ_REG(RN5T618_IRQ_RTC, 0, BIT(2)), + REGMAP_IRQ_REG(RN5T618_IRQ_ADC, 0, BIT(3)), + REGMAP_IRQ_REG(RN5T618_IRQ_GPIO, 0, BIT(4)), + REGMAP_IRQ_REG(RN5T618_IRQ_CHG, 0, BIT(6)), +}; + +static const struct regmap_irq_chip rc5t619_irq_chip = { + .name = "rc5t619", + .irqs = rc5t619_irqs, + .num_irqs = ARRAY_SIZE(rc5t619_irqs), + .num_regs = 1, + .status_base = RN5T618_INTMON, + .mask_base = RN5T618_INTEN, + .mask_invert = true, +}; + static struct rn5t618 *rn5t618_pm_power_off; static struct notifier_block rn5t618_restart_handler; +static int rn5t618_irq_init(struct rn5t618 *rn5t618) +{ + const struct regmap_irq_chip *irq_chip = NULL; + int ret; + + if (!rn5t618->irq) + return 0; + + switch (rn5t618->variant) { + case RC5T619: + irq_chip = &rc5t619_irq_chip; + break; + default: + dev_err(rn5t618->dev, "Currently no IRQ support for variant %d\n", + (int)rn5t618->variant); + return -ENOENT; + } + + ret = devm_regmap_add_irq_chip(rn5t618->dev, rn5t618->regmap, + rn5t618->irq, + IRQF_TRIGGER_FALLING | IRQF_ONESHOT, + 0, irq_chip, &rn5t618->irq_data); + if (ret) + dev_err(rn5t618->dev, "Failed to register IRQ chip\n"); + + return ret; +} + static void rn5t618_trigger_poweroff_sequence(bool repower) { /* disable automatic repower-on */ @@ -106,6 +155,8 @@ static int rn5t618_i2c_probe(struct i2c_client *i2c, i2c_set_clientdata(i2c, priv); priv->variant = (long)of_id->data; + priv->irq = i2c->irq; + priv->dev = &i2c->dev; priv->regmap = devm_regmap_init_i2c(i2c, &rn5t618_regmap_config); if (IS_ERR(priv->regmap)) { @@ -138,7 +189,7 @@ static int rn5t618_i2c_probe(struct i2c_client *i2c, return ret; } - return 0; + return rn5t618_irq_init(priv); } static int rn5t618_i2c_remove(struct i2c_client *i2c) @@ -155,15 +206,40 @@ static int rn5t618_i2c_remove(struct i2c_client *i2c) return 0; } +static int __maybe_unused rn5t618_i2c_suspend(struct device *dev) +{ + struct rn5t618 *priv = dev_get_drvdata(dev); + + if (priv->irq) + disable_irq(priv->irq); + + return 0; +} + +static int __maybe_unused rn5t618_i2c_resume(struct device *dev) +{ + struct rn5t618 *priv = dev_get_drvdata(dev); + + if (priv->irq) + enable_irq(priv->irq); + + return 0; +} + static const struct i2c_device_id rn5t618_i2c_id[] = { { } }; MODULE_DEVICE_TABLE(i2c, rn5t618_i2c_id); +static SIMPLE_DEV_PM_OPS(rn5t618_i2c_dev_pm_ops, + rn5t618_i2c_suspend, + rn5t618_i2c_resume); + static struct i2c_driver rn5t618_i2c_driver = { .driver = { .name = "rn5t618", .of_match_table = of_match_ptr(rn5t618_of_match), + .pm = &rn5t618_i2c_dev_pm_ops, }, .probe = rn5t618_i2c_probe, .remove = rn5t618_i2c_remove, diff --git a/include/linux/mfd/rn5t618.h b/include/linux/mfd/rn5t618.h index d62ef48060b5..739571656f2b 100644 --- a/include/linux/mfd/rn5t618.h +++ b/include/linux/mfd/rn5t618.h @@ -242,9 +242,24 @@ enum { RC5T619, }; +/* RN5T618 IRQ definitions */ +enum { + RN5T618_IRQ_SYS = 0, + RN5T618_IRQ_DCDC, + RN5T618_IRQ_RTC, + RN5T618_IRQ_ADC, + RN5T618_IRQ_GPIO, + RN5T618_IRQ_CHG, + RN5T618_NR_IRQS, +}; + struct rn5t618 { struct regmap *regmap; + struct device *dev; long variant; + + int irq; + struct regmap_irq_chip_data *irq_data; }; #endif /* __LINUX_MFD_RN5T618_H */ From 11027ce6f1d2a20af16b0eae3d21d3ab78089262 Mon Sep 17 00:00:00 2001 From: Andreas Kemnade Date: Fri, 20 Mar 2020 09:11:01 +0100 Subject: [PATCH 226/635] mfd: rn5t618: Add RTC related registers Defines for some RTC related registers were missing, also they were not included in the volatile register list Signed-off-by: Andreas Kemnade Signed-off-by: Lee Jones --- drivers/mfd/rn5t618.c | 2 ++ include/linux/mfd/rn5t618.h | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/drivers/mfd/rn5t618.c b/drivers/mfd/rn5t618.c index b66dc4605d56..7686cc36e8c0 100644 --- a/drivers/mfd/rn5t618.c +++ b/drivers/mfd/rn5t618.c @@ -34,6 +34,8 @@ static bool rn5t618_volatile_reg(struct device *dev, unsigned int reg) case RN5T618_IR_GPF: case RN5T618_MON_IOIN: case RN5T618_INTMON: + case RN5T618_RTC_CTRL1 ... RN5T618_RTC_CTRL2: + case RN5T618_RTC_SECONDS ... RN5T618_RTC_YEAR: return true; default: return false; diff --git a/include/linux/mfd/rn5t618.h b/include/linux/mfd/rn5t618.h index 739571656f2b..fba0df13d9a8 100644 --- a/include/linux/mfd/rn5t618.h +++ b/include/linux/mfd/rn5t618.h @@ -139,6 +139,17 @@ #define RN5T618_INTPOL 0x9c #define RN5T618_INTEN 0x9d #define RN5T618_INTMON 0x9e + +#define RN5T618_RTC_SECONDS 0xA0 +#define RN5T618_RTC_MDAY 0xA4 +#define RN5T618_RTC_MONTH 0xA5 +#define RN5T618_RTC_YEAR 0xA6 +#define RN5T618_RTC_ADJUST 0xA7 +#define RN5T618_RTC_ALARM_Y_SEC 0xA8 +#define RN5T618_RTC_DAL_MONTH 0xAC +#define RN5T618_RTC_CTRL1 0xAE +#define RN5T618_RTC_CTRL2 0xAF + #define RN5T618_PREVINDAC 0xb0 #define RN5T618_BATDAC 0xb1 #define RN5T618_CHGCTL1 0xb3 From bc61676617d3d8da0eec3979a4922e7371ce48af Mon Sep 17 00:00:00 2001 From: Andreas Kemnade Date: Fri, 20 Mar 2020 09:11:02 +0100 Subject: [PATCH 227/635] mfd: rn5t618: Add more subdevices Since the RC5T619 has a RTC, use a separate subdevice list for that. The ADC should be the same as in the RN5T618, according to drivers in the wild, but since it is not tested, the ADC is only added for the RC5T619. Signed-off-by: Andreas Kemnade Signed-off-by: Lee Jones --- drivers/mfd/rn5t618.c | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/drivers/mfd/rn5t618.c b/drivers/mfd/rn5t618.c index 7686cc36e8c0..bc117adede4c 100644 --- a/drivers/mfd/rn5t618.c +++ b/drivers/mfd/rn5t618.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -22,6 +23,13 @@ static const struct mfd_cell rn5t618_cells[] = { { .name = "rn5t618-wdt" }, }; +static const struct mfd_cell rc5t619_cells[] = { + { .name = "rn5t618-adc" }, + { .name = "rn5t618-regulator" }, + { .name = "rc5t619-rtc" }, + { .name = "rn5t618-wdt" }, +}; + static bool rn5t618_volatile_reg(struct device *dev, unsigned int reg) { switch (reg) { @@ -167,8 +175,16 @@ static int rn5t618_i2c_probe(struct i2c_client *i2c, return ret; } - ret = devm_mfd_add_devices(&i2c->dev, -1, rn5t618_cells, - ARRAY_SIZE(rn5t618_cells), NULL, 0, NULL); + if (priv->variant == RC5T619) + ret = devm_mfd_add_devices(&i2c->dev, PLATFORM_DEVID_NONE, + rc5t619_cells, + ARRAY_SIZE(rc5t619_cells), + NULL, 0, NULL); + else + ret = devm_mfd_add_devices(&i2c->dev, PLATFORM_DEVID_NONE, + rn5t618_cells, + ARRAY_SIZE(rn5t618_cells), + NULL, 0, NULL); if (ret) { dev_err(&i2c->dev, "failed to add sub-devices: %d\n", ret); return ret; From 540d1e15393d92288d133fca2ea245d8d38227e6 Mon Sep 17 00:00:00 2001 From: Andreas Kemnade Date: Fri, 20 Mar 2020 09:11:03 +0100 Subject: [PATCH 228/635] rtc: rc5t619: Add Ricoh RC5T619 RTC driver Add an RTC driver for the RTC device on Ricoh MFD RC5T619, which is implemented as a variant of RN5T618. rtc-range output: Testing 2000-02-28 23:59:59. OK Testing 2038-01-19 03:14:07. OK Testing 2069-12-31 23:59:59. OK Testing 2099-12-31 23:59:59. KO RTC_RD_TIME returned 22 (line 138) Testing 2100-02-28 23:59:59. KO RTC_SET_TIME returned 34 (line 122) Testing 2106-02-07 06:28:15. KO RTC_SET_TIME returned 34 (line 122) Testing 2262-04-11 23:47:16. KO RTC_SET_TIME returned 34 (line 122) Signed-off-by: Andreas Kemnade Acked-by: Alexandre Belloni Signed-off-by: Lee Jones --- drivers/rtc/Kconfig | 10 + drivers/rtc/Makefile | 1 + drivers/rtc/rtc-rc5t619.c | 444 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 455 insertions(+) create mode 100644 drivers/rtc/rtc-rc5t619.c diff --git a/drivers/rtc/Kconfig b/drivers/rtc/Kconfig index 34c8b6c7e095..72fed8dc55e8 100644 --- a/drivers/rtc/Kconfig +++ b/drivers/rtc/Kconfig @@ -591,6 +591,16 @@ config RTC_DRV_RC5T583 This driver can also be built as a module. If so, the module will be called rtc-rc5t583. +config RTC_DRV_RC5T619 + tristate "RICOH RC5T619 RTC driver" + depends on MFD_RN5T618 + help + If you say yes here you get support for the RTC on the + RICOH RC5T619 chips. + + This driver can also be built as a module. If so, the module + will be called rtc-rc5t619. + config RTC_DRV_S35390A tristate "Seiko Instruments S-35390A" select BITREVERSE diff --git a/drivers/rtc/Makefile b/drivers/rtc/Makefile index 4ac8f19fb631..7612912cdf00 100644 --- a/drivers/rtc/Makefile +++ b/drivers/rtc/Makefile @@ -137,6 +137,7 @@ obj-$(CONFIG_RTC_DRV_PXA) += rtc-pxa.o obj-$(CONFIG_RTC_DRV_R7301) += rtc-r7301.o obj-$(CONFIG_RTC_DRV_R9701) += rtc-r9701.o obj-$(CONFIG_RTC_DRV_RC5T583) += rtc-rc5t583.o +obj-$(CONFIG_RTC_DRV_RC5T619) += rtc-rc5t619.o obj-$(CONFIG_RTC_DRV_RK808) += rtc-rk808.o obj-$(CONFIG_RTC_DRV_RP5C01) += rtc-rp5c01.o obj-$(CONFIG_RTC_DRV_RS5C313) += rtc-rs5c313.o diff --git a/drivers/rtc/rtc-rc5t619.c b/drivers/rtc/rtc-rc5t619.c new file mode 100644 index 000000000000..24e386ecbc7e --- /dev/null +++ b/drivers/rtc/rtc-rc5t619.c @@ -0,0 +1,444 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * drivers/rtc/rtc-rc5t619.c + * + * Real time clock driver for RICOH RC5T619 power management chip. + * + * Copyright (C) 2019 Andreas Kemnade + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct rc5t619_rtc { + int irq; + struct rtc_device *rtc; + struct rn5t618 *rn5t618; +}; + +#define CTRL1_ALARM_ENABLED 0x40 +#define CTRL1_24HR 0x20 +#define CTRL1_PERIODIC_MASK 0xf + +#define CTRL2_PON 0x10 +#define CTRL2_ALARM_STATUS 0x80 +#define CTRL2_CTFG 0x4 +#define CTRL2_CTC 0x1 + +#define MONTH_CENTFLAG 0x80 +#define HOUR_PMFLAG 0x20 +#define MDAY_DAL_EXT 0x80 + +static uint8_t rtc5t619_12hour_bcd2bin(uint8_t hour) +{ + if (hour & HOUR_PMFLAG) { + hour = bcd2bin(hour & ~HOUR_PMFLAG); + return hour == 12 ? 12 : 12 + hour; + } + + hour = bcd2bin(hour); + return hour == 12 ? 0 : hour; +} + +static uint8_t rtc5t619_12hour_bin2bcd(uint8_t hour) +{ + if (!hour) + return 0x12; + + if (hour < 12) + return bin2bcd(hour); + + if (hour == 12) + return 0x12 | HOUR_PMFLAG; + + return bin2bcd(hour - 12) | HOUR_PMFLAG; +} + +static int rc5t619_rtc_periodic_disable(struct device *dev) +{ + struct rc5t619_rtc *rtc = dev_get_drvdata(dev); + int err; + + /* disable function */ + err = regmap_update_bits(rtc->rn5t618->regmap, + RN5T618_RTC_CTRL1, CTRL1_PERIODIC_MASK, 0); + if (err < 0) + return err; + + /* clear alarm flag and CTFG */ + err = regmap_update_bits(rtc->rn5t618->regmap, RN5T618_RTC_CTRL2, + CTRL2_ALARM_STATUS | CTRL2_CTFG | CTRL2_CTC, + 0); + if (err < 0) + return err; + + return 0; +} + +/* things to be done once after power on */ +static int rc5t619_rtc_pon_setup(struct device *dev) +{ + struct rc5t619_rtc *rtc = dev_get_drvdata(dev); + int err; + unsigned int reg_data; + + err = regmap_read(rtc->rn5t618->regmap, RN5T618_RTC_CTRL2, ®_data); + if (err < 0) + return err; + + /* clear VDET PON */ + reg_data &= ~(CTRL2_PON | CTRL2_CTC | 0x4a); /* 0101-1011 */ + reg_data |= 0x20; /* 0010-0000 */ + err = regmap_write(rtc->rn5t618->regmap, RN5T618_RTC_CTRL2, reg_data); + if (err < 0) + return err; + + /* clearing RTC Adjust register */ + err = regmap_write(rtc->rn5t618->regmap, RN5T618_RTC_ADJUST, 0); + if (err) + return err; + + return regmap_update_bits(rtc->rn5t618->regmap, + RN5T618_RTC_CTRL1, + CTRL1_24HR, CTRL1_24HR); +} + +static int rc5t619_rtc_read_time(struct device *dev, struct rtc_time *tm) +{ + struct rc5t619_rtc *rtc = dev_get_drvdata(dev); + u8 buff[7]; + int err; + int cent_flag; + unsigned int ctrl1; + unsigned int ctrl2; + + err = regmap_read(rtc->rn5t618->regmap, RN5T618_RTC_CTRL2, &ctrl2); + if (err < 0) + return err; + + if (ctrl2 & CTRL2_PON) + return -EINVAL; + + err = regmap_read(rtc->rn5t618->regmap, RN5T618_RTC_CTRL1, &ctrl1); + if (err < 0) + return err; + + err = regmap_bulk_read(rtc->rn5t618->regmap, RN5T618_RTC_SECONDS, + buff, sizeof(buff)); + if (err < 0) + return err; + + if (buff[5] & MONTH_CENTFLAG) + cent_flag = 1; + else + cent_flag = 0; + + tm->tm_sec = bcd2bin(buff[0]); + tm->tm_min = bcd2bin(buff[1]); + + if (ctrl1 & CTRL1_24HR) + tm->tm_hour = bcd2bin(buff[2]); + else + tm->tm_hour = rtc5t619_12hour_bcd2bin(buff[2]); + + tm->tm_wday = bcd2bin(buff[3]); + tm->tm_mday = bcd2bin(buff[4]); + tm->tm_mon = bcd2bin(buff[5] & 0x1f) - 1; /* back to system 0-11 */ + tm->tm_year = bcd2bin(buff[6]) + 100 * cent_flag; + + return 0; +} + +static int rc5t619_rtc_set_time(struct device *dev, struct rtc_time *tm) +{ + struct rc5t619_rtc *rtc = dev_get_drvdata(dev); + u8 buff[7]; + int err; + int cent_flag; + unsigned int ctrl1; + unsigned int ctrl2; + + err = regmap_read(rtc->rn5t618->regmap, RN5T618_RTC_CTRL2, &ctrl2); + if (err < 0) + return err; + + if (ctrl2 & CTRL2_PON) + rc5t619_rtc_pon_setup(dev); + + err = regmap_read(rtc->rn5t618->regmap, RN5T618_RTC_CTRL1, &ctrl1); + if (err < 0) + return err; + + if (tm->tm_year >= 100) + cent_flag = 1; + else + cent_flag = 0; + + buff[0] = bin2bcd(tm->tm_sec); + buff[1] = bin2bcd(tm->tm_min); + + if (ctrl1 & CTRL1_24HR) + buff[2] = bin2bcd(tm->tm_hour); + else + buff[2] = rtc5t619_12hour_bin2bcd(tm->tm_hour); + + buff[3] = bin2bcd(tm->tm_wday); + buff[4] = bin2bcd(tm->tm_mday); + buff[5] = bin2bcd(tm->tm_mon + 1); /* system set 0-11 */ + buff[6] = bin2bcd(tm->tm_year - cent_flag * 100); + + if (cent_flag) + buff[5] |= MONTH_CENTFLAG; + + err = regmap_bulk_write(rtc->rn5t618->regmap, RN5T618_RTC_SECONDS, + buff, sizeof(buff)); + if (err < 0) { + dev_err(dev, "failed to program new time: %d\n", err); + return err; + } + + return 0; +} + +/* 0-disable, 1-enable */ +static int rc5t619_rtc_alarm_enable(struct device *dev, unsigned int enabled) +{ + struct rc5t619_rtc *rtc = dev_get_drvdata(dev); + + return regmap_update_bits(rtc->rn5t618->regmap, + RN5T618_RTC_CTRL1, + CTRL1_ALARM_ENABLED, + enabled ? CTRL1_ALARM_ENABLED : 0); +} + +static int rc5t619_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm) +{ + struct rc5t619_rtc *rtc = dev_get_drvdata(dev); + u8 buff[6]; + unsigned int buff_cent; + int err; + int cent_flag; + unsigned int ctrl1; + + err = regmap_read(rtc->rn5t618->regmap, RN5T618_RTC_CTRL1, &ctrl1); + if (err) + return err; + + err = regmap_read(rtc->rn5t618->regmap, RN5T618_RTC_MONTH, &buff_cent); + if (err < 0) { + dev_err(dev, "failed to read time: %d\n", err); + return err; + } + + if (buff_cent & MONTH_CENTFLAG) + cent_flag = 1; + else + cent_flag = 0; + + err = regmap_bulk_read(rtc->rn5t618->regmap, RN5T618_RTC_ALARM_Y_SEC, + buff, sizeof(buff)); + if (err) + return err; + + buff[3] = buff[3] & 0x3f; + + alrm->time.tm_sec = bcd2bin(buff[0]); + alrm->time.tm_min = bcd2bin(buff[1]); + + if (ctrl1 & CTRL1_24HR) + alrm->time.tm_hour = bcd2bin(buff[2]); + else + alrm->time.tm_hour = rtc5t619_12hour_bcd2bin(buff[2]); + + alrm->time.tm_mday = bcd2bin(buff[3]); + alrm->time.tm_mon = bcd2bin(buff[4]) - 1; + alrm->time.tm_year = bcd2bin(buff[5]) + 100 * cent_flag; + alrm->enabled = !!(ctrl1 & CTRL1_ALARM_ENABLED); + dev_dbg(dev, "read alarm: %ptR\n", &alrm->time); + + return 0; +} + +static int rc5t619_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm) +{ + struct rc5t619_rtc *rtc = dev_get_drvdata(dev); + u8 buff[6]; + int err; + int cent_flag; + unsigned int ctrl1; + + err = regmap_read(rtc->rn5t618->regmap, RN5T618_RTC_CTRL1, &ctrl1); + if (err) + return err; + + err = rc5t619_rtc_alarm_enable(dev, 0); + if (err < 0) + return err; + + if (rtc->irq == -1) + return -EINVAL; + + if (alrm->enabled == 0) + return 0; + + if (alrm->time.tm_year >= 100) + cent_flag = 1; + else + cent_flag = 0; + + alrm->time.tm_mon += 1; + buff[0] = bin2bcd(alrm->time.tm_sec); + buff[1] = bin2bcd(alrm->time.tm_min); + + if (ctrl1 & CTRL1_24HR) + buff[2] = bin2bcd(alrm->time.tm_hour); + else + buff[2] = rtc5t619_12hour_bin2bcd(alrm->time.tm_hour); + + buff[3] = bin2bcd(alrm->time.tm_mday); + buff[4] = bin2bcd(alrm->time.tm_mon); + buff[5] = bin2bcd(alrm->time.tm_year - 100 * cent_flag); + buff[3] |= MDAY_DAL_EXT; + + err = regmap_bulk_write(rtc->rn5t618->regmap, RN5T618_RTC_ALARM_Y_SEC, + buff, sizeof(buff)); + if (err < 0) + return err; + + return rc5t619_rtc_alarm_enable(dev, alrm->enabled); +} + +static const struct rtc_class_ops rc5t619_rtc_ops = { + .read_time = rc5t619_rtc_read_time, + .set_time = rc5t619_rtc_set_time, + .set_alarm = rc5t619_rtc_set_alarm, + .read_alarm = rc5t619_rtc_read_alarm, + .alarm_irq_enable = rc5t619_rtc_alarm_enable, +}; + +static int rc5t619_rtc_alarm_flag_clr(struct device *dev) +{ + struct rc5t619_rtc *rtc = dev_get_drvdata(dev); + + /* clear alarm-D status bits.*/ + return regmap_update_bits(rtc->rn5t618->regmap, + RN5T618_RTC_CTRL2, + CTRL2_ALARM_STATUS | CTRL2_CTC, 0); +} + +static irqreturn_t rc5t619_rtc_irq(int irq, void *data) +{ + struct device *dev = data; + struct rc5t619_rtc *rtc = dev_get_drvdata(dev); + + rc5t619_rtc_alarm_flag_clr(dev); + + rtc_update_irq(rtc->rtc, 1, RTC_IRQF | RTC_AF); + return IRQ_HANDLED; +} + +static int rc5t619_rtc_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct rn5t618 *rn5t618 = dev_get_drvdata(pdev->dev.parent); + struct rc5t619_rtc *rtc; + unsigned int ctrl2; + int err; + + rtc = devm_kzalloc(dev, sizeof(*rtc), GFP_KERNEL); + if (IS_ERR(rtc)) { + err = PTR_ERR(rtc); + return -ENOMEM; + } + + rtc->rn5t618 = rn5t618; + + dev_set_drvdata(dev, rtc); + rtc->irq = -1; + + if (rn5t618->irq_data) + rtc->irq = regmap_irq_get_virq(rn5t618->irq_data, + RN5T618_IRQ_RTC); + + if (rtc->irq < 0) + rtc->irq = -1; + + err = regmap_read(rtc->rn5t618->regmap, RN5T618_RTC_CTRL2, &ctrl2); + if (err < 0) + return err; + + /* disable rtc periodic function */ + err = rc5t619_rtc_periodic_disable(&pdev->dev); + if (err) + return err; + + if (ctrl2 & CTRL2_PON) { + err = rc5t619_rtc_alarm_flag_clr(&pdev->dev); + if (err) + return err; + } + + rtc->rtc = devm_rtc_allocate_device(&pdev->dev); + if (IS_ERR(rtc->rtc)) { + err = PTR_ERR(rtc->rtc); + dev_err(dev, "RTC device register: err %d\n", err); + return err; + } + + rtc->rtc->ops = &rc5t619_rtc_ops; + rtc->rtc->range_min = RTC_TIMESTAMP_BEGIN_1900; + rtc->rtc->range_max = RTC_TIMESTAMP_END_2099; + + /* set interrupt and enable it */ + if (rtc->irq != -1) { + err = devm_request_threaded_irq(&pdev->dev, rtc->irq, NULL, + rc5t619_rtc_irq, + IRQF_ONESHOT, + "rtc-rc5t619", + &pdev->dev); + if (err < 0) { + dev_err(&pdev->dev, "request IRQ:%d fail\n", rtc->irq); + rtc->irq = -1; + + err = rc5t619_rtc_alarm_enable(&pdev->dev, 0); + if (err) + return err; + + } else { + /* enable wake */ + device_init_wakeup(&pdev->dev, 1); + enable_irq_wake(rtc->irq); + } + } else { + /* system don't want to using alarm interrupt, so close it */ + err = rc5t619_rtc_alarm_enable(&pdev->dev, 0); + if (err) + return err; + + dev_warn(&pdev->dev, "rc5t619 interrupt is disabled\n"); + } + + return rtc_register_device(rtc->rtc); +} + +static struct platform_driver rc5t619_rtc_driver = { + .driver = { + .name = "rc5t619-rtc", + }, + .probe = rc5t619_rtc_probe, +}; + +module_platform_driver(rc5t619_rtc_driver); +MODULE_ALIAS("platform:rc5t619-rtc"); +MODULE_DESCRIPTION("RICOH RC5T619 RTC driver"); +MODULE_LICENSE("GPL"); From 87a25333c8c3ba6694f8230a649e162d6efb181a Mon Sep 17 00:00:00 2001 From: Andreas Kemnade Date: Fri, 20 Mar 2020 09:11:04 +0100 Subject: [PATCH 229/635] iio: adc: rn5t618: Add ADC driver for RN5T618/RC5T619 Both chips have an A/D converter capable of measuring things like VBAT, VUSB and analog inputs. Signed-off-by: Andreas Kemnade Reviewed-by: Jonathan Cameron Signed-off-by: Lee Jones --- drivers/iio/adc/Kconfig | 10 ++ drivers/iio/adc/Makefile | 1 + drivers/iio/adc/rn5t618-adc.c | 256 ++++++++++++++++++++++++++++++++++ 3 files changed, 267 insertions(+) create mode 100644 drivers/iio/adc/rn5t618-adc.c diff --git a/drivers/iio/adc/Kconfig b/drivers/iio/adc/Kconfig index 82e33082958c..300904b93656 100644 --- a/drivers/iio/adc/Kconfig +++ b/drivers/iio/adc/Kconfig @@ -783,6 +783,16 @@ config RCAR_GYRO_ADC To compile this driver as a module, choose M here: the module will be called rcar-gyroadc. +config RN5T618_ADC + tristate "ADC for the RN5T618/RC5T619 family of chips" + depends on MFD_RN5T618 + help + Say yes here to build support for the integrated ADC inside the + RN5T618/619 series PMICs: + + This driver can also be built as a module. If so, the module + will be called rn5t618-adc. + config ROCKCHIP_SARADC tristate "Rockchip SARADC driver" depends on ARCH_ROCKCHIP || (ARM && COMPILE_TEST) diff --git a/drivers/iio/adc/Makefile b/drivers/iio/adc/Makefile index 919228900df9..b182d234f45a 100644 --- a/drivers/iio/adc/Makefile +++ b/drivers/iio/adc/Makefile @@ -74,6 +74,7 @@ obj-$(CONFIG_QCOM_VADC_COMMON) += qcom-vadc-common.o obj-$(CONFIG_QCOM_SPMI_VADC) += qcom-spmi-vadc.o obj-$(CONFIG_QCOM_PM8XXX_XOADC) += qcom-pm8xxx-xoadc.o obj-$(CONFIG_RCAR_GYRO_ADC) += rcar-gyroadc.o +obj-$(CONFIG_RN5T618_ADC) += rn5t618-adc.o obj-$(CONFIG_ROCKCHIP_SARADC) += rockchip_saradc.o obj-$(CONFIG_SC27XX_ADC) += sc27xx_adc.o obj-$(CONFIG_SPEAR_ADC) += spear_adc.o diff --git a/drivers/iio/adc/rn5t618-adc.c b/drivers/iio/adc/rn5t618-adc.c new file mode 100644 index 000000000000..f21027e4e26a --- /dev/null +++ b/drivers/iio/adc/rn5t618-adc.c @@ -0,0 +1,256 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * ADC driver for the RICOH RN5T618 power management chip family + * + * Copyright (C) 2019 Andreas Kemnade + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define RN5T618_ADC_CONVERSION_TIMEOUT (msecs_to_jiffies(500)) +#define RN5T618_REFERENCE_VOLT 2500 + +/* mask for selecting channels for single conversion */ +#define RN5T618_ADCCNT3_CHANNEL_MASK 0x7 +/* average 4-time conversion mode */ +#define RN5T618_ADCCNT3_AVG BIT(3) +/* set for starting a single conversion, gets cleared by hw when done */ +#define RN5T618_ADCCNT3_GODONE BIT(4) +/* automatic conversion, period is in ADCCNT2, selected channels are + * in ADCCNT1 + */ +#define RN5T618_ADCCNT3_AUTO BIT(5) +#define RN5T618_ADCEND_IRQ BIT(0) + +struct rn5t618_adc_data { + struct device *dev; + struct rn5t618 *rn5t618; + struct completion conv_completion; + int irq; +}; + +struct rn5t618_channel_ratios { + u16 numerator; + u16 denominator; +}; + +enum rn5t618_channels { + LIMMON = 0, + VBAT, + VADP, + VUSB, + VSYS, + VTHM, + AIN1, + AIN0 +}; + +static const struct rn5t618_channel_ratios rn5t618_ratios[8] = { + [LIMMON] = {50, 32}, /* measured across 20mOhm, amplified by 32 */ + [VBAT] = {2, 1}, + [VADP] = {3, 1}, + [VUSB] = {3, 1}, + [VSYS] = {3, 1}, + [VTHM] = {1, 1}, + [AIN1] = {1, 1}, + [AIN0] = {1, 1}, +}; + +static int rn5t618_read_adc_reg(struct rn5t618 *rn5t618, int reg, u16 *val) +{ + u8 data[2]; + int ret; + + ret = regmap_bulk_read(rn5t618->regmap, reg, data, sizeof(data)); + if (ret < 0) + return ret; + + *val = (data[0] << 4) | (data[1] & 0xF); + + return 0; +} + +static irqreturn_t rn5t618_adc_irq(int irq, void *data) +{ + struct rn5t618_adc_data *adc = data; + unsigned int r = 0; + int ret; + + /* clear low & high threshold irqs */ + regmap_write(adc->rn5t618->regmap, RN5T618_IR_ADC1, 0); + regmap_write(adc->rn5t618->regmap, RN5T618_IR_ADC2, 0); + + ret = regmap_read(adc->rn5t618->regmap, RN5T618_IR_ADC3, &r); + if (ret < 0) + dev_err(adc->dev, "failed to read IRQ status: %d\n", ret); + + regmap_write(adc->rn5t618->regmap, RN5T618_IR_ADC3, 0); + + if (r & RN5T618_ADCEND_IRQ) + complete(&adc->conv_completion); + + return IRQ_HANDLED; +} + +static int rn5t618_adc_read(struct iio_dev *iio_dev, + const struct iio_chan_spec *chan, + int *val, int *val2, long mask) +{ + struct rn5t618_adc_data *adc = iio_priv(iio_dev); + u16 raw; + int ret; + + if (mask == IIO_CHAN_INFO_SCALE) { + *val = RN5T618_REFERENCE_VOLT * + rn5t618_ratios[chan->channel].numerator; + *val2 = rn5t618_ratios[chan->channel].denominator * 4095; + + return IIO_VAL_FRACTIONAL; + } + + /* select channel */ + ret = regmap_update_bits(adc->rn5t618->regmap, RN5T618_ADCCNT3, + RN5T618_ADCCNT3_CHANNEL_MASK, + chan->channel); + if (ret < 0) + return ret; + + ret = regmap_write(adc->rn5t618->regmap, RN5T618_EN_ADCIR3, + RN5T618_ADCEND_IRQ); + if (ret < 0) + return ret; + + ret = regmap_update_bits(adc->rn5t618->regmap, RN5T618_ADCCNT3, + RN5T618_ADCCNT3_AVG, + mask == IIO_CHAN_INFO_AVERAGE_RAW ? + RN5T618_ADCCNT3_AVG : 0); + if (ret < 0) + return ret; + + init_completion(&adc->conv_completion); + /* single conversion */ + ret = regmap_update_bits(adc->rn5t618->regmap, RN5T618_ADCCNT3, + RN5T618_ADCCNT3_GODONE, + RN5T618_ADCCNT3_GODONE); + if (ret < 0) + return ret; + + ret = wait_for_completion_timeout(&adc->conv_completion, + RN5T618_ADC_CONVERSION_TIMEOUT); + if (ret == 0) { + dev_warn(adc->dev, "timeout waiting for adc result\n"); + return -ETIMEDOUT; + } + + ret = rn5t618_read_adc_reg(adc->rn5t618, + RN5T618_ILIMDATAH + 2 * chan->channel, + &raw); + if (ret < 0) + return ret; + + *val = raw; + + return IIO_VAL_INT; +} + +static const struct iio_info rn5t618_adc_iio_info = { + .read_raw = &rn5t618_adc_read, +}; + +#define RN5T618_ADC_CHANNEL(_channel, _type, _name) { \ + .type = _type, \ + .channel = _channel, \ + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | \ + BIT(IIO_CHAN_INFO_AVERAGE_RAW) | \ + BIT(IIO_CHAN_INFO_SCALE), \ + .datasheet_name = _name, \ + .indexed = 1. \ +} + +static const struct iio_chan_spec rn5t618_adc_iio_channels[] = { + RN5T618_ADC_CHANNEL(LIMMON, IIO_CURRENT, "LIMMON"), + RN5T618_ADC_CHANNEL(VBAT, IIO_VOLTAGE, "VBAT"), + RN5T618_ADC_CHANNEL(VADP, IIO_VOLTAGE, "VADP"), + RN5T618_ADC_CHANNEL(VUSB, IIO_VOLTAGE, "VUSB"), + RN5T618_ADC_CHANNEL(VSYS, IIO_VOLTAGE, "VSYS"), + RN5T618_ADC_CHANNEL(VTHM, IIO_VOLTAGE, "VTHM"), + RN5T618_ADC_CHANNEL(AIN1, IIO_VOLTAGE, "AIN1"), + RN5T618_ADC_CHANNEL(AIN0, IIO_VOLTAGE, "AIN0") +}; + +static int rn5t618_adc_probe(struct platform_device *pdev) +{ + int ret; + struct iio_dev *iio_dev; + struct rn5t618_adc_data *adc; + struct rn5t618 *rn5t618 = dev_get_drvdata(pdev->dev.parent); + + iio_dev = devm_iio_device_alloc(&pdev->dev, sizeof(*adc)); + if (!iio_dev) { + dev_err(&pdev->dev, "failed allocating iio device\n"); + return -ENOMEM; + } + + adc = iio_priv(iio_dev); + adc->dev = &pdev->dev; + adc->rn5t618 = rn5t618; + + if (rn5t618->irq_data) + adc->irq = regmap_irq_get_virq(rn5t618->irq_data, + RN5T618_IRQ_ADC); + + if (adc->irq <= 0) { + dev_err(&pdev->dev, "get virq failed\n"); + return -EINVAL; + } + + init_completion(&adc->conv_completion); + + iio_dev->name = dev_name(&pdev->dev); + iio_dev->dev.parent = &pdev->dev; + iio_dev->info = &rn5t618_adc_iio_info; + iio_dev->modes = INDIO_DIRECT_MODE; + iio_dev->channels = rn5t618_adc_iio_channels; + iio_dev->num_channels = ARRAY_SIZE(rn5t618_adc_iio_channels); + + /* stop any auto-conversion */ + ret = regmap_write(rn5t618->regmap, RN5T618_ADCCNT3, 0); + if (ret < 0) + return ret; + + platform_set_drvdata(pdev, iio_dev); + + ret = devm_request_threaded_irq(adc->dev, adc->irq, NULL, + rn5t618_adc_irq, + IRQF_ONESHOT, dev_name(adc->dev), + adc); + if (ret < 0) { + dev_err(adc->dev, "request irq %d failed: %d\n", adc->irq, ret); + return ret; + } + + return devm_iio_device_register(adc->dev, iio_dev); +} + +static struct platform_driver rn5t618_adc_driver = { + .driver = { + .name = "rn5t618-adc", + }, + .probe = rn5t618_adc_probe, +}; + +module_platform_driver(rn5t618_adc_driver); +MODULE_ALIAS("platform:rn5t618-adc"); +MODULE_DESCRIPTION("RICOH RN5T618 ADC driver"); +MODULE_LICENSE("GPL"); From 7858658cdcab7ca8057444b1d24c6a28ddaa8589 Mon Sep 17 00:00:00 2001 From: Andreas Kemnade Date: Fri, 20 Mar 2020 09:11:05 +0100 Subject: [PATCH 230/635] mfd: rn5t618: Cleanup i2c_device_id That list was just empty, so it can be removed if .probe_new instead of .probe is used Suggested-by: Lee Jones Signed-off-by: Andreas Kemnade Signed-off-by: Lee Jones --- drivers/mfd/rn5t618.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/drivers/mfd/rn5t618.c b/drivers/mfd/rn5t618.c index bc117adede4c..232de50562f9 100644 --- a/drivers/mfd/rn5t618.c +++ b/drivers/mfd/rn5t618.c @@ -146,8 +146,7 @@ static const struct of_device_id rn5t618_of_match[] = { }; MODULE_DEVICE_TABLE(of, rn5t618_of_match); -static int rn5t618_i2c_probe(struct i2c_client *i2c, - const struct i2c_device_id *id) +static int rn5t618_i2c_probe(struct i2c_client *i2c) { const struct of_device_id *of_id; struct rn5t618 *priv; @@ -244,11 +243,6 @@ static int __maybe_unused rn5t618_i2c_resume(struct device *dev) return 0; } -static const struct i2c_device_id rn5t618_i2c_id[] = { - { } -}; -MODULE_DEVICE_TABLE(i2c, rn5t618_i2c_id); - static SIMPLE_DEV_PM_OPS(rn5t618_i2c_dev_pm_ops, rn5t618_i2c_suspend, rn5t618_i2c_resume); @@ -259,9 +253,8 @@ static struct i2c_driver rn5t618_i2c_driver = { .of_match_table = of_match_ptr(rn5t618_of_match), .pm = &rn5t618_i2c_dev_pm_ops, }, - .probe = rn5t618_i2c_probe, + .probe_new = rn5t618_i2c_probe, .remove = rn5t618_i2c_remove, - .id_table = rn5t618_i2c_id, }; module_i2c_driver(rn5t618_i2c_driver); From 902baf61adf6b187f0a6b789e70d788ea71ff5bc Mon Sep 17 00:00:00 2001 From: Jacob Pan Date: Thu, 19 Mar 2020 21:32:30 -0700 Subject: [PATCH 231/635] iommu/vt-d: Fix mm reference leak Move canonical address check before mmget_not_zero() to avoid mm reference leak. Fixes: 9d8c3af31607 ("iommu/vt-d: IOMMU Page Request needs to check if address is canonical.") Signed-off-by: Jacob Pan Acked-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel-svm.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c index d7f2a5358900..fc7d78876e02 100644 --- a/drivers/iommu/intel-svm.c +++ b/drivers/iommu/intel-svm.c @@ -611,14 +611,15 @@ static irqreturn_t prq_event_thread(int irq, void *d) * any faults on kernel addresses. */ if (!svm->mm) goto bad_req; - /* If the mm is already defunct, don't handle faults. */ - if (!mmget_not_zero(svm->mm)) - goto bad_req; /* If address is not canonical, return invalid response */ if (!is_canonical_address(address)) goto bad_req; + /* If the mm is already defunct, don't handle faults. */ + if (!mmget_not_zero(svm->mm)) + goto bad_req; + down_read(&svm->mm->mmap_sem); vma = find_extend_vma(svm->mm, address); if (!vma || address < vma->vm_start) From 4a663dae47316ae8b97d5b77025fe7dfd9d3487f Mon Sep 17 00:00:00 2001 From: Jacob Pan Date: Thu, 19 Mar 2020 21:32:31 -0700 Subject: [PATCH 232/635] iommu/vt-d: Add build dependency on IOASID IOASID code is needed by VT-d scalable mode for PASID allocation. Add explicit dependency such that IOASID is built-in whenever Intel IOMMU is enabled. Otherwise, aux domain code will fail when IOMMU is built-in and IOASID is compiled as a module. Fixes: 59a623374dc38 ("iommu/vt-d: Replace Intel specific PASID allocator with IOASID") Signed-off-by: Jacob Pan Acked-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index d2fade984999..25149544d57c 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -188,6 +188,7 @@ config INTEL_IOMMU select NEED_DMA_MAP_STATE select DMAR_TABLE select SWIOTLB + select IOASID help DMA remapping (DMAR) devices support enables independent address translations for Direct Memory Access (DMA) from devices. From 3f84b96c9779c8c2c8f4215a9d08cc6af1d45fdb Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Thu, 26 Mar 2020 10:35:56 +0100 Subject: [PATCH 233/635] iommu/virtio: Fix sparse warning We copied the virtio_iommu_config from the virtio-iommu specification, which declares the fields using little-endian annotations (for example le32). Unfortunately this causes sparse to warn about comparison between little- and cpu-endian, because of the typecheck() in virtio_cread(): drivers/iommu/virtio-iommu.c:1024:9: sparse: sparse: incompatible types in comparison expression (different base types): drivers/iommu/virtio-iommu.c:1024:9: sparse: restricted __le64 * drivers/iommu/virtio-iommu.c:1024:9: sparse: unsigned long long * drivers/iommu/virtio-iommu.c:1036:9: sparse: sparse: incompatible types in comparison expression (different base types): drivers/iommu/virtio-iommu.c:1036:9: sparse: restricted __le64 * drivers/iommu/virtio-iommu.c:1036:9: sparse: unsigned long long * drivers/iommu/virtio-iommu.c:1040:9: sparse: sparse: incompatible types in comparison expression (different base types): drivers/iommu/virtio-iommu.c:1040:9: sparse: restricted __le64 * drivers/iommu/virtio-iommu.c:1040:9: sparse: unsigned long long * drivers/iommu/virtio-iommu.c:1044:9: sparse: sparse: incompatible types in comparison expression (different base types): drivers/iommu/virtio-iommu.c:1044:9: sparse: restricted __le32 * drivers/iommu/virtio-iommu.c:1044:9: sparse: unsigned int * drivers/iommu/virtio-iommu.c:1048:9: sparse: sparse: incompatible types in comparison expression (different base types): drivers/iommu/virtio-iommu.c:1048:9: sparse: restricted __le32 * drivers/iommu/virtio-iommu.c:1048:9: sparse: unsigned int * drivers/iommu/virtio-iommu.c:1052:9: sparse: sparse: incompatible types in comparison expression (different base types): drivers/iommu/virtio-iommu.c:1052:9: sparse: restricted __le32 * drivers/iommu/virtio-iommu.c:1052:9: sparse: unsigned int * Although virtio_cread() does convert virtio-endian (in our case little-endian) to cpu-endian, the typecheck() needs the two arguments to have the same endianness. Do as UAPI headers of other virtio devices do, and remove the endian annotation from the device config. Even though we change the UAPI this shouldn't cause any regression since QEMU, the existing implementation of virtio-iommu that uses this header, already removes the annotations when importing headers. Reported-by: kbuild test robot Signed-off-by: Jean-Philippe Brucker Link: https://lore.kernel.org/r/20200326093558.2641019-2-jean-philippe@linaro.org Signed-off-by: Joerg Roedel --- include/uapi/linux/virtio_iommu.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/include/uapi/linux/virtio_iommu.h b/include/uapi/linux/virtio_iommu.h index 237e36a280cb..48e3c29223b5 100644 --- a/include/uapi/linux/virtio_iommu.h +++ b/include/uapi/linux/virtio_iommu.h @@ -18,24 +18,24 @@ #define VIRTIO_IOMMU_F_MMIO 5 struct virtio_iommu_range_64 { - __le64 start; - __le64 end; + __u64 start; + __u64 end; }; struct virtio_iommu_range_32 { - __le32 start; - __le32 end; + __u32 start; + __u32 end; }; struct virtio_iommu_config { /* Supported page sizes */ - __le64 page_size_mask; + __u64 page_size_mask; /* Supported IOVA range */ struct virtio_iommu_range_64 input_range; /* Max domain ID size */ struct virtio_iommu_range_32 domain_range; /* Probe buffer size */ - __le32 probe_size; + __u32 probe_size; }; /* Request types */ From 7062af3ed2ba451029e3733d9f677c68f5ea9e77 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Thu, 26 Mar 2020 10:35:57 +0100 Subject: [PATCH 234/635] iommu/virtio: Fix freeing of incomplete domains Calling viommu_domain_free() on a domain that hasn't been finalised (not attached to any device, for example) can currently cause an Oops, because we attempt to call ida_free() on ID 0, which may either be unallocated or used by another domain. Only initialise the vdomain->viommu pointer, which denotes a finalised domain, at the end of a successful viommu_domain_finalise(). Fixes: edcd69ab9a32 ("iommu: Add virtio-iommu driver") Reported-by: Eric Auger Signed-off-by: Jean-Philippe Brucker Reviewed-by: Robin Murphy Link: https://lore.kernel.org/r/20200326093558.2641019-3-jean-philippe@linaro.org Signed-off-by: Joerg Roedel --- drivers/iommu/virtio-iommu.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c index 93ff58632452..9d5d4c941874 100644 --- a/drivers/iommu/virtio-iommu.c +++ b/drivers/iommu/virtio-iommu.c @@ -613,18 +613,20 @@ static int viommu_domain_finalise(struct viommu_dev *viommu, int ret; struct viommu_domain *vdomain = to_viommu_domain(domain); - vdomain->viommu = viommu; - vdomain->map_flags = viommu->map_flags; + ret = ida_alloc_range(&viommu->domain_ids, viommu->first_domain, + viommu->last_domain, GFP_KERNEL); + if (ret < 0) + return ret; + + vdomain->id = (unsigned int)ret; domain->pgsize_bitmap = viommu->pgsize_bitmap; domain->geometry = viommu->geometry; - ret = ida_alloc_range(&viommu->domain_ids, viommu->first_domain, - viommu->last_domain, GFP_KERNEL); - if (ret >= 0) - vdomain->id = (unsigned int)ret; + vdomain->map_flags = viommu->map_flags; + vdomain->viommu = viommu; - return ret > 0 ? 0 : ret; + return 0; } static void viommu_domain_free(struct iommu_domain *domain) From 39b3b3c9cac1b4a64a6e546c9faf0c1011d775d4 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Thu, 26 Mar 2020 10:35:58 +0100 Subject: [PATCH 235/635] iommu/virtio: Reject IOMMU page granule larger than PAGE_SIZE We don't currently support IOMMUs with a page granule larger than the system page size. The IOVA allocator has a BUG_ON() in this case, and VFIO has a WARN_ON(). Removing these obstacles ranges doesn't seem possible without major changes to the DMA API and VFIO. Some callers of iommu_map(), for example, want to map multiple page-aligned regions adjacent to each others for scatter-gather purposes. Even in simple DMA API uses, a call to dma_map_page() would let the endpoint access neighbouring memory. And VFIO users cannot ensure that their virtual address buffer is physically contiguous at the IOMMU granule. Rather than triggering the IOVA BUG_ON() on mismatched page sizes, abort the vdomain finalise() with an error message. We could simply abort the viommu probe(), but an upcoming extension to virtio-iommu will allow setting different page masks for each endpoint. Reported-by: Bharat Bhushan Signed-off-by: Jean-Philippe Brucker Reviewed-by: Bharat Bhushan Reviewed-by: Eric Auger Reviewed-by: Robin Murphy Link: https://lore.kernel.org/r/20200326093558.2641019-4-jean-philippe@linaro.org Signed-off-by: Joerg Roedel --- drivers/iommu/virtio-iommu.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c index 9d5d4c941874..1adc73d1f90a 100644 --- a/drivers/iommu/virtio-iommu.c +++ b/drivers/iommu/virtio-iommu.c @@ -607,12 +607,22 @@ static struct iommu_domain *viommu_domain_alloc(unsigned type) return &vdomain->domain; } -static int viommu_domain_finalise(struct viommu_dev *viommu, +static int viommu_domain_finalise(struct viommu_endpoint *vdev, struct iommu_domain *domain) { int ret; + unsigned long viommu_page_size; + struct viommu_dev *viommu = vdev->viommu; struct viommu_domain *vdomain = to_viommu_domain(domain); + viommu_page_size = 1UL << __ffs(viommu->pgsize_bitmap); + if (viommu_page_size > PAGE_SIZE) { + dev_err(vdev->dev, + "granule 0x%lx larger than system page size 0x%lx\n", + viommu_page_size, PAGE_SIZE); + return -EINVAL; + } + ret = ida_alloc_range(&viommu->domain_ids, viommu->first_domain, viommu->last_domain, GFP_KERNEL); if (ret < 0) @@ -659,7 +669,7 @@ static int viommu_attach_dev(struct iommu_domain *domain, struct device *dev) * Properly initialize the domain now that we know which viommu * owns it. */ - ret = viommu_domain_finalise(vdev->viommu, domain); + ret = viommu_domain_finalise(vdev, domain); } else if (vdomain->viommu != vdev->viommu) { dev_err(dev, "cannot attach to foreign vIOMMU\n"); ret = -EXDEV; From 0008d0c3b1ab03b046b04b7bd9d70df1e2fffbfc Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 26 Mar 2020 16:08:26 +0100 Subject: [PATCH 236/635] iommu: Define dev_iommu_fwspec_get() for !CONFIG_IOMMU_API There are users outside of the IOMMU code that need to call that function. Define it for !CONFIG_IOMMU_API too so that compilation does not break. Reported-by: kbuild test robot Signed-off-by: Joerg Roedel Reviewed-by: Jean-Philippe Brucker Link: https://lore.kernel.org/r/20200326150841.10083-2-joro@8bytes.org --- include/linux/iommu.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 4d1ba76c9a64..505163e9702a 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -1073,6 +1073,10 @@ static inline int iommu_sva_unbind_gpasid(struct iommu_domain *domain, return -ENODEV; } +static inline struct iommu_fwspec *dev_iommu_fwspec_get(struct device *dev) +{ + return NULL; +} #endif /* CONFIG_IOMMU_API */ #ifdef CONFIG_IOMMU_DEBUGFS From 6990ec796a7470740e6fb108dd30948b1ac9beb9 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 26 Mar 2020 16:08:27 +0100 Subject: [PATCH 237/635] ACPI/IORT: Remove direct access of dev->iommu_fwspec Use the accessor functions instead of directly dereferencing dev->iommu_fwspec. Signed-off-by: Joerg Roedel Tested-by: Hanjun Guo Reviewed-by: Jean-Philippe Brucker Link: https://lore.kernel.org/r/20200326150841.10083-3-joro@8bytes.org --- drivers/acpi/arm64/iort.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c index ed3d2d1a7ae9..7d04424189df 100644 --- a/drivers/acpi/arm64/iort.c +++ b/drivers/acpi/arm64/iort.c @@ -1015,6 +1015,7 @@ const struct iommu_ops *iort_iommu_configure(struct device *dev) return ops; if (dev_is_pci(dev)) { + struct iommu_fwspec *fwspec; struct pci_bus *bus = to_pci_dev(dev)->bus; struct iort_pci_alias_info info = { .dev = dev }; @@ -1027,8 +1028,9 @@ const struct iommu_ops *iort_iommu_configure(struct device *dev) err = pci_for_each_dma_alias(to_pci_dev(dev), iort_pci_iommu_init, &info); - if (!err && iort_pci_rc_supports_ats(node)) - dev->iommu_fwspec->flags |= IOMMU_FWSPEC_PCI_RC_ATS; + fwspec = dev_iommu_fwspec_get(dev); + if (fwspec && iort_pci_rc_supports_ats(node)) + fwspec->flags |= IOMMU_FWSPEC_PCI_RC_ATS; } else { int i = 0; From d3e240c4bcf0030349b9289d1cbcbd0672b73284 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 26 Mar 2020 16:08:28 +0100 Subject: [PATCH 238/635] drm/msm/mdp5: Remove direct access of dev->iommu_fwspec Use the accessor functions instead of directly dereferencing dev->iommu_fwspec. Signed-off-by: Joerg Roedel Reviewed-by: Jean-Philippe Brucker Link: https://lore.kernel.org/r/20200326150841.10083-4-joro@8bytes.org --- drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c index e43ecd4be10a..1252e1d76340 100644 --- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c +++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c @@ -725,7 +725,7 @@ struct msm_kms *mdp5_kms_init(struct drm_device *dev) if (config->platform.iommu) { iommu_dev = &pdev->dev; - if (!iommu_dev->iommu_fwspec) + if (!dev_iommu_fwspec_get(iommu_dev)) iommu_dev = iommu_dev->parent; aspace = msm_gem_address_space_create(iommu_dev, From 8c3d69237eb2cae01a015fefed4b770284e4ebd4 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 26 Mar 2020 16:08:29 +0100 Subject: [PATCH 239/635] iommu/tegra-gart: Remove direct access of dev->iommu_fwspec Use the accessor functions instead of directly dereferencing dev->iommu_fwspec. Signed-off-by: Joerg Roedel Reviewed-by: Jean-Philippe Brucker Link: https://lore.kernel.org/r/20200326150841.10083-5-joro@8bytes.org --- drivers/iommu/tegra-gart.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/tegra-gart.c b/drivers/iommu/tegra-gart.c index 3fb7ba72507d..db6559e8336f 100644 --- a/drivers/iommu/tegra-gart.c +++ b/drivers/iommu/tegra-gart.c @@ -247,7 +247,7 @@ static int gart_iommu_add_device(struct device *dev) { struct iommu_group *group; - if (!dev->iommu_fwspec) + if (!dev_iommu_fwspec_get(dev)) return -ENODEV; group = iommu_group_get_for_dev(dev); From 045a70426067d6a22e3e5745b55efc18fa75aabf Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 26 Mar 2020 16:08:30 +0100 Subject: [PATCH 240/635] iommu: Rename struct iommu_param to dev_iommu The term dev_iommu aligns better with other existing structures and their accessor functions. Signed-off-by: Joerg Roedel Tested-by: Will Deacon # arm-smmu Reviewed-by: Jean-Philippe Brucker Reviewed-by: Greg Kroah-Hartman Cc: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20200326150841.10083-6-joro@8bytes.org --- drivers/iommu/iommu.c | 28 ++++++++++++++-------------- include/linux/device.h | 6 +++--- include/linux/iommu.h | 4 ++-- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 660eea8d1d2f..15d64a175d92 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -152,9 +152,9 @@ void iommu_device_unregister(struct iommu_device *iommu) } EXPORT_SYMBOL_GPL(iommu_device_unregister); -static struct iommu_param *iommu_get_dev_param(struct device *dev) +static struct dev_iommu *dev_iommu_get(struct device *dev) { - struct iommu_param *param = dev->iommu_param; + struct dev_iommu *param = dev->iommu; if (param) return param; @@ -164,14 +164,14 @@ static struct iommu_param *iommu_get_dev_param(struct device *dev) return NULL; mutex_init(¶m->lock); - dev->iommu_param = param; + dev->iommu = param; return param; } -static void iommu_free_dev_param(struct device *dev) +static void dev_iommu_free(struct device *dev) { - kfree(dev->iommu_param); - dev->iommu_param = NULL; + kfree(dev->iommu); + dev->iommu = NULL; } int iommu_probe_device(struct device *dev) @@ -183,7 +183,7 @@ int iommu_probe_device(struct device *dev) if (!ops) return -EINVAL; - if (!iommu_get_dev_param(dev)) + if (!dev_iommu_get(dev)) return -ENOMEM; if (!try_module_get(ops->owner)) { @@ -200,7 +200,7 @@ int iommu_probe_device(struct device *dev) err_module_put: module_put(ops->owner); err_free_dev_param: - iommu_free_dev_param(dev); + dev_iommu_free(dev); return ret; } @@ -211,9 +211,9 @@ void iommu_release_device(struct device *dev) if (dev->iommu_group) ops->remove_device(dev); - if (dev->iommu_param) { + if (dev->iommu) { module_put(ops->owner); - iommu_free_dev_param(dev); + dev_iommu_free(dev); } } @@ -972,7 +972,7 @@ int iommu_register_device_fault_handler(struct device *dev, iommu_dev_fault_handler_t handler, void *data) { - struct iommu_param *param = dev->iommu_param; + struct dev_iommu *param = dev->iommu; int ret = 0; if (!param) @@ -1015,7 +1015,7 @@ EXPORT_SYMBOL_GPL(iommu_register_device_fault_handler); */ int iommu_unregister_device_fault_handler(struct device *dev) { - struct iommu_param *param = dev->iommu_param; + struct dev_iommu *param = dev->iommu; int ret = 0; if (!param) @@ -1055,7 +1055,7 @@ EXPORT_SYMBOL_GPL(iommu_unregister_device_fault_handler); */ int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt) { - struct iommu_param *param = dev->iommu_param; + struct dev_iommu *param = dev->iommu; struct iommu_fault_event *evt_pending = NULL; struct iommu_fault_param *fparam; int ret = 0; @@ -1104,7 +1104,7 @@ int iommu_page_response(struct device *dev, int ret = -EINVAL; struct iommu_fault_event *evt; struct iommu_fault_page_request *prm; - struct iommu_param *param = dev->iommu_param; + struct dev_iommu *param = dev->iommu; struct iommu_domain *domain = iommu_get_domain_for_dev(dev); if (!domain || !domain->ops->page_response) diff --git a/include/linux/device.h b/include/linux/device.h index 0cd7c647c16c..af621f9fe85b 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -44,7 +44,7 @@ struct iommu_ops; struct iommu_group; struct iommu_fwspec; struct dev_pin_info; -struct iommu_param; +struct dev_iommu; /** * struct subsys_interface - interfaces to device functions @@ -514,7 +514,7 @@ struct dev_links_info { * device (i.e. the bus driver that discovered the device). * @iommu_group: IOMMU group the device belongs to. * @iommu_fwspec: IOMMU-specific properties supplied by firmware. - * @iommu_param: Per device generic IOMMU runtime data + * @iommu: Per device generic IOMMU runtime data * * @offline_disabled: If set, the device is permanently online. * @offline: Set after successful invocation of bus type's .offline(). @@ -614,7 +614,7 @@ struct device { void (*release)(struct device *dev); struct iommu_group *iommu_group; struct iommu_fwspec *iommu_fwspec; - struct iommu_param *iommu_param; + struct dev_iommu *iommu; bool offline_disabled:1; bool offline:1; diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 505163e9702a..843baaa65f10 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -365,7 +365,7 @@ struct iommu_fault_param { }; /** - * struct iommu_param - collection of per-device IOMMU data + * struct dev_iommu - Collection of per-device IOMMU data * * @fault_param: IOMMU detected device fault reporting data * @@ -373,7 +373,7 @@ struct iommu_fault_param { * struct iommu_group *iommu_group; * struct iommu_fwspec *iommu_fwspec; */ -struct iommu_param { +struct dev_iommu { struct mutex lock; struct iommu_fault_param *fault_param; }; From 72acd9df18f12420001f901493c54b7364f34d60 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 26 Mar 2020 16:08:31 +0100 Subject: [PATCH 241/635] iommu: Move iommu_fwspec to struct dev_iommu Move the iommu_fwspec pointer in struct device into struct dev_iommu. This is a step in the effort to reduce the iommu related pointers in struct device to one. Signed-off-by: Joerg Roedel Tested-by: Will Deacon # arm-smmu Reviewed-by: Jean-Philippe Brucker Reviewed-by: Greg Kroah-Hartman Cc: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20200326150841.10083-7-joro@8bytes.org --- drivers/iommu/iommu.c | 3 +++ include/linux/device.h | 3 --- include/linux/iommu.h | 12 ++++++++---- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 15d64a175d92..2b471419e26c 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2405,6 +2405,9 @@ int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode, if (fwspec) return ops == fwspec->ops ? 0 : -EINVAL; + if (!dev_iommu_get(dev)) + return -ENOMEM; + /* Preallocate for the overwhelmingly common case of 1 ID */ fwspec = kzalloc(struct_size(fwspec, ids, 1), GFP_KERNEL); if (!fwspec) diff --git a/include/linux/device.h b/include/linux/device.h index af621f9fe85b..9610d0accd88 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -42,7 +42,6 @@ struct device_node; struct fwnode_handle; struct iommu_ops; struct iommu_group; -struct iommu_fwspec; struct dev_pin_info; struct dev_iommu; @@ -513,7 +512,6 @@ struct dev_links_info { * gone away. This should be set by the allocator of the * device (i.e. the bus driver that discovered the device). * @iommu_group: IOMMU group the device belongs to. - * @iommu_fwspec: IOMMU-specific properties supplied by firmware. * @iommu: Per device generic IOMMU runtime data * * @offline_disabled: If set, the device is permanently online. @@ -613,7 +611,6 @@ struct device { void (*release)(struct device *dev); struct iommu_group *iommu_group; - struct iommu_fwspec *iommu_fwspec; struct dev_iommu *iommu; bool offline_disabled:1; diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 843baaa65f10..d031ddc0596b 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -368,14 +368,15 @@ struct iommu_fault_param { * struct dev_iommu - Collection of per-device IOMMU data * * @fault_param: IOMMU detected device fault reporting data + * @fwspec: IOMMU fwspec data * * TODO: migrate other per device data pointers under iommu_dev_data, e.g. * struct iommu_group *iommu_group; - * struct iommu_fwspec *iommu_fwspec; */ struct dev_iommu { struct mutex lock; - struct iommu_fault_param *fault_param; + struct iommu_fault_param *fault_param; + struct iommu_fwspec *fwspec; }; int iommu_device_register(struct iommu_device *iommu); @@ -614,13 +615,16 @@ const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode); static inline struct iommu_fwspec *dev_iommu_fwspec_get(struct device *dev) { - return dev->iommu_fwspec; + if (dev->iommu) + return dev->iommu->fwspec; + else + return NULL; } static inline void dev_iommu_fwspec_set(struct device *dev, struct iommu_fwspec *fwspec) { - dev->iommu_fwspec = fwspec; + dev->iommu->fwspec = fwspec; } int iommu_probe_device(struct device *dev); From 0b242ebbb7fca818323d02d2bbb08ff7d425f594 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 26 Mar 2020 16:08:32 +0100 Subject: [PATCH 242/635] iommu/arm-smmu: Fix uninitilized variable warning Some unrelated changes in the iommu code caused a new warning to appear in the arm-smmu driver: CC drivers/iommu/arm-smmu.o drivers/iommu/arm-smmu.c: In function 'arm_smmu_add_device': drivers/iommu/arm-smmu.c:1441:2: warning: 'smmu' may be used uninitialized in this function [-Wmaybe-uninitialized] arm_smmu_rpm_put(smmu); ^~~~~~~~~~~~~~~~~~~~~~ The warning is a false positive, but initialize the variable to NULL to get rid of it. Signed-off-by: Joerg Roedel Tested-by: Will Deacon # arm-smmu Reviewed-by: Jean-Philippe Brucker Link: https://lore.kernel.org/r/20200326150841.10083-8-joro@8bytes.org --- drivers/iommu/arm-smmu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 16c4b87af42b..980aae73b45b 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -1383,7 +1383,7 @@ struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode) static int arm_smmu_add_device(struct device *dev) { - struct arm_smmu_device *smmu; + struct arm_smmu_device *smmu = NULL; struct arm_smmu_master_cfg *cfg; struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); int i, ret; From f9867f416ee721141e1664810516b8ebc2563cdd Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 26 Mar 2020 16:08:33 +0100 Subject: [PATCH 243/635] iommu: Introduce accessors for iommu private data Add dev_iommu_priv_get/set() functions to access per-device iommu private data. This makes it easier to move the pointer to a different location. Signed-off-by: Joerg Roedel Tested-by: Will Deacon # arm-smmu Reviewed-by: Jean-Philippe Brucker Link: https://lore.kernel.org/r/20200326150841.10083-9-joro@8bytes.org --- include/linux/iommu.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index d031ddc0596b..49e3173260b3 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -627,6 +627,16 @@ static inline void dev_iommu_fwspec_set(struct device *dev, dev->iommu->fwspec = fwspec; } +static inline void *dev_iommu_priv_get(struct device *dev) +{ + return dev->iommu->fwspec->iommu_priv; +} + +static inline void dev_iommu_priv_set(struct device *dev, void *priv) +{ + dev->iommu->fwspec->iommu_priv = priv; +} + int iommu_probe_device(struct device *dev); void iommu_release_device(struct device *dev); From b7a9662f2c809e9778437aaf0e339279f4c8f3a1 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 26 Mar 2020 16:08:34 +0100 Subject: [PATCH 244/635] iommu/arm-smmu-v3: Use accessor functions for iommu private data Make use of dev_iommu_priv_set/get() functions in the code. Signed-off-by: Joerg Roedel Tested-by: Hanjun Guo Reviewed-by: Jean-Philippe Brucker Link: https://lore.kernel.org/r/20200326150841.10083-10-joro@8bytes.org --- drivers/iommu/arm-smmu-v3.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index aa3ac2a03807..2b68498dfb66 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -2659,7 +2659,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) if (!fwspec) return -ENOENT; - master = fwspec->iommu_priv; + master = dev_iommu_priv_get(dev); smmu = master->smmu; arm_smmu_detach_dev(master); @@ -2795,7 +2795,7 @@ static int arm_smmu_add_device(struct device *dev) if (!fwspec || fwspec->ops != &arm_smmu_ops) return -ENODEV; - if (WARN_ON_ONCE(fwspec->iommu_priv)) + if (WARN_ON_ONCE(dev_iommu_priv_get(dev))) return -EBUSY; smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode); @@ -2810,7 +2810,7 @@ static int arm_smmu_add_device(struct device *dev) master->smmu = smmu; master->sids = fwspec->ids; master->num_sids = fwspec->num_ids; - fwspec->iommu_priv = master; + dev_iommu_priv_set(dev, master); /* Check the SIDs are in range of the SMMU and our stream table */ for (i = 0; i < master->num_sids; i++) { @@ -2852,7 +2852,7 @@ err_unlink: iommu_device_unlink(&smmu->iommu, dev); err_free_master: kfree(master); - fwspec->iommu_priv = NULL; + dev_iommu_priv_set(dev, NULL); return ret; } @@ -2865,7 +2865,7 @@ static void arm_smmu_remove_device(struct device *dev) if (!fwspec || fwspec->ops != &arm_smmu_ops) return; - master = fwspec->iommu_priv; + master = dev_iommu_priv_get(dev); smmu = master->smmu; arm_smmu_detach_dev(master); iommu_group_remove_device(dev); From 2465170f98de5fef0d35bafc4dc29490d925ab36 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 26 Mar 2020 16:08:35 +0100 Subject: [PATCH 245/635] iommu/arm-smmu: Refactor master_cfg/fwspec usage In preparation for restructuring iommu_fwspec, refactor the way we access the arm_smmu_master_cfg private data to be less dependent on the current layout. Signed-off-by: Robin Murphy Signed-off-by: Joerg Roedel Link: https://lore.kernel.org/r/20200326150841.10083-11-joro@8bytes.org --- drivers/iommu/arm-smmu.c | 42 +++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 980aae73b45b..3cef2bfd6f3e 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -98,12 +98,10 @@ struct arm_smmu_master_cfg { s16 smendx[]; }; #define INVALID_SMENDX -1 -#define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv) -#define fwspec_smmu(fw) (__fwspec_cfg(fw)->smmu) -#define fwspec_smendx(fw, i) \ - (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i]) -#define for_each_cfg_sme(fw, i, idx) \ - for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i) +#define cfg_smendx(cfg, fw, i) \ + (i >= fw->num_ids ? INVALID_SMENDX : cfg->smendx[i]) +#define for_each_cfg_sme(cfg, fw, i, idx) \ + for (i = 0; idx = cfg_smendx(cfg, fw, i), i < fw->num_ids; ++i) static bool using_legacy_binding, using_generic_binding; @@ -1069,7 +1067,7 @@ static int arm_smmu_master_alloc_smes(struct device *dev) mutex_lock(&smmu->stream_map_mutex); /* Figure out a viable stream map entry allocation */ - for_each_cfg_sme(fwspec, i, idx) { + for_each_cfg_sme(cfg, fwspec, i, idx) { u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]); u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]); @@ -1100,7 +1098,7 @@ static int arm_smmu_master_alloc_smes(struct device *dev) iommu_group_put(group); /* It worked! Now, poke the actual hardware */ - for_each_cfg_sme(fwspec, i, idx) { + for_each_cfg_sme(cfg, fwspec, i, idx) { arm_smmu_write_sme(smmu, idx); smmu->s2crs[idx].group = group; } @@ -1117,14 +1115,14 @@ out_err: return ret; } -static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec) +static void arm_smmu_master_free_smes(struct arm_smmu_master_cfg *cfg, + struct iommu_fwspec *fwspec) { - struct arm_smmu_device *smmu = fwspec_smmu(fwspec); - struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv; + struct arm_smmu_device *smmu = cfg->smmu; int i, idx; mutex_lock(&smmu->stream_map_mutex); - for_each_cfg_sme(fwspec, i, idx) { + for_each_cfg_sme(cfg, fwspec, i, idx) { if (arm_smmu_free_sme(smmu, idx)) arm_smmu_write_sme(smmu, idx); cfg->smendx[i] = INVALID_SMENDX; @@ -1133,6 +1131,7 @@ static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec) } static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain, + struct arm_smmu_master_cfg *cfg, struct iommu_fwspec *fwspec) { struct arm_smmu_device *smmu = smmu_domain->smmu; @@ -1146,7 +1145,7 @@ static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain, else type = S2CR_TYPE_TRANS; - for_each_cfg_sme(fwspec, i, idx) { + for_each_cfg_sme(cfg, fwspec, i, idx) { if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx) continue; @@ -1162,8 +1161,9 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) { int ret; struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); - struct arm_smmu_device *smmu; struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct arm_smmu_master_cfg *cfg; + struct arm_smmu_device *smmu; if (!fwspec || fwspec->ops != &arm_smmu_ops) { dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n"); @@ -1177,10 +1177,11 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) * domains, just say no (but more politely than by dereferencing NULL). * This should be at least a WARN_ON once that's sorted. */ - if (!fwspec->iommu_priv) + cfg = fwspec->iommu_priv; + if (!cfg) return -ENODEV; - smmu = fwspec_smmu(fwspec); + smmu = cfg->smmu; ret = arm_smmu_rpm_get(smmu); if (ret < 0) @@ -1204,7 +1205,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) } /* Looks ok, so add the device to the domain */ - ret = arm_smmu_domain_add_master(smmu_domain, fwspec); + ret = arm_smmu_domain_add_master(smmu_domain, cfg, fwspec); /* * Setup an autosuspend delay to avoid bouncing runpm state. @@ -1475,7 +1476,7 @@ static void arm_smmu_remove_device(struct device *dev) return; iommu_device_unlink(&smmu->iommu, dev); - arm_smmu_master_free_smes(fwspec); + arm_smmu_master_free_smes(cfg, fwspec); arm_smmu_rpm_put(smmu); @@ -1487,11 +1488,12 @@ static void arm_smmu_remove_device(struct device *dev) static struct iommu_group *arm_smmu_device_group(struct device *dev) { struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); - struct arm_smmu_device *smmu = fwspec_smmu(fwspec); + struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv; + struct arm_smmu_device *smmu = cfg->smmu; struct iommu_group *group = NULL; int i, idx; - for_each_cfg_sme(fwspec, i, idx) { + for_each_cfg_sme(cfg, fwspec, i, idx) { if (group && smmu->s2crs[idx].group && group != smmu->s2crs[idx].group) return ERR_PTR(-EINVAL); From c84500a365b4b4b29b7ec31b8e29a3f97b68cb3e Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 26 Mar 2020 16:08:36 +0100 Subject: [PATCH 246/635] iommu/arm-smmu: Use accessor functions for iommu private data Make use of dev_iommu_priv_set/get() functions and simplify the code where possible with this change. Signed-off-by: Joerg Roedel Tested-by: Will Deacon # arm-smmu Link: https://lore.kernel.org/r/20200326150841.10083-12-joro@8bytes.org --- drivers/iommu/arm-smmu.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 3cef2bfd6f3e..a6a5796e9c41 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -1059,7 +1059,7 @@ static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx) static int arm_smmu_master_alloc_smes(struct device *dev) { struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); - struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv; + struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); struct arm_smmu_device *smmu = cfg->smmu; struct arm_smmu_smr *smrs = smmu->smrs; struct iommu_group *group; @@ -1159,11 +1159,11 @@ static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain, static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) { - int ret; - struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct arm_smmu_master_cfg *cfg; struct arm_smmu_device *smmu; + int ret; if (!fwspec || fwspec->ops != &arm_smmu_ops) { dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n"); @@ -1177,7 +1177,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) * domains, just say no (but more politely than by dereferencing NULL). * This should be at least a WARN_ON once that's sorted. */ - cfg = fwspec->iommu_priv; + cfg = dev_iommu_priv_get(dev); if (!cfg) return -ENODEV; @@ -1430,7 +1430,7 @@ static int arm_smmu_add_device(struct device *dev) goto out_free; cfg->smmu = smmu; - fwspec->iommu_priv = cfg; + dev_iommu_priv_set(dev, cfg); while (i--) cfg->smendx[i] = INVALID_SMENDX; @@ -1468,7 +1468,7 @@ static void arm_smmu_remove_device(struct device *dev) if (!fwspec || fwspec->ops != &arm_smmu_ops) return; - cfg = fwspec->iommu_priv; + cfg = dev_iommu_priv_get(dev); smmu = cfg->smmu; ret = arm_smmu_rpm_get(smmu); @@ -1480,15 +1480,16 @@ static void arm_smmu_remove_device(struct device *dev) arm_smmu_rpm_put(smmu); + dev_iommu_priv_set(dev, NULL); iommu_group_remove_device(dev); - kfree(fwspec->iommu_priv); + kfree(cfg); iommu_fwspec_free(dev); } static struct iommu_group *arm_smmu_device_group(struct device *dev) { + struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); - struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv; struct arm_smmu_device *smmu = cfg->smmu; struct iommu_group *group = NULL; int i, idx; From be568d6d5a5bebfcfe8789ee9560d0acf1b30d51 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 26 Mar 2020 16:08:37 +0100 Subject: [PATCH 247/635] iommu/renesas: Use accessor functions for iommu private data Make use of dev_iommu_priv_set/get() functions. Signed-off-by: Joerg Roedel Reviewed-by: Jean-Philippe Brucker Link: https://lore.kernel.org/r/20200326150841.10083-13-joro@8bytes.org --- drivers/iommu/ipmmu-vmsa.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index ecb3f9464dd5..310cf09feea3 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -89,9 +89,7 @@ static struct ipmmu_vmsa_domain *to_vmsa_domain(struct iommu_domain *dom) static struct ipmmu_vmsa_device *to_ipmmu(struct device *dev) { - struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); - - return fwspec ? fwspec->iommu_priv : NULL; + return dev_iommu_priv_get(dev); } #define TLB_LOOP_TIMEOUT 100 /* 100us */ @@ -727,14 +725,13 @@ static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain, static int ipmmu_init_platform_device(struct device *dev, struct of_phandle_args *args) { - struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct platform_device *ipmmu_pdev; ipmmu_pdev = of_find_device_by_node(args->np); if (!ipmmu_pdev) return -ENODEV; - fwspec->iommu_priv = platform_get_drvdata(ipmmu_pdev); + dev_iommu_priv_set(dev, platform_get_drvdata(ipmmu_pdev)); return 0; } From 3524b5592cad638bd0f65177a7807b686182d90b Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 26 Mar 2020 16:08:38 +0100 Subject: [PATCH 248/635] iommu/mediatek: Use accessor functions for iommu private data Make use of dev_iommu_priv_set/get() functions. Signed-off-by: Joerg Roedel Reviewed-by: Jean-Philippe Brucker Link: https://lore.kernel.org/r/20200326150841.10083-14-joro@8bytes.org --- drivers/iommu/mtk_iommu.c | 13 ++++++------- drivers/iommu/mtk_iommu_v1.c | 14 +++++++------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 95945f467c03..5f4d6df59cf6 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -358,8 +358,8 @@ static void mtk_iommu_domain_free(struct iommu_domain *domain) static int mtk_iommu_attach_device(struct iommu_domain *domain, struct device *dev) { + struct mtk_iommu_data *data = dev_iommu_priv_get(dev); struct mtk_iommu_domain *dom = to_mtk_domain(domain); - struct mtk_iommu_data *data = dev_iommu_fwspec_get(dev)->iommu_priv; if (!data) return -ENODEV; @@ -378,7 +378,7 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, static void mtk_iommu_detach_device(struct iommu_domain *domain, struct device *dev) { - struct mtk_iommu_data *data = dev_iommu_fwspec_get(dev)->iommu_priv; + struct mtk_iommu_data *data = dev_iommu_priv_get(dev); if (!data) return; @@ -450,7 +450,7 @@ static int mtk_iommu_add_device(struct device *dev) if (!fwspec || fwspec->ops != &mtk_iommu_ops) return -ENODEV; /* Not a iommu client device */ - data = fwspec->iommu_priv; + data = dev_iommu_priv_get(dev); iommu_device_link(&data->iommu, dev); group = iommu_group_get_for_dev(dev); @@ -469,7 +469,7 @@ static void mtk_iommu_remove_device(struct device *dev) if (!fwspec || fwspec->ops != &mtk_iommu_ops) return; - data = fwspec->iommu_priv; + data = dev_iommu_priv_get(dev); iommu_device_unlink(&data->iommu, dev); iommu_group_remove_device(dev); @@ -496,7 +496,6 @@ static struct iommu_group *mtk_iommu_device_group(struct device *dev) static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) { - struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct platform_device *m4updev; if (args->args_count != 1) { @@ -505,13 +504,13 @@ static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) return -EINVAL; } - if (!fwspec->iommu_priv) { + if (!dev_iommu_priv_get(dev)) { /* Get the m4u device */ m4updev = of_find_device_by_node(args->np); if (WARN_ON(!m4updev)) return -EINVAL; - fwspec->iommu_priv = platform_get_drvdata(m4updev); + dev_iommu_priv_set(dev, platform_get_drvdata(m4updev)); } return iommu_fwspec_add_ids(dev, args->args, 1); diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c index e93b94ecac45..a31be05601c9 100644 --- a/drivers/iommu/mtk_iommu_v1.c +++ b/drivers/iommu/mtk_iommu_v1.c @@ -263,8 +263,8 @@ static void mtk_iommu_domain_free(struct iommu_domain *domain) static int mtk_iommu_attach_device(struct iommu_domain *domain, struct device *dev) { + struct mtk_iommu_data *data = dev_iommu_priv_get(dev); struct mtk_iommu_domain *dom = to_mtk_domain(domain); - struct mtk_iommu_data *data = dev_iommu_fwspec_get(dev)->iommu_priv; int ret; if (!data) @@ -286,7 +286,7 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, static void mtk_iommu_detach_device(struct iommu_domain *domain, struct device *dev) { - struct mtk_iommu_data *data = dev_iommu_fwspec_get(dev)->iommu_priv; + struct mtk_iommu_data *data = dev_iommu_priv_get(dev); if (!data) return; @@ -387,20 +387,20 @@ static int mtk_iommu_create_mapping(struct device *dev, return -EINVAL; } - if (!fwspec->iommu_priv) { + if (!dev_iommu_priv_get(dev)) { /* Get the m4u device */ m4updev = of_find_device_by_node(args->np); if (WARN_ON(!m4updev)) return -EINVAL; - fwspec->iommu_priv = platform_get_drvdata(m4updev); + dev_iommu_priv_set(dev, platform_get_drvdata(m4updev)); } ret = iommu_fwspec_add_ids(dev, args->args, 1); if (ret) return ret; - data = fwspec->iommu_priv; + data = dev_iommu_priv_get(dev); m4udev = data->dev; mtk_mapping = m4udev->archdata.iommu; if (!mtk_mapping) { @@ -459,7 +459,7 @@ static int mtk_iommu_add_device(struct device *dev) if (err) return err; - data = fwspec->iommu_priv; + data = dev_iommu_priv_get(dev); mtk_mapping = data->dev->archdata.iommu; err = arm_iommu_attach_device(dev, mtk_mapping); if (err) { @@ -478,7 +478,7 @@ static void mtk_iommu_remove_device(struct device *dev) if (!fwspec || fwspec->ops != &mtk_iommu_ops) return; - data = fwspec->iommu_priv; + data = dev_iommu_priv_get(dev); iommu_device_unlink(&data->iommu, dev); iommu_group_remove_device(dev); From 09b5dfff9ad68f3619ea5b656ba5f941226f6afe Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 26 Mar 2020 16:08:39 +0100 Subject: [PATCH 249/635] iommu/qcom: Use accessor functions for iommu private data Make use of dev_iommu_priv_set/get() functions. Signed-off-by: Joerg Roedel Link: https://lore.kernel.org/r/20200326150841.10083-15-joro@8bytes.org --- drivers/iommu/qcom_iommu.c | 61 ++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/drivers/iommu/qcom_iommu.c b/drivers/iommu/qcom_iommu.c index 4328da0b0a9f..824a9e85fac6 100644 --- a/drivers/iommu/qcom_iommu.c +++ b/drivers/iommu/qcom_iommu.c @@ -74,16 +74,19 @@ static struct qcom_iommu_domain *to_qcom_iommu_domain(struct iommu_domain *dom) static const struct iommu_ops qcom_iommu_ops; -static struct qcom_iommu_dev * to_iommu(struct iommu_fwspec *fwspec) +static struct qcom_iommu_dev * to_iommu(struct device *dev) { + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + if (!fwspec || fwspec->ops != &qcom_iommu_ops) return NULL; - return fwspec->iommu_priv; + + return dev_iommu_priv_get(dev); } -static struct qcom_iommu_ctx * to_ctx(struct iommu_fwspec *fwspec, unsigned asid) +static struct qcom_iommu_ctx * to_ctx(struct device *dev, unsigned asid) { - struct qcom_iommu_dev *qcom_iommu = to_iommu(fwspec); + struct qcom_iommu_dev *qcom_iommu = to_iommu(dev); if (!qcom_iommu) return NULL; return qcom_iommu->ctxs[asid - 1]; @@ -115,11 +118,14 @@ iommu_readq(struct qcom_iommu_ctx *ctx, unsigned reg) static void qcom_iommu_tlb_sync(void *cookie) { - struct iommu_fwspec *fwspec = cookie; + struct iommu_fwspec *fwspec; + struct device *dev = cookie; unsigned i; + fwspec = dev_iommu_fwspec_get(dev); + for (i = 0; i < fwspec->num_ids; i++) { - struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]); + struct qcom_iommu_ctx *ctx = to_ctx(dev, fwspec->ids[i]); unsigned int val, ret; iommu_writel(ctx, ARM_SMMU_CB_TLBSYNC, 0); @@ -133,11 +139,14 @@ static void qcom_iommu_tlb_sync(void *cookie) static void qcom_iommu_tlb_inv_context(void *cookie) { - struct iommu_fwspec *fwspec = cookie; + struct device *dev = cookie; + struct iommu_fwspec *fwspec; unsigned i; + fwspec = dev_iommu_fwspec_get(dev); + for (i = 0; i < fwspec->num_ids; i++) { - struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]); + struct qcom_iommu_ctx *ctx = to_ctx(dev, fwspec->ids[i]); iommu_writel(ctx, ARM_SMMU_CB_S1_TLBIASID, ctx->asid); } @@ -147,13 +156,16 @@ static void qcom_iommu_tlb_inv_context(void *cookie) static void qcom_iommu_tlb_inv_range_nosync(unsigned long iova, size_t size, size_t granule, bool leaf, void *cookie) { - struct iommu_fwspec *fwspec = cookie; + struct device *dev = cookie; + struct iommu_fwspec *fwspec; unsigned i, reg; reg = leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA; + fwspec = dev_iommu_fwspec_get(dev); + for (i = 0; i < fwspec->num_ids; i++) { - struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]); + struct qcom_iommu_ctx *ctx = to_ctx(dev, fwspec->ids[i]); size_t s = size; iova = (iova >> 12) << 12; @@ -222,9 +234,10 @@ static irqreturn_t qcom_iommu_fault(int irq, void *dev) static int qcom_iommu_init_domain(struct iommu_domain *domain, struct qcom_iommu_dev *qcom_iommu, - struct iommu_fwspec *fwspec) + struct device *dev) { struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain); + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct io_pgtable_ops *pgtbl_ops; struct io_pgtable_cfg pgtbl_cfg; int i, ret = 0; @@ -243,7 +256,7 @@ static int qcom_iommu_init_domain(struct iommu_domain *domain, }; qcom_domain->iommu = qcom_iommu; - pgtbl_ops = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &pgtbl_cfg, fwspec); + pgtbl_ops = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &pgtbl_cfg, dev); if (!pgtbl_ops) { dev_err(qcom_iommu->dev, "failed to allocate pagetable ops\n"); ret = -ENOMEM; @@ -256,7 +269,7 @@ static int qcom_iommu_init_domain(struct iommu_domain *domain, domain->geometry.force_aperture = true; for (i = 0; i < fwspec->num_ids; i++) { - struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]); + struct qcom_iommu_ctx *ctx = to_ctx(dev, fwspec->ids[i]); if (!ctx->secure_init) { ret = qcom_scm_restore_sec_cfg(qcom_iommu->sec_id, ctx->asid); @@ -363,8 +376,7 @@ static void qcom_iommu_domain_free(struct iommu_domain *domain) static int qcom_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) { - struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); - struct qcom_iommu_dev *qcom_iommu = to_iommu(fwspec); + struct qcom_iommu_dev *qcom_iommu = to_iommu(dev); struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain); int ret; @@ -375,7 +387,7 @@ static int qcom_iommu_attach_dev(struct iommu_domain *domain, struct device *dev /* Ensure that the domain is finalized */ pm_runtime_get_sync(qcom_iommu->dev); - ret = qcom_iommu_init_domain(domain, qcom_iommu, fwspec); + ret = qcom_iommu_init_domain(domain, qcom_iommu, dev); pm_runtime_put_sync(qcom_iommu->dev); if (ret < 0) return ret; @@ -397,9 +409,9 @@ static int qcom_iommu_attach_dev(struct iommu_domain *domain, struct device *dev static void qcom_iommu_detach_dev(struct iommu_domain *domain, struct device *dev) { - struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); - struct qcom_iommu_dev *qcom_iommu = to_iommu(fwspec); struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain); + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + struct qcom_iommu_dev *qcom_iommu = to_iommu(dev); unsigned i; if (WARN_ON(!qcom_domain->iommu)) @@ -407,7 +419,7 @@ static void qcom_iommu_detach_dev(struct iommu_domain *domain, struct device *de pm_runtime_get_sync(qcom_iommu->dev); for (i = 0; i < fwspec->num_ids; i++) { - struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]); + struct qcom_iommu_ctx *ctx = to_ctx(dev, fwspec->ids[i]); /* Disable the context bank: */ iommu_writel(ctx, ARM_SMMU_CB_SCTLR, 0); @@ -514,7 +526,7 @@ static bool qcom_iommu_capable(enum iommu_cap cap) static int qcom_iommu_add_device(struct device *dev) { - struct qcom_iommu_dev *qcom_iommu = to_iommu(dev_iommu_fwspec_get(dev)); + struct qcom_iommu_dev *qcom_iommu = to_iommu(dev); struct iommu_group *group; struct device_link *link; @@ -545,7 +557,7 @@ static int qcom_iommu_add_device(struct device *dev) static void qcom_iommu_remove_device(struct device *dev) { - struct qcom_iommu_dev *qcom_iommu = to_iommu(dev_iommu_fwspec_get(dev)); + struct qcom_iommu_dev *qcom_iommu = to_iommu(dev); if (!qcom_iommu) return; @@ -557,7 +569,6 @@ static void qcom_iommu_remove_device(struct device *dev) static int qcom_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) { - struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct qcom_iommu_dev *qcom_iommu; struct platform_device *iommu_pdev; unsigned asid = args->args[0]; @@ -583,14 +594,14 @@ static int qcom_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) WARN_ON(asid > qcom_iommu->num_ctxs)) return -EINVAL; - if (!fwspec->iommu_priv) { - fwspec->iommu_priv = qcom_iommu; + if (!dev_iommu_priv_get(dev)) { + dev_iommu_priv_set(dev, qcom_iommu); } else { /* make sure devices iommus dt node isn't referring to * multiple different iommu devices. Multiple context * banks are ok, but multiple devices are not: */ - if (WARN_ON(qcom_iommu != fwspec->iommu_priv)) + if (WARN_ON(qcom_iommu != dev_iommu_priv_get(dev))) return -EINVAL; } From a4b6c2af8f65439cd29c1de56489bc6e9244db35 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 26 Mar 2020 16:08:40 +0100 Subject: [PATCH 250/635] iommu/virtio: Use accessor functions for iommu private data Make use of dev_iommu_priv_set/get() functions. Signed-off-by: Joerg Roedel Reviewed-by: Jean-Philippe Brucker Link: https://lore.kernel.org/r/20200326150841.10083-16-joro@8bytes.org --- drivers/iommu/virtio-iommu.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c index cce329d71fba..8ead57f031f5 100644 --- a/drivers/iommu/virtio-iommu.c +++ b/drivers/iommu/virtio-iommu.c @@ -466,7 +466,7 @@ static int viommu_probe_endpoint(struct viommu_dev *viommu, struct device *dev) struct virtio_iommu_req_probe *probe; struct virtio_iommu_probe_property *prop; struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); - struct viommu_endpoint *vdev = fwspec->iommu_priv; + struct viommu_endpoint *vdev = dev_iommu_priv_get(dev); if (!fwspec->num_ids) return -EINVAL; @@ -648,7 +648,7 @@ static int viommu_attach_dev(struct iommu_domain *domain, struct device *dev) int ret = 0; struct virtio_iommu_req_attach req; struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); - struct viommu_endpoint *vdev = fwspec->iommu_priv; + struct viommu_endpoint *vdev = dev_iommu_priv_get(dev); struct viommu_domain *vdomain = to_viommu_domain(domain); mutex_lock(&vdomain->mutex); @@ -807,8 +807,7 @@ static void viommu_iotlb_sync(struct iommu_domain *domain, static void viommu_get_resv_regions(struct device *dev, struct list_head *head) { struct iommu_resv_region *entry, *new_entry, *msi = NULL; - struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); - struct viommu_endpoint *vdev = fwspec->iommu_priv; + struct viommu_endpoint *vdev = dev_iommu_priv_get(dev); int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; list_for_each_entry(entry, &vdev->resv_regions, list) { @@ -876,7 +875,7 @@ static int viommu_add_device(struct device *dev) vdev->dev = dev; vdev->viommu = viommu; INIT_LIST_HEAD(&vdev->resv_regions); - fwspec->iommu_priv = vdev; + dev_iommu_priv_set(dev, vdev); if (viommu->probe_size) { /* Get additional information for this endpoint */ @@ -920,7 +919,7 @@ static void viommu_remove_device(struct device *dev) if (!fwspec || fwspec->ops != &viommu_ops) return; - vdev = fwspec->iommu_priv; + vdev = dev_iommu_priv_get(dev); iommu_group_remove_device(dev); iommu_device_unlink(&vdev->viommu->iommu, dev); From 986d5ecc56999800a5d112a70e88522d9212aefd Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 26 Mar 2020 16:08:41 +0100 Subject: [PATCH 251/635] iommu: Move fwspec->iommu_priv to struct dev_iommu Move the pointer for iommu private data from struct iommu_fwspec to struct dev_iommu. Signed-off-by: Joerg Roedel Tested-by: Will Deacon # arm-smmu Reviewed-by: Jean-Philippe Brucker Link: https://lore.kernel.org/r/20200326150841.10083-17-joro@8bytes.org --- include/linux/iommu.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 49e3173260b3..7ef8b0bda695 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -369,6 +369,7 @@ struct iommu_fault_param { * * @fault_param: IOMMU detected device fault reporting data * @fwspec: IOMMU fwspec data + * @priv: IOMMU Driver private data * * TODO: migrate other per device data pointers under iommu_dev_data, e.g. * struct iommu_group *iommu_group; @@ -377,6 +378,7 @@ struct dev_iommu { struct mutex lock; struct iommu_fault_param *fault_param; struct iommu_fwspec *fwspec; + void *priv; }; int iommu_device_register(struct iommu_device *iommu); @@ -589,7 +591,6 @@ struct iommu_group *fsl_mc_device_group(struct device *dev); struct iommu_fwspec { const struct iommu_ops *ops; struct fwnode_handle *iommu_fwnode; - void *iommu_priv; u32 flags; u32 num_pasid_bits; unsigned int num_ids; @@ -629,12 +630,12 @@ static inline void dev_iommu_fwspec_set(struct device *dev, static inline void *dev_iommu_priv_get(struct device *dev) { - return dev->iommu->fwspec->iommu_priv; + return dev->iommu->priv; } static inline void dev_iommu_priv_set(struct device *dev, void *priv) { - dev->iommu->fwspec->iommu_priv = priv; + dev->iommu->priv = priv; } int iommu_probe_device(struct device *dev); From b1e7396a1d0e6af6806337fdaaa44098d6b3343c Mon Sep 17 00:00:00 2001 From: Nicolas Saenz Julienne Date: Thu, 26 Mar 2020 13:20:01 +0100 Subject: [PATCH 252/635] drm/vc4: Fix HDMI mode validation Current mode validation impedes setting up some video modes which should be supported otherwise. Namely 1920x1200@60Hz. Fix this by lowering the minimum HDMI state machine clock to pixel clock ratio allowed. Fixes: 32e823c63e90 ("drm/vc4: Reject HDMI modes with too high of clocks.") Reported-by: Stefan Wahren Suggested-by: Dave Stevenson Signed-off-by: Nicolas Saenz Julienne Signed-off-by: Maxime Ripard Link: https://patchwork.freedesktop.org/patch/msgid/20200326122001.22215-1-nsaenzjulienne@suse.de --- drivers/gpu/drm/vc4/vc4_hdmi.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.c b/drivers/gpu/drm/vc4/vc4_hdmi.c index cea18dc15f77..340719238753 100644 --- a/drivers/gpu/drm/vc4/vc4_hdmi.c +++ b/drivers/gpu/drm/vc4/vc4_hdmi.c @@ -681,11 +681,23 @@ static enum drm_mode_status vc4_hdmi_encoder_mode_valid(struct drm_encoder *crtc, const struct drm_display_mode *mode) { - /* HSM clock must be 108% of the pixel clock. Additionally, - * the AXI clock needs to be at least 25% of pixel clock, but - * HSM ends up being the limiting factor. + /* + * As stated in RPi's vc4 firmware "HDMI state machine (HSM) clock must + * be faster than pixel clock, infinitesimally faster, tested in + * simulation. Otherwise, exact value is unimportant for HDMI + * operation." This conflicts with bcm2835's vc4 documentation, which + * states HSM's clock has to be at least 108% of the pixel clock. + * + * Real life tests reveal that vc4's firmware statement holds up, and + * users are able to use pixel clocks closer to HSM's, namely for + * 1920x1200@60Hz. So it was decided to have leave a 1% margin between + * both clocks. Which, for RPi0-3 implies a maximum pixel clock of + * 162MHz. + * + * Additionally, the AXI clock needs to be at least 25% of + * pixel clock, but HSM ends up being the limiting factor. */ - if (mode->clock > HSM_CLOCK_FREQ / (1000 * 108 / 100)) + if (mode->clock > HSM_CLOCK_FREQ / (1000 * 101 / 100)) return MODE_CLOCK_HIGH; return MODE_OK; From 62a89501a3bde310ba339cc90bb05879528d84a0 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Wed, 12 Feb 2020 11:12:35 -0500 Subject: [PATCH 253/635] xprtrdma: Enhance MR-related trace points Two changes: - Show the number of SG entries that were mapped. This helps debug DMA-related problems. - Record the MR's resource ID instead of its memory address. This groups each MR with its associated rdma-tool output, and reduces needless exposure of memory addresses. Signed-off-by: Chuck Lever Signed-off-by: Anna Schumaker --- include/trace/events/rpcrdma.h | 56 ++++++++++++++++++---------------- net/sunrpc/xprtrdma/frwr_ops.c | 2 +- 2 files changed, 31 insertions(+), 27 deletions(-) diff --git a/include/trace/events/rpcrdma.h b/include/trace/events/rpcrdma.h index c0e4c93324f5..87f4461ab108 100644 --- a/include/trace/events/rpcrdma.h +++ b/include/trace/events/rpcrdma.h @@ -228,20 +228,20 @@ DECLARE_EVENT_CLASS(xprtrdma_frwr_done, TP_ARGS(wc, frwr), TP_STRUCT__entry( - __field(const void *, mr) + __field(u32, mr_id) __field(unsigned int, status) __field(unsigned int, vendor_err) ), TP_fast_assign( - __entry->mr = container_of(frwr, struct rpcrdma_mr, frwr); + __entry->mr_id = frwr->fr_mr->res.id; __entry->status = wc->status; __entry->vendor_err = __entry->status ? wc->vendor_err : 0; ), TP_printk( - "mr=%p: %s (%u/0x%x)", - __entry->mr, rdma_show_wc_status(__entry->status), + "mr.id=%u: %s (%u/0x%x)", + __entry->mr_id, rdma_show_wc_status(__entry->status), __entry->status, __entry->vendor_err ) ); @@ -274,7 +274,8 @@ DECLARE_EVENT_CLASS(xprtrdma_mr, TP_ARGS(mr), TP_STRUCT__entry( - __field(const void *, mr) + __field(u32, mr_id) + __field(int, nents) __field(u32, handle) __field(u32, length) __field(u64, offset) @@ -282,15 +283,16 @@ DECLARE_EVENT_CLASS(xprtrdma_mr, ), TP_fast_assign( - __entry->mr = mr; + __entry->mr_id = mr->frwr.fr_mr->res.id; + __entry->nents = mr->mr_nents; __entry->handle = mr->mr_handle; __entry->length = mr->mr_length; __entry->offset = mr->mr_offset; __entry->dir = mr->mr_dir; ), - TP_printk("mr=%p %u@0x%016llx:0x%08x (%s)", - __entry->mr, __entry->length, + TP_printk("mr.id=%u nents=%d %u@0x%016llx:0x%08x (%s)", + __entry->mr_id, __entry->nents, __entry->length, (unsigned long long)__entry->offset, __entry->handle, xprtrdma_show_direction(__entry->dir) ) @@ -920,17 +922,17 @@ TRACE_EVENT(xprtrdma_frwr_alloc, TP_ARGS(mr, rc), TP_STRUCT__entry( - __field(const void *, mr) + __field(u32, mr_id) __field(int, rc) ), TP_fast_assign( - __entry->mr = mr; - __entry->rc = rc; + __entry->mr_id = mr->frwr.fr_mr->res.id; + __entry->rc = rc; ), - TP_printk("mr=%p: rc=%d", - __entry->mr, __entry->rc + TP_printk("mr.id=%u: rc=%d", + __entry->mr_id, __entry->rc ) ); @@ -943,7 +945,8 @@ TRACE_EVENT(xprtrdma_frwr_dereg, TP_ARGS(mr, rc), TP_STRUCT__entry( - __field(const void *, mr) + __field(u32, mr_id) + __field(int, nents) __field(u32, handle) __field(u32, length) __field(u64, offset) @@ -952,7 +955,8 @@ TRACE_EVENT(xprtrdma_frwr_dereg, ), TP_fast_assign( - __entry->mr = mr; + __entry->mr_id = mr->frwr.fr_mr->res.id; + __entry->nents = mr->mr_nents; __entry->handle = mr->mr_handle; __entry->length = mr->mr_length; __entry->offset = mr->mr_offset; @@ -960,8 +964,8 @@ TRACE_EVENT(xprtrdma_frwr_dereg, __entry->rc = rc; ), - TP_printk("mr=%p %u@0x%016llx:0x%08x (%s): rc=%d", - __entry->mr, __entry->length, + TP_printk("mr.id=%u nents=%d %u@0x%016llx:0x%08x (%s): rc=%d", + __entry->mr_id, __entry->nents, __entry->length, (unsigned long long)__entry->offset, __entry->handle, xprtrdma_show_direction(__entry->dir), __entry->rc @@ -977,21 +981,21 @@ TRACE_EVENT(xprtrdma_frwr_sgerr, TP_ARGS(mr, sg_nents), TP_STRUCT__entry( - __field(const void *, mr) + __field(u32, mr_id) __field(u64, addr) __field(u32, dir) __field(int, nents) ), TP_fast_assign( - __entry->mr = mr; + __entry->mr_id = mr->frwr.fr_mr->res.id; __entry->addr = mr->mr_sg->dma_address; __entry->dir = mr->mr_dir; __entry->nents = sg_nents; ), - TP_printk("mr=%p dma addr=0x%llx (%s) sg_nents=%d", - __entry->mr, __entry->addr, + TP_printk("mr.id=%u DMA addr=0x%llx (%s) sg_nents=%d", + __entry->mr_id, __entry->addr, xprtrdma_show_direction(__entry->dir), __entry->nents ) @@ -1006,7 +1010,7 @@ TRACE_EVENT(xprtrdma_frwr_maperr, TP_ARGS(mr, num_mapped), TP_STRUCT__entry( - __field(const void *, mr) + __field(u32, mr_id) __field(u64, addr) __field(u32, dir) __field(int, num_mapped) @@ -1014,15 +1018,15 @@ TRACE_EVENT(xprtrdma_frwr_maperr, ), TP_fast_assign( - __entry->mr = mr; + __entry->mr_id = mr->frwr.fr_mr->res.id; __entry->addr = mr->mr_sg->dma_address; __entry->dir = mr->mr_dir; __entry->num_mapped = num_mapped; __entry->nents = mr->mr_nents; ), - TP_printk("mr=%p dma addr=0x%llx (%s) nents=%d of %d", - __entry->mr, __entry->addr, + TP_printk("mr.id=%u DMA addr=0x%llx (%s) nents=%d of %d", + __entry->mr_id, __entry->addr, xprtrdma_show_direction(__entry->dir), __entry->num_mapped, __entry->nents ) @@ -1031,7 +1035,7 @@ TRACE_EVENT(xprtrdma_frwr_maperr, DEFINE_MR_EVENT(localinv); DEFINE_MR_EVENT(map); DEFINE_MR_EVENT(unmap); -DEFINE_MR_EVENT(remoteinv); +DEFINE_MR_EVENT(reminv); DEFINE_MR_EVENT(recycle); TRACE_EVENT(xprtrdma_dma_maperr, diff --git a/net/sunrpc/xprtrdma/frwr_ops.c b/net/sunrpc/xprtrdma/frwr_ops.c index 125297c9aa3e..0dc799553a08 100644 --- a/net/sunrpc/xprtrdma/frwr_ops.c +++ b/net/sunrpc/xprtrdma/frwr_ops.c @@ -419,7 +419,7 @@ void frwr_reminv(struct rpcrdma_rep *rep, struct list_head *mrs) list_for_each_entry(mr, mrs, mr_list) if (mr->mr_handle == rep->rr_inv_rkey) { list_del_init(&mr->mr_list); - trace_xprtrdma_mr_remoteinv(mr); + trace_xprtrdma_mr_reminv(mr); rpcrdma_mr_put(mr); break; /* only one invalidated MR per RPC */ } From 85cd8e2b78eea7374927750ffec60bf047f8f90b Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Fri, 21 Feb 2020 17:00:12 -0500 Subject: [PATCH 254/635] xprtrdma: Invoke rpcrdma_ep_create() in the connect worker Refactor rpcrdma_ep_create(), rpcrdma_ep_disconnect(), and rpcrdma_ep_destroy(). rpcrdma_ep_create will be invoked at connect time instead of at transport set-up time. It will be responsible for allocating per- connection resources. In this patch it allocates the CQs and creates a QP. More to come. rpcrdma_ep_destroy() is the inverse functionality that is invoked at disconnect time. It will be responsible for releasing the CQs and QP. These changes should be safe to do because both connect and disconnect is guaranteed to be serialized by the transport send lock. This takes us another step closer to resolving the address and route only at connect time so that connection failover to another device will work correctly. Signed-off-by: Chuck Lever Signed-off-by: Anna Schumaker --- net/sunrpc/xprtrdma/transport.c | 10 +-- net/sunrpc/xprtrdma/verbs.c | 147 ++++++++++++-------------------- net/sunrpc/xprtrdma/xprt_rdma.h | 2 - 3 files changed, 56 insertions(+), 103 deletions(-) diff --git a/net/sunrpc/xprtrdma/transport.c b/net/sunrpc/xprtrdma/transport.c index 3cfeba68ee9a..d915524a8e68 100644 --- a/net/sunrpc/xprtrdma/transport.c +++ b/net/sunrpc/xprtrdma/transport.c @@ -284,7 +284,7 @@ xprt_rdma_destroy(struct rpc_xprt *xprt) cancel_delayed_work_sync(&r_xprt->rx_connect_worker); - rpcrdma_ep_destroy(r_xprt); + rpcrdma_ep_disconnect(&r_xprt->rx_ep, &r_xprt->rx_ia); rpcrdma_buffer_destroy(&r_xprt->rx_buf); rpcrdma_ia_close(&r_xprt->rx_ia); @@ -351,13 +351,9 @@ xprt_setup_rdma(struct xprt_create *args) if (rc) goto out1; - rc = rpcrdma_ep_create(new_xprt); - if (rc) - goto out2; - rc = rpcrdma_buffer_create(new_xprt); if (rc) - goto out3; + goto out2; if (!try_module_get(THIS_MODULE)) goto out4; @@ -375,8 +371,6 @@ xprt_setup_rdma(struct xprt_create *args) out4: rpcrdma_buffer_destroy(&new_xprt->rx_buf); rc = -ENODEV; -out3: - rpcrdma_ep_destroy(new_xprt); out2: rpcrdma_ia_close(&new_xprt->rx_ia); out1: diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c index 353f61ac8d51..042e6cc4f767 100644 --- a/net/sunrpc/xprtrdma/verbs.c +++ b/net/sunrpc/xprtrdma/verbs.c @@ -84,6 +84,7 @@ static void rpcrdma_rep_destroy(struct rpcrdma_rep *rep); static void rpcrdma_reps_unmap(struct rpcrdma_xprt *r_xprt); static void rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt); static void rpcrdma_mrs_destroy(struct rpcrdma_xprt *r_xprt); +static void rpcrdma_ep_destroy(struct rpcrdma_xprt *r_xprt); static struct rpcrdma_regbuf * rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction, gfp_t flags); @@ -391,32 +392,17 @@ rpcrdma_ia_remove(struct rpcrdma_ia *ia) { struct rpcrdma_xprt *r_xprt = container_of(ia, struct rpcrdma_xprt, rx_ia); - struct rpcrdma_ep *ep = &r_xprt->rx_ep; - /* This is similar to rpcrdma_ep_destroy, but: - * - Don't cancel the connect worker. - * - Don't call rpcrdma_ep_disconnect, which waits - * for another conn upcall, which will deadlock. - * - rdma_disconnect is unneeded, the underlying - * connection is already gone. - */ - if (ia->ri_id->qp) { + if (ia->ri_id->qp) rpcrdma_xprt_drain(r_xprt); - rdma_destroy_qp(ia->ri_id); - ia->ri_id->qp = NULL; - } - ib_free_cq(ep->rep_attr.recv_cq); - ep->rep_attr.recv_cq = NULL; - ib_free_cq(ep->rep_attr.send_cq); - ep->rep_attr.send_cq = NULL; - /* The ULP is responsible for ensuring all DMA - * mappings and MRs are gone. - */ rpcrdma_reps_unmap(r_xprt); rpcrdma_reqs_reset(r_xprt); rpcrdma_mrs_destroy(r_xprt); rpcrdma_sendctxs_destroy(r_xprt); + + rpcrdma_ep_destroy(r_xprt); + ib_dealloc_pd(ia->ri_pd); ia->ri_pd = NULL; @@ -434,11 +420,8 @@ rpcrdma_ia_remove(struct rpcrdma_ia *ia) void rpcrdma_ia_close(struct rpcrdma_ia *ia) { - if (ia->ri_id != NULL && !IS_ERR(ia->ri_id)) { - if (ia->ri_id->qp) - rdma_destroy_qp(ia->ri_id); + if (ia->ri_id && !IS_ERR(ia->ri_id)) rdma_destroy_id(ia->ri_id); - } ia->ri_id = NULL; /* If the pd is still busy, xprtrdma missed freeing a resource */ @@ -447,25 +430,19 @@ rpcrdma_ia_close(struct rpcrdma_ia *ia) ia->ri_pd = NULL; } -/** - * rpcrdma_ep_create - Create unconnected endpoint - * @r_xprt: transport to instantiate - * - * Returns zero on success, or a negative errno. - */ -int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt) +static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt, + struct rdma_cm_id *id) { struct rpcrdma_ep *ep = &r_xprt->rx_ep; struct rpcrdma_ia *ia = &r_xprt->rx_ia; struct rpcrdma_connect_private *pmsg = &ep->rep_cm_private; - struct ib_cq *sendcq, *recvcq; int rc; ep->rep_max_requests = r_xprt->rx_xprt.max_reqs; ep->rep_inline_send = xprt_rdma_max_inline_write; ep->rep_inline_recv = xprt_rdma_max_inline_read; - rc = frwr_query_device(r_xprt, ia->ri_id->device); + rc = frwr_query_device(r_xprt, id->device); if (rc) return rc; r_xprt->rx_buf.rb_max_requests = cpu_to_be32(ep->rep_max_requests); @@ -491,25 +468,22 @@ int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt) init_waitqueue_head(&ep->rep_connect_wait); ep->rep_receive_count = 0; - sendcq = ib_alloc_cq_any(ia->ri_id->device, r_xprt, - ep->rep_attr.cap.max_send_wr + 1, - IB_POLL_WORKQUEUE); - if (IS_ERR(sendcq)) { - rc = PTR_ERR(sendcq); - goto out1; + ep->rep_attr.send_cq = ib_alloc_cq_any(id->device, r_xprt, + ep->rep_attr.cap.max_send_wr, + IB_POLL_WORKQUEUE); + if (IS_ERR(ep->rep_attr.send_cq)) { + rc = PTR_ERR(ep->rep_attr.send_cq); + goto out_destroy; } - recvcq = ib_alloc_cq_any(ia->ri_id->device, NULL, - ep->rep_attr.cap.max_recv_wr + 1, - IB_POLL_WORKQUEUE); - if (IS_ERR(recvcq)) { - rc = PTR_ERR(recvcq); - goto out2; + ep->rep_attr.recv_cq = ib_alloc_cq_any(id->device, NULL, + ep->rep_attr.cap.max_recv_wr, + IB_POLL_WORKQUEUE); + if (IS_ERR(ep->rep_attr.recv_cq)) { + rc = PTR_ERR(ep->rep_attr.recv_cq); + goto out_destroy; } - ep->rep_attr.send_cq = sendcq; - ep->rep_attr.recv_cq = recvcq; - /* Initialize cma parameters */ memset(&ep->rep_remote_cma, 0, sizeof(ep->rep_remote_cma)); @@ -525,7 +499,7 @@ int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt) /* Client offers RDMA Read but does not initiate */ ep->rep_remote_cma.initiator_depth = 0; ep->rep_remote_cma.responder_resources = - min_t(int, U8_MAX, ia->ri_id->device->attrs.max_qp_rd_atom); + min_t(int, U8_MAX, id->device->attrs.max_qp_rd_atom); /* Limit transport retries so client can detect server * GID changes quickly. RPC layer handles re-establishing @@ -540,45 +514,41 @@ int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt) ep->rep_remote_cma.flow_control = 0; ep->rep_remote_cma.rnr_retry_count = 0; + rc = rdma_create_qp(id, ia->ri_pd, &ep->rep_attr); + if (rc) + goto out_destroy; return 0; -out2: - ib_free_cq(sendcq); -out1: +out_destroy: + rpcrdma_ep_destroy(r_xprt); return rc; } -/** - * rpcrdma_ep_destroy - Disconnect and destroy endpoint. - * @r_xprt: transport instance to shut down - * - */ -void rpcrdma_ep_destroy(struct rpcrdma_xprt *r_xprt) +static void rpcrdma_ep_destroy(struct rpcrdma_xprt *r_xprt) { struct rpcrdma_ep *ep = &r_xprt->rx_ep; struct rpcrdma_ia *ia = &r_xprt->rx_ia; if (ia->ri_id && ia->ri_id->qp) { - rpcrdma_ep_disconnect(ep, ia); rdma_destroy_qp(ia->ri_id); ia->ri_id->qp = NULL; } if (ep->rep_attr.recv_cq) ib_free_cq(ep->rep_attr.recv_cq); + ep->rep_attr.recv_cq = NULL; if (ep->rep_attr.send_cq) ib_free_cq(ep->rep_attr.send_cq); + ep->rep_attr.send_cq = NULL; } /* Re-establish a connection after a device removal event. * Unlike a normal reconnection, a fresh PD and a new set * of MRs and buffers is needed. */ -static int rpcrdma_ep_recreate_xprt(struct rpcrdma_xprt *r_xprt, - struct ib_qp_init_attr *qp_init_attr) +static int rpcrdma_ep_recreate_xprt(struct rpcrdma_xprt *r_xprt) { struct rpcrdma_ia *ia = &r_xprt->rx_ia; - struct rpcrdma_ep *ep = &r_xprt->rx_ep; int rc, err; trace_xprtrdma_reinsert(r_xprt); @@ -587,39 +557,24 @@ static int rpcrdma_ep_recreate_xprt(struct rpcrdma_xprt *r_xprt, if (rpcrdma_ia_open(r_xprt)) goto out1; - rc = -ENOMEM; - err = rpcrdma_ep_create(r_xprt); - if (err) { - pr_err("rpcrdma: rpcrdma_ep_create returned %d\n", err); - goto out2; - } - memcpy(qp_init_attr, &ep->rep_attr, sizeof(*qp_init_attr)); - rc = -ENETUNREACH; - err = rdma_create_qp(ia->ri_id, ia->ri_pd, qp_init_attr); - if (err) { - pr_err("rpcrdma: rdma_create_qp returned %d\n", err); - goto out3; - } + err = rpcrdma_ep_create(r_xprt, ia->ri_id); + if (err) + goto out2; return 0; -out3: - rpcrdma_ep_destroy(r_xprt); out2: rpcrdma_ia_close(ia); out1: return rc; } -static int rpcrdma_ep_reconnect(struct rpcrdma_xprt *r_xprt, - struct ib_qp_init_attr *qp_init_attr) +static int rpcrdma_ep_reconnect(struct rpcrdma_xprt *r_xprt) { struct rpcrdma_ia *ia = &r_xprt->rx_ia; struct rdma_cm_id *id, *old; int err, rc; - rpcrdma_ep_disconnect(&r_xprt->rx_ep, ia); - rc = -EHOSTUNREACH; id = rpcrdma_create_id(r_xprt, ia); if (IS_ERR(id)) @@ -640,15 +595,14 @@ static int rpcrdma_ep_reconnect(struct rpcrdma_xprt *r_xprt, goto out_destroy; } - err = rdma_create_qp(id, ia->ri_pd, qp_init_attr); + err = rpcrdma_ep_create(r_xprt, id); if (err) goto out_destroy; - /* Atomically replace the transport's ID and QP. */ + /* Atomically replace the transport's ID. */ rc = 0; old = ia->ri_id; ia->ri_id = id; - rdma_destroy_qp(old); out_destroy: rdma_destroy_id(old); @@ -665,26 +619,25 @@ rpcrdma_ep_connect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia) struct rpcrdma_xprt *r_xprt = container_of(ia, struct rpcrdma_xprt, rx_ia); struct rpc_xprt *xprt = &r_xprt->rx_xprt; - struct ib_qp_init_attr qp_init_attr; int rc; retry: - memcpy(&qp_init_attr, &ep->rep_attr, sizeof(qp_init_attr)); switch (ep->rep_connected) { case 0: - rc = rdma_create_qp(ia->ri_id, ia->ri_pd, &qp_init_attr); - if (rc) { - rc = -ENETUNREACH; + rc = -ENETUNREACH; + if (rpcrdma_ep_create(r_xprt, ia->ri_id)) goto out_noupdate; - } break; case -ENODEV: - rc = rpcrdma_ep_recreate_xprt(r_xprt, &qp_init_attr); + rc = rpcrdma_ep_recreate_xprt(r_xprt); if (rc) goto out_noupdate; break; + case 1: + rpcrdma_ep_disconnect(ep, ia); + /* fall through */ default: - rc = rpcrdma_ep_reconnect(r_xprt, &qp_init_attr); + rc = rpcrdma_ep_reconnect(r_xprt); if (rc) goto out; } @@ -742,10 +695,14 @@ rpcrdma_ep_disconnect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia) { struct rpcrdma_xprt *r_xprt = container_of(ep, struct rpcrdma_xprt, rx_ep); + struct rdma_cm_id *id = ia->ri_id; int rc; + if (!id) + goto out; + /* returns without wait if ID is not connected */ - rc = rdma_disconnect(ia->ri_id); + rc = rdma_disconnect(id); if (!rc) wait_event_interruptible(ep->rep_connect_wait, ep->rep_connected != 1); @@ -753,10 +710,14 @@ rpcrdma_ep_disconnect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia) ep->rep_connected = rc; trace_xprtrdma_disconnect(r_xprt, rc); - rpcrdma_xprt_drain(r_xprt); + if (id->qp) + rpcrdma_xprt_drain(r_xprt); +out: rpcrdma_reqs_reset(r_xprt); rpcrdma_mrs_destroy(r_xprt); rpcrdma_sendctxs_destroy(r_xprt); + + rpcrdma_ep_destroy(r_xprt); } /* Fixed-size circular FIFO queue. This implementation is wait-free and diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h index 37d5080c250b..9a536319557e 100644 --- a/net/sunrpc/xprtrdma/xprt_rdma.h +++ b/net/sunrpc/xprtrdma/xprt_rdma.h @@ -464,8 +464,6 @@ void rpcrdma_ia_close(struct rpcrdma_ia *); /* * Endpoint calls - xprtrdma/verbs.c */ -int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt); -void rpcrdma_ep_destroy(struct rpcrdma_xprt *r_xprt); int rpcrdma_ep_connect(struct rpcrdma_ep *, struct rpcrdma_ia *); void rpcrdma_ep_disconnect(struct rpcrdma_ep *, struct rpcrdma_ia *); From 253a51622fb03425b611e709e34f1ea70949a61f Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Fri, 21 Feb 2020 17:00:17 -0500 Subject: [PATCH 255/635] xprtrdma: Refactor frwr_init_mr() Clean up: prepare for combining the rpcrdma_ia and rpcrdma_ep structures. Take the opportunity to rename the function to be consistent with the "subsystem _ object _ verb" naming scheme. Signed-off-by: Chuck Lever Signed-off-by: Anna Schumaker --- net/sunrpc/xprtrdma/frwr_ops.c | 10 ++++++---- net/sunrpc/xprtrdma/verbs.c | 4 +--- net/sunrpc/xprtrdma/xprt_rdma.h | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/net/sunrpc/xprtrdma/frwr_ops.c b/net/sunrpc/xprtrdma/frwr_ops.c index 0dc799553a08..2c96b624a77d 100644 --- a/net/sunrpc/xprtrdma/frwr_ops.c +++ b/net/sunrpc/xprtrdma/frwr_ops.c @@ -52,7 +52,7 @@ /** * frwr_release_mr - Destroy one MR - * @mr: MR allocated by frwr_init_mr + * @mr: MR allocated by frwr_mr_init * */ void frwr_release_mr(struct rpcrdma_mr *mr) @@ -106,15 +106,16 @@ void frwr_reset(struct rpcrdma_req *req) } /** - * frwr_init_mr - Initialize one MR - * @ia: interface adapter + * frwr_mr_init - Initialize one MR + * @r_xprt: controlling transport instance * @mr: generic MR to prepare for FRWR * * Returns zero if successful. Otherwise a negative errno * is returned. */ -int frwr_init_mr(struct rpcrdma_ia *ia, struct rpcrdma_mr *mr) +int frwr_mr_init(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr *mr) { + struct rpcrdma_ia *ia = &r_xprt->rx_ia; unsigned int depth = ia->ri_max_frwr_depth; struct scatterlist *sg; struct ib_mr *frmr; @@ -128,6 +129,7 @@ int frwr_init_mr(struct rpcrdma_ia *ia, struct rpcrdma_mr *mr) if (!sg) goto out_list_err; + mr->mr_xprt = r_xprt; mr->frwr.fr_mr = frmr; mr->mr_dir = DMA_NONE; INIT_LIST_HEAD(&mr->mr_list); diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c index 042e6cc4f767..02ce3d548825 100644 --- a/net/sunrpc/xprtrdma/verbs.c +++ b/net/sunrpc/xprtrdma/verbs.c @@ -896,14 +896,12 @@ rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt) if (!mr) break; - rc = frwr_init_mr(ia, mr); + rc = frwr_mr_init(r_xprt, mr); if (rc) { kfree(mr); break; } - mr->mr_xprt = r_xprt; - spin_lock(&buf->rb_lock); rpcrdma_mr_push(mr, &buf->rb_mrs); list_add(&mr->mr_all, &buf->rb_all_mrs); diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h index 9a536319557e..9e3e9a82cb9f 100644 --- a/net/sunrpc/xprtrdma/xprt_rdma.h +++ b/net/sunrpc/xprtrdma/xprt_rdma.h @@ -536,7 +536,7 @@ rpcrdma_data_dir(bool writing) void frwr_reset(struct rpcrdma_req *req); int frwr_query_device(struct rpcrdma_xprt *r_xprt, const struct ib_device *device); -int frwr_init_mr(struct rpcrdma_ia *ia, struct rpcrdma_mr *mr); +int frwr_mr_init(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr *mr); void frwr_release_mr(struct rpcrdma_mr *mr); struct rpcrdma_mr_seg *frwr_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg, From 97d0de8812a10a66510ff95f8fe6e8d3053fd2ca Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Fri, 21 Feb 2020 17:00:23 -0500 Subject: [PATCH 256/635] xprtrdma: Clean up the post_send path Clean up: Simplify the synopses of functions in the post_send path by combining the struct rpcrdma_ia and struct rpcrdma_ep arguments. Signed-off-by: Chuck Lever Signed-off-by: Anna Schumaker --- net/sunrpc/xprtrdma/backchannel.c | 2 +- net/sunrpc/xprtrdma/frwr_ops.c | 14 +++++++++----- net/sunrpc/xprtrdma/transport.c | 2 +- net/sunrpc/xprtrdma/verbs.c | 13 +++++-------- net/sunrpc/xprtrdma/xprt_rdma.h | 5 ++--- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/net/sunrpc/xprtrdma/backchannel.c b/net/sunrpc/xprtrdma/backchannel.c index 1a0ae0c61353..4b43910a6ed2 100644 --- a/net/sunrpc/xprtrdma/backchannel.c +++ b/net/sunrpc/xprtrdma/backchannel.c @@ -115,7 +115,7 @@ int xprt_rdma_bc_send_reply(struct rpc_rqst *rqst) if (rc < 0) goto failed_marshal; - if (rpcrdma_ep_post(&r_xprt->rx_ia, &r_xprt->rx_ep, req)) + if (rpcrdma_post_sends(r_xprt, req)) goto drop_connection; return 0; diff --git a/net/sunrpc/xprtrdma/frwr_ops.c b/net/sunrpc/xprtrdma/frwr_ops.c index 2c96b624a77d..a1b5c8024cca 100644 --- a/net/sunrpc/xprtrdma/frwr_ops.c +++ b/net/sunrpc/xprtrdma/frwr_ops.c @@ -374,18 +374,22 @@ static void frwr_wc_fastreg(struct ib_cq *cq, struct ib_wc *wc) } /** - * frwr_send - post Send WR containing the RPC Call message - * @ia: interface adapter - * @req: Prepared RPC Call + * frwr_send - post Send WRs containing the RPC Call message + * @r_xprt: controlling transport instance + * @req: prepared RPC Call * * For FRWR, chain any FastReg WRs to the Send WR. Only a * single ib_post_send call is needed to register memory * and then post the Send WR. * - * Returns the result of ib_post_send. + * Returns the return code from ib_post_send. + * + * Caller must hold the transport send lock to ensure that the + * pointers to the transport's rdma_cm_id and QP are stable. */ -int frwr_send(struct rpcrdma_ia *ia, struct rpcrdma_req *req) +int frwr_send(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) { + struct rpcrdma_ia *ia = &r_xprt->rx_ia; struct ib_send_wr *post_wr; struct rpcrdma_mr *mr; diff --git a/net/sunrpc/xprtrdma/transport.c b/net/sunrpc/xprtrdma/transport.c index d915524a8e68..8934c24a5701 100644 --- a/net/sunrpc/xprtrdma/transport.c +++ b/net/sunrpc/xprtrdma/transport.c @@ -688,7 +688,7 @@ xprt_rdma_send_request(struct rpc_rqst *rqst) goto drop_connection; rqst->rq_xtime = ktime_get(); - if (rpcrdma_ep_post(&r_xprt->rx_ia, &r_xprt->rx_ep, req)) + if (rpcrdma_post_sends(r_xprt, req)) goto drop_connection; rqst->rq_xmit_bytes_sent += rqst->rq_snd_buf.len; diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c index 02ce3d548825..8fd6682d2646 100644 --- a/net/sunrpc/xprtrdma/verbs.c +++ b/net/sunrpc/xprtrdma/verbs.c @@ -1461,20 +1461,17 @@ static void rpcrdma_regbuf_free(struct rpcrdma_regbuf *rb) } /** - * rpcrdma_ep_post - Post WRs to a transport's Send Queue - * @ia: transport's device information - * @ep: transport's RDMA endpoint information + * rpcrdma_post_sends - Post WRs to a transport's Send Queue + * @r_xprt: controlling transport instance * @req: rpcrdma_req containing the Send WR to post * * Returns 0 if the post was successful, otherwise -ENOTCONN * is returned. */ -int -rpcrdma_ep_post(struct rpcrdma_ia *ia, - struct rpcrdma_ep *ep, - struct rpcrdma_req *req) +int rpcrdma_post_sends(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) { struct ib_send_wr *send_wr = &req->rl_wr; + struct rpcrdma_ep *ep = &r_xprt->rx_ep; int rc; if (!ep->rep_send_count || kref_read(&req->rl_kref) > 1) { @@ -1485,7 +1482,7 @@ rpcrdma_ep_post(struct rpcrdma_ia *ia, --ep->rep_send_count; } - rc = frwr_send(ia, req); + rc = frwr_send(r_xprt, req); trace_xprtrdma_post_send(req, rc); if (rc) return -ENOTCONN; diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h index 9e3e9a82cb9f..82ec4c25432f 100644 --- a/net/sunrpc/xprtrdma/xprt_rdma.h +++ b/net/sunrpc/xprtrdma/xprt_rdma.h @@ -467,8 +467,7 @@ void rpcrdma_ia_close(struct rpcrdma_ia *); int rpcrdma_ep_connect(struct rpcrdma_ep *, struct rpcrdma_ia *); void rpcrdma_ep_disconnect(struct rpcrdma_ep *, struct rpcrdma_ia *); -int rpcrdma_ep_post(struct rpcrdma_ia *, struct rpcrdma_ep *, - struct rpcrdma_req *); +int rpcrdma_post_sends(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req); void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp); /* @@ -542,7 +541,7 @@ struct rpcrdma_mr_seg *frwr_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg, int nsegs, bool writing, __be32 xid, struct rpcrdma_mr *mr); -int frwr_send(struct rpcrdma_ia *ia, struct rpcrdma_req *req); +int frwr_send(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req); void frwr_reminv(struct rpcrdma_rep *rep, struct list_head *mrs); void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req); void frwr_unmap_async(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req); From 9144a803df6ca4185238ca343dbb65d8137c036e Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Fri, 21 Feb 2020 17:00:28 -0500 Subject: [PATCH 257/635] xprtrdma: Refactor rpcrdma_ep_connect() and rpcrdma_ep_disconnect() Clean up: Simplify the synopses of functions in the connect and disconnect paths in preparation for combining the rpcrdma_ia and struct rpcrdma_ep structures. Signed-off-by: Chuck Lever Signed-off-by: Anna Schumaker --- net/sunrpc/xprtrdma/transport.c | 6 +++--- net/sunrpc/xprtrdma/verbs.c | 30 +++++++++++++++--------------- net/sunrpc/xprtrdma/xprt_rdma.h | 4 ++-- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/net/sunrpc/xprtrdma/transport.c b/net/sunrpc/xprtrdma/transport.c index 8934c24a5701..6349e6c98b57 100644 --- a/net/sunrpc/xprtrdma/transport.c +++ b/net/sunrpc/xprtrdma/transport.c @@ -240,7 +240,7 @@ xprt_rdma_connect_worker(struct work_struct *work) struct rpc_xprt *xprt = &r_xprt->rx_xprt; int rc; - rc = rpcrdma_ep_connect(&r_xprt->rx_ep, &r_xprt->rx_ia); + rc = rpcrdma_xprt_connect(r_xprt); xprt_clear_connecting(xprt); if (r_xprt->rx_ep.rep_connected > 0) { xprt->stat.connect_count++; @@ -284,7 +284,7 @@ xprt_rdma_destroy(struct rpc_xprt *xprt) cancel_delayed_work_sync(&r_xprt->rx_connect_worker); - rpcrdma_ep_disconnect(&r_xprt->rx_ep, &r_xprt->rx_ia); + rpcrdma_xprt_disconnect(r_xprt); rpcrdma_buffer_destroy(&r_xprt->rx_buf); rpcrdma_ia_close(&r_xprt->rx_ia); @@ -409,7 +409,7 @@ void xprt_rdma_close(struct rpc_xprt *xprt) if (ep->rep_connected == -ENODEV) return; - rpcrdma_ep_disconnect(ep, ia); + rpcrdma_xprt_disconnect(r_xprt); out: xprt->reestablish_timeout = 0; diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c index 8fd6682d2646..f361213a8157 100644 --- a/net/sunrpc/xprtrdma/verbs.c +++ b/net/sunrpc/xprtrdma/verbs.c @@ -610,15 +610,17 @@ out: return rc; } -/* - * Connect unconnected endpoint. +/** + * rpcrdma_xprt_connect - Connect an unconnected transport + * @r_xprt: controlling transport instance + * + * Returns 0 on success or a negative errno. */ -int -rpcrdma_ep_connect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia) +int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt) { - struct rpcrdma_xprt *r_xprt = container_of(ia, struct rpcrdma_xprt, - rx_ia); struct rpc_xprt *xprt = &r_xprt->rx_xprt; + struct rpcrdma_ep *ep = &r_xprt->rx_ep; + struct rpcrdma_ia *ia = &r_xprt->rx_ia; int rc; retry: @@ -634,7 +636,7 @@ retry: goto out_noupdate; break; case 1: - rpcrdma_ep_disconnect(ep, ia); + rpcrdma_xprt_disconnect(r_xprt); /* fall through */ default: rc = rpcrdma_ep_reconnect(r_xprt); @@ -668,7 +670,7 @@ retry: rc = rpcrdma_reqs_setup(r_xprt); if (rc) { - rpcrdma_ep_disconnect(ep, ia); + rpcrdma_xprt_disconnect(r_xprt); goto out; } rpcrdma_mrs_create(r_xprt); @@ -683,18 +685,16 @@ out_noupdate: } /** - * rpcrdma_ep_disconnect - Disconnect underlying transport - * @ep: endpoint to disconnect - * @ia: associated interface adapter + * rpcrdma_xprt_disconnect - Disconnect underlying transport + * @r_xprt: controlling transport instance * * Caller serializes. Either the transport send lock is held, * or we're being called to destroy the transport. */ -void -rpcrdma_ep_disconnect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia) +void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt) { - struct rpcrdma_xprt *r_xprt = container_of(ep, struct rpcrdma_xprt, - rx_ep); + struct rpcrdma_ep *ep = &r_xprt->rx_ep; + struct rpcrdma_ia *ia = &r_xprt->rx_ia; struct rdma_cm_id *id = ia->ri_id; int rc; diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h index 82ec4c25432f..9ead06b1d8a4 100644 --- a/net/sunrpc/xprtrdma/xprt_rdma.h +++ b/net/sunrpc/xprtrdma/xprt_rdma.h @@ -464,8 +464,8 @@ void rpcrdma_ia_close(struct rpcrdma_ia *); /* * Endpoint calls - xprtrdma/verbs.c */ -int rpcrdma_ep_connect(struct rpcrdma_ep *, struct rpcrdma_ia *); -void rpcrdma_ep_disconnect(struct rpcrdma_ep *, struct rpcrdma_ia *); +int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt); +void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt); int rpcrdma_post_sends(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req); void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp); From 9ba373ee24ea07b60b8c5041cafe88d5796bacef Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Fri, 21 Feb 2020 17:00:33 -0500 Subject: [PATCH 258/635] xprtrdma: Allocate Protection Domain in rpcrdma_ep_create() Make a Protection Domain (PD) a per-connection resource rather than a per-transport resource. In other words, when the connection terminates, the PD is destroyed. Thus there is one less HW resource that remains allocated to a transport after a connection is closed. Signed-off-by: Chuck Lever Signed-off-by: Anna Schumaker --- net/sunrpc/xprtrdma/verbs.c | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c index f361213a8157..36fe7baea014 100644 --- a/net/sunrpc/xprtrdma/verbs.c +++ b/net/sunrpc/xprtrdma/verbs.c @@ -363,14 +363,6 @@ rpcrdma_ia_open(struct rpcrdma_xprt *xprt) rc = PTR_ERR(ia->ri_id); goto out_err; } - - ia->ri_pd = ib_alloc_pd(ia->ri_id->device, 0); - if (IS_ERR(ia->ri_pd)) { - rc = PTR_ERR(ia->ri_pd); - pr_err("rpcrdma: ib_alloc_pd() returned %d\n", rc); - goto out_err; - } - return 0; out_err: @@ -403,9 +395,6 @@ rpcrdma_ia_remove(struct rpcrdma_ia *ia) rpcrdma_ep_destroy(r_xprt); - ib_dealloc_pd(ia->ri_pd); - ia->ri_pd = NULL; - /* Allow waiters to continue */ complete(&ia->ri_remove_done); @@ -423,11 +412,6 @@ rpcrdma_ia_close(struct rpcrdma_ia *ia) if (ia->ri_id && !IS_ERR(ia->ri_id)) rdma_destroy_id(ia->ri_id); ia->ri_id = NULL; - - /* If the pd is still busy, xprtrdma missed freeing a resource */ - if (ia->ri_pd && !IS_ERR(ia->ri_pd)) - ib_dealloc_pd(ia->ri_pd); - ia->ri_pd = NULL; } static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt, @@ -514,6 +498,12 @@ static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt, ep->rep_remote_cma.flow_control = 0; ep->rep_remote_cma.rnr_retry_count = 0; + ia->ri_pd = ib_alloc_pd(id->device, 0); + if (IS_ERR(ia->ri_pd)) { + rc = PTR_ERR(ia->ri_pd); + goto out_destroy; + } + rc = rdma_create_qp(id, ia->ri_pd, &ep->rep_attr); if (rc) goto out_destroy; @@ -540,6 +530,10 @@ static void rpcrdma_ep_destroy(struct rpcrdma_xprt *r_xprt) if (ep->rep_attr.send_cq) ib_free_cq(ep->rep_attr.send_cq); ep->rep_attr.send_cq = NULL; + + if (ia->ri_pd) + ib_dealloc_pd(ia->ri_pd); + ia->ri_pd = NULL; } /* Re-establish a connection after a device removal event. From 81fe0c57f4e136375f3bcda74af413f82a34a1bb Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Fri, 21 Feb 2020 17:00:38 -0500 Subject: [PATCH 259/635] xprtrdma: Invoke rpcrdma_ia_open in the connect worker Move rdma_cm_id creation into rpcrdma_ep_create() so that it is now responsible for allocating all per-connection hardware resources. With this clean-up, all three arms of the switch statement in rpcrdma_ep_connect are exactly the same now, thus the switch can be removed. Because device removal behaves a little differently than disconnection, there is a little more work to be done before rpcrdma_ep_destroy() can release the connection's rdma_cm_id. So it is not quite symmetrical with rpcrdma_ep_create() yet. Signed-off-by: Chuck Lever Signed-off-by: Anna Schumaker --- include/trace/events/rpcrdma.h | 1 - net/sunrpc/xprtrdma/transport.c | 7 -- net/sunrpc/xprtrdma/verbs.c | 153 +++++--------------------------- net/sunrpc/xprtrdma/xprt_rdma.h | 2 - 4 files changed, 20 insertions(+), 143 deletions(-) diff --git a/include/trace/events/rpcrdma.h b/include/trace/events/rpcrdma.h index 87f4461ab108..ba37c47b51e8 100644 --- a/include/trace/events/rpcrdma.h +++ b/include/trace/events/rpcrdma.h @@ -415,7 +415,6 @@ DEFINE_CONN_EVENT(disconnect); DEFINE_RXPRT_EVENT(xprtrdma_create); DEFINE_RXPRT_EVENT(xprtrdma_op_destroy); DEFINE_RXPRT_EVENT(xprtrdma_remove); -DEFINE_RXPRT_EVENT(xprtrdma_reinsert); DEFINE_RXPRT_EVENT(xprtrdma_op_inject_dsc); DEFINE_RXPRT_EVENT(xprtrdma_op_close); DEFINE_RXPRT_EVENT(xprtrdma_op_setport); diff --git a/net/sunrpc/xprtrdma/transport.c b/net/sunrpc/xprtrdma/transport.c index 6349e6c98b57..745dfd149637 100644 --- a/net/sunrpc/xprtrdma/transport.c +++ b/net/sunrpc/xprtrdma/transport.c @@ -286,7 +286,6 @@ xprt_rdma_destroy(struct rpc_xprt *xprt) rpcrdma_xprt_disconnect(r_xprt); rpcrdma_buffer_destroy(&r_xprt->rx_buf); - rpcrdma_ia_close(&r_xprt->rx_ia); xprt_rdma_free_addresses(xprt); xprt_free(xprt); @@ -347,10 +346,6 @@ xprt_setup_rdma(struct xprt_create *args) xprt_rdma_format_addresses(xprt, sap); new_xprt = rpcx_to_rdmax(xprt); - rc = rpcrdma_ia_open(new_xprt); - if (rc) - goto out1; - rc = rpcrdma_buffer_create(new_xprt); if (rc) goto out2; @@ -372,8 +367,6 @@ out4: rpcrdma_buffer_destroy(&new_xprt->rx_buf); rc = -ENODEV; out2: - rpcrdma_ia_close(&new_xprt->rx_ia); -out1: trace_xprtrdma_op_destroy(new_xprt); xprt_rdma_free_addresses(xprt); xprt_free(xprt); diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c index 36fe7baea014..3df20f355579 100644 --- a/net/sunrpc/xprtrdma/verbs.c +++ b/net/sunrpc/xprtrdma/verbs.c @@ -345,31 +345,6 @@ out: * Exported functions. */ -/** - * rpcrdma_ia_open - Open and initialize an Interface Adapter. - * @xprt: transport with IA to (re)initialize - * - * Returns 0 on success, negative errno if an appropriate - * Interface Adapter could not be found and opened. - */ -int -rpcrdma_ia_open(struct rpcrdma_xprt *xprt) -{ - struct rpcrdma_ia *ia = &xprt->rx_ia; - int rc; - - ia->ri_id = rpcrdma_create_id(xprt, ia); - if (IS_ERR(ia->ri_id)) { - rc = PTR_ERR(ia->ri_id); - goto out_err; - } - return 0; - -out_err: - rpcrdma_ia_close(ia); - return rc; -} - /** * rpcrdma_ia_remove - Handle device driver unload * @ia: interface adapter being removed @@ -401,34 +376,26 @@ rpcrdma_ia_remove(struct rpcrdma_ia *ia) trace_xprtrdma_remove(r_xprt); } -/** - * rpcrdma_ia_close - Clean up/close an IA. - * @ia: interface adapter to close - * - */ -void -rpcrdma_ia_close(struct rpcrdma_ia *ia) -{ - if (ia->ri_id && !IS_ERR(ia->ri_id)) - rdma_destroy_id(ia->ri_id); - ia->ri_id = NULL; -} - -static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt, - struct rdma_cm_id *id) +static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt) { struct rpcrdma_ep *ep = &r_xprt->rx_ep; struct rpcrdma_ia *ia = &r_xprt->rx_ia; struct rpcrdma_connect_private *pmsg = &ep->rep_cm_private; + struct rdma_cm_id *id; int rc; + id = rpcrdma_create_id(r_xprt, ia); + if (IS_ERR(id)) + return PTR_ERR(id); + ep->rep_max_requests = r_xprt->rx_xprt.max_reqs; ep->rep_inline_send = xprt_rdma_max_inline_write; ep->rep_inline_recv = xprt_rdma_max_inline_read; rc = frwr_query_device(r_xprt, id->device); if (rc) - return rc; + goto out_destroy; + r_xprt->rx_buf.rb_max_requests = cpu_to_be32(ep->rep_max_requests); ep->rep_attr.event_handler = rpcrdma_qp_event_handler; @@ -507,10 +474,12 @@ static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt, rc = rdma_create_qp(id, ia->ri_pd, &ep->rep_attr); if (rc) goto out_destroy; + ia->ri_id = id; return 0; out_destroy: rpcrdma_ep_destroy(r_xprt); + rdma_destroy_id(id); return rc; } @@ -536,79 +505,8 @@ static void rpcrdma_ep_destroy(struct rpcrdma_xprt *r_xprt) ia->ri_pd = NULL; } -/* Re-establish a connection after a device removal event. - * Unlike a normal reconnection, a fresh PD and a new set - * of MRs and buffers is needed. - */ -static int rpcrdma_ep_recreate_xprt(struct rpcrdma_xprt *r_xprt) -{ - struct rpcrdma_ia *ia = &r_xprt->rx_ia; - int rc, err; - - trace_xprtrdma_reinsert(r_xprt); - - rc = -EHOSTUNREACH; - if (rpcrdma_ia_open(r_xprt)) - goto out1; - - rc = -ENETUNREACH; - err = rpcrdma_ep_create(r_xprt, ia->ri_id); - if (err) - goto out2; - return 0; - -out2: - rpcrdma_ia_close(ia); -out1: - return rc; -} - -static int rpcrdma_ep_reconnect(struct rpcrdma_xprt *r_xprt) -{ - struct rpcrdma_ia *ia = &r_xprt->rx_ia; - struct rdma_cm_id *id, *old; - int err, rc; - - rc = -EHOSTUNREACH; - id = rpcrdma_create_id(r_xprt, ia); - if (IS_ERR(id)) - goto out; - - /* As long as the new ID points to the same device as the - * old ID, we can reuse the transport's existing PD and all - * previously allocated MRs. Also, the same device means - * the transport's previous DMA mappings are still valid. - * - * This is a sanity check only. There should be no way these - * point to two different devices here. - */ - old = id; - rc = -ENETUNREACH; - if (ia->ri_id->device != id->device) { - pr_err("rpcrdma: can't reconnect on different device!\n"); - goto out_destroy; - } - - err = rpcrdma_ep_create(r_xprt, id); - if (err) - goto out_destroy; - - /* Atomically replace the transport's ID. */ - rc = 0; - old = ia->ri_id; - ia->ri_id = id; - -out_destroy: - rdma_destroy_id(old); -out: - return rc; -} - -/** - * rpcrdma_xprt_connect - Connect an unconnected transport - * @r_xprt: controlling transport instance - * - * Returns 0 on success or a negative errno. +/* + * Connect unconnected endpoint. */ int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt) { @@ -618,25 +516,10 @@ int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt) int rc; retry: - switch (ep->rep_connected) { - case 0: - rc = -ENETUNREACH; - if (rpcrdma_ep_create(r_xprt, ia->ri_id)) - goto out_noupdate; - break; - case -ENODEV: - rc = rpcrdma_ep_recreate_xprt(r_xprt); - if (rc) - goto out_noupdate; - break; - case 1: - rpcrdma_xprt_disconnect(r_xprt); - /* fall through */ - default: - rc = rpcrdma_ep_reconnect(r_xprt); - if (rc) - goto out; - } + rpcrdma_xprt_disconnect(r_xprt); + rc = rpcrdma_ep_create(r_xprt); + if (rc) + goto out_noupdate; ep->rep_connected = 0; xprt_clear_connected(xprt); @@ -712,6 +595,10 @@ out: rpcrdma_sendctxs_destroy(r_xprt); rpcrdma_ep_destroy(r_xprt); + + if (ia->ri_id) + rdma_destroy_id(ia->ri_id); + ia->ri_id = NULL; } /* Fixed-size circular FIFO queue. This implementation is wait-free and diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h index 9ead06b1d8a4..8be1b70b71a2 100644 --- a/net/sunrpc/xprtrdma/xprt_rdma.h +++ b/net/sunrpc/xprtrdma/xprt_rdma.h @@ -457,9 +457,7 @@ extern unsigned int xprt_rdma_memreg_strategy; /* * Interface Adapter calls - xprtrdma/verbs.c */ -int rpcrdma_ia_open(struct rpcrdma_xprt *xprt); void rpcrdma_ia_remove(struct rpcrdma_ia *ia); -void rpcrdma_ia_close(struct rpcrdma_ia *); /* * Endpoint calls - xprtrdma/verbs.c From 897b7be9bca0caa27cdf7520bdc7689abe989a53 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Fri, 21 Feb 2020 17:00:44 -0500 Subject: [PATCH 260/635] xprtrdma: Remove rpcrdma_ia::ri_flags Clean up: The upper layer serializes calls to xprt_rdma_close, so there is no need for an atomic bit operation, saving 8 bytes in rpcrdma_ia. This enables merging rpcrdma_ia_remove directly into the disconnect logic. Signed-off-by: Chuck Lever Signed-off-by: Anna Schumaker --- net/sunrpc/xprtrdma/transport.c | 15 --------- net/sunrpc/xprtrdma/verbs.c | 55 ++++++++------------------------- net/sunrpc/xprtrdma/xprt_rdma.h | 10 ------ 3 files changed, 13 insertions(+), 67 deletions(-) diff --git a/net/sunrpc/xprtrdma/transport.c b/net/sunrpc/xprtrdma/transport.c index 745dfd149637..d7b7dab0aeb6 100644 --- a/net/sunrpc/xprtrdma/transport.c +++ b/net/sunrpc/xprtrdma/transport.c @@ -385,26 +385,11 @@ out2: void xprt_rdma_close(struct rpc_xprt *xprt) { struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt); - struct rpcrdma_ep *ep = &r_xprt->rx_ep; - struct rpcrdma_ia *ia = &r_xprt->rx_ia; - - might_sleep(); trace_xprtrdma_op_close(r_xprt); - /* Prevent marshaling and sending of new requests */ - xprt_clear_connected(xprt); - - if (test_and_clear_bit(RPCRDMA_IAF_REMOVING, &ia->ri_flags)) { - rpcrdma_ia_remove(ia); - goto out; - } - - if (ep->rep_connected == -ENODEV) - return; rpcrdma_xprt_disconnect(r_xprt); -out: xprt->reestablish_timeout = 0; ++xprt->connect_cookie; xprt_disconnect_done(xprt); diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c index 3df20f355579..a7f46bbbf017 100644 --- a/net/sunrpc/xprtrdma/verbs.c +++ b/net/sunrpc/xprtrdma/verbs.c @@ -250,12 +250,11 @@ rpcrdma_cm_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event) rpcrdma_addrstr(r_xprt), rpcrdma_portstr(r_xprt)); #endif init_completion(&ia->ri_remove_done); - set_bit(RPCRDMA_IAF_REMOVING, &ia->ri_flags); ep->rep_connected = -ENODEV; xprt_force_disconnect(xprt); wait_for_completion(&ia->ri_remove_done); + trace_xprtrdma_remove(r_xprt); - ia->ri_id = NULL; /* Return 1 to ensure the core destroys the id. */ return 1; case RDMA_CM_EVENT_ESTABLISHED: @@ -345,37 +344,6 @@ out: * Exported functions. */ -/** - * rpcrdma_ia_remove - Handle device driver unload - * @ia: interface adapter being removed - * - * Divest transport H/W resources associated with this adapter, - * but allow it to be restored later. - * - * Caller must hold the transport send lock. - */ -void -rpcrdma_ia_remove(struct rpcrdma_ia *ia) -{ - struct rpcrdma_xprt *r_xprt = container_of(ia, struct rpcrdma_xprt, - rx_ia); - - if (ia->ri_id->qp) - rpcrdma_xprt_drain(r_xprt); - - rpcrdma_reps_unmap(r_xprt); - rpcrdma_reqs_reset(r_xprt); - rpcrdma_mrs_destroy(r_xprt); - rpcrdma_sendctxs_destroy(r_xprt); - - rpcrdma_ep_destroy(r_xprt); - - /* Allow waiters to continue */ - complete(&ia->ri_remove_done); - - trace_xprtrdma_remove(r_xprt); -} - static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt) { struct rpcrdma_ep *ep = &r_xprt->rx_ep; @@ -573,12 +541,13 @@ void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt) struct rpcrdma_ep *ep = &r_xprt->rx_ep; struct rpcrdma_ia *ia = &r_xprt->rx_ia; struct rdma_cm_id *id = ia->ri_id; - int rc; + int rc, status = ep->rep_connected; + + might_sleep(); if (!id) - goto out; + return; - /* returns without wait if ID is not connected */ rc = rdma_disconnect(id); if (!rc) wait_event_interruptible(ep->rep_connect_wait, @@ -589,15 +558,17 @@ void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt) if (id->qp) rpcrdma_xprt_drain(r_xprt); -out: + rpcrdma_reps_unmap(r_xprt); rpcrdma_reqs_reset(r_xprt); rpcrdma_mrs_destroy(r_xprt); rpcrdma_sendctxs_destroy(r_xprt); rpcrdma_ep_destroy(r_xprt); - if (ia->ri_id) - rdma_destroy_id(ia->ri_id); + if (status == -ENODEV) + complete(&ia->ri_remove_done); + else + rdma_destroy_id(id); ia->ri_id = NULL; } @@ -815,10 +786,10 @@ void rpcrdma_mrs_refresh(struct rpcrdma_xprt *r_xprt) struct rpcrdma_buffer *buf = &r_xprt->rx_buf; struct rpcrdma_ep *ep = &r_xprt->rx_ep; - /* If there is no underlying device, it's no use to - * wake the refresh worker. + /* If there is no underlying connection, it's no use + * to wake the refresh worker. */ - if (ep->rep_connected != -ENODEV) { + if (ep->rep_connected == 1) { /* The work is scheduled on a WQ_MEM_RECLAIM * workqueue in order to prevent MR allocation * from recursing into NFS during direct reclaim. diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h index 8be1b70b71a2..d2a0f125f7a8 100644 --- a/net/sunrpc/xprtrdma/xprt_rdma.h +++ b/net/sunrpc/xprtrdma/xprt_rdma.h @@ -75,15 +75,10 @@ struct rpcrdma_ia { unsigned int ri_max_frwr_depth; bool ri_implicit_roundup; enum ib_mr_type ri_mrtype; - unsigned long ri_flags; struct completion ri_done; struct completion ri_remove_done; }; -enum { - RPCRDMA_IAF_REMOVING = 0, -}; - /* * RDMA Endpoint -- one per transport instance */ @@ -454,11 +449,6 @@ extern int xprt_rdma_pad_optimize; */ extern unsigned int xprt_rdma_memreg_strategy; -/* - * Interface Adapter calls - xprtrdma/verbs.c - */ -void rpcrdma_ia_remove(struct rpcrdma_ia *ia); - /* * Endpoint calls - xprtrdma/verbs.c */ From d6ccebf956338ea015d7d54c4a4c9c17605707cb Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Fri, 21 Feb 2020 17:00:49 -0500 Subject: [PATCH 261/635] xprtrdma: Disconnect on flushed completion Completion errors after a disconnect often occur much sooner than a CM_DISCONNECT event. Use this to try to detect connection loss more quickly. Note that other kernel ULPs do take care to disconnect explicitly when a WR is flushed. Signed-off-by: Chuck Lever Signed-off-by: Anna Schumaker --- include/trace/events/rpcrdma.h | 3 ++- net/sunrpc/xprtrdma/frwr_ops.c | 24 ++++++++++++++------- net/sunrpc/xprtrdma/verbs.c | 37 +++++++++++++++++++++++++-------- net/sunrpc/xprtrdma/xprt_rdma.h | 1 + 4 files changed, 47 insertions(+), 18 deletions(-) diff --git a/include/trace/events/rpcrdma.h b/include/trace/events/rpcrdma.h index ba37c47b51e8..cfbe28ad2614 100644 --- a/include/trace/events/rpcrdma.h +++ b/include/trace/events/rpcrdma.h @@ -109,7 +109,7 @@ DECLARE_EVENT_CLASS(xprtrdma_connect_class, __assign_str(port, rpcrdma_portstr(r_xprt)); ), - TP_printk("peer=[%s]:%s r_xprt=%p: rc=%d connect status=%d", + TP_printk("peer=[%s]:%s r_xprt=%p: rc=%d connection status=%d", __get_str(addr), __get_str(port), __entry->r_xprt, __entry->rc, __entry->connect_status ) @@ -411,6 +411,7 @@ TRACE_EVENT(xprtrdma_inline_thresh, DEFINE_CONN_EVENT(connect); DEFINE_CONN_EVENT(disconnect); +DEFINE_CONN_EVENT(flush_dct); DEFINE_RXPRT_EVENT(xprtrdma_create); DEFINE_RXPRT_EVENT(xprtrdma_op_destroy); diff --git a/net/sunrpc/xprtrdma/frwr_ops.c b/net/sunrpc/xprtrdma/frwr_ops.c index a1b5c8024cca..b482fac7be89 100644 --- a/net/sunrpc/xprtrdma/frwr_ops.c +++ b/net/sunrpc/xprtrdma/frwr_ops.c @@ -358,8 +358,8 @@ out_mapmr_err: /** * frwr_wc_fastreg - Invoked by RDMA provider for a flushed FastReg WC - * @cq: completion queue (ignored) - * @wc: completed WR + * @cq: completion queue + * @wc: WCE for a completed FastReg WR * */ static void frwr_wc_fastreg(struct ib_cq *cq, struct ib_wc *wc) @@ -371,6 +371,8 @@ static void frwr_wc_fastreg(struct ib_cq *cq, struct ib_wc *wc) /* WARNING: Only wr_cqe and status are reliable at this point */ trace_xprtrdma_wc_fastreg(wc, frwr); /* The MR will get recycled when the associated req is retransmitted */ + + rpcrdma_flush_disconnect(cq, wc); } /** @@ -441,8 +443,8 @@ static void __frwr_release_mr(struct ib_wc *wc, struct rpcrdma_mr *mr) /** * frwr_wc_localinv - Invoked by RDMA provider for a LOCAL_INV WC - * @cq: completion queue (ignored) - * @wc: completed WR + * @cq: completion queue + * @wc: WCE for a completed LocalInv WR * */ static void frwr_wc_localinv(struct ib_cq *cq, struct ib_wc *wc) @@ -455,12 +457,14 @@ static void frwr_wc_localinv(struct ib_cq *cq, struct ib_wc *wc) /* WARNING: Only wr_cqe and status are reliable at this point */ trace_xprtrdma_wc_li(wc, frwr); __frwr_release_mr(wc, mr); + + rpcrdma_flush_disconnect(cq, wc); } /** * frwr_wc_localinv_wake - Invoked by RDMA provider for a LOCAL_INV WC - * @cq: completion queue (ignored) - * @wc: completed WR + * @cq: completion queue + * @wc: WCE for a completed LocalInv WR * * Awaken anyone waiting for an MR to finish being fenced. */ @@ -475,6 +479,8 @@ static void frwr_wc_localinv_wake(struct ib_cq *cq, struct ib_wc *wc) trace_xprtrdma_wc_li_wake(wc, frwr); __frwr_release_mr(wc, mr); complete(&frwr->fr_linv_done); + + rpcrdma_flush_disconnect(cq, wc); } /** @@ -562,8 +568,8 @@ void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) /** * frwr_wc_localinv_done - Invoked by RDMA provider for a signaled LOCAL_INV WC - * @cq: completion queue (ignored) - * @wc: completed WR + * @cq: completion queue + * @wc: WCE for a completed LocalInv WR * */ static void frwr_wc_localinv_done(struct ib_cq *cq, struct ib_wc *wc) @@ -581,6 +587,8 @@ static void frwr_wc_localinv_done(struct ib_cq *cq, struct ib_wc *wc) /* Ensure @rep is generated before __frwr_release_mr */ smp_rmb(); rpcrdma_complete_rqst(rep); + + rpcrdma_flush_disconnect(cq, wc); } /** diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c index a7f46bbbf017..dfe680e3234a 100644 --- a/net/sunrpc/xprtrdma/verbs.c +++ b/net/sunrpc/xprtrdma/verbs.c @@ -128,14 +128,32 @@ rpcrdma_qp_event_handler(struct ib_event *event, void *context) trace_xprtrdma_qp_event(r_xprt, event); } +/** + * rpcrdma_flush_disconnect - Disconnect on flushed completion + * @cq: completion queue + * @wc: work completion entry + * + * Must be called in process context. + */ +void rpcrdma_flush_disconnect(struct ib_cq *cq, struct ib_wc *wc) +{ + struct rpcrdma_xprt *r_xprt = cq->cq_context; + struct rpc_xprt *xprt = &r_xprt->rx_xprt; + + if (wc->status != IB_WC_SUCCESS && r_xprt->rx_ep.rep_connected == 1) { + r_xprt->rx_ep.rep_connected = -ECONNABORTED; + trace_xprtrdma_flush_dct(r_xprt, wc->status); + xprt_force_disconnect(xprt); + } +} + /** * rpcrdma_wc_send - Invoked by RDMA provider for each polled Send WC * @cq: completion queue - * @wc: completed WR + * @wc: WCE for a completed Send WR * */ -static void -rpcrdma_wc_send(struct ib_cq *cq, struct ib_wc *wc) +static void rpcrdma_wc_send(struct ib_cq *cq, struct ib_wc *wc) { struct ib_cqe *cqe = wc->wr_cqe; struct rpcrdma_sendctx *sc = @@ -144,21 +162,21 @@ rpcrdma_wc_send(struct ib_cq *cq, struct ib_wc *wc) /* WARNING: Only wr_cqe and status are reliable at this point */ trace_xprtrdma_wc_send(sc, wc); rpcrdma_sendctx_put_locked((struct rpcrdma_xprt *)cq->cq_context, sc); + rpcrdma_flush_disconnect(cq, wc); } /** * rpcrdma_wc_receive - Invoked by RDMA provider for each polled Receive WC - * @cq: completion queue (ignored) - * @wc: completed WR + * @cq: completion queue + * @wc: WCE for a completed Receive WR * */ -static void -rpcrdma_wc_receive(struct ib_cq *cq, struct ib_wc *wc) +static void rpcrdma_wc_receive(struct ib_cq *cq, struct ib_wc *wc) { struct ib_cqe *cqe = wc->wr_cqe; struct rpcrdma_rep *rep = container_of(cqe, struct rpcrdma_rep, rr_cqe); - struct rpcrdma_xprt *r_xprt = rep->rr_rxprt; + struct rpcrdma_xprt *r_xprt = cq->cq_context; /* WARNING: Only wr_cqe and status are reliable at this point */ trace_xprtrdma_wc_receive(wc); @@ -179,6 +197,7 @@ rpcrdma_wc_receive(struct ib_cq *cq, struct ib_wc *wc) return; out_flushed: + rpcrdma_flush_disconnect(cq, wc); rpcrdma_rep_destroy(rep); } @@ -395,7 +414,7 @@ static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt) goto out_destroy; } - ep->rep_attr.recv_cq = ib_alloc_cq_any(id->device, NULL, + ep->rep_attr.recv_cq = ib_alloc_cq_any(id->device, r_xprt, ep->rep_attr.cap.max_recv_wr, IB_POLL_WORKQUEUE); if (IS_ERR(ep->rep_attr.recv_cq)) { diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h index d2a0f125f7a8..8a3ac9d7ee81 100644 --- a/net/sunrpc/xprtrdma/xprt_rdma.h +++ b/net/sunrpc/xprtrdma/xprt_rdma.h @@ -452,6 +452,7 @@ extern unsigned int xprt_rdma_memreg_strategy; /* * Endpoint calls - xprtrdma/verbs.c */ +void rpcrdma_flush_disconnect(struct ib_cq *cq, struct ib_wc *wc); int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt); void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt); From 93aa8e0a9de80e1df2be17158a3469285e572b39 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Fri, 21 Feb 2020 17:00:54 -0500 Subject: [PATCH 262/635] xprtrdma: Merge struct rpcrdma_ia into struct rpcrdma_ep I eventually want to allocate rpcrdma_ep separately from struct rpcrdma_xprt so that on occasion there can be more than one ep per xprt. The new struct rpcrdma_ep will contain all the fields currently in rpcrdma_ia and in rpcrdma_ep. This is all the device and CM settings for the connection, in addition to per-connection settings negotiated with the remote. Take this opportunity to rename the existing ep fields from rep_* to re_* to disambiguate these from struct rpcrdma_rep. Signed-off-by: Chuck Lever Signed-off-by: Anna Schumaker --- include/trace/events/rpcrdma.h | 12 +- net/sunrpc/xprtrdma/backchannel.c | 4 +- net/sunrpc/xprtrdma/frwr_ops.c | 106 ++++++------ net/sunrpc/xprtrdma/rpc_rdma.c | 31 ++-- net/sunrpc/xprtrdma/transport.c | 9 +- net/sunrpc/xprtrdma/verbs.c | 279 +++++++++++++++--------------- net/sunrpc/xprtrdma/xprt_rdma.h | 60 +++---- 7 files changed, 243 insertions(+), 258 deletions(-) diff --git a/include/trace/events/rpcrdma.h b/include/trace/events/rpcrdma.h index cfbe28ad2614..843269f0e291 100644 --- a/include/trace/events/rpcrdma.h +++ b/include/trace/events/rpcrdma.h @@ -104,7 +104,7 @@ DECLARE_EVENT_CLASS(xprtrdma_connect_class, TP_fast_assign( __entry->r_xprt = r_xprt; __entry->rc = rc; - __entry->connect_status = r_xprt->rx_ep.rep_connected; + __entry->connect_status = r_xprt->rx_ep.re_connect_status; __assign_str(addr, rpcrdma_addrstr(r_xprt)); __assign_str(port, rpcrdma_portstr(r_xprt)); ), @@ -394,10 +394,10 @@ TRACE_EVENT(xprtrdma_inline_thresh, const struct rpcrdma_ep *ep = &r_xprt->rx_ep; __entry->r_xprt = r_xprt; - __entry->inline_send = ep->rep_inline_send; - __entry->inline_recv = ep->rep_inline_recv; - __entry->max_send = ep->rep_max_inline_send; - __entry->max_recv = ep->rep_max_inline_recv; + __entry->inline_send = ep->re_inline_send; + __entry->inline_recv = ep->re_inline_recv; + __entry->max_send = ep->re_max_inline_send; + __entry->max_recv = ep->re_max_inline_recv; __assign_str(addr, rpcrdma_addrstr(r_xprt)); __assign_str(port, rpcrdma_portstr(r_xprt)); ), @@ -803,7 +803,7 @@ TRACE_EVENT(xprtrdma_post_recvs, __entry->r_xprt = r_xprt; __entry->count = count; __entry->status = status; - __entry->posted = r_xprt->rx_ep.rep_receive_count; + __entry->posted = r_xprt->rx_ep.re_receive_count; __assign_str(addr, rpcrdma_addrstr(r_xprt)); __assign_str(port, rpcrdma_portstr(r_xprt)); ), diff --git a/net/sunrpc/xprtrdma/backchannel.c b/net/sunrpc/xprtrdma/backchannel.c index 4b43910a6ed2..4b20102cf060 100644 --- a/net/sunrpc/xprtrdma/backchannel.c +++ b/net/sunrpc/xprtrdma/backchannel.c @@ -47,7 +47,7 @@ size_t xprt_rdma_bc_maxpayload(struct rpc_xprt *xprt) struct rpcrdma_ep *ep = &r_xprt->rx_ep; size_t maxmsg; - maxmsg = min_t(unsigned int, ep->rep_inline_send, ep->rep_inline_recv); + maxmsg = min_t(unsigned int, ep->re_inline_send, ep->re_inline_recv); maxmsg = min_t(unsigned int, maxmsg, PAGE_SIZE); return maxmsg - RPCRDMA_HDRLEN_MIN; } @@ -190,7 +190,7 @@ create_req: if (xprt->bc_alloc_count >= RPCRDMA_BACKWARD_WRS) return NULL; - size = min_t(size_t, r_xprt->rx_ep.rep_inline_recv, PAGE_SIZE); + size = min_t(size_t, r_xprt->rx_ep.re_inline_recv, PAGE_SIZE); req = rpcrdma_req_create(r_xprt, size, GFP_KERNEL); if (!req) return NULL; diff --git a/net/sunrpc/xprtrdma/frwr_ops.c b/net/sunrpc/xprtrdma/frwr_ops.c index b482fac7be89..19bf422f010b 100644 --- a/net/sunrpc/xprtrdma/frwr_ops.c +++ b/net/sunrpc/xprtrdma/frwr_ops.c @@ -74,7 +74,7 @@ static void frwr_mr_recycle(struct rpcrdma_mr *mr) if (mr->mr_dir != DMA_NONE) { trace_xprtrdma_mr_unmap(mr); - ib_dma_unmap_sg(r_xprt->rx_ia.ri_id->device, + ib_dma_unmap_sg(r_xprt->rx_ep.re_id->device, mr->mr_sg, mr->mr_nents, mr->mr_dir); mr->mr_dir = DMA_NONE; } @@ -115,13 +115,13 @@ void frwr_reset(struct rpcrdma_req *req) */ int frwr_mr_init(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr *mr) { - struct rpcrdma_ia *ia = &r_xprt->rx_ia; - unsigned int depth = ia->ri_max_frwr_depth; + struct rpcrdma_ep *ep = &r_xprt->rx_ep; + unsigned int depth = ep->re_max_fr_depth; struct scatterlist *sg; struct ib_mr *frmr; int rc; - frmr = ib_alloc_mr(ia->ri_pd, ia->ri_mrtype, depth); + frmr = ib_alloc_mr(ep->re_pd, ep->re_mrtype, depth); if (IS_ERR(frmr)) goto out_mr_err; @@ -151,29 +151,24 @@ out_list_err: /** * frwr_query_device - Prepare a transport for use with FRWR - * @r_xprt: controlling transport instance + * @ep: endpoint to fill in * @device: RDMA device to query * * On success, sets: - * ep->rep_attr - * ep->rep_max_requests - * ia->ri_max_rdma_segs - * - * And these FRWR-related fields: - * ia->ri_max_frwr_depth - * ia->ri_mrtype + * ep->re_attr + * ep->re_max_requests + * ep->re_max_rdma_segs + * ep->re_max_fr_depth + * ep->re_mrtype * * Return values: * On success, returns zero. * %-EINVAL - the device does not support FRWR memory registration * %-ENOMEM - the device is not sufficiently capable for NFS/RDMA */ -int frwr_query_device(struct rpcrdma_xprt *r_xprt, - const struct ib_device *device) +int frwr_query_device(struct rpcrdma_ep *ep, const struct ib_device *device) { const struct ib_device_attr *attrs = &device->attrs; - struct rpcrdma_ia *ia = &r_xprt->rx_ia; - struct rpcrdma_ep *ep = &r_xprt->rx_ep; int max_qp_wr, depth, delta; unsigned int max_sge; @@ -190,23 +185,23 @@ int frwr_query_device(struct rpcrdma_xprt *r_xprt, pr_err("rpcrdma: HCA provides only %u send SGEs\n", max_sge); return -ENOMEM; } - ep->rep_attr.cap.max_send_sge = max_sge; - ep->rep_attr.cap.max_recv_sge = 1; + ep->re_attr.cap.max_send_sge = max_sge; + ep->re_attr.cap.max_recv_sge = 1; - ia->ri_mrtype = IB_MR_TYPE_MEM_REG; + ep->re_mrtype = IB_MR_TYPE_MEM_REG; if (attrs->device_cap_flags & IB_DEVICE_SG_GAPS_REG) - ia->ri_mrtype = IB_MR_TYPE_SG_GAPS; + ep->re_mrtype = IB_MR_TYPE_SG_GAPS; /* Quirk: Some devices advertise a large max_fast_reg_page_list_len * capability, but perform optimally when the MRs are not larger * than a page. */ if (attrs->max_sge_rd > RPCRDMA_MAX_HDR_SEGS) - ia->ri_max_frwr_depth = attrs->max_sge_rd; + ep->re_max_fr_depth = attrs->max_sge_rd; else - ia->ri_max_frwr_depth = attrs->max_fast_reg_page_list_len; - if (ia->ri_max_frwr_depth > RPCRDMA_MAX_DATA_SEGS) - ia->ri_max_frwr_depth = RPCRDMA_MAX_DATA_SEGS; + ep->re_max_fr_depth = attrs->max_fast_reg_page_list_len; + if (ep->re_max_fr_depth > RPCRDMA_MAX_DATA_SEGS) + ep->re_max_fr_depth = RPCRDMA_MAX_DATA_SEGS; /* Add room for frwr register and invalidate WRs. * 1. FRWR reg WR for head @@ -222,11 +217,11 @@ int frwr_query_device(struct rpcrdma_xprt *r_xprt, /* Calculate N if the device max FRWR depth is smaller than * RPCRDMA_MAX_DATA_SEGS. */ - if (ia->ri_max_frwr_depth < RPCRDMA_MAX_DATA_SEGS) { - delta = RPCRDMA_MAX_DATA_SEGS - ia->ri_max_frwr_depth; + if (ep->re_max_fr_depth < RPCRDMA_MAX_DATA_SEGS) { + delta = RPCRDMA_MAX_DATA_SEGS - ep->re_max_fr_depth; do { depth += 2; /* FRWR reg + invalidate */ - delta -= ia->ri_max_frwr_depth; + delta -= ep->re_max_fr_depth; } while (delta > 0); } @@ -235,34 +230,34 @@ int frwr_query_device(struct rpcrdma_xprt *r_xprt, max_qp_wr -= 1; if (max_qp_wr < RPCRDMA_MIN_SLOT_TABLE) return -ENOMEM; - if (ep->rep_max_requests > max_qp_wr) - ep->rep_max_requests = max_qp_wr; - ep->rep_attr.cap.max_send_wr = ep->rep_max_requests * depth; - if (ep->rep_attr.cap.max_send_wr > max_qp_wr) { - ep->rep_max_requests = max_qp_wr / depth; - if (!ep->rep_max_requests) + if (ep->re_max_requests > max_qp_wr) + ep->re_max_requests = max_qp_wr; + ep->re_attr.cap.max_send_wr = ep->re_max_requests * depth; + if (ep->re_attr.cap.max_send_wr > max_qp_wr) { + ep->re_max_requests = max_qp_wr / depth; + if (!ep->re_max_requests) return -ENOMEM; - ep->rep_attr.cap.max_send_wr = ep->rep_max_requests * depth; + ep->re_attr.cap.max_send_wr = ep->re_max_requests * depth; } - ep->rep_attr.cap.max_send_wr += RPCRDMA_BACKWARD_WRS; - ep->rep_attr.cap.max_send_wr += 1; /* for ib_drain_sq */ - ep->rep_attr.cap.max_recv_wr = ep->rep_max_requests; - ep->rep_attr.cap.max_recv_wr += RPCRDMA_BACKWARD_WRS; - ep->rep_attr.cap.max_recv_wr += 1; /* for ib_drain_rq */ + ep->re_attr.cap.max_send_wr += RPCRDMA_BACKWARD_WRS; + ep->re_attr.cap.max_send_wr += 1; /* for ib_drain_sq */ + ep->re_attr.cap.max_recv_wr = ep->re_max_requests; + ep->re_attr.cap.max_recv_wr += RPCRDMA_BACKWARD_WRS; + ep->re_attr.cap.max_recv_wr += 1; /* for ib_drain_rq */ - ia->ri_max_rdma_segs = - DIV_ROUND_UP(RPCRDMA_MAX_DATA_SEGS, ia->ri_max_frwr_depth); + ep->re_max_rdma_segs = + DIV_ROUND_UP(RPCRDMA_MAX_DATA_SEGS, ep->re_max_fr_depth); /* Reply chunks require segments for head and tail buffers */ - ia->ri_max_rdma_segs += 2; - if (ia->ri_max_rdma_segs > RPCRDMA_MAX_HDR_SEGS) - ia->ri_max_rdma_segs = RPCRDMA_MAX_HDR_SEGS; + ep->re_max_rdma_segs += 2; + if (ep->re_max_rdma_segs > RPCRDMA_MAX_HDR_SEGS) + ep->re_max_rdma_segs = RPCRDMA_MAX_HDR_SEGS; /* Ensure the underlying device is capable of conveying the * largest r/wsize NFS will ask for. This guarantees that * failing over from one RDMA device to another will not * break NFS I/O. */ - if ((ia->ri_max_rdma_segs * ia->ri_max_frwr_depth) < RPCRDMA_MAX_SEGS) + if ((ep->re_max_rdma_segs * ep->re_max_fr_depth) < RPCRDMA_MAX_SEGS) return -ENOMEM; return 0; @@ -288,14 +283,14 @@ struct rpcrdma_mr_seg *frwr_map(struct rpcrdma_xprt *r_xprt, int nsegs, bool writing, __be32 xid, struct rpcrdma_mr *mr) { - struct rpcrdma_ia *ia = &r_xprt->rx_ia; + struct rpcrdma_ep *ep = &r_xprt->rx_ep; struct ib_reg_wr *reg_wr; int i, n, dma_nents; struct ib_mr *ibmr; u8 key; - if (nsegs > ia->ri_max_frwr_depth) - nsegs = ia->ri_max_frwr_depth; + if (nsegs > ep->re_max_fr_depth) + nsegs = ep->re_max_fr_depth; for (i = 0; i < nsegs;) { if (seg->mr_page) sg_set_page(&mr->mr_sg[i], @@ -308,7 +303,7 @@ struct rpcrdma_mr_seg *frwr_map(struct rpcrdma_xprt *r_xprt, ++seg; ++i; - if (ia->ri_mrtype == IB_MR_TYPE_SG_GAPS) + if (ep->re_mrtype == IB_MR_TYPE_SG_GAPS) continue; if ((i < nsegs && offset_in_page(seg->mr_offset)) || offset_in_page((seg-1)->mr_offset + (seg-1)->mr_len)) @@ -317,7 +312,7 @@ struct rpcrdma_mr_seg *frwr_map(struct rpcrdma_xprt *r_xprt, mr->mr_dir = rpcrdma_data_dir(writing); mr->mr_nents = i; - dma_nents = ib_dma_map_sg(ia->ri_id->device, mr->mr_sg, mr->mr_nents, + dma_nents = ib_dma_map_sg(ep->re_id->device, mr->mr_sg, mr->mr_nents, mr->mr_dir); if (!dma_nents) goto out_dmamap_err; @@ -391,7 +386,6 @@ static void frwr_wc_fastreg(struct ib_cq *cq, struct ib_wc *wc) */ int frwr_send(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) { - struct rpcrdma_ia *ia = &r_xprt->rx_ia; struct ib_send_wr *post_wr; struct rpcrdma_mr *mr; @@ -411,7 +405,7 @@ int frwr_send(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) post_wr = &frwr->fr_regwr.wr; } - return ib_post_send(ia->ri_id->qp, post_wr, NULL); + return ib_post_send(r_xprt->rx_ep.re_id->qp, post_wr, NULL); } /** @@ -538,10 +532,10 @@ void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) /* Transport disconnect drains the receive CQ before it * replaces the QP. The RPC reply handler won't call us - * unless ri_id->qp is a valid pointer. + * unless re_id->qp is a valid pointer. */ bad_wr = NULL; - rc = ib_post_send(r_xprt->rx_ia.ri_id->qp, first, &bad_wr); + rc = ib_post_send(r_xprt->rx_ep.re_id->qp, first, &bad_wr); /* The final LOCAL_INV WR in the chain is supposed to * do the wake. If it was never posted, the wake will @@ -643,10 +637,10 @@ void frwr_unmap_async(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) /* Transport disconnect drains the receive CQ before it * replaces the QP. The RPC reply handler won't call us - * unless ri_id->qp is a valid pointer. + * unless re_id->qp is a valid pointer. */ bad_wr = NULL; - rc = ib_post_send(r_xprt->rx_ia.ri_id->qp, first, &bad_wr); + rc = ib_post_send(r_xprt->rx_ep.re_id->qp, first, &bad_wr); if (!rc) return; diff --git a/net/sunrpc/xprtrdma/rpc_rdma.c b/net/sunrpc/xprtrdma/rpc_rdma.c index 28020ec104d4..ad7e6b0187bd 100644 --- a/net/sunrpc/xprtrdma/rpc_rdma.c +++ b/net/sunrpc/xprtrdma/rpc_rdma.c @@ -103,21 +103,20 @@ static unsigned int rpcrdma_max_reply_header_size(unsigned int maxsegs) /** * rpcrdma_set_max_header_sizes - Initialize inline payload sizes - * @r_xprt: transport instance to initialize + * @ep: endpoint to initialize * * The max_inline fields contain the maximum size of an RPC message * so the marshaling code doesn't have to repeat this calculation * for every RPC. */ -void rpcrdma_set_max_header_sizes(struct rpcrdma_xprt *r_xprt) +void rpcrdma_set_max_header_sizes(struct rpcrdma_ep *ep) { - unsigned int maxsegs = r_xprt->rx_ia.ri_max_rdma_segs; - struct rpcrdma_ep *ep = &r_xprt->rx_ep; + unsigned int maxsegs = ep->re_max_rdma_segs; - ep->rep_max_inline_send = - ep->rep_inline_send - rpcrdma_max_call_header_size(maxsegs); - ep->rep_max_inline_recv = - ep->rep_inline_recv - rpcrdma_max_reply_header_size(maxsegs); + ep->re_max_inline_send = + ep->re_inline_send - rpcrdma_max_call_header_size(maxsegs); + ep->re_max_inline_recv = + ep->re_inline_recv - rpcrdma_max_reply_header_size(maxsegs); } /* The client can send a request inline as long as the RPCRDMA header @@ -134,7 +133,7 @@ static bool rpcrdma_args_inline(struct rpcrdma_xprt *r_xprt, struct xdr_buf *xdr = &rqst->rq_snd_buf; unsigned int count, remaining, offset; - if (xdr->len > r_xprt->rx_ep.rep_max_inline_send) + if (xdr->len > r_xprt->rx_ep.re_max_inline_send) return false; if (xdr->page_len) { @@ -145,7 +144,7 @@ static bool rpcrdma_args_inline(struct rpcrdma_xprt *r_xprt, remaining -= min_t(unsigned int, PAGE_SIZE - offset, remaining); offset = 0; - if (++count > r_xprt->rx_ep.rep_attr.cap.max_send_sge) + if (++count > r_xprt->rx_ep.re_attr.cap.max_send_sge) return false; } } @@ -162,7 +161,7 @@ static bool rpcrdma_args_inline(struct rpcrdma_xprt *r_xprt, static bool rpcrdma_results_inline(struct rpcrdma_xprt *r_xprt, struct rpc_rqst *rqst) { - return rqst->rq_rcv_buf.buflen <= r_xprt->rx_ep.rep_max_inline_recv; + return rqst->rq_rcv_buf.buflen <= r_xprt->rx_ep.re_max_inline_recv; } /* The client is required to provide a Reply chunk if the maximum @@ -176,7 +175,7 @@ rpcrdma_nonpayload_inline(const struct rpcrdma_xprt *r_xprt, const struct xdr_buf *buf = &rqst->rq_rcv_buf; return (buf->head[0].iov_len + buf->tail[0].iov_len) < - r_xprt->rx_ep.rep_max_inline_recv; + r_xprt->rx_ep.re_max_inline_recv; } /* Split @vec on page boundaries into SGEs. FMR registers pages, not @@ -255,7 +254,7 @@ rpcrdma_convert_iovs(struct rpcrdma_xprt *r_xprt, struct xdr_buf *xdrbuf, /* When encoding a Read chunk, the tail iovec contains an * XDR pad and may be omitted. */ - if (type == rpcrdma_readch && r_xprt->rx_ia.ri_implicit_roundup) + if (type == rpcrdma_readch && r_xprt->rx_ep.re_implicit_roundup) goto out; /* When encoding a Write chunk, some servers need to see an @@ -263,7 +262,7 @@ rpcrdma_convert_iovs(struct rpcrdma_xprt *r_xprt, struct xdr_buf *xdrbuf, * layer provides space in the tail iovec that may be used * for this purpose. */ - if (type == rpcrdma_writech && r_xprt->rx_ia.ri_implicit_roundup) + if (type == rpcrdma_writech && r_xprt->rx_ep.re_implicit_roundup) goto out; if (xdrbuf->tail[0].iov_len) @@ -1476,8 +1475,8 @@ void rpcrdma_reply_handler(struct rpcrdma_rep *rep) if (credits == 0) credits = 1; /* don't deadlock */ - else if (credits > r_xprt->rx_ep.rep_max_requests) - credits = r_xprt->rx_ep.rep_max_requests; + else if (credits > r_xprt->rx_ep.re_max_requests) + credits = r_xprt->rx_ep.re_max_requests; if (buf->rb_credits != credits) rpcrdma_update_cwnd(r_xprt, credits); rpcrdma_post_recvs(r_xprt, false); diff --git a/net/sunrpc/xprtrdma/transport.c b/net/sunrpc/xprtrdma/transport.c index d7b7dab0aeb6..4352fd6e9817 100644 --- a/net/sunrpc/xprtrdma/transport.c +++ b/net/sunrpc/xprtrdma/transport.c @@ -238,11 +238,12 @@ xprt_rdma_connect_worker(struct work_struct *work) struct rpcrdma_xprt *r_xprt = container_of(work, struct rpcrdma_xprt, rx_connect_worker.work); struct rpc_xprt *xprt = &r_xprt->rx_xprt; + struct rpcrdma_ep *ep = &r_xprt->rx_ep; int rc; rc = rpcrdma_xprt_connect(r_xprt); xprt_clear_connecting(xprt); - if (r_xprt->rx_ep.rep_connected > 0) { + if (ep->re_connect_status > 0) { xprt->stat.connect_count++; xprt->stat.connect_time += (long)jiffies - xprt->stat.connect_start; @@ -265,7 +266,7 @@ xprt_rdma_inject_disconnect(struct rpc_xprt *xprt) struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt); trace_xprtrdma_op_inject_dsc(r_xprt); - rdma_disconnect(r_xprt->rx_ia.ri_id); + rdma_disconnect(r_xprt->rx_ep.re_id); } /** @@ -355,6 +356,7 @@ xprt_setup_rdma(struct xprt_create *args) INIT_DELAYED_WORK(&new_xprt->rx_connect_worker, xprt_rdma_connect_worker); + xprt->max_payload = RPCRDMA_MAX_DATA_SEGS << PAGE_SHIFT; dprintk("RPC: %s: %s:%s\n", __func__, @@ -489,10 +491,11 @@ static void xprt_rdma_connect(struct rpc_xprt *xprt, struct rpc_task *task) { struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt); + struct rpcrdma_ep *ep = &r_xprt->rx_ep; unsigned long delay; delay = 0; - if (r_xprt->rx_ep.rep_connected != 0) { + if (ep->re_connect_status != 0) { delay = xprt_reconnect_delay(xprt); xprt_reconnect_backoff(xprt, RPCRDMA_INIT_REEST_TO); } diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c index dfe680e3234a..10826982ddf8 100644 --- a/net/sunrpc/xprtrdma/verbs.c +++ b/net/sunrpc/xprtrdma/verbs.c @@ -97,17 +97,17 @@ static void rpcrdma_regbuf_free(struct rpcrdma_regbuf *rb); */ static void rpcrdma_xprt_drain(struct rpcrdma_xprt *r_xprt) { - struct rpcrdma_ia *ia = &r_xprt->rx_ia; + struct rdma_cm_id *id = r_xprt->rx_ep.re_id; /* Flush Receives, then wait for deferred Reply work * to complete. */ - ib_drain_rq(ia->ri_id->qp); + ib_drain_rq(id->qp); /* Deferred Reply processing might have scheduled * local invalidations. */ - ib_drain_sq(ia->ri_id->qp); + ib_drain_sq(id->qp); } /** @@ -140,8 +140,9 @@ void rpcrdma_flush_disconnect(struct ib_cq *cq, struct ib_wc *wc) struct rpcrdma_xprt *r_xprt = cq->cq_context; struct rpc_xprt *xprt = &r_xprt->rx_xprt; - if (wc->status != IB_WC_SUCCESS && r_xprt->rx_ep.rep_connected == 1) { - r_xprt->rx_ep.rep_connected = -ECONNABORTED; + if (wc->status != IB_WC_SUCCESS && + r_xprt->rx_ep.re_connect_status == 1) { + r_xprt->rx_ep.re_connect_status = -ECONNABORTED; trace_xprtrdma_flush_dct(r_xprt, wc->status); xprt_force_disconnect(xprt); } @@ -180,7 +181,7 @@ static void rpcrdma_wc_receive(struct ib_cq *cq, struct ib_wc *wc) /* WARNING: Only wr_cqe and status are reliable at this point */ trace_xprtrdma_wc_receive(wc); - --r_xprt->rx_ep.rep_receive_count; + --r_xprt->rx_ep.re_receive_count; if (wc->status != IB_WC_SUCCESS) goto out_flushed; @@ -209,24 +210,24 @@ static void rpcrdma_update_cm_private(struct rpcrdma_xprt *r_xprt, unsigned int rsize, wsize; /* Default settings for RPC-over-RDMA Version One */ - r_xprt->rx_ia.ri_implicit_roundup = xprt_rdma_pad_optimize; + ep->re_implicit_roundup = xprt_rdma_pad_optimize; rsize = RPCRDMA_V1_DEF_INLINE_SIZE; wsize = RPCRDMA_V1_DEF_INLINE_SIZE; if (pmsg && pmsg->cp_magic == rpcrdma_cmp_magic && pmsg->cp_version == RPCRDMA_CMP_VERSION) { - r_xprt->rx_ia.ri_implicit_roundup = true; + ep->re_implicit_roundup = true; rsize = rpcrdma_decode_buffer_size(pmsg->cp_send_size); wsize = rpcrdma_decode_buffer_size(pmsg->cp_recv_size); } - if (rsize < ep->rep_inline_recv) - ep->rep_inline_recv = rsize; - if (wsize < ep->rep_inline_send) - ep->rep_inline_send = wsize; + if (rsize < ep->re_inline_recv) + ep->re_inline_recv = rsize; + if (wsize < ep->re_inline_send) + ep->re_inline_send = wsize; - rpcrdma_set_max_header_sizes(r_xprt); + rpcrdma_set_max_header_sizes(ep); } /** @@ -241,7 +242,6 @@ static int rpcrdma_cm_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event) { struct rpcrdma_xprt *r_xprt = id->context; - struct rpcrdma_ia *ia = &r_xprt->rx_ia; struct rpcrdma_ep *ep = &r_xprt->rx_ep; struct rpc_xprt *xprt = &r_xprt->rx_xprt; @@ -251,57 +251,57 @@ rpcrdma_cm_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event) switch (event->event) { case RDMA_CM_EVENT_ADDR_RESOLVED: case RDMA_CM_EVENT_ROUTE_RESOLVED: - ia->ri_async_rc = 0; - complete(&ia->ri_done); + ep->re_async_rc = 0; + complete(&ep->re_done); return 0; case RDMA_CM_EVENT_ADDR_ERROR: - ia->ri_async_rc = -EPROTO; - complete(&ia->ri_done); + ep->re_async_rc = -EPROTO; + complete(&ep->re_done); return 0; case RDMA_CM_EVENT_ROUTE_ERROR: - ia->ri_async_rc = -ENETUNREACH; - complete(&ia->ri_done); + ep->re_async_rc = -ENETUNREACH; + complete(&ep->re_done); return 0; case RDMA_CM_EVENT_DEVICE_REMOVAL: #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) pr_info("rpcrdma: removing device %s for %s:%s\n", - ia->ri_id->device->name, + ep->re_id->device->name, rpcrdma_addrstr(r_xprt), rpcrdma_portstr(r_xprt)); #endif - init_completion(&ia->ri_remove_done); - ep->rep_connected = -ENODEV; + init_completion(&ep->re_remove_done); + ep->re_connect_status = -ENODEV; xprt_force_disconnect(xprt); - wait_for_completion(&ia->ri_remove_done); + wait_for_completion(&ep->re_remove_done); trace_xprtrdma_remove(r_xprt); /* Return 1 to ensure the core destroys the id. */ return 1; case RDMA_CM_EVENT_ESTABLISHED: ++xprt->connect_cookie; - ep->rep_connected = 1; + ep->re_connect_status = 1; rpcrdma_update_cm_private(r_xprt, &event->param.conn); trace_xprtrdma_inline_thresh(r_xprt); - wake_up_all(&ep->rep_connect_wait); + wake_up_all(&ep->re_connect_wait); break; case RDMA_CM_EVENT_CONNECT_ERROR: - ep->rep_connected = -ENOTCONN; + ep->re_connect_status = -ENOTCONN; goto disconnected; case RDMA_CM_EVENT_UNREACHABLE: - ep->rep_connected = -ENETUNREACH; + ep->re_connect_status = -ENETUNREACH; goto disconnected; case RDMA_CM_EVENT_REJECTED: dprintk("rpcrdma: connection to %s:%s rejected: %s\n", rpcrdma_addrstr(r_xprt), rpcrdma_portstr(r_xprt), rdma_reject_msg(id, event->status)); - ep->rep_connected = -ECONNREFUSED; + ep->re_connect_status = -ECONNREFUSED; if (event->status == IB_CM_REJ_STALE_CONN) - ep->rep_connected = -EAGAIN; + ep->re_connect_status = -EAGAIN; goto disconnected; case RDMA_CM_EVENT_DISCONNECTED: - ep->rep_connected = -ECONNABORTED; + ep->re_connect_status = -ECONNABORTED; disconnected: xprt_force_disconnect(xprt); - wake_up_all(&ep->rep_connect_wait); + wake_up_all(&ep->re_connect_wait); break; default: break; @@ -309,46 +309,46 @@ disconnected: dprintk("RPC: %s: %s:%s on %s/frwr: %s\n", __func__, rpcrdma_addrstr(r_xprt), rpcrdma_portstr(r_xprt), - ia->ri_id->device->name, rdma_event_msg(event->event)); + ep->re_id->device->name, rdma_event_msg(event->event)); return 0; } -static struct rdma_cm_id * -rpcrdma_create_id(struct rpcrdma_xprt *xprt, struct rpcrdma_ia *ia) +static struct rdma_cm_id *rpcrdma_create_id(struct rpcrdma_xprt *r_xprt, + struct rpcrdma_ep *ep) { unsigned long wtimeout = msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1; + struct rpc_xprt *xprt = &r_xprt->rx_xprt; struct rdma_cm_id *id; int rc; - init_completion(&ia->ri_done); + init_completion(&ep->re_done); - id = rdma_create_id(xprt->rx_xprt.xprt_net, rpcrdma_cm_event_handler, - xprt, RDMA_PS_TCP, IB_QPT_RC); + id = rdma_create_id(xprt->xprt_net, rpcrdma_cm_event_handler, r_xprt, + RDMA_PS_TCP, IB_QPT_RC); if (IS_ERR(id)) return id; - ia->ri_async_rc = -ETIMEDOUT; - rc = rdma_resolve_addr(id, NULL, - (struct sockaddr *)&xprt->rx_xprt.addr, + ep->re_async_rc = -ETIMEDOUT; + rc = rdma_resolve_addr(id, NULL, (struct sockaddr *)&xprt->addr, RDMA_RESOLVE_TIMEOUT); if (rc) goto out; - rc = wait_for_completion_interruptible_timeout(&ia->ri_done, wtimeout); + rc = wait_for_completion_interruptible_timeout(&ep->re_done, wtimeout); if (rc < 0) goto out; - rc = ia->ri_async_rc; + rc = ep->re_async_rc; if (rc) goto out; - ia->ri_async_rc = -ETIMEDOUT; + ep->re_async_rc = -ETIMEDOUT; rc = rdma_resolve_route(id, RDMA_RESOLVE_TIMEOUT); if (rc) goto out; - rc = wait_for_completion_interruptible_timeout(&ia->ri_done, wtimeout); + rc = wait_for_completion_interruptible_timeout(&ep->re_done, wtimeout); if (rc < 0) goto out; - rc = ia->ri_async_rc; + rc = ep->re_async_rc; if (rc) goto out; @@ -366,102 +366,101 @@ out: static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt) { struct rpcrdma_ep *ep = &r_xprt->rx_ep; - struct rpcrdma_ia *ia = &r_xprt->rx_ia; - struct rpcrdma_connect_private *pmsg = &ep->rep_cm_private; + struct rpcrdma_connect_private *pmsg = &ep->re_cm_private; struct rdma_cm_id *id; int rc; - id = rpcrdma_create_id(r_xprt, ia); + id = rpcrdma_create_id(r_xprt, ep); if (IS_ERR(id)) return PTR_ERR(id); - ep->rep_max_requests = r_xprt->rx_xprt.max_reqs; - ep->rep_inline_send = xprt_rdma_max_inline_write; - ep->rep_inline_recv = xprt_rdma_max_inline_read; - - rc = frwr_query_device(r_xprt, id->device); + ep->re_max_requests = r_xprt->rx_xprt.max_reqs; + ep->re_inline_send = xprt_rdma_max_inline_write; + ep->re_inline_recv = xprt_rdma_max_inline_read; + rc = frwr_query_device(ep, id->device); if (rc) goto out_destroy; - r_xprt->rx_buf.rb_max_requests = cpu_to_be32(ep->rep_max_requests); + r_xprt->rx_buf.rb_max_requests = cpu_to_be32(ep->re_max_requests); - ep->rep_attr.event_handler = rpcrdma_qp_event_handler; - ep->rep_attr.qp_context = ep; - ep->rep_attr.srq = NULL; - ep->rep_attr.cap.max_inline_data = 0; - ep->rep_attr.sq_sig_type = IB_SIGNAL_REQ_WR; - ep->rep_attr.qp_type = IB_QPT_RC; - ep->rep_attr.port_num = ~0; + ep->re_attr.event_handler = rpcrdma_qp_event_handler; + ep->re_attr.qp_context = ep; + ep->re_attr.srq = NULL; + ep->re_attr.cap.max_inline_data = 0; + ep->re_attr.sq_sig_type = IB_SIGNAL_REQ_WR; + ep->re_attr.qp_type = IB_QPT_RC; + ep->re_attr.port_num = ~0; dprintk("RPC: %s: requested max: dtos: send %d recv %d; " "iovs: send %d recv %d\n", __func__, - ep->rep_attr.cap.max_send_wr, - ep->rep_attr.cap.max_recv_wr, - ep->rep_attr.cap.max_send_sge, - ep->rep_attr.cap.max_recv_sge); + ep->re_attr.cap.max_send_wr, + ep->re_attr.cap.max_recv_wr, + ep->re_attr.cap.max_send_sge, + ep->re_attr.cap.max_recv_sge); - ep->rep_send_batch = ep->rep_max_requests >> 3; - ep->rep_send_count = ep->rep_send_batch; - init_waitqueue_head(&ep->rep_connect_wait); - ep->rep_receive_count = 0; + ep->re_send_batch = ep->re_max_requests >> 3; + ep->re_send_count = ep->re_send_batch; + init_waitqueue_head(&ep->re_connect_wait); - ep->rep_attr.send_cq = ib_alloc_cq_any(id->device, r_xprt, - ep->rep_attr.cap.max_send_wr, - IB_POLL_WORKQUEUE); - if (IS_ERR(ep->rep_attr.send_cq)) { - rc = PTR_ERR(ep->rep_attr.send_cq); + ep->re_attr.send_cq = ib_alloc_cq_any(id->device, r_xprt, + ep->re_attr.cap.max_send_wr, + IB_POLL_WORKQUEUE); + if (IS_ERR(ep->re_attr.send_cq)) { + rc = PTR_ERR(ep->re_attr.send_cq); goto out_destroy; } - ep->rep_attr.recv_cq = ib_alloc_cq_any(id->device, r_xprt, - ep->rep_attr.cap.max_recv_wr, - IB_POLL_WORKQUEUE); - if (IS_ERR(ep->rep_attr.recv_cq)) { - rc = PTR_ERR(ep->rep_attr.recv_cq); + ep->re_attr.recv_cq = ib_alloc_cq_any(id->device, r_xprt, + ep->re_attr.cap.max_recv_wr, + IB_POLL_WORKQUEUE); + if (IS_ERR(ep->re_attr.recv_cq)) { + rc = PTR_ERR(ep->re_attr.recv_cq); goto out_destroy; } + ep->re_receive_count = 0; /* Initialize cma parameters */ - memset(&ep->rep_remote_cma, 0, sizeof(ep->rep_remote_cma)); + memset(&ep->re_remote_cma, 0, sizeof(ep->re_remote_cma)); /* Prepare RDMA-CM private message */ pmsg->cp_magic = rpcrdma_cmp_magic; pmsg->cp_version = RPCRDMA_CMP_VERSION; pmsg->cp_flags |= RPCRDMA_CMP_F_SND_W_INV_OK; - pmsg->cp_send_size = rpcrdma_encode_buffer_size(ep->rep_inline_send); - pmsg->cp_recv_size = rpcrdma_encode_buffer_size(ep->rep_inline_recv); - ep->rep_remote_cma.private_data = pmsg; - ep->rep_remote_cma.private_data_len = sizeof(*pmsg); + pmsg->cp_send_size = rpcrdma_encode_buffer_size(ep->re_inline_send); + pmsg->cp_recv_size = rpcrdma_encode_buffer_size(ep->re_inline_recv); + ep->re_remote_cma.private_data = pmsg; + ep->re_remote_cma.private_data_len = sizeof(*pmsg); /* Client offers RDMA Read but does not initiate */ - ep->rep_remote_cma.initiator_depth = 0; - ep->rep_remote_cma.responder_resources = + ep->re_remote_cma.initiator_depth = 0; + ep->re_remote_cma.responder_resources = min_t(int, U8_MAX, id->device->attrs.max_qp_rd_atom); /* Limit transport retries so client can detect server * GID changes quickly. RPC layer handles re-establishing * transport connection and retransmission. */ - ep->rep_remote_cma.retry_count = 6; + ep->re_remote_cma.retry_count = 6; /* RPC-over-RDMA handles its own flow control. In addition, * make all RNR NAKs visible so we know that RPC-over-RDMA * flow control is working correctly (no NAKs should be seen). */ - ep->rep_remote_cma.flow_control = 0; - ep->rep_remote_cma.rnr_retry_count = 0; + ep->re_remote_cma.flow_control = 0; + ep->re_remote_cma.rnr_retry_count = 0; - ia->ri_pd = ib_alloc_pd(id->device, 0); - if (IS_ERR(ia->ri_pd)) { - rc = PTR_ERR(ia->ri_pd); + ep->re_pd = ib_alloc_pd(id->device, 0); + if (IS_ERR(ep->re_pd)) { + rc = PTR_ERR(ep->re_pd); goto out_destroy; } - rc = rdma_create_qp(id, ia->ri_pd, &ep->rep_attr); + rc = rdma_create_qp(id, ep->re_pd, &ep->re_attr); if (rc) goto out_destroy; - ia->ri_id = id; + + ep->re_id = id; return 0; out_destroy: @@ -473,23 +472,22 @@ out_destroy: static void rpcrdma_ep_destroy(struct rpcrdma_xprt *r_xprt) { struct rpcrdma_ep *ep = &r_xprt->rx_ep; - struct rpcrdma_ia *ia = &r_xprt->rx_ia; - if (ia->ri_id && ia->ri_id->qp) { - rdma_destroy_qp(ia->ri_id); - ia->ri_id->qp = NULL; + if (ep->re_id && ep->re_id->qp) { + rdma_destroy_qp(ep->re_id); + ep->re_id->qp = NULL; } - if (ep->rep_attr.recv_cq) - ib_free_cq(ep->rep_attr.recv_cq); - ep->rep_attr.recv_cq = NULL; - if (ep->rep_attr.send_cq) - ib_free_cq(ep->rep_attr.send_cq); - ep->rep_attr.send_cq = NULL; + if (ep->re_attr.recv_cq) + ib_free_cq(ep->re_attr.recv_cq); + ep->re_attr.recv_cq = NULL; + if (ep->re_attr.send_cq) + ib_free_cq(ep->re_attr.send_cq); + ep->re_attr.send_cq = NULL; - if (ia->ri_pd) - ib_dealloc_pd(ia->ri_pd); - ia->ri_pd = NULL; + if (ep->re_pd) + ib_dealloc_pd(ep->re_pd); + ep->re_pd = NULL; } /* @@ -499,7 +497,6 @@ int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt) { struct rpc_xprt *xprt = &r_xprt->rx_xprt; struct rpcrdma_ep *ep = &r_xprt->rx_ep; - struct rpcrdma_ia *ia = &r_xprt->rx_ia; int rc; retry: @@ -508,7 +505,7 @@ retry: if (rc) goto out_noupdate; - ep->rep_connected = 0; + ep->re_connect_status = 0; xprt_clear_connected(xprt); rpcrdma_reset_cwnd(r_xprt); @@ -518,17 +515,18 @@ retry: if (rc) goto out; - rc = rdma_connect(ia->ri_id, &ep->rep_remote_cma); + rc = rdma_connect(ep->re_id, &ep->re_remote_cma); if (rc) goto out; if (xprt->reestablish_timeout < RPCRDMA_INIT_REEST_TO) xprt->reestablish_timeout = RPCRDMA_INIT_REEST_TO; - wait_event_interruptible(ep->rep_connect_wait, ep->rep_connected != 0); - if (ep->rep_connected <= 0) { - if (ep->rep_connected == -EAGAIN) + wait_event_interruptible(ep->re_connect_wait, + ep->re_connect_status != 0); + if (ep->re_connect_status <= 0) { + if (ep->re_connect_status == -EAGAIN) goto retry; - rc = ep->rep_connected; + rc = ep->re_connect_status; goto out; } @@ -541,7 +539,7 @@ retry: out: if (rc) - ep->rep_connected = rc; + ep->re_connect_status = rc; out_noupdate: trace_xprtrdma_connect(r_xprt, rc); @@ -558,9 +556,8 @@ out_noupdate: void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt) { struct rpcrdma_ep *ep = &r_xprt->rx_ep; - struct rpcrdma_ia *ia = &r_xprt->rx_ia; - struct rdma_cm_id *id = ia->ri_id; - int rc, status = ep->rep_connected; + struct rdma_cm_id *id = ep->re_id; + int rc, status = ep->re_connect_status; might_sleep(); @@ -569,10 +566,10 @@ void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt) rc = rdma_disconnect(id); if (!rc) - wait_event_interruptible(ep->rep_connect_wait, - ep->rep_connected != 1); + wait_event_interruptible(ep->re_connect_wait, + ep->re_connect_status != 1); else - ep->rep_connected = rc; + ep->re_connect_status = rc; trace_xprtrdma_disconnect(r_xprt, rc); if (id->qp) @@ -585,10 +582,10 @@ void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt) rpcrdma_ep_destroy(r_xprt); if (status == -ENODEV) - complete(&ia->ri_remove_done); + complete(&ep->re_remove_done); else rdma_destroy_id(id); - ia->ri_id = NULL; + ep->re_id = NULL; } /* Fixed-size circular FIFO queue. This implementation is wait-free and @@ -625,7 +622,7 @@ static struct rpcrdma_sendctx *rpcrdma_sendctx_create(struct rpcrdma_ep *ep) { struct rpcrdma_sendctx *sc; - sc = kzalloc(struct_size(sc, sc_sges, ep->rep_attr.cap.max_send_sge), + sc = kzalloc(struct_size(sc, sc_sges, ep->re_attr.cap.max_send_sge), GFP_KERNEL); if (!sc) return NULL; @@ -645,7 +642,7 @@ static int rpcrdma_sendctxs_create(struct rpcrdma_xprt *r_xprt) * the ->send_request call to fail temporarily before too many * Sends are posted. */ - i = r_xprt->rx_ep.rep_max_requests + RPCRDMA_MAX_BC_REQUESTS; + i = r_xprt->rx_ep.re_max_requests + RPCRDMA_MAX_BC_REQUESTS; buf->rb_sc_ctxs = kcalloc(i, sizeof(sc), GFP_KERNEL); if (!buf->rb_sc_ctxs) return -ENOMEM; @@ -756,10 +753,10 @@ static void rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt) { struct rpcrdma_buffer *buf = &r_xprt->rx_buf; - struct rpcrdma_ia *ia = &r_xprt->rx_ia; + struct rpcrdma_ep *ep = &r_xprt->rx_ep; unsigned int count; - for (count = 0; count < ia->ri_max_rdma_segs; count++) { + for (count = 0; count < ep->re_max_rdma_segs; count++) { struct rpcrdma_mr *mr; int rc; @@ -808,7 +805,7 @@ void rpcrdma_mrs_refresh(struct rpcrdma_xprt *r_xprt) /* If there is no underlying connection, it's no use * to wake the refresh worker. */ - if (ep->rep_connected == 1) { + if (ep->re_connect_status == 1) { /* The work is scheduled on a WQ_MEM_RECLAIM * workqueue in order to prevent MR allocation * from recursing into NFS during direct reclaim. @@ -872,7 +869,7 @@ int rpcrdma_req_setup(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) /* Compute maximum header buffer size in bytes */ maxhdrsize = rpcrdma_fixed_maxsz + 3 + - r_xprt->rx_ia.ri_max_rdma_segs * rpcrdma_readchunk_maxsz; + r_xprt->rx_ep.re_max_rdma_segs * rpcrdma_readchunk_maxsz; maxhdrsize *= sizeof(__be32); rb = rpcrdma_regbuf_alloc(__roundup_pow_of_two(maxhdrsize), DMA_TO_DEVICE, GFP_KERNEL); @@ -950,7 +947,7 @@ struct rpcrdma_rep *rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt, if (rep == NULL) goto out; - rep->rr_rdmabuf = rpcrdma_regbuf_alloc(r_xprt->rx_ep.rep_inline_recv, + rep->rr_rdmabuf = rpcrdma_regbuf_alloc(r_xprt->rx_ep.re_inline_recv, DMA_FROM_DEVICE, GFP_KERNEL); if (!rep->rr_rdmabuf) goto out_free; @@ -1175,7 +1172,7 @@ void rpcrdma_mr_put(struct rpcrdma_mr *mr) if (mr->mr_dir != DMA_NONE) { trace_xprtrdma_mr_unmap(mr); - ib_dma_unmap_sg(r_xprt->rx_ia.ri_id->device, + ib_dma_unmap_sg(r_xprt->rx_ep.re_id->device, mr->mr_sg, mr->mr_nents, mr->mr_dir); mr->mr_dir = DMA_NONE; } @@ -1293,7 +1290,7 @@ bool rpcrdma_regbuf_realloc(struct rpcrdma_regbuf *rb, size_t size, gfp_t flags) bool __rpcrdma_regbuf_dma_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_regbuf *rb) { - struct ib_device *device = r_xprt->rx_ia.ri_id->device; + struct ib_device *device = r_xprt->rx_ep.re_id->device; if (rb->rg_direction == DMA_NONE) return false; @@ -1306,7 +1303,7 @@ bool __rpcrdma_regbuf_dma_map(struct rpcrdma_xprt *r_xprt, } rb->rg_device = device; - rb->rg_iov.lkey = r_xprt->rx_ia.ri_pd->local_dma_lkey; + rb->rg_iov.lkey = r_xprt->rx_ep.re_pd->local_dma_lkey; return true; } @@ -1345,12 +1342,12 @@ int rpcrdma_post_sends(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) struct rpcrdma_ep *ep = &r_xprt->rx_ep; int rc; - if (!ep->rep_send_count || kref_read(&req->rl_kref) > 1) { + if (!ep->re_send_count || kref_read(&req->rl_kref) > 1) { send_wr->send_flags |= IB_SEND_SIGNALED; - ep->rep_send_count = ep->rep_send_batch; + ep->re_send_count = ep->re_send_batch; } else { send_wr->send_flags &= ~IB_SEND_SIGNALED; - --ep->rep_send_count; + --ep->re_send_count; } rc = frwr_send(r_xprt, req); @@ -1378,9 +1375,9 @@ void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp) count = 0; needed = buf->rb_credits + (buf->rb_bc_srv_max_requests << 1); - if (likely(ep->rep_receive_count > needed)) + if (likely(ep->re_receive_count > needed)) goto out; - needed -= ep->rep_receive_count; + needed -= ep->re_receive_count; if (!temp) needed += RPCRDMA_MAX_RECV_BATCH; @@ -1406,7 +1403,7 @@ void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp) if (!wr) goto out; - rc = ib_post_recv(r_xprt->rx_ia.ri_id->qp, wr, + rc = ib_post_recv(r_xprt->rx_ep.re_id->qp, wr, (const struct ib_recv_wr **)&bad_wr); out: trace_xprtrdma_post_recvs(r_xprt, count, rc); @@ -1420,6 +1417,6 @@ out: --count; } } - ep->rep_receive_count += count; + ep->re_receive_count += count; return; } diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h index 8a3ac9d7ee81..f3c0b826c9ed 100644 --- a/net/sunrpc/xprtrdma/xprt_rdma.h +++ b/net/sunrpc/xprtrdma/xprt_rdma.h @@ -65,38 +65,32 @@ #define RPCRDMA_IDLE_DISC_TO (5U * 60 * HZ) /* - * Interface Adapter -- one per transport instance + * RDMA Endpoint -- connection endpoint details */ -struct rpcrdma_ia { - struct rdma_cm_id *ri_id; - struct ib_pd *ri_pd; - int ri_async_rc; - unsigned int ri_max_rdma_segs; - unsigned int ri_max_frwr_depth; - bool ri_implicit_roundup; - enum ib_mr_type ri_mrtype; - struct completion ri_done; - struct completion ri_remove_done; -}; - -/* - * RDMA Endpoint -- one per transport instance - */ - struct rpcrdma_ep { - unsigned int rep_send_count; - unsigned int rep_send_batch; - unsigned int rep_max_inline_send; - unsigned int rep_max_inline_recv; - int rep_connected; - struct ib_qp_init_attr rep_attr; - wait_queue_head_t rep_connect_wait; - struct rpcrdma_connect_private rep_cm_private; - struct rdma_conn_param rep_remote_cma; - unsigned int rep_max_requests; /* depends on device */ - unsigned int rep_inline_send; /* negotiated */ - unsigned int rep_inline_recv; /* negotiated */ - int rep_receive_count; + struct rdma_cm_id *re_id; + struct ib_pd *re_pd; + unsigned int re_max_rdma_segs; + unsigned int re_max_fr_depth; + bool re_implicit_roundup; + enum ib_mr_type re_mrtype; + struct completion re_done; + struct completion re_remove_done; + unsigned int re_send_count; + unsigned int re_send_batch; + unsigned int re_max_inline_send; + unsigned int re_max_inline_recv; + int re_async_rc; + int re_connect_status; + struct ib_qp_init_attr re_attr; + wait_queue_head_t re_connect_wait; + struct rpcrdma_connect_private + re_cm_private; + struct rdma_conn_param re_remote_cma; + int re_receive_count; + unsigned int re_max_requests; /* depends on device */ + unsigned int re_inline_send; /* negotiated */ + unsigned int re_inline_recv; /* negotiated */ }; /* Pre-allocate extra Work Requests for handling backward receives @@ -417,7 +411,6 @@ struct rpcrdma_stats { */ struct rpcrdma_xprt { struct rpc_xprt rx_xprt; - struct rpcrdma_ia rx_ia; struct rpcrdma_ep rx_ep; struct rpcrdma_buffer rx_buf; struct delayed_work rx_connect_worker; @@ -522,8 +515,7 @@ rpcrdma_data_dir(bool writing) /* Memory registration calls xprtrdma/frwr_ops.c */ void frwr_reset(struct rpcrdma_req *req); -int frwr_query_device(struct rpcrdma_xprt *r_xprt, - const struct ib_device *device); +int frwr_query_device(struct rpcrdma_ep *ep, const struct ib_device *device); int frwr_mr_init(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr *mr); void frwr_release_mr(struct rpcrdma_mr *mr); struct rpcrdma_mr_seg *frwr_map(struct rpcrdma_xprt *r_xprt, @@ -555,7 +547,7 @@ int rpcrdma_prepare_send_sges(struct rpcrdma_xprt *r_xprt, enum rpcrdma_chunktype rtype); void rpcrdma_sendctx_unmap(struct rpcrdma_sendctx *sc); int rpcrdma_marshal_req(struct rpcrdma_xprt *r_xprt, struct rpc_rqst *rqst); -void rpcrdma_set_max_header_sizes(struct rpcrdma_xprt *); +void rpcrdma_set_max_header_sizes(struct rpcrdma_ep *ep); void rpcrdma_reset_cwnd(struct rpcrdma_xprt *r_xprt); void rpcrdma_complete_rqst(struct rpcrdma_rep *rep); void rpcrdma_reply_handler(struct rpcrdma_rep *rep); From 745b734c9bb80559b8ca64ae688901afefc1c3fd Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Fri, 21 Feb 2020 17:01:00 -0500 Subject: [PATCH 263/635] xprtrdma: Extract sockaddr from struct rdma_cm_id rpcrdma_cm_event_handler() is always passed an @id pointer that is valid. However, in a subsequent patch, we won't be able to extract an r_xprt in every case. So instead of using the r_xprt's presentation address strings, extract them from struct rdma_cm_id. Signed-off-by: Chuck Lever Signed-off-by: Anna Schumaker --- include/trace/events/rpcrdma.h | 78 +++++++++++++++++++++++----------- net/sunrpc/xprtrdma/verbs.c | 35 ++++++--------- 2 files changed, 67 insertions(+), 46 deletions(-) diff --git a/include/trace/events/rpcrdma.h b/include/trace/events/rpcrdma.h index 843269f0e291..295f75b9b796 100644 --- a/include/trace/events/rpcrdma.h +++ b/include/trace/events/rpcrdma.h @@ -375,47 +375,74 @@ TRACE_EVENT(xprtrdma_cm_event, TRACE_EVENT(xprtrdma_inline_thresh, TP_PROTO( - const struct rpcrdma_xprt *r_xprt + const struct rpcrdma_ep *ep ), - TP_ARGS(r_xprt), + TP_ARGS(ep), TP_STRUCT__entry( - __field(const void *, r_xprt) __field(unsigned int, inline_send) __field(unsigned int, inline_recv) __field(unsigned int, max_send) __field(unsigned int, max_recv) - __string(addr, rpcrdma_addrstr(r_xprt)) - __string(port, rpcrdma_portstr(r_xprt)) + __array(unsigned char, srcaddr, sizeof(struct sockaddr_in6)) + __array(unsigned char, dstaddr, sizeof(struct sockaddr_in6)) ), TP_fast_assign( - const struct rpcrdma_ep *ep = &r_xprt->rx_ep; + const struct rdma_cm_id *id = ep->re_id; - __entry->r_xprt = r_xprt; __entry->inline_send = ep->re_inline_send; __entry->inline_recv = ep->re_inline_recv; __entry->max_send = ep->re_max_inline_send; __entry->max_recv = ep->re_max_inline_recv; - __assign_str(addr, rpcrdma_addrstr(r_xprt)); - __assign_str(port, rpcrdma_portstr(r_xprt)); + memcpy(__entry->srcaddr, &id->route.addr.src_addr, + sizeof(struct sockaddr_in6)); + memcpy(__entry->dstaddr, &id->route.addr.dst_addr, + sizeof(struct sockaddr_in6)); ), - TP_printk("peer=[%s]:%s r_xprt=%p neg send/recv=%u/%u, calc send/recv=%u/%u", - __get_str(addr), __get_str(port), __entry->r_xprt, + TP_printk("%pISpc -> %pISpc neg send/recv=%u/%u, calc send/recv=%u/%u", + __entry->srcaddr, __entry->dstaddr, __entry->inline_send, __entry->inline_recv, __entry->max_send, __entry->max_recv ) ); +TRACE_EVENT(xprtrdma_remove, + TP_PROTO( + const struct rpcrdma_ep *ep + ), + + TP_ARGS(ep), + + TP_STRUCT__entry( + __array(unsigned char, srcaddr, sizeof(struct sockaddr_in6)) + __array(unsigned char, dstaddr, sizeof(struct sockaddr_in6)) + __string(name, ep->re_id->device->name) + ), + + TP_fast_assign( + const struct rdma_cm_id *id = ep->re_id; + + memcpy(__entry->srcaddr, &id->route.addr.src_addr, + sizeof(struct sockaddr_in6)); + memcpy(__entry->dstaddr, &id->route.addr.dst_addr, + sizeof(struct sockaddr_in6)); + __assign_str(name, id->device->name); + ), + + TP_printk("%pISpc -> %pISpc device=%s", + __entry->srcaddr, __entry->dstaddr, __get_str(name) + ) +); + DEFINE_CONN_EVENT(connect); DEFINE_CONN_EVENT(disconnect); DEFINE_CONN_EVENT(flush_dct); DEFINE_RXPRT_EVENT(xprtrdma_create); DEFINE_RXPRT_EVENT(xprtrdma_op_destroy); -DEFINE_RXPRT_EVENT(xprtrdma_remove); DEFINE_RXPRT_EVENT(xprtrdma_op_inject_dsc); DEFINE_RXPRT_EVENT(xprtrdma_op_close); DEFINE_RXPRT_EVENT(xprtrdma_op_setport); @@ -482,32 +509,33 @@ TRACE_EVENT(xprtrdma_op_set_cto, TRACE_EVENT(xprtrdma_qp_event, TP_PROTO( - const struct rpcrdma_xprt *r_xprt, + const struct rpcrdma_ep *ep, const struct ib_event *event ), - TP_ARGS(r_xprt, event), + TP_ARGS(ep, event), TP_STRUCT__entry( - __field(const void *, r_xprt) - __field(unsigned int, event) + __field(unsigned long, event) __string(name, event->device->name) - __string(addr, rpcrdma_addrstr(r_xprt)) - __string(port, rpcrdma_portstr(r_xprt)) + __array(unsigned char, srcaddr, sizeof(struct sockaddr_in6)) + __array(unsigned char, dstaddr, sizeof(struct sockaddr_in6)) ), TP_fast_assign( - __entry->r_xprt = r_xprt; + const struct rdma_cm_id *id = ep->re_id; + __entry->event = event->event; __assign_str(name, event->device->name); - __assign_str(addr, rpcrdma_addrstr(r_xprt)); - __assign_str(port, rpcrdma_portstr(r_xprt)); + memcpy(__entry->srcaddr, &id->route.addr.src_addr, + sizeof(struct sockaddr_in6)); + memcpy(__entry->dstaddr, &id->route.addr.dst_addr, + sizeof(struct sockaddr_in6)); ), - TP_printk("peer=[%s]:%s r_xprt=%p: dev %s: %s (%u)", - __get_str(addr), __get_str(port), __entry->r_xprt, - __get_str(name), rdma_show_ib_event(__entry->event), - __entry->event + TP_printk("%pISpc -> %pISpc device=%s %s (%lu)", + __entry->srcaddr, __entry->dstaddr, __get_str(name), + rdma_show_ib_event(__entry->event), __entry->event ) ); diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c index 10826982ddf8..37d07072bdbf 100644 --- a/net/sunrpc/xprtrdma/verbs.c +++ b/net/sunrpc/xprtrdma/verbs.c @@ -116,16 +116,14 @@ static void rpcrdma_xprt_drain(struct rpcrdma_xprt *r_xprt) * @context: ep that owns QP where event occurred * * Called from the RDMA provider (device driver) possibly in an interrupt - * context. + * context. The QP is always destroyed before the ID, so the ID will be + * reliably available when this handler is invoked. */ -static void -rpcrdma_qp_event_handler(struct ib_event *event, void *context) +static void rpcrdma_qp_event_handler(struct ib_event *event, void *context) { struct rpcrdma_ep *ep = context; - struct rpcrdma_xprt *r_xprt = container_of(ep, struct rpcrdma_xprt, - rx_ep); - trace_xprtrdma_qp_event(r_xprt, event); + trace_xprtrdma_qp_event(ep, event); } /** @@ -202,11 +200,10 @@ out_flushed: rpcrdma_rep_destroy(rep); } -static void rpcrdma_update_cm_private(struct rpcrdma_xprt *r_xprt, +static void rpcrdma_update_cm_private(struct rpcrdma_ep *ep, struct rdma_conn_param *param) { const struct rpcrdma_connect_private *pmsg = param->private_data; - struct rpcrdma_ep *ep = &r_xprt->rx_ep; unsigned int rsize, wsize; /* Default settings for RPC-over-RDMA Version One */ @@ -241,6 +238,7 @@ static void rpcrdma_update_cm_private(struct rpcrdma_xprt *r_xprt, static int rpcrdma_cm_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event) { + struct sockaddr *sap = (struct sockaddr *)&id->route.addr.dst_addr; struct rpcrdma_xprt *r_xprt = id->context; struct rpcrdma_ep *ep = &r_xprt->rx_ep; struct rpc_xprt *xprt = &r_xprt->rx_xprt; @@ -263,24 +261,21 @@ rpcrdma_cm_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event) complete(&ep->re_done); return 0; case RDMA_CM_EVENT_DEVICE_REMOVAL: -#if IS_ENABLED(CONFIG_SUNRPC_DEBUG) - pr_info("rpcrdma: removing device %s for %s:%s\n", - ep->re_id->device->name, - rpcrdma_addrstr(r_xprt), rpcrdma_portstr(r_xprt)); -#endif + pr_info("rpcrdma: removing device %s for %pISpc\n", + ep->re_id->device->name, sap); init_completion(&ep->re_remove_done); ep->re_connect_status = -ENODEV; xprt_force_disconnect(xprt); wait_for_completion(&ep->re_remove_done); - trace_xprtrdma_remove(r_xprt); + trace_xprtrdma_remove(ep); /* Return 1 to ensure the core destroys the id. */ return 1; case RDMA_CM_EVENT_ESTABLISHED: ++xprt->connect_cookie; ep->re_connect_status = 1; - rpcrdma_update_cm_private(r_xprt, &event->param.conn); - trace_xprtrdma_inline_thresh(r_xprt); + rpcrdma_update_cm_private(ep, &event->param.conn); + trace_xprtrdma_inline_thresh(ep); wake_up_all(&ep->re_connect_wait); break; case RDMA_CM_EVENT_CONNECT_ERROR: @@ -290,9 +285,8 @@ rpcrdma_cm_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event) ep->re_connect_status = -ENETUNREACH; goto disconnected; case RDMA_CM_EVENT_REJECTED: - dprintk("rpcrdma: connection to %s:%s rejected: %s\n", - rpcrdma_addrstr(r_xprt), rpcrdma_portstr(r_xprt), - rdma_reject_msg(id, event->status)); + dprintk("rpcrdma: connection to %pISpc rejected: %s\n", + sap, rdma_reject_msg(id, event->status)); ep->re_connect_status = -ECONNREFUSED; if (event->status == IB_CM_REJ_STALE_CONN) ep->re_connect_status = -EAGAIN; @@ -307,8 +301,7 @@ disconnected: break; } - dprintk("RPC: %s: %s:%s on %s/frwr: %s\n", __func__, - rpcrdma_addrstr(r_xprt), rpcrdma_portstr(r_xprt), + dprintk("RPC: %s: %pISpc on %s/frwr: %s\n", __func__, sap, ep->re_id->device->name, rdma_event_msg(event->event)); return 0; } From e28ce90083f032ca0e8ea03478f5b6a38f5930f7 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Fri, 21 Feb 2020 17:01:05 -0500 Subject: [PATCH 264/635] xprtrdma: kmalloc rpcrdma_ep separate from rpcrdma_xprt Change the rpcrdma_xprt_disconnect() function so that it no longer waits for the DISCONNECTED event. This prevents blocking if the remote is unresponsive. In rpcrdma_xprt_disconnect(), the transport's rpcrdma_ep is detached. Upon return from rpcrdma_xprt_disconnect(), the transport (r_xprt) is ready immediately for a new connection. The RDMA_CM_DEVICE_REMOVAL and RDMA_CM_DISCONNECTED events are now handled almost identically. However, because the lifetimes of rpcrdma_xprt structures and rpcrdma_ep structures are now independent, creating an rpcrdma_ep needs to take a module ref count. The ep now owns most of the hardware resources for a transport. Also, a kref is needed to ensure that rpcrdma_ep sticks around long enough for the cm_event_handler to finish. Signed-off-by: Chuck Lever Signed-off-by: Anna Schumaker --- include/trace/events/rpcrdma.h | 63 +--------- net/sunrpc/xprtrdma/backchannel.c | 4 +- net/sunrpc/xprtrdma/frwr_ops.c | 12 +- net/sunrpc/xprtrdma/rpc_rdma.c | 17 +-- net/sunrpc/xprtrdma/transport.c | 37 +++--- net/sunrpc/xprtrdma/verbs.c | 194 ++++++++++++++++-------------- net/sunrpc/xprtrdma/xprt_rdma.h | 7 +- 7 files changed, 143 insertions(+), 191 deletions(-) diff --git a/include/trace/events/rpcrdma.h b/include/trace/events/rpcrdma.h index 295f75b9b796..81b87428f166 100644 --- a/include/trace/events/rpcrdma.h +++ b/include/trace/events/rpcrdma.h @@ -104,7 +104,7 @@ DECLARE_EVENT_CLASS(xprtrdma_connect_class, TP_fast_assign( __entry->r_xprt = r_xprt; __entry->rc = rc; - __entry->connect_status = r_xprt->rx_ep.re_connect_status; + __entry->connect_status = r_xprt->rx_ep->re_connect_status; __assign_str(addr, rpcrdma_addrstr(r_xprt)); __assign_str(port, rpcrdma_portstr(r_xprt)); ), @@ -342,37 +342,6 @@ DECLARE_EVENT_CLASS(xprtrdma_cb_event, ** Connection events **/ -TRACE_EVENT(xprtrdma_cm_event, - TP_PROTO( - const struct rpcrdma_xprt *r_xprt, - struct rdma_cm_event *event - ), - - TP_ARGS(r_xprt, event), - - TP_STRUCT__entry( - __field(const void *, r_xprt) - __field(unsigned int, event) - __field(int, status) - __string(addr, rpcrdma_addrstr(r_xprt)) - __string(port, rpcrdma_portstr(r_xprt)) - ), - - TP_fast_assign( - __entry->r_xprt = r_xprt; - __entry->event = event->event; - __entry->status = event->status; - __assign_str(addr, rpcrdma_addrstr(r_xprt)); - __assign_str(port, rpcrdma_portstr(r_xprt)); - ), - - TP_printk("peer=[%s]:%s r_xprt=%p: %s (%u/%d)", - __get_str(addr), __get_str(port), - __entry->r_xprt, rdma_show_cm_event(__entry->event), - __entry->event, __entry->status - ) -); - TRACE_EVENT(xprtrdma_inline_thresh, TP_PROTO( const struct rpcrdma_ep *ep @@ -409,34 +378,6 @@ TRACE_EVENT(xprtrdma_inline_thresh, ) ); -TRACE_EVENT(xprtrdma_remove, - TP_PROTO( - const struct rpcrdma_ep *ep - ), - - TP_ARGS(ep), - - TP_STRUCT__entry( - __array(unsigned char, srcaddr, sizeof(struct sockaddr_in6)) - __array(unsigned char, dstaddr, sizeof(struct sockaddr_in6)) - __string(name, ep->re_id->device->name) - ), - - TP_fast_assign( - const struct rdma_cm_id *id = ep->re_id; - - memcpy(__entry->srcaddr, &id->route.addr.src_addr, - sizeof(struct sockaddr_in6)); - memcpy(__entry->dstaddr, &id->route.addr.dst_addr, - sizeof(struct sockaddr_in6)); - __assign_str(name, id->device->name); - ), - - TP_printk("%pISpc -> %pISpc device=%s", - __entry->srcaddr, __entry->dstaddr, __get_str(name) - ) -); - DEFINE_CONN_EVENT(connect); DEFINE_CONN_EVENT(disconnect); DEFINE_CONN_EVENT(flush_dct); @@ -831,7 +772,7 @@ TRACE_EVENT(xprtrdma_post_recvs, __entry->r_xprt = r_xprt; __entry->count = count; __entry->status = status; - __entry->posted = r_xprt->rx_ep.re_receive_count; + __entry->posted = r_xprt->rx_ep->re_receive_count; __assign_str(addr, rpcrdma_addrstr(r_xprt)); __assign_str(port, rpcrdma_portstr(r_xprt)); ), diff --git a/net/sunrpc/xprtrdma/backchannel.c b/net/sunrpc/xprtrdma/backchannel.c index 4b20102cf060..c92c1aac270a 100644 --- a/net/sunrpc/xprtrdma/backchannel.c +++ b/net/sunrpc/xprtrdma/backchannel.c @@ -44,7 +44,7 @@ int xprt_rdma_bc_setup(struct rpc_xprt *xprt, unsigned int reqs) size_t xprt_rdma_bc_maxpayload(struct rpc_xprt *xprt) { struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt); - struct rpcrdma_ep *ep = &r_xprt->rx_ep; + struct rpcrdma_ep *ep = r_xprt->rx_ep; size_t maxmsg; maxmsg = min_t(unsigned int, ep->re_inline_send, ep->re_inline_recv); @@ -190,7 +190,7 @@ create_req: if (xprt->bc_alloc_count >= RPCRDMA_BACKWARD_WRS) return NULL; - size = min_t(size_t, r_xprt->rx_ep.re_inline_recv, PAGE_SIZE); + size = min_t(size_t, r_xprt->rx_ep->re_inline_recv, PAGE_SIZE); req = rpcrdma_req_create(r_xprt, size, GFP_KERNEL); if (!req) return NULL; diff --git a/net/sunrpc/xprtrdma/frwr_ops.c b/net/sunrpc/xprtrdma/frwr_ops.c index 19bf422f010b..ef997880e17a 100644 --- a/net/sunrpc/xprtrdma/frwr_ops.c +++ b/net/sunrpc/xprtrdma/frwr_ops.c @@ -74,7 +74,7 @@ static void frwr_mr_recycle(struct rpcrdma_mr *mr) if (mr->mr_dir != DMA_NONE) { trace_xprtrdma_mr_unmap(mr); - ib_dma_unmap_sg(r_xprt->rx_ep.re_id->device, + ib_dma_unmap_sg(r_xprt->rx_ep->re_id->device, mr->mr_sg, mr->mr_nents, mr->mr_dir); mr->mr_dir = DMA_NONE; } @@ -115,7 +115,7 @@ void frwr_reset(struct rpcrdma_req *req) */ int frwr_mr_init(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr *mr) { - struct rpcrdma_ep *ep = &r_xprt->rx_ep; + struct rpcrdma_ep *ep = r_xprt->rx_ep; unsigned int depth = ep->re_max_fr_depth; struct scatterlist *sg; struct ib_mr *frmr; @@ -283,7 +283,7 @@ struct rpcrdma_mr_seg *frwr_map(struct rpcrdma_xprt *r_xprt, int nsegs, bool writing, __be32 xid, struct rpcrdma_mr *mr) { - struct rpcrdma_ep *ep = &r_xprt->rx_ep; + struct rpcrdma_ep *ep = r_xprt->rx_ep; struct ib_reg_wr *reg_wr; int i, n, dma_nents; struct ib_mr *ibmr; @@ -405,7 +405,7 @@ int frwr_send(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) post_wr = &frwr->fr_regwr.wr; } - return ib_post_send(r_xprt->rx_ep.re_id->qp, post_wr, NULL); + return ib_post_send(r_xprt->rx_ep->re_id->qp, post_wr, NULL); } /** @@ -535,7 +535,7 @@ void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) * unless re_id->qp is a valid pointer. */ bad_wr = NULL; - rc = ib_post_send(r_xprt->rx_ep.re_id->qp, first, &bad_wr); + rc = ib_post_send(r_xprt->rx_ep->re_id->qp, first, &bad_wr); /* The final LOCAL_INV WR in the chain is supposed to * do the wake. If it was never posted, the wake will @@ -640,7 +640,7 @@ void frwr_unmap_async(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) * unless re_id->qp is a valid pointer. */ bad_wr = NULL; - rc = ib_post_send(r_xprt->rx_ep.re_id->qp, first, &bad_wr); + rc = ib_post_send(r_xprt->rx_ep->re_id->qp, first, &bad_wr); if (!rc) return; diff --git a/net/sunrpc/xprtrdma/rpc_rdma.c b/net/sunrpc/xprtrdma/rpc_rdma.c index ad7e6b0187bd..d1af48e0139c 100644 --- a/net/sunrpc/xprtrdma/rpc_rdma.c +++ b/net/sunrpc/xprtrdma/rpc_rdma.c @@ -131,9 +131,10 @@ static bool rpcrdma_args_inline(struct rpcrdma_xprt *r_xprt, struct rpc_rqst *rqst) { struct xdr_buf *xdr = &rqst->rq_snd_buf; + struct rpcrdma_ep *ep = r_xprt->rx_ep; unsigned int count, remaining, offset; - if (xdr->len > r_xprt->rx_ep.re_max_inline_send) + if (xdr->len > ep->re_max_inline_send) return false; if (xdr->page_len) { @@ -144,7 +145,7 @@ static bool rpcrdma_args_inline(struct rpcrdma_xprt *r_xprt, remaining -= min_t(unsigned int, PAGE_SIZE - offset, remaining); offset = 0; - if (++count > r_xprt->rx_ep.re_attr.cap.max_send_sge) + if (++count > ep->re_attr.cap.max_send_sge) return false; } } @@ -161,7 +162,7 @@ static bool rpcrdma_args_inline(struct rpcrdma_xprt *r_xprt, static bool rpcrdma_results_inline(struct rpcrdma_xprt *r_xprt, struct rpc_rqst *rqst) { - return rqst->rq_rcv_buf.buflen <= r_xprt->rx_ep.re_max_inline_recv; + return rqst->rq_rcv_buf.buflen <= r_xprt->rx_ep->re_max_inline_recv; } /* The client is required to provide a Reply chunk if the maximum @@ -175,7 +176,7 @@ rpcrdma_nonpayload_inline(const struct rpcrdma_xprt *r_xprt, const struct xdr_buf *buf = &rqst->rq_rcv_buf; return (buf->head[0].iov_len + buf->tail[0].iov_len) < - r_xprt->rx_ep.re_max_inline_recv; + r_xprt->rx_ep->re_max_inline_recv; } /* Split @vec on page boundaries into SGEs. FMR registers pages, not @@ -254,7 +255,7 @@ rpcrdma_convert_iovs(struct rpcrdma_xprt *r_xprt, struct xdr_buf *xdrbuf, /* When encoding a Read chunk, the tail iovec contains an * XDR pad and may be omitted. */ - if (type == rpcrdma_readch && r_xprt->rx_ep.re_implicit_roundup) + if (type == rpcrdma_readch && r_xprt->rx_ep->re_implicit_roundup) goto out; /* When encoding a Write chunk, some servers need to see an @@ -262,7 +263,7 @@ rpcrdma_convert_iovs(struct rpcrdma_xprt *r_xprt, struct xdr_buf *xdrbuf, * layer provides space in the tail iovec that may be used * for this purpose. */ - if (type == rpcrdma_writech && r_xprt->rx_ep.re_implicit_roundup) + if (type == rpcrdma_writech && r_xprt->rx_ep->re_implicit_roundup) goto out; if (xdrbuf->tail[0].iov_len) @@ -1475,8 +1476,8 @@ void rpcrdma_reply_handler(struct rpcrdma_rep *rep) if (credits == 0) credits = 1; /* don't deadlock */ - else if (credits > r_xprt->rx_ep.re_max_requests) - credits = r_xprt->rx_ep.re_max_requests; + else if (credits > r_xprt->rx_ep->re_max_requests) + credits = r_xprt->rx_ep->re_max_requests; if (buf->rb_credits != credits) rpcrdma_update_cwnd(r_xprt, credits); rpcrdma_post_recvs(r_xprt, false); diff --git a/net/sunrpc/xprtrdma/transport.c b/net/sunrpc/xprtrdma/transport.c index 4352fd6e9817..659da37020a4 100644 --- a/net/sunrpc/xprtrdma/transport.c +++ b/net/sunrpc/xprtrdma/transport.c @@ -238,12 +238,12 @@ xprt_rdma_connect_worker(struct work_struct *work) struct rpcrdma_xprt *r_xprt = container_of(work, struct rpcrdma_xprt, rx_connect_worker.work); struct rpc_xprt *xprt = &r_xprt->rx_xprt; - struct rpcrdma_ep *ep = &r_xprt->rx_ep; int rc; rc = rpcrdma_xprt_connect(r_xprt); xprt_clear_connecting(xprt); - if (ep->re_connect_status > 0) { + if (r_xprt->rx_ep && r_xprt->rx_ep->re_connect_status > 0) { + xprt->connect_cookie++; xprt->stat.connect_count++; xprt->stat.connect_time += (long)jiffies - xprt->stat.connect_start; @@ -266,7 +266,7 @@ xprt_rdma_inject_disconnect(struct rpc_xprt *xprt) struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt); trace_xprtrdma_op_inject_dsc(r_xprt); - rdma_disconnect(r_xprt->rx_ep.re_id); + rdma_disconnect(r_xprt->rx_ep->re_id); } /** @@ -316,10 +316,15 @@ xprt_setup_rdma(struct xprt_create *args) if (args->addrlen > sizeof(xprt->addr)) return ERR_PTR(-EBADF); + if (!try_module_get(THIS_MODULE)) + return ERR_PTR(-EIO); + xprt = xprt_alloc(args->net, sizeof(struct rpcrdma_xprt), 0, xprt_rdma_slot_table_entries); - if (!xprt) + if (!xprt) { + module_put(THIS_MODULE); return ERR_PTR(-ENOMEM); + } xprt->timeout = &xprt_rdma_default_timeout; xprt->connect_timeout = xprt->timeout->to_initval; @@ -348,11 +353,12 @@ xprt_setup_rdma(struct xprt_create *args) new_xprt = rpcx_to_rdmax(xprt); rc = rpcrdma_buffer_create(new_xprt); - if (rc) - goto out2; - - if (!try_module_get(THIS_MODULE)) - goto out4; + if (rc) { + xprt_rdma_free_addresses(xprt); + xprt_free(xprt); + module_put(THIS_MODULE); + return ERR_PTR(rc); + } INIT_DELAYED_WORK(&new_xprt->rx_connect_worker, xprt_rdma_connect_worker); @@ -364,15 +370,6 @@ xprt_setup_rdma(struct xprt_create *args) xprt->address_strings[RPC_DISPLAY_PORT]); trace_xprtrdma_create(new_xprt); return xprt; - -out4: - rpcrdma_buffer_destroy(&new_xprt->rx_buf); - rc = -ENODEV; -out2: - trace_xprtrdma_op_destroy(new_xprt); - xprt_rdma_free_addresses(xprt); - xprt_free(xprt); - return ERR_PTR(rc); } /** @@ -491,11 +488,11 @@ static void xprt_rdma_connect(struct rpc_xprt *xprt, struct rpc_task *task) { struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt); - struct rpcrdma_ep *ep = &r_xprt->rx_ep; + struct rpcrdma_ep *ep = r_xprt->rx_ep; unsigned long delay; delay = 0; - if (ep->re_connect_status != 0) { + if (ep && ep->re_connect_status != 0) { delay = xprt_reconnect_delay(xprt); xprt_reconnect_backoff(xprt, RPCRDMA_INIT_REEST_TO); } diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c index 37d07072bdbf..cdd84c09df10 100644 --- a/net/sunrpc/xprtrdma/verbs.c +++ b/net/sunrpc/xprtrdma/verbs.c @@ -84,7 +84,7 @@ static void rpcrdma_rep_destroy(struct rpcrdma_rep *rep); static void rpcrdma_reps_unmap(struct rpcrdma_xprt *r_xprt); static void rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt); static void rpcrdma_mrs_destroy(struct rpcrdma_xprt *r_xprt); -static void rpcrdma_ep_destroy(struct rpcrdma_xprt *r_xprt); +static int rpcrdma_ep_destroy(struct rpcrdma_ep *ep); static struct rpcrdma_regbuf * rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction, gfp_t flags); @@ -97,7 +97,7 @@ static void rpcrdma_regbuf_free(struct rpcrdma_regbuf *rb); */ static void rpcrdma_xprt_drain(struct rpcrdma_xprt *r_xprt) { - struct rdma_cm_id *id = r_xprt->rx_ep.re_id; + struct rdma_cm_id *id = r_xprt->rx_ep->re_id; /* Flush Receives, then wait for deferred Reply work * to complete. @@ -139,8 +139,8 @@ void rpcrdma_flush_disconnect(struct ib_cq *cq, struct ib_wc *wc) struct rpc_xprt *xprt = &r_xprt->rx_xprt; if (wc->status != IB_WC_SUCCESS && - r_xprt->rx_ep.re_connect_status == 1) { - r_xprt->rx_ep.re_connect_status = -ECONNABORTED; + r_xprt->rx_ep->re_connect_status == 1) { + r_xprt->rx_ep->re_connect_status = -ECONNABORTED; trace_xprtrdma_flush_dct(r_xprt, wc->status); xprt_force_disconnect(xprt); } @@ -179,7 +179,7 @@ static void rpcrdma_wc_receive(struct ib_cq *cq, struct ib_wc *wc) /* WARNING: Only wr_cqe and status are reliable at this point */ trace_xprtrdma_wc_receive(wc); - --r_xprt->rx_ep.re_receive_count; + --r_xprt->rx_ep->re_receive_count; if (wc->status != IB_WC_SUCCESS) goto out_flushed; @@ -239,13 +239,11 @@ static int rpcrdma_cm_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event) { struct sockaddr *sap = (struct sockaddr *)&id->route.addr.dst_addr; - struct rpcrdma_xprt *r_xprt = id->context; - struct rpcrdma_ep *ep = &r_xprt->rx_ep; - struct rpc_xprt *xprt = &r_xprt->rx_xprt; + struct rpcrdma_ep *ep = id->context; + struct rpc_xprt *xprt = ep->re_xprt; might_sleep(); - trace_xprtrdma_cm_event(r_xprt, event); switch (event->event) { case RDMA_CM_EVENT_ADDR_RESOLVED: case RDMA_CM_EVENT_ROUTE_RESOLVED: @@ -263,16 +261,13 @@ rpcrdma_cm_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event) case RDMA_CM_EVENT_DEVICE_REMOVAL: pr_info("rpcrdma: removing device %s for %pISpc\n", ep->re_id->device->name, sap); - init_completion(&ep->re_remove_done); + /* fall through */ + case RDMA_CM_EVENT_ADDR_CHANGE: ep->re_connect_status = -ENODEV; xprt_force_disconnect(xprt); - wait_for_completion(&ep->re_remove_done); - trace_xprtrdma_remove(ep); - - /* Return 1 to ensure the core destroys the id. */ - return 1; + goto disconnected; case RDMA_CM_EVENT_ESTABLISHED: - ++xprt->connect_cookie; + kref_get(&ep->re_kref); ep->re_connect_status = 1; rpcrdma_update_cm_private(ep, &event->param.conn); trace_xprtrdma_inline_thresh(ep); @@ -294,9 +289,7 @@ rpcrdma_cm_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event) case RDMA_CM_EVENT_DISCONNECTED: ep->re_connect_status = -ECONNABORTED; disconnected: - xprt_force_disconnect(xprt); - wake_up_all(&ep->re_connect_wait); - break; + return rpcrdma_ep_destroy(ep); default: break; } @@ -316,7 +309,7 @@ static struct rdma_cm_id *rpcrdma_create_id(struct rpcrdma_xprt *r_xprt, init_completion(&ep->re_done); - id = rdma_create_id(xprt->xprt_net, rpcrdma_cm_event_handler, r_xprt, + id = rdma_create_id(xprt->xprt_net, rpcrdma_cm_event_handler, ep, RDMA_PS_TCP, IB_QPT_RC); if (IS_ERR(id)) return id; @@ -352,25 +345,66 @@ out: return ERR_PTR(rc); } -/* - * Exported functions. +static void rpcrdma_ep_put(struct kref *kref) +{ + struct rpcrdma_ep *ep = container_of(kref, struct rpcrdma_ep, re_kref); + + if (ep->re_id->qp) { + rdma_destroy_qp(ep->re_id); + ep->re_id->qp = NULL; + } + + if (ep->re_attr.recv_cq) + ib_free_cq(ep->re_attr.recv_cq); + ep->re_attr.recv_cq = NULL; + if (ep->re_attr.send_cq) + ib_free_cq(ep->re_attr.send_cq); + ep->re_attr.send_cq = NULL; + + if (ep->re_pd) + ib_dealloc_pd(ep->re_pd); + ep->re_pd = NULL; + + kfree(ep); + module_put(THIS_MODULE); +} + +/* Returns: + * %0 if @ep still has a positive kref count, or + * %1 if @ep was destroyed successfully. */ +static int rpcrdma_ep_destroy(struct rpcrdma_ep *ep) +{ + return kref_put(&ep->re_kref, rpcrdma_ep_put); +} static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt) { - struct rpcrdma_ep *ep = &r_xprt->rx_ep; - struct rpcrdma_connect_private *pmsg = &ep->re_cm_private; + struct rpcrdma_connect_private *pmsg; + struct ib_device *device; struct rdma_cm_id *id; + struct rpcrdma_ep *ep; int rc; + ep = kzalloc(sizeof(*ep), GFP_NOFS); + if (!ep) + return -EAGAIN; + ep->re_xprt = &r_xprt->rx_xprt; + kref_init(&ep->re_kref); + id = rpcrdma_create_id(r_xprt, ep); - if (IS_ERR(id)) - return PTR_ERR(id); + if (IS_ERR(id)) { + rc = PTR_ERR(id); + goto out_free; + } + __module_get(THIS_MODULE); + device = id->device; + ep->re_id = id; ep->re_max_requests = r_xprt->rx_xprt.max_reqs; ep->re_inline_send = xprt_rdma_max_inline_write; ep->re_inline_recv = xprt_rdma_max_inline_read; - rc = frwr_query_device(ep, id->device); + rc = frwr_query_device(ep, device); if (rc) goto out_destroy; @@ -396,7 +430,7 @@ static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt) ep->re_send_count = ep->re_send_batch; init_waitqueue_head(&ep->re_connect_wait); - ep->re_attr.send_cq = ib_alloc_cq_any(id->device, r_xprt, + ep->re_attr.send_cq = ib_alloc_cq_any(device, r_xprt, ep->re_attr.cap.max_send_wr, IB_POLL_WORKQUEUE); if (IS_ERR(ep->re_attr.send_cq)) { @@ -404,7 +438,7 @@ static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt) goto out_destroy; } - ep->re_attr.recv_cq = ib_alloc_cq_any(id->device, r_xprt, + ep->re_attr.recv_cq = ib_alloc_cq_any(device, r_xprt, ep->re_attr.cap.max_recv_wr, IB_POLL_WORKQUEUE); if (IS_ERR(ep->re_attr.recv_cq)) { @@ -417,6 +451,7 @@ static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt) memset(&ep->re_remote_cma, 0, sizeof(ep->re_remote_cma)); /* Prepare RDMA-CM private message */ + pmsg = &ep->re_cm_private; pmsg->cp_magic = rpcrdma_cmp_magic; pmsg->cp_version = RPCRDMA_CMP_VERSION; pmsg->cp_flags |= RPCRDMA_CMP_F_SND_W_INV_OK; @@ -428,7 +463,7 @@ static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt) /* Client offers RDMA Read but does not initiate */ ep->re_remote_cma.initiator_depth = 0; ep->re_remote_cma.responder_resources = - min_t(int, U8_MAX, id->device->attrs.max_qp_rd_atom); + min_t(int, U8_MAX, device->attrs.max_qp_rd_atom); /* Limit transport retries so client can detect server * GID changes quickly. RPC layer handles re-establishing @@ -443,7 +478,7 @@ static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt) ep->re_remote_cma.flow_control = 0; ep->re_remote_cma.rnr_retry_count = 0; - ep->re_pd = ib_alloc_pd(id->device, 0); + ep->re_pd = ib_alloc_pd(device, 0); if (IS_ERR(ep->re_pd)) { rc = PTR_ERR(ep->re_pd); goto out_destroy; @@ -453,50 +488,36 @@ static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt) if (rc) goto out_destroy; - ep->re_id = id; + r_xprt->rx_ep = ep; return 0; out_destroy: - rpcrdma_ep_destroy(r_xprt); + rpcrdma_ep_destroy(ep); rdma_destroy_id(id); +out_free: + kfree(ep); + r_xprt->rx_ep = NULL; return rc; } -static void rpcrdma_ep_destroy(struct rpcrdma_xprt *r_xprt) -{ - struct rpcrdma_ep *ep = &r_xprt->rx_ep; - - if (ep->re_id && ep->re_id->qp) { - rdma_destroy_qp(ep->re_id); - ep->re_id->qp = NULL; - } - - if (ep->re_attr.recv_cq) - ib_free_cq(ep->re_attr.recv_cq); - ep->re_attr.recv_cq = NULL; - if (ep->re_attr.send_cq) - ib_free_cq(ep->re_attr.send_cq); - ep->re_attr.send_cq = NULL; - - if (ep->re_pd) - ib_dealloc_pd(ep->re_pd); - ep->re_pd = NULL; -} - -/* - * Connect unconnected endpoint. +/** + * rpcrdma_xprt_connect - Connect an unconnected transport + * @r_xprt: controlling transport instance + * + * Returns 0 on success or a negative errno. */ int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt) { struct rpc_xprt *xprt = &r_xprt->rx_xprt; - struct rpcrdma_ep *ep = &r_xprt->rx_ep; + struct rpcrdma_ep *ep; int rc; retry: rpcrdma_xprt_disconnect(r_xprt); rc = rpcrdma_ep_create(r_xprt); if (rc) - goto out_noupdate; + return rc; + ep = r_xprt->rx_ep; ep->re_connect_status = 0; xprt_clear_connected(xprt); @@ -533,8 +554,6 @@ retry: out: if (rc) ep->re_connect_status = rc; - -out_noupdate: trace_xprtrdma_connect(r_xprt, rc); return rc; } @@ -545,40 +564,33 @@ out_noupdate: * * Caller serializes. Either the transport send lock is held, * or we're being called to destroy the transport. + * + * On return, @r_xprt is completely divested of all hardware + * resources and prepared for the next ->connect operation. */ void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt) { - struct rpcrdma_ep *ep = &r_xprt->rx_ep; - struct rdma_cm_id *id = ep->re_id; - int rc, status = ep->re_connect_status; + struct rpcrdma_ep *ep = r_xprt->rx_ep; + struct rdma_cm_id *id; + int rc; - might_sleep(); - - if (!id) + if (!ep) return; + id = ep->re_id; rc = rdma_disconnect(id); - if (!rc) - wait_event_interruptible(ep->re_connect_wait, - ep->re_connect_status != 1); - else - ep->re_connect_status = rc; trace_xprtrdma_disconnect(r_xprt, rc); - if (id->qp) - rpcrdma_xprt_drain(r_xprt); + rpcrdma_xprt_drain(r_xprt); rpcrdma_reps_unmap(r_xprt); rpcrdma_reqs_reset(r_xprt); rpcrdma_mrs_destroy(r_xprt); rpcrdma_sendctxs_destroy(r_xprt); - rpcrdma_ep_destroy(r_xprt); - - if (status == -ENODEV) - complete(&ep->re_remove_done); - else + if (rpcrdma_ep_destroy(ep)) rdma_destroy_id(id); - ep->re_id = NULL; + + r_xprt->rx_ep = NULL; } /* Fixed-size circular FIFO queue. This implementation is wait-free and @@ -635,14 +647,14 @@ static int rpcrdma_sendctxs_create(struct rpcrdma_xprt *r_xprt) * the ->send_request call to fail temporarily before too many * Sends are posted. */ - i = r_xprt->rx_ep.re_max_requests + RPCRDMA_MAX_BC_REQUESTS; + i = r_xprt->rx_ep->re_max_requests + RPCRDMA_MAX_BC_REQUESTS; buf->rb_sc_ctxs = kcalloc(i, sizeof(sc), GFP_KERNEL); if (!buf->rb_sc_ctxs) return -ENOMEM; buf->rb_sc_last = i - 1; for (i = 0; i <= buf->rb_sc_last; i++) { - sc = rpcrdma_sendctx_create(&r_xprt->rx_ep); + sc = rpcrdma_sendctx_create(r_xprt->rx_ep); if (!sc) return -ENOMEM; @@ -746,7 +758,7 @@ static void rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt) { struct rpcrdma_buffer *buf = &r_xprt->rx_buf; - struct rpcrdma_ep *ep = &r_xprt->rx_ep; + struct rpcrdma_ep *ep = r_xprt->rx_ep; unsigned int count; for (count = 0; count < ep->re_max_rdma_segs; count++) { @@ -793,7 +805,7 @@ rpcrdma_mr_refresh_worker(struct work_struct *work) void rpcrdma_mrs_refresh(struct rpcrdma_xprt *r_xprt) { struct rpcrdma_buffer *buf = &r_xprt->rx_buf; - struct rpcrdma_ep *ep = &r_xprt->rx_ep; + struct rpcrdma_ep *ep = r_xprt->rx_ep; /* If there is no underlying connection, it's no use * to wake the refresh worker. @@ -862,7 +874,7 @@ int rpcrdma_req_setup(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) /* Compute maximum header buffer size in bytes */ maxhdrsize = rpcrdma_fixed_maxsz + 3 + - r_xprt->rx_ep.re_max_rdma_segs * rpcrdma_readchunk_maxsz; + r_xprt->rx_ep->re_max_rdma_segs * rpcrdma_readchunk_maxsz; maxhdrsize *= sizeof(__be32); rb = rpcrdma_regbuf_alloc(__roundup_pow_of_two(maxhdrsize), DMA_TO_DEVICE, GFP_KERNEL); @@ -940,7 +952,7 @@ struct rpcrdma_rep *rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt, if (rep == NULL) goto out; - rep->rr_rdmabuf = rpcrdma_regbuf_alloc(r_xprt->rx_ep.re_inline_recv, + rep->rr_rdmabuf = rpcrdma_regbuf_alloc(r_xprt->rx_ep->re_inline_recv, DMA_FROM_DEVICE, GFP_KERNEL); if (!rep->rr_rdmabuf) goto out_free; @@ -1165,7 +1177,7 @@ void rpcrdma_mr_put(struct rpcrdma_mr *mr) if (mr->mr_dir != DMA_NONE) { trace_xprtrdma_mr_unmap(mr); - ib_dma_unmap_sg(r_xprt->rx_ep.re_id->device, + ib_dma_unmap_sg(r_xprt->rx_ep->re_id->device, mr->mr_sg, mr->mr_nents, mr->mr_dir); mr->mr_dir = DMA_NONE; } @@ -1283,7 +1295,7 @@ bool rpcrdma_regbuf_realloc(struct rpcrdma_regbuf *rb, size_t size, gfp_t flags) bool __rpcrdma_regbuf_dma_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_regbuf *rb) { - struct ib_device *device = r_xprt->rx_ep.re_id->device; + struct ib_device *device = r_xprt->rx_ep->re_id->device; if (rb->rg_direction == DMA_NONE) return false; @@ -1296,7 +1308,7 @@ bool __rpcrdma_regbuf_dma_map(struct rpcrdma_xprt *r_xprt, } rb->rg_device = device; - rb->rg_iov.lkey = r_xprt->rx_ep.re_pd->local_dma_lkey; + rb->rg_iov.lkey = r_xprt->rx_ep->re_pd->local_dma_lkey; return true; } @@ -1332,7 +1344,7 @@ static void rpcrdma_regbuf_free(struct rpcrdma_regbuf *rb) int rpcrdma_post_sends(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) { struct ib_send_wr *send_wr = &req->rl_wr; - struct rpcrdma_ep *ep = &r_xprt->rx_ep; + struct rpcrdma_ep *ep = r_xprt->rx_ep; int rc; if (!ep->re_send_count || kref_read(&req->rl_kref) > 1) { @@ -1359,7 +1371,7 @@ int rpcrdma_post_sends(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp) { struct rpcrdma_buffer *buf = &r_xprt->rx_buf; - struct rpcrdma_ep *ep = &r_xprt->rx_ep; + struct rpcrdma_ep *ep = r_xprt->rx_ep; struct ib_recv_wr *wr, *bad_wr; struct rpcrdma_rep *rep; int needed, count, rc; @@ -1396,7 +1408,7 @@ void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp) if (!wr) goto out; - rc = ib_post_recv(r_xprt->rx_ep.re_id->qp, wr, + rc = ib_post_recv(ep->re_id->qp, wr, (const struct ib_recv_wr **)&bad_wr); out: trace_xprtrdma_post_recvs(r_xprt, count, rc); diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h index f3c0b826c9ed..0a16fdb09b2c 100644 --- a/net/sunrpc/xprtrdma/xprt_rdma.h +++ b/net/sunrpc/xprtrdma/xprt_rdma.h @@ -68,6 +68,7 @@ * RDMA Endpoint -- connection endpoint details */ struct rpcrdma_ep { + struct kref re_kref; struct rdma_cm_id *re_id; struct ib_pd *re_pd; unsigned int re_max_rdma_segs; @@ -75,7 +76,6 @@ struct rpcrdma_ep { bool re_implicit_roundup; enum ib_mr_type re_mrtype; struct completion re_done; - struct completion re_remove_done; unsigned int re_send_count; unsigned int re_send_batch; unsigned int re_max_inline_send; @@ -83,7 +83,8 @@ struct rpcrdma_ep { int re_async_rc; int re_connect_status; struct ib_qp_init_attr re_attr; - wait_queue_head_t re_connect_wait; + wait_queue_head_t re_connect_wait; + struct rpc_xprt *re_xprt; struct rpcrdma_connect_private re_cm_private; struct rdma_conn_param re_remote_cma; @@ -411,7 +412,7 @@ struct rpcrdma_stats { */ struct rpcrdma_xprt { struct rpc_xprt rx_xprt; - struct rpcrdma_ep rx_ep; + struct rpcrdma_ep *rx_ep; struct rpcrdma_buffer rx_buf; struct delayed_work rx_connect_worker; struct rpc_timeout rx_timeout; From 83552eacdfc06378b0993a3705833c1fd0aead4b Mon Sep 17 00:00:00 2001 From: Miklos Szeredi Date: Fri, 27 Mar 2020 16:51:02 +0100 Subject: [PATCH 265/635] ovl: fix WARN_ON nlink drop to zero Changes to underlying layers should not cause WARN_ON(), but this repro does: mkdir w l u mnt sudo mount -t overlay -o workdir=w,lowerdir=l,upperdir=u overlay mnt touch mnt/h ln u/h u/k rm -rf mnt/k rm -rf mnt/h dmesg ------------[ cut here ]------------ WARNING: CPU: 1 PID: 116244 at fs/inode.c:302 drop_nlink+0x28/0x40 After upper hardlinks were added while overlay is mounted, unlinking all overlay hardlinks drops overlay nlink to zero before all upper inodes are unlinked. After unlink/rename prevent i_nlink from going to zero if there are still hashed aliases (i.e. cached hard links to the victim) remaining. Reported-by: Phasip Signed-off-by: Miklos Szeredi --- fs/overlayfs/dir.c | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/fs/overlayfs/dir.c b/fs/overlayfs/dir.c index c91b5aae8e32..279009dee366 100644 --- a/fs/overlayfs/dir.c +++ b/fs/overlayfs/dir.c @@ -822,6 +822,28 @@ static bool ovl_pure_upper(struct dentry *dentry) !ovl_test_flag(OVL_WHITEOUTS, d_inode(dentry)); } +static void ovl_drop_nlink(struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + struct dentry *alias; + + /* Try to find another, hashed alias */ + spin_lock(&inode->i_lock); + hlist_for_each_entry(alias, &inode->i_dentry, d_u.d_alias) { + if (alias != dentry && !d_unhashed(alias)) + break; + } + spin_unlock(&inode->i_lock); + + /* + * Changes to underlying layers may cause i_nlink to lose sync with + * reality. In this case prevent the link count from going to zero + * prematurely. + */ + if (inode->i_nlink > !!alias) + drop_nlink(inode); +} + static int ovl_do_remove(struct dentry *dentry, bool is_dir) { int err; @@ -859,7 +881,7 @@ static int ovl_do_remove(struct dentry *dentry, bool is_dir) if (is_dir) clear_nlink(dentry->d_inode); else - drop_nlink(dentry->d_inode); + ovl_drop_nlink(dentry); } ovl_nlink_end(dentry); @@ -1204,7 +1226,7 @@ static int ovl_rename(struct inode *olddir, struct dentry *old, if (new_is_dir) clear_nlink(d_inode(new)); else - drop_nlink(d_inode(new)); + ovl_drop_nlink(new); } ovl_dir_modified(old->d_parent, ovl_type_origin(old) || From 4d314f7859dc1925ee28b4d4e74f6f3a80e6f34b Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Fri, 21 Feb 2020 16:34:43 +0200 Subject: [PATCH 266/635] ovl: use a private non-persistent ino pool There is no reason to deplete the system's global get_next_ino() pool for overlay non-persistent inode numbers and there is no reason at all to allocate non-persistent inode numbers for non-directories. For non-directories, it is much better to leave i_ino the same as real i_ino, to be consistent with st_ino/d_ino. Signed-off-by: Amir Goldstein Signed-off-by: Miklos Szeredi --- fs/overlayfs/inode.c | 15 ++++++++++++--- fs/overlayfs/ovl_entry.h | 2 ++ fs/overlayfs/super.c | 1 + 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/fs/overlayfs/inode.c b/fs/overlayfs/inode.c index 956c6f9c9c11..8130ae54a332 100644 --- a/fs/overlayfs/inode.c +++ b/fs/overlayfs/inode.c @@ -561,6 +561,15 @@ static inline void ovl_lockdep_annotate_inode_mutex_key(struct inode *inode) #endif } +static void ovl_next_ino(struct inode *inode) +{ + struct ovl_fs *ofs = inode->i_sb->s_fs_info; + + inode->i_ino = atomic_long_inc_return(&ofs->last_ino); + if (unlikely(!inode->i_ino)) + inode->i_ino = atomic_long_inc_return(&ofs->last_ino); +} + static void ovl_map_ino(struct inode *inode, unsigned long ino, int fsid) { int xinobits = ovl_xino_bits(inode->i_sb); @@ -572,12 +581,12 @@ static void ovl_map_ino(struct inode *inode, unsigned long ino, int fsid) * consistent with d_ino and st_ino values. An i_ino value inconsistent * with d_ino also causes nfsd readdirplus to fail. */ + inode->i_ino = ino; if (ovl_same_dev(inode->i_sb)) { - inode->i_ino = ino; if (xinobits && fsid && !(ino >> (64 - xinobits))) inode->i_ino |= (unsigned long)fsid << (64 - xinobits); - } else { - inode->i_ino = get_next_ino(); + } else if (S_ISDIR(inode->i_mode)) { + ovl_next_ino(inode); } } diff --git a/fs/overlayfs/ovl_entry.h b/fs/overlayfs/ovl_entry.h index 89015ea822e7..5762d802fe01 100644 --- a/fs/overlayfs/ovl_entry.h +++ b/fs/overlayfs/ovl_entry.h @@ -75,6 +75,8 @@ struct ovl_fs { struct inode *indexdir_trap; /* -1: disabled, 0: same fs, 1..32: number of unused ino bits */ int xino_mode; + /* For allocation of non-persistent inode numbers */ + atomic_long_t last_ino; }; static inline struct ovl_fs *OVL_FS(struct super_block *sb) diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index ac013ed97a53..1f1d40fc5e33 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -1736,6 +1736,7 @@ static int ovl_fill_super(struct super_block *sb, void *data, int silent) sb->s_stack_depth = 0; sb->s_maxbytes = MAX_LFS_FILESIZE; + atomic_long_set(&ofs->last_ino, 1); /* Assume underlaying fs uses 32bit inodes unless proven otherwise */ if (ofs->config.xino != OVL_XINO_OFF) { ofs->xino_mode = BITS_PER_LONG - 32; From dfe51d47b7eeb5642ed92558b62eeff558f00eda Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Fri, 21 Feb 2020 16:34:44 +0200 Subject: [PATCH 267/635] ovl: avoid possible inode number collisions with xino=on When xino feature is enabled and a real directory inode number overflows the lower xino bits, we cannot map this directory inode number to a unique and persistent inode number and we fall back to the real inode st_ino and overlay st_dev. The real inode st_ino with high bits may collide with a lower inode number on overlay st_dev that was mapped using xino. To avoid possible collision with legitimate xino values, map a non persistent inode number to a dedicated range in the xino address space. The dedicated range is created by adding one more bit to the number of reserved high xino bits. We could have added just one more fsid, but that would have had the undesired effect of changing persistent overlay inode numbers on kernel or require more complex xino mapping code. Signed-off-by: Amir Goldstein Signed-off-by: Miklos Szeredi --- fs/overlayfs/inode.c | 39 +++++++++++++++++++++++++++++---------- fs/overlayfs/readdir.c | 10 ++++++++-- fs/overlayfs/super.c | 13 ++++++++----- 3 files changed, 45 insertions(+), 17 deletions(-) diff --git a/fs/overlayfs/inode.c b/fs/overlayfs/inode.c index 8130ae54a332..2d6251a4caaf 100644 --- a/fs/overlayfs/inode.c +++ b/fs/overlayfs/inode.c @@ -79,6 +79,7 @@ static int ovl_map_dev_ino(struct dentry *dentry, struct kstat *stat, int fsid) { bool samefs = ovl_same_fs(dentry->d_sb); unsigned int xinobits = ovl_xino_bits(dentry->d_sb); + unsigned int xinoshift = 64 - xinobits; if (samefs) { /* @@ -89,20 +90,20 @@ static int ovl_map_dev_ino(struct dentry *dentry, struct kstat *stat, int fsid) stat->dev = dentry->d_sb->s_dev; return 0; } else if (xinobits) { - unsigned int shift = 64 - xinobits; /* * All inode numbers of underlying fs should not be using the * high xinobits, so we use high xinobits to partition the * overlay st_ino address space. The high bits holds the fsid - * (upper fsid is 0). This way overlay inode numbers are unique - * and all inodes use overlay st_dev. Inode numbers are also - * persistent for a given layer configuration. + * (upper fsid is 0). The lowest xinobit is reserved for mapping + * the non-peresistent inode numbers range in case of overflow. + * This way all overlay inode numbers are unique and use the + * overlay st_dev. */ - if (stat->ino >> shift) { + if (unlikely(stat->ino >> xinoshift)) { pr_warn_ratelimited("inode number too big (%pd2, ino=%llu, xinobits=%d)\n", dentry, stat->ino, xinobits); } else { - stat->ino |= ((u64)fsid) << shift; + stat->ino |= ((u64)fsid) << (xinoshift + 1); stat->dev = dentry->d_sb->s_dev; return 0; } @@ -573,6 +574,7 @@ static void ovl_next_ino(struct inode *inode) static void ovl_map_ino(struct inode *inode, unsigned long ino, int fsid) { int xinobits = ovl_xino_bits(inode->i_sb); + unsigned int xinoshift = 64 - xinobits; /* * When d_ino is consistent with st_ino (samefs or i_ino has enough @@ -582,11 +584,28 @@ static void ovl_map_ino(struct inode *inode, unsigned long ino, int fsid) * with d_ino also causes nfsd readdirplus to fail. */ inode->i_ino = ino; - if (ovl_same_dev(inode->i_sb)) { - if (xinobits && fsid && !(ino >> (64 - xinobits))) - inode->i_ino |= (unsigned long)fsid << (64 - xinobits); - } else if (S_ISDIR(inode->i_mode)) { + if (ovl_same_fs(inode->i_sb)) { + return; + } else if (xinobits && likely(!(ino >> xinoshift))) { + inode->i_ino |= (unsigned long)fsid << (xinoshift + 1); + return; + } + + /* + * For directory inodes on non-samefs with xino disabled or xino + * overflow, we allocate a non-persistent inode number, to be used for + * resolving st_ino collisions in ovl_map_dev_ino(). + * + * To avoid ino collision with legitimate xino values from upper + * layer (fsid 0), use the lowest xinobit to map the non + * persistent inode numbers to the unified st_ino address space. + */ + if (S_ISDIR(inode->i_mode)) { ovl_next_ino(inode); + if (xinobits) { + inode->i_ino &= ~0UL >> xinobits; + inode->i_ino |= 1UL << xinoshift; + } } } diff --git a/fs/overlayfs/readdir.c b/fs/overlayfs/readdir.c index 40ac9ce2465a..6325dcc4c48b 100644 --- a/fs/overlayfs/readdir.c +++ b/fs/overlayfs/readdir.c @@ -440,13 +440,19 @@ static struct ovl_dir_cache *ovl_cache_get(struct dentry *dentry) static u64 ovl_remap_lower_ino(u64 ino, int xinobits, int fsid, const char *name, int namelen) { - if (ino >> (64 - xinobits)) { + unsigned int xinoshift = 64 - xinobits; + + if (unlikely(ino >> xinoshift)) { pr_warn_ratelimited("d_ino too big (%.*s, ino=%llu, xinobits=%d)\n", namelen, name, ino, xinobits); return ino; } - return ino | ((u64)fsid) << (64 - xinobits); + /* + * The lowest xinobit is reserved for mapping the non-peresistent inode + * numbers range, but this range is only exposed via st_ino, not here. + */ + return ino | ((u64)fsid) << (xinoshift + 1); } /* diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index 1f1d40fc5e33..53dd094f970e 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -1483,7 +1483,8 @@ static int ovl_get_layers(struct super_block *sb, struct ovl_fs *ofs, * free high bits in underlying fs to hold the unique fsid. * If overlayfs does encounter underlying inodes using the high xino * bits reserved for fsid, it emits a warning and uses the original - * inode number. + * inode number or a non persistent inode number allocated from a + * dedicated range. */ if (ofs->numfs - !ofs->upper_mnt == 1) { if (ofs->config.xino == OVL_XINO_ON) @@ -1494,11 +1495,13 @@ static int ovl_get_layers(struct super_block *sb, struct ovl_fs *ofs, } else if (ofs->config.xino == OVL_XINO_ON && ofs->xino_mode < 0) { /* * This is a roundup of number of bits needed for encoding - * fsid, where fsid 0 is reserved for upper fs even with - * lower only overlay. + * fsid, where fsid 0 is reserved for upper fs (even with + * lower only overlay) +1 extra bit is reserved for the non + * persistent inode number range that is used for resolving + * xino lower bits overflow. */ - BUILD_BUG_ON(ilog2(OVL_MAX_STACK) > 31); - ofs->xino_mode = ilog2(ofs->numfs - 1) + 1; + BUILD_BUG_ON(ilog2(OVL_MAX_STACK) > 30); + ofs->xino_mode = ilog2(ofs->numfs - 1) + 2; } if (ofs->xino_mode > 0) { From 926e94d79baf76ce7a4d26c3116b6d85a77a475b Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Fri, 21 Feb 2020 16:34:45 +0200 Subject: [PATCH 268/635] ovl: enable xino automatically in more cases So far, with xino=auto, we only enable xino if we know that all underlying filesystem use 32bit inode numbers. When users configure overlay with xino=auto, they already declare that they are ready to handle 64bit inode number from overlay. It is a very common case, that underlying filesystem uses 64bit ino, but rarely or never uses the high inode number bits (e.g. tmpfs, xfs). Leaving it for the users to declare high ino bits are unused with xino=on is not a recipe for many users to enjoy the benefits of xino. There appears to be very little reason not to enable xino when users declare xino=auto even if we do not know how many bits underlying filesystem uses for inode numbers. In the worst case of xino bits overflow by real inode number, we already fall back to the non-xino behavior - real inode number with unique pseudo dev or to non persistent inode number and overlay st_dev (for directories). The only annoyance from auto enabling xino is that xino bits overflow emits a warning to kmsg. Suppress those warnings unless users explicitly asked for xino=on, suggesting that they expected high ino bits to be unused by underlying filesystem. Signed-off-by: Amir Goldstein Signed-off-by: Miklos Szeredi --- fs/overlayfs/inode.c | 8 ++++---- fs/overlayfs/overlayfs.h | 16 ++++++++++++++++ fs/overlayfs/readdir.c | 15 ++++++++++----- fs/overlayfs/super.c | 12 +++--------- 4 files changed, 33 insertions(+), 18 deletions(-) diff --git a/fs/overlayfs/inode.c b/fs/overlayfs/inode.c index 2d6251a4caaf..b0d42ece4d7c 100644 --- a/fs/overlayfs/inode.c +++ b/fs/overlayfs/inode.c @@ -99,13 +99,13 @@ static int ovl_map_dev_ino(struct dentry *dentry, struct kstat *stat, int fsid) * This way all overlay inode numbers are unique and use the * overlay st_dev. */ - if (unlikely(stat->ino >> xinoshift)) { - pr_warn_ratelimited("inode number too big (%pd2, ino=%llu, xinobits=%d)\n", - dentry, stat->ino, xinobits); - } else { + if (likely(!(stat->ino >> xinoshift))) { stat->ino |= ((u64)fsid) << (xinoshift + 1); stat->dev = dentry->d_sb->s_dev; return 0; + } else if (ovl_xino_warn(dentry->d_sb)) { + pr_warn_ratelimited("inode number too big (%pd2, ino=%llu, xinobits=%d)\n", + dentry, stat->ino, xinobits); } } diff --git a/fs/overlayfs/overlayfs.h b/fs/overlayfs/overlayfs.h index e81fea86139e..e6f3670146ed 100644 --- a/fs/overlayfs/overlayfs.h +++ b/fs/overlayfs/overlayfs.h @@ -48,6 +48,12 @@ enum ovl_entry_flag { OVL_E_CONNECTED, }; +enum { + OVL_XINO_OFF, + OVL_XINO_AUTO, + OVL_XINO_ON, +}; + /* * The tuple (fh,uuid) is a universal unique identifier for a copy up origin, * where: @@ -301,6 +307,16 @@ static inline bool ovl_is_impuredir(struct dentry *dentry) return ovl_check_dir_xattr(dentry, OVL_XATTR_IMPURE); } +/* + * With xino=auto, we do best effort to keep all inodes on same st_dev and + * d_ino consistent with st_ino. + * With xino=on, we do the same effort but we warn if we failed. + */ +static inline bool ovl_xino_warn(struct super_block *sb) +{ + return OVL_FS(sb)->config.xino == OVL_XINO_ON; +} + /* All layers on same fs? */ static inline bool ovl_same_fs(struct super_block *sb) { diff --git a/fs/overlayfs/readdir.c b/fs/overlayfs/readdir.c index 6325dcc4c48b..e452ff7d583d 100644 --- a/fs/overlayfs/readdir.c +++ b/fs/overlayfs/readdir.c @@ -438,13 +438,15 @@ static struct ovl_dir_cache *ovl_cache_get(struct dentry *dentry) /* Map inode number to lower fs unique range */ static u64 ovl_remap_lower_ino(u64 ino, int xinobits, int fsid, - const char *name, int namelen) + const char *name, int namelen, bool warn) { unsigned int xinoshift = 64 - xinobits; if (unlikely(ino >> xinoshift)) { - pr_warn_ratelimited("d_ino too big (%.*s, ino=%llu, xinobits=%d)\n", - namelen, name, ino, xinobits); + if (warn) { + pr_warn_ratelimited("d_ino too big (%.*s, ino=%llu, xinobits=%d)\n", + namelen, name, ino, xinobits); + } return ino; } @@ -521,7 +523,8 @@ get: } else if (xinobits && !OVL_TYPE_UPPER(type)) { ino = ovl_remap_lower_ino(ino, xinobits, ovl_layer_lower(this)->fsid, - p->name, p->len); + p->name, p->len, + ovl_xino_warn(dir->d_sb)); } out: @@ -651,6 +654,7 @@ struct ovl_readdir_translate { u64 parent_ino; int fsid; int xinobits; + bool xinowarn; }; static int ovl_fill_real(struct dir_context *ctx, const char *name, @@ -671,7 +675,7 @@ static int ovl_fill_real(struct dir_context *ctx, const char *name, ino = p->ino; } else if (rdt->xinobits) { ino = ovl_remap_lower_ino(ino, rdt->xinobits, rdt->fsid, - name, namelen); + name, namelen, rdt->xinowarn); } return orig_ctx->actor(orig_ctx, name, namelen, offset, ino, d_type); @@ -702,6 +706,7 @@ static int ovl_iterate_real(struct file *file, struct dir_context *ctx) .ctx.actor = ovl_fill_real, .orig_ctx = ctx, .xinobits = ovl_xino_bits(dir->d_sb), + .xinowarn = ovl_xino_warn(dir->d_sb), }; if (rdt.xinobits && lower_layer) diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index 53dd094f970e..732ad5495c92 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -317,12 +317,6 @@ static const char *ovl_redirect_mode_def(void) return ovl_redirect_dir_def ? "on" : "off"; } -enum { - OVL_XINO_OFF, - OVL_XINO_AUTO, - OVL_XINO_ON, -}; - static const char * const ovl_xino_str[] = { "off", "auto", @@ -1479,8 +1473,8 @@ static int ovl_get_layers(struct super_block *sb, struct ovl_fs *ofs, /* * When all layers on same fs, overlay can use real inode numbers. - * With mount option "xino=on", mounter declares that there are enough - * free high bits in underlying fs to hold the unique fsid. + * With mount option "xino=", mounter declares that there are + * enough free high bits in underlying fs to hold the unique fsid. * If overlayfs does encounter underlying inodes using the high xino * bits reserved for fsid, it emits a warning and uses the original * inode number or a non persistent inode number allocated from a @@ -1492,7 +1486,7 @@ static int ovl_get_layers(struct super_block *sb, struct ovl_fs *ofs, ofs->xino_mode = 0; } else if (ofs->config.xino == OVL_XINO_OFF) { ofs->xino_mode = -1; - } else if (ofs->config.xino == OVL_XINO_ON && ofs->xino_mode < 0) { + } else if (ofs->xino_mode < 0) { /* * This is a roundup of number of bits needed for encoding * fsid, where fsid 0 is reserved for upper fs (even with From 2eda9eaa6d7ec129150df4c7b7be65f27ac47346 Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Fri, 21 Feb 2020 16:34:46 +0200 Subject: [PATCH 269/635] ovl: document xino expected behavior Summarize the inode properties of different configurations in a table. Signed-off-by: Amir Goldstein Signed-off-by: Miklos Szeredi --- Documentation/filesystems/overlayfs.rst | 38 +++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/Documentation/filesystems/overlayfs.rst b/Documentation/filesystems/overlayfs.rst index e398fdf7353e..c9d2bf96b02d 100644 --- a/Documentation/filesystems/overlayfs.rst +++ b/Documentation/filesystems/overlayfs.rst @@ -40,13 +40,46 @@ On 64bit systems, even if all overlay layers are not on the same underlying filesystem, the same compliant behavior could be achieved with the "xino" feature. The "xino" feature composes a unique object identifier from the real object st_ino and an underlying fsid index. + If all underlying filesystems support NFS file handles and export file handles with 32bit inode number encoding (e.g. ext4), overlay filesystem will use the high inode number bits for fsid. Even when the underlying filesystem uses 64bit inode numbers, users can still enable the "xino" feature with the "-o xino=on" overlay mount option. That is useful for the case of underlying filesystems like xfs and tmpfs, which use 64bit inode -numbers, but are very unlikely to use the high inode number bit. +numbers, but are very unlikely to use the high inode number bits. In case +the underlying inode number does overflow into the high xino bits, overlay +filesystem will fall back to the non xino behavior for that inode. + +The following table summarizes what can be expected in different overlay +configurations. + +Inode properties +```````````````` + ++--------------+------------+------------+-----------------+----------------+ +|Configuration | Persistent | Uniform | st_ino == d_ino | d_ino == i_ino | +| | st_ino | st_dev | | [*] | ++==============+=====+======+=====+======+========+========+========+=======+ +| | dir | !dir | dir | !dir | dir + !dir | dir | !dir | ++--------------+-----+------+-----+------+--------+--------+--------+-------+ +| All layers | Y | Y | Y | Y | Y | Y | Y | Y | +| on same fs | | | | | | | | | ++--------------+-----+------+-----+------+--------+--------+--------+-------+ +| Layers not | N | Y | Y | N | N | Y | N | Y | +| on same fs, | | | | | | | | | +| xino=off | | | | | | | | | ++--------------+-----+------+-----+------+--------+--------+--------+-------+ +| xino=on/auto | Y | Y | Y | Y | Y | Y | Y | Y | +| | | | | | | | | | ++--------------+-----+------+-----+------+--------+--------+--------+-------+ +| xino=on/auto,| N | Y | Y | N | N | Y | N | Y | +| ino overflow | | | | | | | | | ++--------------+-----+------+-----+------+--------+--------+--------+-------+ + +[*] nfsd v3 readdirplus verifies d_ino == i_ino. i_ino is exposed via several +/proc files, such as /proc/locks and /proc/self/fdinfo/ of an inotify +file descriptor. Upper and Lower @@ -427,7 +460,8 @@ guarantee that the values of st_ino and st_dev returned by stat(2) and the value of d_ino returned by readdir(3) will act like on a normal filesystem. E.g. the value of st_dev may be different for two objects in the same overlay filesystem and the value of st_ino for directory objects may not be -persistent and could change even while the overlay filesystem is mounted. +persistent and could change even while the overlay filesystem is mounted, as +summarized in the `Inode properties`_ table above. Changes to underlying filesystems From b2057c64017e96f1f9471fe49569508711f7663f Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Thu, 26 Mar 2020 22:09:51 +0100 Subject: [PATCH 270/635] platform/chrome: chromeos_laptop: make I2C API conversion complete When converting to i2c_new_scanned_device(), it was overlooked that a conversion to i2c_new_client_device() was also needed. Fix it. Fixes: c82ebf1bf738 ("platform/chrome: chromeos_laptop: Convert to i2c_new_scanned_device") Signed-off-by: Wolfram Sang Signed-off-by: Enric Balletbo i Serra --- drivers/platform/chrome/chromeos_laptop.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/platform/chrome/chromeos_laptop.c b/drivers/platform/chrome/chromeos_laptop.c index 4f3651fcd9fe..472a03daa869 100644 --- a/drivers/platform/chrome/chromeos_laptop.c +++ b/drivers/platform/chrome/chromeos_laptop.c @@ -103,7 +103,7 @@ chromes_laptop_instantiate_i2c_device(struct i2c_adapter *adapter, pr_debug("%d-%02x is probed at %02x\n", adapter->nr, info->addr, dummy->addr); i2c_unregister_device(dummy); - client = i2c_new_device(adapter, info); + client = i2c_new_client_device(adapter, info); } } From c21e7168848d4ff4158120dbd4464f0d5cfb1456 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Thu, 19 Mar 2020 13:36:36 -0400 Subject: [PATCH 271/635] NFSv4/pnfs: Support a list of commit arrays in struct pnfs_ds_commit_info When we have multiple layout segments with different lists of mirrored data, we need to track the commits on a per layout segment basis. This patch adds a list to support this tracking in struct pnfs_ds_commit_info. Signed-off-by: Trond Myklebust --- fs/nfs/direct.c | 1 + fs/nfs/filelayout/filelayout.c | 5 ++++- fs/nfs/flexfilelayout/flexfilelayout.c | 1 + fs/nfs/pnfs.h | 11 +++++++++++ include/linux/nfs_xdr.h | 1 + 5 files changed, 18 insertions(+), 1 deletion(-) diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c index ade2435551c8..f9a73febce02 100644 --- a/fs/nfs/direct.c +++ b/fs/nfs/direct.c @@ -305,6 +305,7 @@ static inline struct nfs_direct_req *nfs_direct_req_alloc(void) kref_get(&dreq->kref); init_completion(&dreq->completion); INIT_LIST_HEAD(&dreq->mds_cinfo.list); + pnfs_init_ds_commit_info(&dreq->ds_cinfo); dreq->verf.committed = NFS_INVALID_STABLE_HOW; /* not set yet */ INIT_WORK(&dreq->work, nfs_direct_write_schedule_work); spin_lock_init(&dreq->lock); diff --git a/fs/nfs/filelayout/filelayout.c b/fs/nfs/filelayout/filelayout.c index bd234394a87c..b051d5d320ba 100644 --- a/fs/nfs/filelayout/filelayout.c +++ b/fs/nfs/filelayout/filelayout.c @@ -1140,7 +1140,10 @@ filelayout_alloc_layout_hdr(struct inode *inode, gfp_t gfp_flags) struct nfs4_filelayout *flo; flo = kzalloc(sizeof(*flo), gfp_flags); - return flo != NULL ? &flo->generic_hdr : NULL; + if (flo == NULL) + return NULL; + pnfs_init_ds_commit_info(&flo->commit_info); + return &flo->generic_hdr; } static void diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c index 19728206e9c6..c9e79c8e62cd 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.c +++ b/fs/nfs/flexfilelayout/flexfilelayout.c @@ -48,6 +48,7 @@ ff_layout_alloc_layout_hdr(struct inode *inode, gfp_t gfp_flags) ffl = kzalloc(sizeof(*ffl), gfp_flags); if (ffl) { + pnfs_init_ds_commit_info(&ffl->commit_info); INIT_LIST_HEAD(&ffl->error_list); INIT_LIST_HEAD(&ffl->mirrors); ffl->last_report_time = ktime_get(); diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h index f6b1099aa151..b293afb48d04 100644 --- a/fs/nfs/pnfs.h +++ b/fs/nfs/pnfs.h @@ -462,6 +462,12 @@ pnfs_get_ds_info(struct inode *inode) return ld->get_ds_info(inode); } +static inline void +pnfs_init_ds_commit_info(struct pnfs_ds_commit_info *fl_cinfo) +{ + INIT_LIST_HEAD(&fl_cinfo->commits); +} + static inline void pnfs_generic_mark_devid_invalid(struct nfs4_deviceid_node *node) { @@ -759,6 +765,11 @@ pnfs_get_ds_info(struct inode *inode) return NULL; } +static inline void +pnfs_init_ds_commit_info(struct pnfs_ds_commit_info *fl_cinfo) +{ +} + static inline bool pnfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg, struct nfs_commit_info *cinfo, u32 ds_commit_idx) diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index e91c917c9c1c..9946787eda72 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -1280,6 +1280,7 @@ struct pnfs_commit_array { }; struct pnfs_ds_commit_info { + struct list_head commits; unsigned int nwritten; unsigned int ncommitting; unsigned int nbuckets; From a8e3765e5178fb6a2a9f14d133d0b3c23a7c1e6d Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Thu, 19 Mar 2020 13:41:08 -0400 Subject: [PATCH 272/635] NFSv4/pNFS: Scan the full list of commit arrays when committing Add support for scanning the full list of per-layout segment commit arrays to pnfs_generic_scan_commit_lists() Signed-off-by: Trond Myklebust --- fs/nfs/pnfs_nfs.c | 50 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c index c8518ce3a4ef..81fd85e66fd9 100644 --- a/fs/nfs/pnfs_nfs.c +++ b/fs/nfs/pnfs_nfs.c @@ -118,10 +118,14 @@ pnfs_free_commit_array(struct pnfs_commit_array *p) } EXPORT_SYMBOL_GPL(pnfs_free_commit_array); +/* + * Locks the nfs_page requests for commit and moves them to + * @bucket->committing. + */ static int -pnfs_generic_scan_ds_commit_list(struct pnfs_commit_bucket *bucket, - struct nfs_commit_info *cinfo, - int max) +pnfs_bucket_scan_ds_commit_list(struct pnfs_commit_bucket *bucket, + struct nfs_commit_info *cinfo, + int max) { struct list_head *src = &bucket->written; struct list_head *dst = &bucket->committing; @@ -142,20 +146,44 @@ pnfs_generic_scan_ds_commit_list(struct pnfs_commit_bucket *bucket, return ret; } +static int pnfs_bucket_scan_array(struct nfs_commit_info *cinfo, + struct pnfs_commit_bucket *buckets, + unsigned int nbuckets, + int max) +{ + unsigned int i; + int rv = 0, cnt; + + for (i = 0; i < nbuckets && max != 0; i++) { + cnt = pnfs_bucket_scan_ds_commit_list(&buckets[i], cinfo, max); + rv += cnt; + max -= cnt; + } + return rv; +} + /* Move reqs from written to committing lists, returning count * of number moved. */ -int pnfs_generic_scan_commit_lists(struct nfs_commit_info *cinfo, - int max) +int pnfs_generic_scan_commit_lists(struct nfs_commit_info *cinfo, int max) { - int i, rv = 0, cnt; + struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds; + struct pnfs_commit_array *array; + int rv = 0, cnt; - lockdep_assert_held(&NFS_I(cinfo->inode)->commit_mutex); - for (i = 0; i < cinfo->ds->nbuckets && max != 0; i++) { - cnt = pnfs_generic_scan_ds_commit_list(&cinfo->ds->buckets[i], - cinfo, max); - max -= cnt; + cnt = pnfs_bucket_scan_array(cinfo, fl_cinfo->buckets, + fl_cinfo->nbuckets, max); + rv += cnt; + max -= cnt; + if (!max) + return rv; + list_for_each_entry(array, &fl_cinfo->commits, cinfo_list) { + cnt = pnfs_bucket_scan_array(cinfo, array->buckets, + array->nbuckets, max); rv += cnt; + max -= cnt; + if (!max) + break; } return rv; } From fce9ed0302180336b60a1e14fcda8dc887fd5070 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Thu, 19 Mar 2020 15:48:42 -0400 Subject: [PATCH 273/635] pNFS: Support per-layout segment commits in pnfs_generic_recover_commit_reqs() Add support for scanning the full list of per-layout segment commit arrays to pnfs_generic_recover_commit_reqs(). Signed-off-by: Trond Myklebust --- fs/nfs/pnfs_nfs.c | 41 +++++++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c index 81fd85e66fd9..f16bd6d0e830 100644 --- a/fs/nfs/pnfs_nfs.c +++ b/fs/nfs/pnfs_nfs.c @@ -189,22 +189,23 @@ int pnfs_generic_scan_commit_lists(struct nfs_commit_info *cinfo, int max) } EXPORT_SYMBOL_GPL(pnfs_generic_scan_commit_lists); -/* Pull everything off the committing lists and dump into @dst. */ -void pnfs_generic_recover_commit_reqs(struct list_head *dst, - struct nfs_commit_info *cinfo) +static unsigned int +pnfs_bucket_recover_commit_reqs(struct list_head *dst, + struct pnfs_commit_bucket *buckets, + unsigned int nbuckets, + struct nfs_commit_info *cinfo) { struct pnfs_commit_bucket *b; struct pnfs_layout_segment *freeme; - int nwritten; - int i; + unsigned int nwritten, ret = 0; + unsigned int i; - lockdep_assert_held(&NFS_I(cinfo->inode)->commit_mutex); restart: - for (i = 0, b = cinfo->ds->buckets; i < cinfo->ds->nbuckets; i++, b++) { + for (i = 0, b = buckets; i < nbuckets; i++, b++) { nwritten = nfs_scan_commit_list(&b->written, dst, cinfo, 0); if (!nwritten) continue; - cinfo->ds->nwritten -= nwritten; + ret += nwritten; if (list_empty(&b->written)) { freeme = b->wlseg; b->wlseg = NULL; @@ -212,6 +213,30 @@ restart: goto restart; } } + return ret; +} + +/* Pull everything off the committing lists and dump into @dst. */ +void pnfs_generic_recover_commit_reqs(struct list_head *dst, + struct nfs_commit_info *cinfo) +{ + struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds; + struct pnfs_commit_array *array; + unsigned int nwritten; + + lockdep_assert_held(&NFS_I(cinfo->inode)->commit_mutex); + nwritten = pnfs_bucket_recover_commit_reqs(dst, + fl_cinfo->buckets, + fl_cinfo->nbuckets, + cinfo); + fl_cinfo->nwritten -= nwritten; + list_for_each_entry(array, &fl_cinfo->commits, cinfo_list) { + nwritten = pnfs_bucket_recover_commit_reqs(dst, + array->buckets, + array->nbuckets, + cinfo); + fl_cinfo->nwritten -= nwritten; + } } EXPORT_SYMBOL_GPL(pnfs_generic_recover_commit_reqs); From 0cb1f6df8a63b51f276f94d94957d7a7ca757667 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Thu, 19 Mar 2020 19:31:00 -0400 Subject: [PATCH 274/635] pNFS: Support per-layout segment commits in pnfs_generic_commit_pagelist() Add support for scanning the full list of per-layout segment commit arrays to pnfs_generic_commit_pagelist(). Signed-off-by: Trond Myklebust --- fs/nfs/pnfs_nfs.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c index f16bd6d0e830..f895a28b1e26 100644 --- a/fs/nfs/pnfs_nfs.c +++ b/fs/nfs/pnfs_nfs.c @@ -322,6 +322,20 @@ out_error: return nreq; } +static unsigned int +pnfs_alloc_ds_commits_list(struct list_head *list, + struct pnfs_ds_commit_info *fl_cinfo, + struct nfs_commit_info *cinfo) +{ + struct pnfs_commit_array *array; + unsigned int ret = 0; + + list_for_each_entry(array, &fl_cinfo->commits, cinfo_list) + ret += pnfs_bucket_alloc_ds_commits(list, array->buckets, + array->nbuckets, cinfo); + return ret; +} + /* This follows nfs_commit_list pretty closely */ int pnfs_generic_commit_pagelist(struct inode *inode, struct list_head *mds_pages, @@ -345,6 +359,8 @@ pnfs_generic_commit_pagelist(struct inode *inode, struct list_head *mds_pages, nreq += pnfs_bucket_alloc_ds_commits(&list, fl_cinfo->buckets, fl_cinfo->nbuckets, cinfo); + + nreq += pnfs_alloc_ds_commits_list(&list, fl_cinfo, cinfo); if (nreq == 0) goto out; From 18f412969687ded8f1debd21da758b041993e974 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Fri, 20 Mar 2020 17:08:02 -0400 Subject: [PATCH 275/635] NFS/pNFS: Allow O_DIRECT to release the DS commitinfo Add a pNFS callback to allow the O_DIRECT code to release the DS commitinfo when freeing the dreq. Signed-off-by: Trond Myklebust --- fs/nfs/direct.c | 1 + fs/nfs/pnfs.h | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c index f9a73febce02..7ef7f71ae315 100644 --- a/fs/nfs/direct.c +++ b/fs/nfs/direct.c @@ -317,6 +317,7 @@ static void nfs_direct_req_free(struct kref *kref) { struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref); + pnfs_release_ds_info(&dreq->ds_cinfo, dreq->inode); nfs_free_pnfs_ds_cinfo(&dreq->ds_cinfo); if (dreq->l_ctx != NULL) nfs_put_lock_context(dreq->l_ctx); diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h index b293afb48d04..2ec97b419b56 100644 --- a/fs/nfs/pnfs.h +++ b/fs/nfs/pnfs.h @@ -149,6 +149,8 @@ struct pnfs_layoutdriver_type { const struct nfs_pageio_ops *pg_write_ops; struct pnfs_ds_commit_info *(*get_ds_info) (struct inode *inode); + void (*release_ds_info)(struct pnfs_ds_commit_info *, + struct inode *inode); void (*mark_request_commit) (struct nfs_page *req, struct pnfs_layout_segment *lseg, struct nfs_commit_info *cinfo, @@ -468,6 +470,15 @@ pnfs_init_ds_commit_info(struct pnfs_ds_commit_info *fl_cinfo) INIT_LIST_HEAD(&fl_cinfo->commits); } +static inline void +pnfs_release_ds_info(struct pnfs_ds_commit_info *fl_cinfo, struct inode *inode) +{ + struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; + + if (ld != NULL && ld->release_ds_info != NULL) + ld->release_ds_info(fl_cinfo, inode); +} + static inline void pnfs_generic_mark_devid_invalid(struct nfs4_deviceid_node *node) { @@ -770,6 +781,11 @@ pnfs_init_ds_commit_info(struct pnfs_ds_commit_info *fl_cinfo) { } +static inline void +pnfs_release_ds_info(struct pnfs_ds_commit_info *fl_cinfo, struct inode *inode) +{ +} + static inline bool pnfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg, struct nfs_commit_info *cinfo, u32 ds_commit_idx) From fb5f7f20cdb91f8ef985aef09fa2217c49c38396 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Sat, 21 Mar 2020 09:36:13 -0400 Subject: [PATCH 276/635] NFS: commit errors should be fatal Fix the O_DIRECT code to avoid retries if the COMMIT fails with a fatal error. Signed-off-by: Trond Myklebust --- fs/nfs/direct.c | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c index 7ef7f71ae315..f7bf1181b690 100644 --- a/fs/nfs/direct.c +++ b/fs/nfs/direct.c @@ -94,6 +94,7 @@ struct nfs_direct_req { #define NFS_ODIRECT_RESCHED_WRITES (2) /* write verification failed */ /* for read */ #define NFS_ODIRECT_SHOULD_DIRTY (3) /* dirty user-space page after read */ +#define NFS_ODIRECT_DONE INT_MAX /* write verification failed */ struct nfs_writeverf verf; /* unstable write verifier */ }; @@ -678,8 +679,17 @@ static void nfs_direct_commit_complete(struct nfs_commit_data *data) struct nfs_page *req; int status = data->task.tk_status; + if (status < 0) { + /* Errors in commit are fatal */ + dreq->error = status; + dreq->max_count = 0; + dreq->count = 0; + dreq->flags = NFS_ODIRECT_DONE; + } else if (dreq->flags == NFS_ODIRECT_DONE) + status = dreq->error; + nfs_init_cinfo_from_dreq(&cinfo, dreq); - if (status < 0 || nfs_direct_cmp_commit_data_verf(dreq, data)) + if (nfs_direct_cmp_commit_data_verf(dreq, data)) dreq->flags = NFS_ODIRECT_RESCHED_WRITES; while (!list_empty(&data->pages)) { @@ -708,7 +718,8 @@ static void nfs_direct_resched_write(struct nfs_commit_info *cinfo, struct nfs_direct_req *dreq = cinfo->dreq; spin_lock(&dreq->lock); - dreq->flags = NFS_ODIRECT_RESCHED_WRITES; + if (dreq->flags != NFS_ODIRECT_DONE) + dreq->flags = NFS_ODIRECT_RESCHED_WRITES; spin_unlock(&dreq->lock); nfs_mark_request_commit(req, NULL, cinfo, 0); } @@ -731,6 +742,22 @@ static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq) nfs_direct_write_reschedule(dreq); } +static void nfs_direct_write_clear_reqs(struct nfs_direct_req *dreq) +{ + struct nfs_commit_info cinfo; + struct nfs_page *req; + LIST_HEAD(reqs); + + nfs_init_cinfo_from_dreq(&cinfo, dreq); + nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo); + + while (!list_empty(&reqs)) { + req = nfs_list_entry(reqs.next); + nfs_list_remove_request(req); + nfs_unlock_and_release_request(req); + } +} + static void nfs_direct_write_schedule_work(struct work_struct *work) { struct nfs_direct_req *dreq = container_of(work, struct nfs_direct_req, work); @@ -745,6 +772,7 @@ static void nfs_direct_write_schedule_work(struct work_struct *work) nfs_direct_write_reschedule(dreq); break; default: + nfs_direct_write_clear_reqs(dreq); nfs_zap_mapping(dreq->inode, dreq->inode->i_mapping); nfs_direct_complete(dreq); } From 1f28476dcb98797e838a0c1dd6eae2fda213dd81 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Sat, 21 Mar 2020 09:27:46 -0400 Subject: [PATCH 277/635] NFS: Fix O_DIRECT commit verifier handling Instead of trying to save the commit verifiers and checking them against previous writes, adopt the same strategy as for buffered writes, of just checking the verifiers at commit time. Signed-off-by: Trond Myklebust --- fs/nfs/direct.c | 135 +++++----------------------------------------- fs/nfs/internal.h | 8 +++ fs/nfs/write.c | 3 +- 3 files changed, 22 insertions(+), 124 deletions(-) diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c index f7bf1181b690..4ee26465b510 100644 --- a/fs/nfs/direct.c +++ b/fs/nfs/direct.c @@ -95,7 +95,6 @@ struct nfs_direct_req { /* for read */ #define NFS_ODIRECT_SHOULD_DIRTY (3) /* dirty user-space page after read */ #define NFS_ODIRECT_DONE INT_MAX /* write verification failed */ - struct nfs_writeverf verf; /* unstable write verifier */ }; static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops; @@ -152,106 +151,6 @@ nfs_direct_count_bytes(struct nfs_direct_req *dreq, dreq->count = dreq_len; } -/* - * nfs_direct_select_verf - select the right verifier - * @dreq - direct request possibly spanning multiple servers - * @ds_clp - nfs_client of data server or NULL if MDS / non-pnfs - * @commit_idx - commit bucket index for the DS - * - * returns the correct verifier to use given the role of the server - */ -static struct nfs_writeverf * -nfs_direct_select_verf(struct nfs_direct_req *dreq, - struct nfs_client *ds_clp, - int commit_idx) -{ - struct nfs_writeverf *verfp = &dreq->verf; - -#ifdef CONFIG_NFS_V4_1 - /* - * pNFS is in use, use the DS verf except commit_through_mds is set - * for layout segment where nbuckets is zero. - */ - if (ds_clp && dreq->ds_cinfo.nbuckets > 0) { - if (commit_idx >= 0 && commit_idx < dreq->ds_cinfo.nbuckets) - verfp = &dreq->ds_cinfo.buckets[commit_idx].direct_verf; - else - WARN_ON_ONCE(1); - } -#endif - return verfp; -} - - -/* - * nfs_direct_set_hdr_verf - set the write/commit verifier - * @dreq - direct request possibly spanning multiple servers - * @hdr - pageio header to validate against previously seen verfs - * - * Set the server's (MDS or DS) "seen" verifier - */ -static void nfs_direct_set_hdr_verf(struct nfs_direct_req *dreq, - struct nfs_pgio_header *hdr) -{ - struct nfs_writeverf *verfp; - - verfp = nfs_direct_select_verf(dreq, hdr->ds_clp, hdr->ds_commit_idx); - WARN_ON_ONCE(verfp->committed >= 0); - memcpy(verfp, &hdr->verf, sizeof(struct nfs_writeverf)); - WARN_ON_ONCE(verfp->committed < 0); -} - -static int nfs_direct_cmp_verf(const struct nfs_writeverf *v1, - const struct nfs_writeverf *v2) -{ - return nfs_write_verifier_cmp(&v1->verifier, &v2->verifier); -} - -/* - * nfs_direct_cmp_hdr_verf - compare verifier for pgio header - * @dreq - direct request possibly spanning multiple servers - * @hdr - pageio header to validate against previously seen verf - * - * set the server's "seen" verf if not initialized. - * returns result of comparison between @hdr->verf and the "seen" - * verf of the server used by @hdr (DS or MDS) - */ -static int nfs_direct_set_or_cmp_hdr_verf(struct nfs_direct_req *dreq, - struct nfs_pgio_header *hdr) -{ - struct nfs_writeverf *verfp; - - verfp = nfs_direct_select_verf(dreq, hdr->ds_clp, hdr->ds_commit_idx); - if (verfp->committed < 0) { - nfs_direct_set_hdr_verf(dreq, hdr); - return 0; - } - return nfs_direct_cmp_verf(verfp, &hdr->verf); -} - -/* - * nfs_direct_cmp_commit_data_verf - compare verifier for commit data - * @dreq - direct request possibly spanning multiple servers - * @data - commit data to validate against previously seen verf - * - * returns result of comparison between @data->verf and the verf of - * the server used by @data (DS or MDS) - */ -static int nfs_direct_cmp_commit_data_verf(struct nfs_direct_req *dreq, - struct nfs_commit_data *data) -{ - struct nfs_writeverf *verfp; - - verfp = nfs_direct_select_verf(dreq, data->ds_clp, - data->ds_commit_index); - - /* verifier not set so always fail */ - if (verfp->committed < 0 || data->res.verf->committed <= NFS_UNSTABLE) - return 1; - - return nfs_direct_cmp_verf(verfp, data->res.verf); -} - /** * nfs_direct_IO - NFS address space operation for direct I/O * @iocb: target I/O control block @@ -307,7 +206,6 @@ static inline struct nfs_direct_req *nfs_direct_req_alloc(void) init_completion(&dreq->completion); INIT_LIST_HEAD(&dreq->mds_cinfo.list); pnfs_init_ds_commit_info(&dreq->ds_cinfo); - dreq->verf.committed = NFS_INVALID_STABLE_HOW; /* not set yet */ INIT_WORK(&dreq->work, nfs_direct_write_schedule_work); spin_lock_init(&dreq->lock); @@ -637,7 +535,6 @@ static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq) dreq->max_count = 0; list_for_each_entry(req, &reqs, wb_list) dreq->max_count += req->wb_bytes; - dreq->verf.committed = NFS_INVALID_STABLE_HOW; nfs_clear_pnfs_ds_commit_verifiers(&dreq->ds_cinfo); get_dreq(dreq); @@ -674,6 +571,7 @@ static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq) static void nfs_direct_commit_complete(struct nfs_commit_data *data) { + const struct nfs_writeverf *verf = data->res.verf; struct nfs_direct_req *dreq = data->dreq; struct nfs_commit_info cinfo; struct nfs_page *req; @@ -689,21 +587,19 @@ static void nfs_direct_commit_complete(struct nfs_commit_data *data) status = dreq->error; nfs_init_cinfo_from_dreq(&cinfo, dreq); - if (nfs_direct_cmp_commit_data_verf(dreq, data)) - dreq->flags = NFS_ODIRECT_RESCHED_WRITES; while (!list_empty(&data->pages)) { req = nfs_list_entry(data->pages.next); nfs_list_remove_request(req); - if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES) { + if (status >= 0 && !nfs_write_match_verf(verf, req)) { + dreq->flags = NFS_ODIRECT_RESCHED_WRITES; /* * Despite the reboot, the write was successful, * so reset wb_nio. */ req->wb_nio = 0; - /* Note the rewrite will go through mds */ nfs_mark_request_commit(req, NULL, &cinfo, 0); - } else + } else /* Error or match */ nfs_release_request(req); nfs_unlock_and_release_request(req); } @@ -799,20 +695,15 @@ static void nfs_direct_write_completion(struct nfs_pgio_header *hdr) } nfs_direct_count_bytes(dreq, hdr); - if (hdr->good_bytes != 0) { - if (nfs_write_need_commit(hdr)) { - if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES) - request_commit = true; - else if (dreq->flags == 0) { - nfs_direct_set_hdr_verf(dreq, hdr); - request_commit = true; - dreq->flags = NFS_ODIRECT_DO_COMMIT; - } else if (dreq->flags == NFS_ODIRECT_DO_COMMIT) { - request_commit = true; - if (nfs_direct_set_or_cmp_hdr_verf(dreq, hdr)) - dreq->flags = - NFS_ODIRECT_RESCHED_WRITES; - } + if (hdr->good_bytes != 0 && nfs_write_need_commit(hdr)) { + switch (dreq->flags) { + case 0: + dreq->flags = NFS_ODIRECT_DO_COMMIT; + request_commit = true; + break; + case NFS_ODIRECT_RESCHED_WRITES: + case NFS_ODIRECT_DO_COMMIT: + request_commit = true; } } spin_unlock(&dreq->lock); diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h index 3b6fa9edc9b5..6542411c020f 100644 --- a/fs/nfs/internal.h +++ b/fs/nfs/internal.h @@ -544,6 +544,14 @@ nfs_write_verifier_cmp(const struct nfs_write_verifier *v1, return memcmp(v1->data, v2->data, sizeof(v1->data)); } +static inline bool +nfs_write_match_verf(const struct nfs_writeverf *verf, + struct nfs_page *req) +{ + return verf->committed > NFS_UNSTABLE && + !nfs_write_verifier_cmp(&req->wb_verf, &verf->verifier); +} + /* unlink.c */ extern struct rpc_task * nfs_async_rename(struct inode *old_dir, struct inode *new_dir, diff --git a/fs/nfs/write.c b/fs/nfs/write.c index 1f8108f5a041..03b7f64f7c4f 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -1874,8 +1874,7 @@ static void nfs_commit_release_pages(struct nfs_commit_data *data) /* Okay, COMMIT succeeded, apparently. Check the verifier * returned by the server against all stored verfs. */ - if (verf->committed > NFS_UNSTABLE && - !nfs_write_verifier_cmp(&req->wb_verf, &verf->verifier)) { + if (nfs_write_match_verf(verf, req)) { /* We have a match */ if (req->wb_page) nfs_inode_remove_request(req); From e3b9f7e60b71d3a652ab80665e724d22d61dd629 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Fri, 20 Mar 2020 19:06:48 -0400 Subject: [PATCH 278/635] NFS/pNFS: Support commit arrays in nfs_clear_pnfs_ds_commit_verifiers() Add support for scanning the full list of per-layout segment commit arrays to nfs_clear_pnfs_ds_commit_verifiers(). Signed-off-by: Trond Myklebust --- fs/nfs/internal.h | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h index 6542411c020f..4a1adad3740f 100644 --- a/fs/nfs/internal.h +++ b/fs/nfs/internal.h @@ -517,13 +517,26 @@ int nfs_filemap_write_and_wait_range(struct address_space *mapping, loff_t lstart, loff_t lend); #ifdef CONFIG_NFS_V4_1 +static inline void +pnfs_bucket_clear_pnfs_ds_commit_verifiers(struct pnfs_commit_bucket *buckets, + unsigned int nbuckets) +{ + unsigned int i; + + for (i = 0; i < nbuckets; i++) + buckets[i].direct_verf.committed = NFS_INVALID_STABLE_HOW; +} static inline void nfs_clear_pnfs_ds_commit_verifiers(struct pnfs_ds_commit_info *cinfo) { - int i; + struct pnfs_commit_array *array; - for (i = 0; i < cinfo->nbuckets; i++) - cinfo->buckets[i].direct_verf.committed = NFS_INVALID_STABLE_HOW; + pnfs_bucket_clear_pnfs_ds_commit_verifiers(cinfo->buckets, + cinfo->nbuckets); + + list_for_each_entry(array, &cinfo->commits, cinfo_list) + pnfs_bucket_clear_pnfs_ds_commit_verifiers(array->buckets, + array->nbuckets); } #else static inline From a9901899b649dc80ef75c14d6d78059cae14def7 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Fri, 20 Mar 2020 16:04:06 -0400 Subject: [PATCH 279/635] pNFS: Add infrastructure for cleaning up per-layout commit structures Ensure that both the file and flexfiles layout types clean up when freeing the layout segments. Signed-off-by: Trond Myklebust --- fs/nfs/filelayout/filelayout.c | 16 +++++ fs/nfs/flexfilelayout/flexfilelayout.c | 11 ++++ fs/nfs/internal.h | 4 +- fs/nfs/pnfs.c | 1 + fs/nfs/pnfs.h | 4 ++ fs/nfs/pnfs_nfs.c | 88 +++++++++++++++++++++++++- include/linux/nfs_xdr.h | 1 + 7 files changed, 121 insertions(+), 4 deletions(-) diff --git a/fs/nfs/filelayout/filelayout.c b/fs/nfs/filelayout/filelayout.c index b051d5d320ba..ffc5e2af1776 100644 --- a/fs/nfs/filelayout/filelayout.c +++ b/fs/nfs/filelayout/filelayout.c @@ -750,11 +750,16 @@ filelayout_free_lseg(struct pnfs_layout_segment *lseg) /* This assumes a single RW lseg */ if (lseg->pls_range.iomode == IOMODE_RW) { struct nfs4_filelayout *flo; + struct inode *inode; flo = FILELAYOUT_FROM_HDR(lseg->pls_layout); + inode = flo->generic_hdr.plh_inode; + spin_lock(&inode->i_lock); flo->commit_info.nbuckets = 0; kfree(flo->commit_info.buckets); flo->commit_info.buckets = NULL; + pnfs_generic_ds_cinfo_release_lseg(&flo->commit_info, lseg); + spin_unlock(&inode->i_lock); } _filelayout_free_lseg(fl); } @@ -1163,6 +1168,16 @@ filelayout_get_ds_info(struct inode *inode) return &FILELAYOUT_FROM_HDR(layout)->commit_info; } +static void +filelayout_release_ds_info(struct pnfs_ds_commit_info *fl_cinfo, + struct inode *inode) +{ + spin_lock(&inode->i_lock); + pnfs_generic_ds_cinfo_destroy(fl_cinfo); + spin_unlock(&inode->i_lock); +} + + static struct pnfs_layoutdriver_type filelayout_type = { .id = LAYOUT_NFSV4_1_FILES, .name = "LAYOUT_NFSV4_1_FILES", @@ -1176,6 +1191,7 @@ static struct pnfs_layoutdriver_type filelayout_type = { .pg_read_ops = &filelayout_pg_read_ops, .pg_write_ops = &filelayout_pg_write_ops, .get_ds_info = &filelayout_get_ds_info, + .release_ds_info = filelayout_release_ds_info, .mark_request_commit = filelayout_mark_request_commit, .clear_request_commit = pnfs_generic_clear_request_commit, .scan_commit_lists = pnfs_generic_scan_commit_lists, diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c index c9e79c8e62cd..8e1393d75cbc 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.c +++ b/fs/nfs/flexfilelayout/flexfilelayout.c @@ -580,6 +580,7 @@ ff_layout_free_lseg(struct pnfs_layout_segment *lseg) kfree(ffl->commit_info.buckets); ffl->commit_info.buckets = NULL; } + pnfs_generic_ds_cinfo_release_lseg(&ffl->commit_info, lseg); spin_unlock(&inode->i_lock); } _ff_layout_free_lseg(fls); @@ -2003,6 +2004,15 @@ ff_layout_get_ds_info(struct inode *inode) return &FF_LAYOUT_FROM_HDR(layout)->commit_info; } +static void +ff_layout_release_ds_info(struct pnfs_ds_commit_info *fl_cinfo, + struct inode *inode) +{ + spin_lock(&inode->i_lock); + pnfs_generic_ds_cinfo_destroy(fl_cinfo); + spin_unlock(&inode->i_lock); +} + static void ff_layout_free_deviceid_node(struct nfs4_deviceid_node *d) { @@ -2503,6 +2513,7 @@ static struct pnfs_layoutdriver_type flexfilelayout_type = { .pg_read_ops = &ff_layout_pg_read_ops, .pg_write_ops = &ff_layout_pg_write_ops, .get_ds_info = ff_layout_get_ds_info, + .release_ds_info = ff_layout_release_ds_info, .free_deviceid_node = ff_layout_free_deviceid_node, .mark_request_commit = pnfs_layout_mark_request_commit, .clear_request_commit = pnfs_generic_clear_request_commit, diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h index 4a1adad3740f..683146a51599 100644 --- a/fs/nfs/internal.h +++ b/fs/nfs/internal.h @@ -534,9 +534,11 @@ void nfs_clear_pnfs_ds_commit_verifiers(struct pnfs_ds_commit_info *cinfo) pnfs_bucket_clear_pnfs_ds_commit_verifiers(cinfo->buckets, cinfo->nbuckets); - list_for_each_entry(array, &cinfo->commits, cinfo_list) + rcu_read_lock(); + list_for_each_entry_rcu(array, &cinfo->commits, cinfo_list) pnfs_bucket_clear_pnfs_ds_commit_verifiers(array->buckets, array->nbuckets); + rcu_read_unlock(); } #else static inline diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index 6b25117fca5f..eba18f137fb0 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -506,6 +506,7 @@ pnfs_init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg, { INIT_LIST_HEAD(&lseg->pls_list); INIT_LIST_HEAD(&lseg->pls_lc_list); + INIT_LIST_HEAD(&lseg->pls_commits); refcount_set(&lseg->pls_refcount, 1); set_bit(NFS_LSEG_VALID, &lseg->pls_flags); lseg->pls_layout = lo; diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h index 2ec97b419b56..6c48bd7b4640 100644 --- a/fs/nfs/pnfs.h +++ b/fs/nfs/pnfs.h @@ -66,6 +66,7 @@ struct nfs4_pnfs_ds { struct pnfs_layout_segment { struct list_head pls_list; struct list_head pls_lc_list; + struct list_head pls_commits; struct pnfs_layout_range pls_range; refcount_t pls_refcount; u32 pls_seq; @@ -370,6 +371,9 @@ void nfs4_deviceid_purge_client(const struct nfs_client *); /* pnfs_nfs.c */ struct pnfs_commit_array *pnfs_alloc_commit_array(size_t n, gfp_t gfp_flags); void pnfs_free_commit_array(struct pnfs_commit_array *p); +void pnfs_generic_ds_cinfo_release_lseg(struct pnfs_ds_commit_info *fl_cinfo, + struct pnfs_layout_segment *lseg); +void pnfs_generic_ds_cinfo_destroy(struct pnfs_ds_commit_info *fl_cinfo); void pnfs_generic_clear_request_commit(struct nfs_page *req, struct nfs_commit_info *cinfo); diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c index f895a28b1e26..edad251a6a48 100644 --- a/fs/nfs/pnfs_nfs.c +++ b/fs/nfs/pnfs_nfs.c @@ -118,6 +118,67 @@ pnfs_free_commit_array(struct pnfs_commit_array *p) } EXPORT_SYMBOL_GPL(pnfs_free_commit_array); +static void +pnfs_release_commit_array_locked(struct pnfs_commit_array *array) +{ + list_del_rcu(&array->cinfo_list); + list_del(&array->lseg_list); + pnfs_free_commit_array(array); +} + +static void +pnfs_put_commit_array_locked(struct pnfs_commit_array *array) +{ + if (refcount_dec_and_test(&array->refcount)) + pnfs_release_commit_array_locked(array); +} + +static void +pnfs_put_commit_array(struct pnfs_commit_array *array, struct inode *inode) +{ + if (refcount_dec_and_lock(&array->refcount, &inode->i_lock)) { + pnfs_release_commit_array_locked(array); + spin_unlock(&inode->i_lock); + } +} + +static struct pnfs_commit_array * +pnfs_get_commit_array(struct pnfs_commit_array *array) +{ + if (refcount_inc_not_zero(&array->refcount)) + return array; + return NULL; +} + +static void +pnfs_remove_and_free_commit_array(struct pnfs_commit_array *array) +{ + array->lseg = NULL; + list_del_init(&array->lseg_list); + pnfs_put_commit_array_locked(array); +} + +void +pnfs_generic_ds_cinfo_release_lseg(struct pnfs_ds_commit_info *fl_cinfo, + struct pnfs_layout_segment *lseg) +{ + struct pnfs_commit_array *array, *tmp; + + list_for_each_entry_safe(array, tmp, &lseg->pls_commits, lseg_list) + pnfs_remove_and_free_commit_array(array); +} +EXPORT_SYMBOL_GPL(pnfs_generic_ds_cinfo_release_lseg); + +void +pnfs_generic_ds_cinfo_destroy(struct pnfs_ds_commit_info *fl_cinfo) +{ + struct pnfs_commit_array *array, *tmp; + + list_for_each_entry_safe(array, tmp, &fl_cinfo->commits, cinfo_list) + pnfs_remove_and_free_commit_array(array); +} +EXPORT_SYMBOL_GPL(pnfs_generic_ds_cinfo_destroy); + /* * Locks the nfs_page requests for commit and moves them to * @bucket->committing. @@ -177,14 +238,21 @@ int pnfs_generic_scan_commit_lists(struct nfs_commit_info *cinfo, int max) max -= cnt; if (!max) return rv; - list_for_each_entry(array, &fl_cinfo->commits, cinfo_list) { + rcu_read_lock(); + list_for_each_entry_rcu(array, &fl_cinfo->commits, cinfo_list) { + if (!array->lseg || !pnfs_get_commit_array(array)) + continue; + rcu_read_unlock(); cnt = pnfs_bucket_scan_array(cinfo, array->buckets, array->nbuckets, max); + rcu_read_lock(); + pnfs_put_commit_array(array, cinfo->inode); rv += cnt; max -= cnt; if (!max) break; } + rcu_read_unlock(); return rv; } EXPORT_SYMBOL_GPL(pnfs_generic_scan_commit_lists); @@ -230,13 +298,20 @@ void pnfs_generic_recover_commit_reqs(struct list_head *dst, fl_cinfo->nbuckets, cinfo); fl_cinfo->nwritten -= nwritten; - list_for_each_entry(array, &fl_cinfo->commits, cinfo_list) { + rcu_read_lock(); + list_for_each_entry_rcu(array, &fl_cinfo->commits, cinfo_list) { + if (!array->lseg || !pnfs_get_commit_array(array)) + continue; + rcu_read_unlock(); nwritten = pnfs_bucket_recover_commit_reqs(dst, array->buckets, array->nbuckets, cinfo); + rcu_read_lock(); + pnfs_put_commit_array(array, cinfo->inode); fl_cinfo->nwritten -= nwritten; } + rcu_read_unlock(); } EXPORT_SYMBOL_GPL(pnfs_generic_recover_commit_reqs); @@ -330,9 +405,16 @@ pnfs_alloc_ds_commits_list(struct list_head *list, struct pnfs_commit_array *array; unsigned int ret = 0; - list_for_each_entry(array, &fl_cinfo->commits, cinfo_list) + rcu_read_lock(); + list_for_each_entry_rcu(array, &fl_cinfo->commits, cinfo_list) { + if (!array->lseg || !pnfs_get_commit_array(array)) + continue; + rcu_read_unlock(); ret += pnfs_bucket_alloc_ds_commits(list, array->buckets, array->nbuckets, cinfo); + rcu_read_lock(); + pnfs_put_commit_array(array, cinfo->inode); + } return ret; } diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index 9946787eda72..33be2ee2a248 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -1275,6 +1275,7 @@ struct pnfs_commit_array { struct list_head lseg_list; struct pnfs_layout_segment *lseg; struct rcu_head rcu; + refcount_t refcount; unsigned int nbuckets; struct pnfs_commit_bucket buckets[]; }; From ba827c9abb0b413a4c87b68c87d39b310fc01101 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Fri, 20 Mar 2020 18:34:33 -0400 Subject: [PATCH 280/635] pNFS: Enable per-layout segment commit structures Enable adding and lookup of per-layout segment commits in filelayout and flexfilelayout. Signed-off-by: Trond Myklebust --- fs/nfs/filelayout/filelayout.c | 21 +++++++ fs/nfs/flexfilelayout/flexfilelayout.c | 19 +++++++ fs/nfs/pnfs.h | 6 ++ fs/nfs/pnfs_nfs.c | 77 ++++++++++++++++++++++++-- 4 files changed, 117 insertions(+), 6 deletions(-) diff --git a/fs/nfs/filelayout/filelayout.c b/fs/nfs/filelayout/filelayout.c index ffc5e2af1776..e3cf42c91d80 100644 --- a/fs/nfs/filelayout/filelayout.c +++ b/fs/nfs/filelayout/filelayout.c @@ -1168,6 +1168,26 @@ filelayout_get_ds_info(struct inode *inode) return &FILELAYOUT_FROM_HDR(layout)->commit_info; } +static void +filelayout_setup_ds_info(struct pnfs_ds_commit_info *fl_cinfo, + struct pnfs_layout_segment *lseg) +{ + struct nfs4_filelayout_segment *fl = FILELAYOUT_LSEG(lseg); + struct inode *inode = lseg->pls_layout->plh_inode; + struct pnfs_commit_array *array, *new; + unsigned int size = (fl->stripe_type == STRIPE_SPARSE) ? + fl->dsaddr->ds_num : fl->dsaddr->stripe_count; + + new = pnfs_alloc_commit_array(size, GFP_NOIO); + if (new) { + spin_lock(&inode->i_lock); + array = pnfs_add_commit_array(fl_cinfo, new, lseg); + spin_unlock(&inode->i_lock); + if (array != new) + pnfs_free_commit_array(new); + } +} + static void filelayout_release_ds_info(struct pnfs_ds_commit_info *fl_cinfo, struct inode *inode) @@ -1191,6 +1211,7 @@ static struct pnfs_layoutdriver_type filelayout_type = { .pg_read_ops = &filelayout_pg_read_ops, .pg_write_ops = &filelayout_pg_write_ops, .get_ds_info = &filelayout_get_ds_info, + .setup_ds_info = filelayout_setup_ds_info, .release_ds_info = filelayout_release_ds_info, .mark_request_commit = filelayout_mark_request_commit, .clear_request_commit = pnfs_generic_clear_request_commit, diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c index 8e1393d75cbc..f343a241906a 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.c +++ b/fs/nfs/flexfilelayout/flexfilelayout.c @@ -2004,6 +2004,24 @@ ff_layout_get_ds_info(struct inode *inode) return &FF_LAYOUT_FROM_HDR(layout)->commit_info; } +static void +ff_layout_setup_ds_info(struct pnfs_ds_commit_info *fl_cinfo, + struct pnfs_layout_segment *lseg) +{ + struct nfs4_ff_layout_segment *flseg = FF_LAYOUT_LSEG(lseg); + struct inode *inode = lseg->pls_layout->plh_inode; + struct pnfs_commit_array *array, *new; + + new = pnfs_alloc_commit_array(flseg->mirror_array_cnt, GFP_NOIO); + if (new) { + spin_lock(&inode->i_lock); + array = pnfs_add_commit_array(fl_cinfo, new, lseg); + spin_unlock(&inode->i_lock); + if (array != new) + pnfs_free_commit_array(new); + } +} + static void ff_layout_release_ds_info(struct pnfs_ds_commit_info *fl_cinfo, struct inode *inode) @@ -2513,6 +2531,7 @@ static struct pnfs_layoutdriver_type flexfilelayout_type = { .pg_read_ops = &ff_layout_pg_read_ops, .pg_write_ops = &ff_layout_pg_write_ops, .get_ds_info = ff_layout_get_ds_info, + .setup_ds_info = ff_layout_setup_ds_info, .release_ds_info = ff_layout_release_ds_info, .free_deviceid_node = ff_layout_free_deviceid_node, .mark_request_commit = pnfs_layout_mark_request_commit, diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h index 6c48bd7b4640..9647045a60c2 100644 --- a/fs/nfs/pnfs.h +++ b/fs/nfs/pnfs.h @@ -150,6 +150,8 @@ struct pnfs_layoutdriver_type { const struct nfs_pageio_ops *pg_write_ops; struct pnfs_ds_commit_info *(*get_ds_info) (struct inode *inode); + void (*setup_ds_info)(struct pnfs_ds_commit_info *, + struct pnfs_layout_segment *); void (*release_ds_info)(struct pnfs_ds_commit_info *, struct inode *inode); void (*mark_request_commit) (struct nfs_page *req, @@ -371,6 +373,10 @@ void nfs4_deviceid_purge_client(const struct nfs_client *); /* pnfs_nfs.c */ struct pnfs_commit_array *pnfs_alloc_commit_array(size_t n, gfp_t gfp_flags); void pnfs_free_commit_array(struct pnfs_commit_array *p); +struct pnfs_commit_array *pnfs_add_commit_array(struct pnfs_ds_commit_info *, + struct pnfs_commit_array *, + struct pnfs_layout_segment *); + void pnfs_generic_ds_cinfo_release_lseg(struct pnfs_ds_commit_info *fl_cinfo, struct pnfs_layout_segment *lseg); void pnfs_generic_ds_cinfo_destroy(struct pnfs_ds_commit_info *fl_cinfo); diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c index edad251a6a48..5b426a090ee3 100644 --- a/fs/nfs/pnfs_nfs.c +++ b/fs/nfs/pnfs_nfs.c @@ -118,6 +118,66 @@ pnfs_free_commit_array(struct pnfs_commit_array *p) } EXPORT_SYMBOL_GPL(pnfs_free_commit_array); +static struct pnfs_commit_array * +pnfs_find_commit_array_by_lseg(struct pnfs_ds_commit_info *fl_cinfo, + struct pnfs_layout_segment *lseg) +{ + struct pnfs_commit_array *array; + + list_for_each_entry_rcu(array, &fl_cinfo->commits, cinfo_list) { + if (array->lseg == lseg) + return array; + } + return NULL; +} + +struct pnfs_commit_array * +pnfs_add_commit_array(struct pnfs_ds_commit_info *fl_cinfo, + struct pnfs_commit_array *new, + struct pnfs_layout_segment *lseg) +{ + struct pnfs_commit_array *array; + + array = pnfs_find_commit_array_by_lseg(fl_cinfo, lseg); + if (array) + return array; + new->lseg = lseg; + refcount_set(&new->refcount, 1); + list_add_rcu(&new->cinfo_list, &fl_cinfo->commits); + list_add(&new->lseg_list, &lseg->pls_commits); + return new; +} +EXPORT_SYMBOL_GPL(pnfs_add_commit_array); + +static void +pnfs_setup_ds_info(struct pnfs_ds_commit_info *fl_cinfo, + struct pnfs_layout_segment *lseg) +{ + struct inode *inode = lseg->pls_layout->plh_inode; + struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; + + if (ld->setup_ds_info != NULL) + ld->setup_ds_info(fl_cinfo, lseg); +} + +static struct pnfs_commit_array * +pnfs_lookup_commit_array(struct pnfs_ds_commit_info *fl_cinfo, + struct pnfs_layout_segment *lseg) +{ + struct pnfs_commit_array *array; + + rcu_read_lock(); + array = pnfs_find_commit_array_by_lseg(fl_cinfo, lseg); + if (!array) { + rcu_read_unlock(); + pnfs_setup_ds_info(fl_cinfo, lseg); + rcu_read_lock(); + array = pnfs_find_commit_array_by_lseg(fl_cinfo, lseg); + } + rcu_read_unlock(); + return array; +} + static void pnfs_release_commit_array_locked(struct pnfs_commit_array *array) { @@ -1082,17 +1142,18 @@ pnfs_layout_mark_request_commit(struct nfs_page *req, u32 ds_commit_idx) { struct list_head *list; + struct pnfs_commit_array *array; struct pnfs_commit_bucket *buckets; mutex_lock(&NFS_I(cinfo->inode)->commit_mutex); - buckets = cinfo->ds->buckets; + array = pnfs_lookup_commit_array(cinfo->ds, lseg); + if (!array) + goto out_resched; + buckets = array->buckets; list = &buckets[ds_commit_idx].written; if (list_empty(list)) { - if (!pnfs_is_valid_lseg(lseg)) { - mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex); - cinfo->completion_ops->resched_write(cinfo, req); - return; - } + if (!pnfs_is_valid_lseg(lseg)) + goto out_resched; /* Non-empty buckets hold a reference on the lseg. That ref * is normally transferred to the COMMIT call and released * there. It could also be released if the last req is pulled @@ -1108,6 +1169,10 @@ pnfs_layout_mark_request_commit(struct nfs_page *req, nfs_request_add_commit_list_locked(req, list, cinfo); mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex); nfs_mark_page_unstable(req->wb_page, cinfo); + return; +out_resched: + mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex); + cinfo->completion_ops->resched_write(cinfo, req); } EXPORT_SYMBOL_GPL(pnfs_layout_mark_request_commit); From fb6b53ba40a7c51a5347aeabaa32546efae7fba4 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Fri, 20 Mar 2020 19:24:19 -0400 Subject: [PATCH 281/635] NFS/pNFS: Add a helper pnfs_generic_search_commit_reqs() Lift filelayout_search_commit_reqs() into the generic pnfs/nfs code, and add support for commit arrays. Signed-off-by: Trond Myklebust --- fs/nfs/filelayout/filelayout.c | 32 +-------------------- fs/nfs/pnfs.h | 2 ++ fs/nfs/pnfs_nfs.c | 51 ++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 31 deletions(-) diff --git a/fs/nfs/filelayout/filelayout.c b/fs/nfs/filelayout/filelayout.c index e3cf42c91d80..795508054a4d 100644 --- a/fs/nfs/filelayout/filelayout.c +++ b/fs/nfs/filelayout/filelayout.c @@ -1083,36 +1083,6 @@ out_err: return -EAGAIN; } -/* filelayout_search_commit_reqs - Search lists in @cinfo for the head reqest - * for @page - * @cinfo - commit info for current inode - * @page - page to search for matching head request - * - * Returns a the head request if one is found, otherwise returns NULL. - */ -static struct nfs_page * -filelayout_search_commit_reqs(struct nfs_commit_info *cinfo, struct page *page) -{ - struct nfs_page *freq, *t; - struct pnfs_commit_bucket *b; - int i; - - /* Linearly search the commit lists for each bucket until a matching - * request is found */ - for (i = 0, b = cinfo->ds->buckets; i < cinfo->ds->nbuckets; i++, b++) { - list_for_each_entry_safe(freq, t, &b->written, wb_list) { - if (freq->wb_page == page) - return freq->wb_head; - } - list_for_each_entry_safe(freq, t, &b->committing, wb_list) { - if (freq->wb_page == page) - return freq->wb_head; - } - } - - return NULL; -} - static int filelayout_commit_pagelist(struct inode *inode, struct list_head *mds_pages, int how, struct nfs_commit_info *cinfo) @@ -1217,7 +1187,7 @@ static struct pnfs_layoutdriver_type filelayout_type = { .clear_request_commit = pnfs_generic_clear_request_commit, .scan_commit_lists = pnfs_generic_scan_commit_lists, .recover_commit_reqs = pnfs_generic_recover_commit_reqs, - .search_commit_reqs = filelayout_search_commit_reqs, + .search_commit_reqs = pnfs_generic_search_commit_reqs, .commit_pagelist = filelayout_commit_pagelist, .read_pagelist = filelayout_read_pagelist, .write_pagelist = filelayout_write_pagelist, diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h index 9647045a60c2..faed9be6e479 100644 --- a/fs/nfs/pnfs.h +++ b/fs/nfs/pnfs.h @@ -388,6 +388,8 @@ void pnfs_generic_prepare_to_resend_writes(struct nfs_commit_data *data); void pnfs_generic_rw_release(void *data); void pnfs_generic_recover_commit_reqs(struct list_head *dst, struct nfs_commit_info *cinfo); +struct nfs_page *pnfs_generic_search_commit_reqs(struct nfs_commit_info *cinfo, + struct page *page); int pnfs_generic_commit_pagelist(struct inode *inode, struct list_head *mds_pages, int how, diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c index 5b426a090ee3..9b55919e64ac 100644 --- a/fs/nfs/pnfs_nfs.c +++ b/fs/nfs/pnfs_nfs.c @@ -375,6 +375,57 @@ void pnfs_generic_recover_commit_reqs(struct list_head *dst, } EXPORT_SYMBOL_GPL(pnfs_generic_recover_commit_reqs); +static struct nfs_page * +pnfs_bucket_search_commit_reqs(struct pnfs_commit_bucket *buckets, + unsigned int nbuckets, struct page *page) +{ + struct nfs_page *req; + struct pnfs_commit_bucket *b; + unsigned int i; + + /* Linearly search the commit lists for each bucket until a matching + * request is found */ + for (i = 0, b = buckets; i < nbuckets; i++, b++) { + list_for_each_entry(req, &b->written, wb_list) { + if (req->wb_page == page) + return req->wb_head; + } + list_for_each_entry(req, &b->committing, wb_list) { + if (req->wb_page == page) + return req->wb_head; + } + } + return NULL; +} + +/* pnfs_generic_search_commit_reqs - Search lists in @cinfo for the head reqest + * for @page + * @cinfo - commit info for current inode + * @page - page to search for matching head request + * + * Returns a the head request if one is found, otherwise returns NULL. + */ +struct nfs_page * +pnfs_generic_search_commit_reqs(struct nfs_commit_info *cinfo, struct page *page) +{ + struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds; + struct pnfs_commit_array *array; + struct nfs_page *req; + + req = pnfs_bucket_search_commit_reqs(fl_cinfo->buckets, + fl_cinfo->nbuckets, page); + if (req) + return req; + list_for_each_entry(array, &fl_cinfo->commits, cinfo_list) { + req = pnfs_bucket_search_commit_reqs(array->buckets, + array->nbuckets, page); + if (req) + return req; + } + return NULL; +} +EXPORT_SYMBOL_GPL(pnfs_generic_search_commit_reqs); + static struct pnfs_layout_segment * pnfs_bucket_get_committing(struct list_head *head, struct pnfs_commit_bucket *bucket, From 0aa647b7369dd29de0789c321111b2e4668c46b2 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Sat, 21 Mar 2020 09:50:05 -0400 Subject: [PATCH 282/635] NFS: Remove bucket array from struct pnfs_ds_commit_info Remove the unused bucket array in struct pnfs_ds_commit_info. Signed-off-by: Trond Myklebust --- fs/nfs/direct.c | 1 - fs/nfs/filelayout/filelayout.c | 75 +------------------------ fs/nfs/flexfilelayout/flexfilelayout.c | 76 -------------------------- fs/nfs/internal.h | 3 - fs/nfs/pnfs_nfs.c | 18 ------ include/linux/nfs_xdr.h | 13 ----- 6 files changed, 1 insertion(+), 185 deletions(-) diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c index 4ee26465b510..61f93a0fb0e0 100644 --- a/fs/nfs/direct.c +++ b/fs/nfs/direct.c @@ -217,7 +217,6 @@ static void nfs_direct_req_free(struct kref *kref) struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref); pnfs_release_ds_info(&dreq->ds_cinfo, dreq->inode); - nfs_free_pnfs_ds_cinfo(&dreq->ds_cinfo); if (dreq->l_ctx != NULL) nfs_put_lock_context(dreq->l_ctx); if (dreq->ctx != NULL) diff --git a/fs/nfs/filelayout/filelayout.c b/fs/nfs/filelayout/filelayout.c index 795508054a4d..854f350e2599 100644 --- a/fs/nfs/filelayout/filelayout.c +++ b/fs/nfs/filelayout/filelayout.c @@ -755,72 +755,12 @@ filelayout_free_lseg(struct pnfs_layout_segment *lseg) flo = FILELAYOUT_FROM_HDR(lseg->pls_layout); inode = flo->generic_hdr.plh_inode; spin_lock(&inode->i_lock); - flo->commit_info.nbuckets = 0; - kfree(flo->commit_info.buckets); - flo->commit_info.buckets = NULL; pnfs_generic_ds_cinfo_release_lseg(&flo->commit_info, lseg); spin_unlock(&inode->i_lock); } _filelayout_free_lseg(fl); } -static int -filelayout_alloc_commit_info(struct pnfs_layout_segment *lseg, - struct nfs_commit_info *cinfo, - gfp_t gfp_flags) -{ - struct nfs4_filelayout_segment *fl = FILELAYOUT_LSEG(lseg); - struct pnfs_commit_bucket *buckets; - int size, i; - - if (fl->commit_through_mds) - return 0; - - size = (fl->stripe_type == STRIPE_SPARSE) ? - fl->dsaddr->ds_num : fl->dsaddr->stripe_count; - - if (cinfo->ds->nbuckets >= size) { - /* This assumes there is only one IOMODE_RW lseg. What - * we really want to do is have a layout_hdr level - * dictionary of keys, each - * associated with a struct list_head, populated by calls - * to filelayout_write_pagelist(). - * */ - return 0; - } - - buckets = kcalloc(size, sizeof(struct pnfs_commit_bucket), - gfp_flags); - if (!buckets) - return -ENOMEM; - for (i = 0; i < size; i++) { - INIT_LIST_HEAD(&buckets[i].written); - INIT_LIST_HEAD(&buckets[i].committing); - /* mark direct verifier as unset */ - buckets[i].direct_verf.committed = NFS_INVALID_STABLE_HOW; - } - - spin_lock(&cinfo->inode->i_lock); - if (cinfo->ds->nbuckets >= size) - goto out; - for (i = 0; i < cinfo->ds->nbuckets; i++) { - list_splice(&cinfo->ds->buckets[i].written, - &buckets[i].written); - list_splice(&cinfo->ds->buckets[i].committing, - &buckets[i].committing); - buckets[i].direct_verf.committed = - cinfo->ds->buckets[i].direct_verf.committed; - buckets[i].wlseg = cinfo->ds->buckets[i].wlseg; - buckets[i].clseg = cinfo->ds->buckets[i].clseg; - } - swap(cinfo->ds->buckets, buckets); - cinfo->ds->nbuckets = size; -out: - spin_unlock(&cinfo->inode->i_lock); - kfree(buckets); - return 0; -} - static struct pnfs_layout_segment * filelayout_alloc_lseg(struct pnfs_layout_hdr *layoutid, struct nfs4_layoutget_res *lgr, @@ -943,9 +883,6 @@ static void filelayout_pg_init_write(struct nfs_pageio_descriptor *pgio, struct nfs_page *req) { - struct nfs_commit_info cinfo; - int status; - pnfs_generic_pg_check_layout(pgio); if (!pgio->pg_lseg) { pgio->pg_lseg = fl_pnfs_update_layout(pgio->pg_inode, @@ -964,17 +901,7 @@ filelayout_pg_init_write(struct nfs_pageio_descriptor *pgio, /* If no lseg, fall back to write through mds */ if (pgio->pg_lseg == NULL) - goto out_mds; - nfs_init_cinfo(&cinfo, pgio->pg_inode, pgio->pg_dreq); - status = filelayout_alloc_commit_info(pgio->pg_lseg, &cinfo, GFP_NOFS); - if (status < 0) { - pnfs_put_lseg(pgio->pg_lseg); - pgio->pg_lseg = NULL; - goto out_mds; - } - return; -out_mds: - nfs_pageio_reset_write_mds(pgio); + nfs_pageio_reset_write_mds(pgio); } static const struct nfs_pageio_ops filelayout_pg_read_ops = { diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c index f343a241906a..1a4e36d07eab 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.c +++ b/fs/nfs/flexfilelayout/flexfilelayout.c @@ -550,17 +550,6 @@ out_err_free: goto out_free_page; } -static bool ff_layout_has_rw_segments(struct pnfs_layout_hdr *layout) -{ - struct pnfs_layout_segment *lseg; - - list_for_each_entry(lseg, &layout->plh_segs, pls_list) - if (lseg->pls_range.iomode == IOMODE_RW) - return true; - - return false; -} - static void ff_layout_free_lseg(struct pnfs_layout_segment *lseg) { @@ -575,24 +564,12 @@ ff_layout_free_lseg(struct pnfs_layout_segment *lseg) ffl = FF_LAYOUT_FROM_HDR(lseg->pls_layout); inode = ffl->generic_hdr.plh_inode; spin_lock(&inode->i_lock); - if (!ff_layout_has_rw_segments(lseg->pls_layout)) { - ffl->commit_info.nbuckets = 0; - kfree(ffl->commit_info.buckets); - ffl->commit_info.buckets = NULL; - } pnfs_generic_ds_cinfo_release_lseg(&ffl->commit_info, lseg); spin_unlock(&inode->i_lock); } _ff_layout_free_lseg(fls); } -/* Return 1 until we have multiple lsegs support */ -static int -ff_layout_get_lseg_count(struct nfs4_ff_layout_segment *fls) -{ - return 1; -} - static void nfs4_ff_start_busy_timer(struct nfs4_ff_busy_timer *timer, ktime_t now) { @@ -737,52 +714,6 @@ nfs4_ff_layout_stat_io_end_write(struct rpc_task *task, spin_unlock(&mirror->lock); } -static int -ff_layout_alloc_commit_info(struct pnfs_layout_segment *lseg, - struct nfs_commit_info *cinfo, - gfp_t gfp_flags) -{ - struct nfs4_ff_layout_segment *fls = FF_LAYOUT_LSEG(lseg); - struct pnfs_commit_bucket *buckets; - int size; - - if (cinfo->ds->nbuckets != 0) { - /* This assumes there is only one RW lseg per file. - * To support multiple lseg per file, we need to - * change struct pnfs_commit_bucket to allow dynamic - * increasing nbuckets. - */ - return 0; - } - - size = ff_layout_get_lseg_count(fls) * FF_LAYOUT_MIRROR_COUNT(lseg); - - buckets = kcalloc(size, sizeof(struct pnfs_commit_bucket), - gfp_flags); - if (!buckets) - return -ENOMEM; - else { - int i; - - spin_lock(&cinfo->inode->i_lock); - if (cinfo->ds->nbuckets != 0) - kfree(buckets); - else { - cinfo->ds->buckets = buckets; - cinfo->ds->nbuckets = size; - for (i = 0; i < size; i++) { - INIT_LIST_HEAD(&buckets[i].written); - INIT_LIST_HEAD(&buckets[i].committing); - /* mark direct verifier as unset */ - buckets[i].direct_verf.committed = - NFS_INVALID_STABLE_HOW; - } - } - spin_unlock(&cinfo->inode->i_lock); - return 0; - } -} - static void ff_layout_mark_ds_unreachable(struct pnfs_layout_segment *lseg, int idx) { @@ -944,10 +875,8 @@ ff_layout_pg_init_write(struct nfs_pageio_descriptor *pgio, { struct nfs4_ff_layout_mirror *mirror; struct nfs_pgio_mirror *pgm; - struct nfs_commit_info cinfo; struct nfs4_pnfs_ds *ds; int i; - int status; retry: pnfs_generic_pg_check_layout(pgio); @@ -969,11 +898,6 @@ retry: if (pgio->pg_lseg == NULL) goto out_mds; - nfs_init_cinfo(&cinfo, pgio->pg_inode, pgio->pg_dreq); - status = ff_layout_alloc_commit_info(pgio->pg_lseg, &cinfo, GFP_NOFS); - if (status < 0) - goto out_mds; - /* Use a direct mapping of ds_idx to pgio mirror_idx */ if (WARN_ON_ONCE(pgio->pg_mirror_count != FF_LAYOUT_MIRROR_COUNT(pgio->pg_lseg))) diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h index 683146a51599..78f317fac940 100644 --- a/fs/nfs/internal.h +++ b/fs/nfs/internal.h @@ -531,9 +531,6 @@ void nfs_clear_pnfs_ds_commit_verifiers(struct pnfs_ds_commit_info *cinfo) { struct pnfs_commit_array *array; - pnfs_bucket_clear_pnfs_ds_commit_verifiers(cinfo->buckets, - cinfo->nbuckets); - rcu_read_lock(); list_for_each_entry_rcu(array, &cinfo->commits, cinfo_list) pnfs_bucket_clear_pnfs_ds_commit_verifiers(array->buckets, diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c index 9b55919e64ac..20f12f3cbe38 100644 --- a/fs/nfs/pnfs_nfs.c +++ b/fs/nfs/pnfs_nfs.c @@ -292,12 +292,6 @@ int pnfs_generic_scan_commit_lists(struct nfs_commit_info *cinfo, int max) struct pnfs_commit_array *array; int rv = 0, cnt; - cnt = pnfs_bucket_scan_array(cinfo, fl_cinfo->buckets, - fl_cinfo->nbuckets, max); - rv += cnt; - max -= cnt; - if (!max) - return rv; rcu_read_lock(); list_for_each_entry_rcu(array, &fl_cinfo->commits, cinfo_list) { if (!array->lseg || !pnfs_get_commit_array(array)) @@ -353,11 +347,6 @@ void pnfs_generic_recover_commit_reqs(struct list_head *dst, unsigned int nwritten; lockdep_assert_held(&NFS_I(cinfo->inode)->commit_mutex); - nwritten = pnfs_bucket_recover_commit_reqs(dst, - fl_cinfo->buckets, - fl_cinfo->nbuckets, - cinfo); - fl_cinfo->nwritten -= nwritten; rcu_read_lock(); list_for_each_entry_rcu(array, &fl_cinfo->commits, cinfo_list) { if (!array->lseg || !pnfs_get_commit_array(array)) @@ -412,10 +401,6 @@ pnfs_generic_search_commit_reqs(struct nfs_commit_info *cinfo, struct page *page struct pnfs_commit_array *array; struct nfs_page *req; - req = pnfs_bucket_search_commit_reqs(fl_cinfo->buckets, - fl_cinfo->nbuckets, page); - if (req) - return req; list_for_each_entry(array, &fl_cinfo->commits, cinfo_list) { req = pnfs_bucket_search_commit_reqs(array->buckets, array->nbuckets, page); @@ -550,9 +535,6 @@ pnfs_generic_commit_pagelist(struct inode *inode, struct list_head *mds_pages, nreq++; } - nreq += pnfs_bucket_alloc_ds_commits(&list, fl_cinfo->buckets, - fl_cinfo->nbuckets, cinfo); - nreq += pnfs_alloc_ds_commits_list(&list, fl_cinfo, cinfo); if (nreq == 0) goto out; diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index 33be2ee2a248..2903597ec88c 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -1284,8 +1284,6 @@ struct pnfs_ds_commit_info { struct list_head commits; unsigned int nwritten; unsigned int ncommitting; - unsigned int nbuckets; - struct pnfs_commit_bucket *buckets; }; struct nfs41_state_protection { @@ -1396,22 +1394,11 @@ struct nfs41_free_stateid_res { unsigned int status; }; -static inline void -nfs_free_pnfs_ds_cinfo(struct pnfs_ds_commit_info *cinfo) -{ - kfree(cinfo->buckets); -} - #else struct pnfs_ds_commit_info { }; -static inline void -nfs_free_pnfs_ds_cinfo(struct pnfs_ds_commit_info *cinfo) -{ -} - #endif /* CONFIG_NFS_V4_1 */ #ifdef CONFIG_NFS_V4_2 From 9c455a8c1e146dac3a6d1405fe6a7096177b9546 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Sat, 21 Mar 2020 11:13:05 -0400 Subject: [PATCH 283/635] NFS/pNFS: Clean up pNFS commit operations Move the pNFS commit related operations into a separate structure that can be carried by the pnfs_ds_commit_info. Signed-off-by: Trond Myklebust --- fs/nfs/direct.c | 6 +- fs/nfs/filelayout/filelayout.c | 20 +++-- fs/nfs/flexfilelayout/flexfilelayout.c | 19 +++-- fs/nfs/pnfs.h | 110 ++++++++++++++++--------- fs/nfs/pnfs_nfs.c | 13 +-- include/linux/nfs_xdr.h | 1 + 6 files changed, 98 insertions(+), 71 deletions(-) diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c index 61f93a0fb0e0..51ab4627c4d6 100644 --- a/fs/nfs/direct.c +++ b/fs/nfs/direct.c @@ -511,10 +511,7 @@ nfs_direct_write_scan_commit_list(struct inode *inode, struct nfs_commit_info *cinfo) { mutex_lock(&NFS_I(cinfo->inode)->commit_mutex); -#ifdef CONFIG_NFS_V4_1 - if (cinfo->ds != NULL && cinfo->ds->nwritten != 0) - NFS_SERVER(inode)->pnfs_curr_ld->recover_commit_reqs(list, cinfo); -#endif + pnfs_recover_commit_reqs(list, cinfo); nfs_scan_commit_list(&cinfo->mds->list, list, cinfo, 0); mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex); } @@ -917,6 +914,7 @@ ssize_t nfs_file_direct_write(struct kiocb *iocb, struct iov_iter *iter) dreq->l_ctx = l_ctx; if (!is_sync_kiocb(iocb)) dreq->iocb = iocb; + pnfs_init_ds_commit_info_ops(&dreq->ds_cinfo, inode); nfs_start_io_direct(inode); diff --git a/fs/nfs/filelayout/filelayout.c b/fs/nfs/filelayout/filelayout.c index 854f350e2599..a13e69009f19 100644 --- a/fs/nfs/filelayout/filelayout.c +++ b/fs/nfs/filelayout/filelayout.c @@ -49,6 +49,7 @@ MODULE_AUTHOR("Dean Hildebrand "); MODULE_DESCRIPTION("The NFSv4 file layout driver"); #define FILELAYOUT_POLL_RETRY_MAX (15*HZ) +static const struct pnfs_commit_ops filelayout_commit_ops; static loff_t filelayout_get_dense_offset(struct nfs4_filelayout_segment *flseg, @@ -1045,6 +1046,7 @@ filelayout_alloc_layout_hdr(struct inode *inode, gfp_t gfp_flags) if (flo == NULL) return NULL; pnfs_init_ds_commit_info(&flo->commit_info); + flo->commit_info.ops = &filelayout_commit_ops; return &flo->generic_hdr; } @@ -1094,6 +1096,16 @@ filelayout_release_ds_info(struct pnfs_ds_commit_info *fl_cinfo, spin_unlock(&inode->i_lock); } +static const struct pnfs_commit_ops filelayout_commit_ops = { + .setup_ds_info = filelayout_setup_ds_info, + .release_ds_info = filelayout_release_ds_info, + .mark_request_commit = filelayout_mark_request_commit, + .clear_request_commit = pnfs_generic_clear_request_commit, + .scan_commit_lists = pnfs_generic_scan_commit_lists, + .recover_commit_reqs = pnfs_generic_recover_commit_reqs, + .search_commit_reqs = pnfs_generic_search_commit_reqs, + .commit_pagelist = filelayout_commit_pagelist, +}; static struct pnfs_layoutdriver_type filelayout_type = { .id = LAYOUT_NFSV4_1_FILES, @@ -1108,14 +1120,6 @@ static struct pnfs_layoutdriver_type filelayout_type = { .pg_read_ops = &filelayout_pg_read_ops, .pg_write_ops = &filelayout_pg_write_ops, .get_ds_info = &filelayout_get_ds_info, - .setup_ds_info = filelayout_setup_ds_info, - .release_ds_info = filelayout_release_ds_info, - .mark_request_commit = filelayout_mark_request_commit, - .clear_request_commit = pnfs_generic_clear_request_commit, - .scan_commit_lists = pnfs_generic_scan_commit_lists, - .recover_commit_reqs = pnfs_generic_recover_commit_reqs, - .search_commit_reqs = pnfs_generic_search_commit_reqs, - .commit_pagelist = filelayout_commit_pagelist, .read_pagelist = filelayout_read_pagelist, .write_pagelist = filelayout_write_pagelist, .alloc_deviceid_node = filelayout_alloc_deviceid_node, diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c index 1a4e36d07eab..d37883a2b51f 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.c +++ b/fs/nfs/flexfilelayout/flexfilelayout.c @@ -32,6 +32,7 @@ static unsigned short io_maxretrans; +static const struct pnfs_commit_ops ff_layout_commit_ops; static void ff_layout_read_record_layoutstats_done(struct rpc_task *task, struct nfs_pgio_header *hdr); static int ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo, @@ -52,6 +53,7 @@ ff_layout_alloc_layout_hdr(struct inode *inode, gfp_t gfp_flags) INIT_LIST_HEAD(&ffl->error_list); INIT_LIST_HEAD(&ffl->mirrors); ffl->last_report_time = ktime_get(); + ffl->commit_info.ops = &ff_layout_commit_ops; return &ffl->generic_hdr; } else return NULL; @@ -2440,6 +2442,16 @@ ff_layout_set_layoutdriver(struct nfs_server *server, return 0; } +static const struct pnfs_commit_ops ff_layout_commit_ops = { + .setup_ds_info = ff_layout_setup_ds_info, + .release_ds_info = ff_layout_release_ds_info, + .mark_request_commit = pnfs_layout_mark_request_commit, + .clear_request_commit = pnfs_generic_clear_request_commit, + .scan_commit_lists = pnfs_generic_scan_commit_lists, + .recover_commit_reqs = pnfs_generic_recover_commit_reqs, + .commit_pagelist = ff_layout_commit_pagelist, +}; + static struct pnfs_layoutdriver_type flexfilelayout_type = { .id = LAYOUT_FLEX_FILES, .name = "LAYOUT_FLEX_FILES", @@ -2455,14 +2467,7 @@ static struct pnfs_layoutdriver_type flexfilelayout_type = { .pg_read_ops = &ff_layout_pg_read_ops, .pg_write_ops = &ff_layout_pg_write_ops, .get_ds_info = ff_layout_get_ds_info, - .setup_ds_info = ff_layout_setup_ds_info, - .release_ds_info = ff_layout_release_ds_info, .free_deviceid_node = ff_layout_free_deviceid_node, - .mark_request_commit = pnfs_layout_mark_request_commit, - .clear_request_commit = pnfs_generic_clear_request_commit, - .scan_commit_lists = pnfs_generic_scan_commit_lists, - .recover_commit_reqs = pnfs_generic_recover_commit_reqs, - .commit_pagelist = ff_layout_commit_pagelist, .read_pagelist = ff_layout_read_pagelist, .write_pagelist = ff_layout_write_pagelist, .alloc_deviceid_node = ff_layout_alloc_deviceid_node, diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h index faed9be6e479..b32025553f26 100644 --- a/fs/nfs/pnfs.h +++ b/fs/nfs/pnfs.h @@ -150,26 +150,6 @@ struct pnfs_layoutdriver_type { const struct nfs_pageio_ops *pg_write_ops; struct pnfs_ds_commit_info *(*get_ds_info) (struct inode *inode); - void (*setup_ds_info)(struct pnfs_ds_commit_info *, - struct pnfs_layout_segment *); - void (*release_ds_info)(struct pnfs_ds_commit_info *, - struct inode *inode); - void (*mark_request_commit) (struct nfs_page *req, - struct pnfs_layout_segment *lseg, - struct nfs_commit_info *cinfo, - u32 ds_commit_idx); - void (*clear_request_commit) (struct nfs_page *req, - struct nfs_commit_info *cinfo); - int (*scan_commit_lists) (struct nfs_commit_info *cinfo, - int max); - void (*recover_commit_reqs) (struct list_head *list, - struct nfs_commit_info *cinfo); - struct nfs_page * (*search_commit_reqs)(struct nfs_commit_info *cinfo, - struct page *page); - int (*commit_pagelist)(struct inode *inode, - struct list_head *mds_pages, - int how, - struct nfs_commit_info *cinfo); int (*sync)(struct inode *inode, bool datasync); @@ -192,6 +172,29 @@ struct pnfs_layoutdriver_type { int (*prepare_layoutstats) (struct nfs42_layoutstat_args *args); }; +struct pnfs_commit_ops { + void (*setup_ds_info)(struct pnfs_ds_commit_info *, + struct pnfs_layout_segment *); + void (*release_ds_info)(struct pnfs_ds_commit_info *, + struct inode *inode); + int (*commit_pagelist)(struct inode *inode, + struct list_head *mds_pages, + int how, + struct nfs_commit_info *cinfo); + void (*mark_request_commit) (struct nfs_page *req, + struct pnfs_layout_segment *lseg, + struct nfs_commit_info *cinfo, + u32 ds_commit_idx); + void (*clear_request_commit) (struct nfs_page *req, + struct nfs_commit_info *cinfo); + int (*scan_commit_lists) (struct nfs_commit_info *cinfo, + int max); + void (*recover_commit_reqs) (struct list_head *list, + struct nfs_commit_info *cinfo); + struct nfs_page * (*search_commit_reqs)(struct nfs_commit_info *cinfo, + struct page *page); +}; + struct pnfs_layout_hdr { refcount_t plh_refcount; atomic_t plh_outstanding; /* number of RPCs out */ @@ -461,9 +464,11 @@ static inline int pnfs_commit_list(struct inode *inode, struct list_head *mds_pages, int how, struct nfs_commit_info *cinfo) { - if (cinfo->ds == NULL || cinfo->ds->ncommitting == 0) + struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds; + + if (fl_cinfo == NULL || fl_cinfo->ncommitting == 0) return PNFS_NOT_ATTEMPTED; - return NFS_SERVER(inode)->pnfs_curr_ld->commit_pagelist(inode, mds_pages, how, cinfo); + return fl_cinfo->ops->commit_pagelist(inode, mds_pages, how, cinfo); } static inline struct pnfs_ds_commit_info * @@ -476,19 +481,26 @@ pnfs_get_ds_info(struct inode *inode) return ld->get_ds_info(inode); } +static inline void +pnfs_init_ds_commit_info_ops(struct pnfs_ds_commit_info *fl_cinfo, struct inode *inode) +{ + struct pnfs_ds_commit_info *inode_cinfo = pnfs_get_ds_info(inode); + if (inode_cinfo != NULL) + fl_cinfo->ops = inode_cinfo->ops; +} + static inline void pnfs_init_ds_commit_info(struct pnfs_ds_commit_info *fl_cinfo) { INIT_LIST_HEAD(&fl_cinfo->commits); + fl_cinfo->ops = NULL; } static inline void pnfs_release_ds_info(struct pnfs_ds_commit_info *fl_cinfo, struct inode *inode) { - struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; - - if (ld != NULL && ld->release_ds_info != NULL) - ld->release_ds_info(fl_cinfo, inode); + if (fl_cinfo->ops != NULL && fl_cinfo->ops->release_ds_info != NULL) + fl_cinfo->ops->release_ds_info(fl_cinfo, inode); } static inline void @@ -501,24 +513,22 @@ static inline bool pnfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg, struct nfs_commit_info *cinfo, u32 ds_commit_idx) { - struct inode *inode = d_inode(nfs_req_openctx(req)->dentry); - struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; + struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds; - if (lseg == NULL || ld->mark_request_commit == NULL) + if (!lseg || !fl_cinfo->ops->mark_request_commit) return false; - ld->mark_request_commit(req, lseg, cinfo, ds_commit_idx); + fl_cinfo->ops->mark_request_commit(req, lseg, cinfo, ds_commit_idx); return true; } static inline bool pnfs_clear_request_commit(struct nfs_page *req, struct nfs_commit_info *cinfo) { - struct inode *inode = d_inode(nfs_req_openctx(req)->dentry); - struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; + struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds; - if (ld == NULL || ld->clear_request_commit == NULL) + if (!fl_cinfo || !fl_cinfo->ops || !fl_cinfo->ops->clear_request_commit) return false; - ld->clear_request_commit(req, cinfo); + fl_cinfo->ops->clear_request_commit(req, cinfo); return true; } @@ -526,21 +536,31 @@ static inline int pnfs_scan_commit_lists(struct inode *inode, struct nfs_commit_info *cinfo, int max) { - if (cinfo->ds == NULL || cinfo->ds->nwritten == 0) + struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds; + + if (!fl_cinfo || fl_cinfo->nwritten == 0) return 0; - else - return NFS_SERVER(inode)->pnfs_curr_ld->scan_commit_lists(cinfo, max); + return fl_cinfo->ops->scan_commit_lists(cinfo, max); +} + +static inline void +pnfs_recover_commit_reqs(struct list_head *head, struct nfs_commit_info *cinfo) +{ + struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds; + + if (fl_cinfo && fl_cinfo->nwritten != 0) + fl_cinfo->ops->recover_commit_reqs(head, cinfo); } static inline struct nfs_page * pnfs_search_commit_reqs(struct inode *inode, struct nfs_commit_info *cinfo, struct page *page) { - struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; + struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds; - if (ld == NULL || ld->search_commit_reqs == NULL) + if (!fl_cinfo->ops || !fl_cinfo->ops->search_commit_reqs) return NULL; - return ld->search_commit_reqs(cinfo, page); + return fl_cinfo->ops->search_commit_reqs(cinfo, page); } /* Should the pNFS client commit and return the layout upon a setattr */ @@ -788,6 +808,11 @@ pnfs_get_ds_info(struct inode *inode) return NULL; } +static inline void +pnfs_init_ds_commit_info_ops(struct pnfs_ds_commit_info *fl_cinfo, struct inode *inode) +{ +} + static inline void pnfs_init_ds_commit_info(struct pnfs_ds_commit_info *fl_cinfo) { @@ -818,6 +843,11 @@ pnfs_scan_commit_lists(struct inode *inode, struct nfs_commit_info *cinfo, return 0; } +static inline void +pnfs_recover_commit_reqs(struct list_head *head, struct nfs_commit_info *cinfo) +{ +} + static inline struct nfs_page * pnfs_search_commit_reqs(struct inode *inode, struct nfs_commit_info *cinfo, struct page *page) diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c index 20f12f3cbe38..06df2e6663dc 100644 --- a/fs/nfs/pnfs_nfs.c +++ b/fs/nfs/pnfs_nfs.c @@ -149,17 +149,6 @@ pnfs_add_commit_array(struct pnfs_ds_commit_info *fl_cinfo, } EXPORT_SYMBOL_GPL(pnfs_add_commit_array); -static void -pnfs_setup_ds_info(struct pnfs_ds_commit_info *fl_cinfo, - struct pnfs_layout_segment *lseg) -{ - struct inode *inode = lseg->pls_layout->plh_inode; - struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; - - if (ld->setup_ds_info != NULL) - ld->setup_ds_info(fl_cinfo, lseg); -} - static struct pnfs_commit_array * pnfs_lookup_commit_array(struct pnfs_ds_commit_info *fl_cinfo, struct pnfs_layout_segment *lseg) @@ -170,7 +159,7 @@ pnfs_lookup_commit_array(struct pnfs_ds_commit_info *fl_cinfo, array = pnfs_find_commit_array_by_lseg(fl_cinfo, lseg); if (!array) { rcu_read_unlock(); - pnfs_setup_ds_info(fl_cinfo, lseg); + fl_cinfo->ops->setup_ds_info(fl_cinfo, lseg); rcu_read_lock(); array = pnfs_find_commit_array_by_lseg(fl_cinfo, lseg); } diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index 2903597ec88c..adbbeae9ce5b 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -1284,6 +1284,7 @@ struct pnfs_ds_commit_info { struct list_head commits; unsigned int nwritten; unsigned int ncommitting; + const struct pnfs_commit_ops *ops; }; struct nfs41_state_protection { From c84bea59449aaa699a0600a50f59d441cc1d4501 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Sun, 22 Mar 2020 14:47:38 -0400 Subject: [PATCH 284/635] NFS/pNFS: Simplify bucket layout segment reference counting Signed-off-by: Trond Myklebust --- fs/nfs/pnfs_nfs.c | 39 ++++++++++++++++++++------------------- include/linux/nfs_xdr.h | 3 +-- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c index 06df2e6663dc..abf16fc98346 100644 --- a/fs/nfs/pnfs_nfs.c +++ b/fs/nfs/pnfs_nfs.c @@ -59,6 +59,17 @@ void pnfs_generic_commit_release(void *calldata) } EXPORT_SYMBOL_GPL(pnfs_generic_commit_release); +static struct pnfs_layout_segment * +pnfs_free_bucket_lseg(struct pnfs_commit_bucket *bucket) +{ + if (list_empty(&bucket->committing) && list_empty(&bucket->written)) { + struct pnfs_layout_segment *freeme = bucket->lseg; + bucket->lseg = NULL; + return freeme; + } + return NULL; +} + /* The generic layer is about to remove the req from the commit list. * If this will make the bucket empty, it will need to put the lseg reference. * Note this must be called holding nfsi->commit_mutex @@ -78,8 +89,7 @@ pnfs_generic_clear_request_commit(struct nfs_page *req, bucket = list_first_entry(&req->wb_list, struct pnfs_commit_bucket, written); - freeme = bucket->wlseg; - bucket->wlseg = NULL; + freeme = pnfs_free_bucket_lseg(bucket); } out: nfs_request_remove_commit_list(req, cinfo); @@ -103,8 +113,7 @@ pnfs_alloc_commit_array(size_t n, gfp_t gfp_flags) for (b = &p->buckets[0]; n != 0; b++, n--) { INIT_LIST_HEAD(&b->written); INIT_LIST_HEAD(&b->committing); - b->wlseg = NULL; - b->clseg = NULL; + b->lseg = NULL; b->direct_verf.committed = NFS_INVALID_STABLE_HOW; } return p; @@ -246,12 +255,6 @@ pnfs_bucket_scan_ds_commit_list(struct pnfs_commit_bucket *bucket, if (ret) { cinfo->ds->nwritten -= ret; cinfo->ds->ncommitting += ret; - if (bucket->clseg == NULL) - bucket->clseg = pnfs_get_lseg(bucket->wlseg); - if (list_empty(src)) { - pnfs_put_lseg(bucket->wlseg); - bucket->wlseg = NULL; - } } return ret; } @@ -317,9 +320,8 @@ restart: if (!nwritten) continue; ret += nwritten; - if (list_empty(&b->written)) { - freeme = b->wlseg; - b->wlseg = NULL; + freeme = pnfs_free_bucket_lseg(b); + if (freeme) { pnfs_put_lseg(freeme); goto restart; } @@ -405,15 +407,12 @@ pnfs_bucket_get_committing(struct list_head *head, struct pnfs_commit_bucket *bucket, struct nfs_commit_info *cinfo) { - struct pnfs_layout_segment *freeme; struct list_head *pos; list_for_each(pos, &bucket->committing) cinfo->ds->ncommitting--; list_splice_init(&bucket->committing, head); - freeme = bucket->clseg; - bucket->clseg = NULL; - return freeme; + return pnfs_free_bucket_lseg(bucket); } static struct nfs_commit_data * @@ -425,6 +424,8 @@ pnfs_bucket_fetch_commitdata(struct pnfs_commit_bucket *bucket, if (!data) return NULL; data->lseg = pnfs_bucket_get_committing(&data->pages, bucket, cinfo); + if (!data->lseg) + data->lseg = pnfs_get_lseg(bucket->lseg); return data; } @@ -1182,8 +1183,8 @@ pnfs_layout_mark_request_commit(struct nfs_page *req, * off due to a rewrite, in which case it will be done in * pnfs_common_clear_request_commit */ - WARN_ON_ONCE(buckets[ds_commit_idx].wlseg != NULL); - buckets[ds_commit_idx].wlseg = pnfs_get_lseg(lseg); + if (!buckets[ds_commit_idx].lseg) + buckets[ds_commit_idx].lseg = pnfs_get_lseg(lseg); } set_bit(PG_COMMIT_TO_DS, &req->wb_flags); cinfo->ds->nwritten++; diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index adbbeae9ce5b..7bbb1f6fc1b1 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -1265,8 +1265,7 @@ struct nfstime4 { struct pnfs_commit_bucket { struct list_head written; struct list_head committing; - struct pnfs_layout_segment *wlseg; - struct pnfs_layout_segment *clseg; + struct pnfs_layout_segment *lseg; struct nfs_writeverf direct_verf; }; From e18c18ebd7c128346b532729792e21d97eeb15b0 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Sun, 22 Mar 2020 16:08:55 -0400 Subject: [PATCH 285/635] NFS/pNFS: Fix pnfs_layout_mark_request_commit() invalid layout segment handling Fix up pnfs_layout_mark_request_commit() to alway reschedule the write if the layout segment is invalid. Also minor cleanup. Signed-off-by: Trond Myklebust --- fs/nfs/pnfs_nfs.c | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c index abf16fc98346..25f135572fc8 100644 --- a/fs/nfs/pnfs_nfs.c +++ b/fs/nfs/pnfs_nfs.c @@ -1166,26 +1166,22 @@ pnfs_layout_mark_request_commit(struct nfs_page *req, { struct list_head *list; struct pnfs_commit_array *array; - struct pnfs_commit_bucket *buckets; + struct pnfs_commit_bucket *bucket; mutex_lock(&NFS_I(cinfo->inode)->commit_mutex); array = pnfs_lookup_commit_array(cinfo->ds, lseg); - if (!array) + if (!array || !pnfs_is_valid_lseg(lseg)) goto out_resched; - buckets = array->buckets; - list = &buckets[ds_commit_idx].written; - if (list_empty(list)) { - if (!pnfs_is_valid_lseg(lseg)) - goto out_resched; - /* Non-empty buckets hold a reference on the lseg. That ref - * is normally transferred to the COMMIT call and released - * there. It could also be released if the last req is pulled - * off due to a rewrite, in which case it will be done in - * pnfs_common_clear_request_commit - */ - if (!buckets[ds_commit_idx].lseg) - buckets[ds_commit_idx].lseg = pnfs_get_lseg(lseg); - } + bucket = &array->buckets[ds_commit_idx]; + list = &bucket->written; + /* Non-empty buckets hold a reference on the lseg. That ref + * is normally transferred to the COMMIT call and released + * there. It could also be released if the last req is pulled + * off due to a rewrite, in which case it will be done in + * pnfs_common_clear_request_commit + */ + if (!bucket->lseg) + bucket->lseg = pnfs_get_lseg(lseg); set_bit(PG_COMMIT_TO_DS, &req->wb_flags); cinfo->ds->nwritten++; From 660d1eb22301c290945450414fb0be5ae25526f1 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 23 Mar 2020 15:40:20 -0400 Subject: [PATCH 286/635] pNFS/flexfile: Don't merge layout segments if the mirrors don't match Check that the number of mirrors, and the mirror information matches before deciding to merge layout segments in pNFS/flexfiles. Signed-off-by: Trond Myklebust --- fs/nfs/flexfilelayout/flexfilelayout.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c index d37883a2b51f..3221001f2ea1 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.c +++ b/fs/nfs/flexfilelayout/flexfilelayout.c @@ -283,6 +283,23 @@ static void _ff_layout_free_lseg(struct nfs4_ff_layout_segment *fls) } } +static bool +ff_lseg_match_mirrors(struct pnfs_layout_segment *l1, + struct pnfs_layout_segment *l2) +{ + const struct nfs4_ff_layout_segment *fl1 = FF_LAYOUT_LSEG(l1); + const struct nfs4_ff_layout_segment *fl2 = FF_LAYOUT_LSEG(l1); + u32 i; + + if (fl1->mirror_array_cnt != fl2->mirror_array_cnt) + return false; + for (i = 0; i < fl1->mirror_array_cnt; i++) { + if (fl1->mirror_array[i] != fl2->mirror_array[i]) + return false; + } + return true; +} + static bool ff_lseg_range_is_after(const struct pnfs_layout_range *l1, const struct pnfs_layout_range *l2) @@ -318,6 +335,8 @@ ff_lseg_merge(struct pnfs_layout_segment *new, new->pls_range.length); if (new_end < old->pls_range.offset) return false; + if (!ff_lseg_match_mirrors(new, old)) + return false; /* Mergeable: copy info from 'old' to 'new' */ if (new_end < old_end) From e1e54ab710f8da922cd44651463bf8bb61114b5a Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 23 Mar 2020 14:48:23 -0400 Subject: [PATCH 287/635] pNFS/flexfiles: Check the layout segment range before doing I/O When starting to read or write with a layout segment, check that the range matches our request. Signed-off-by: Trond Myklebust --- fs/nfs/flexfilelayout/flexfilelayout.c | 12 ++++++++++-- fs/nfs/pnfs.c | 3 ++- fs/nfs/pnfs.h | 1 + 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c index 3221001f2ea1..2b45807a5221 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.c +++ b/fs/nfs/flexfilelayout/flexfilelayout.c @@ -830,6 +830,14 @@ ff_layout_pg_get_read(struct nfs_pageio_descriptor *pgio, } } +static void +ff_layout_pg_check_layout(struct nfs_pageio_descriptor *pgio, + struct nfs_page *req) +{ + pnfs_generic_pg_check_layout(pgio); + pnfs_generic_pg_check_range(pgio, req); +} + static void ff_layout_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req) @@ -840,7 +848,7 @@ ff_layout_pg_init_read(struct nfs_pageio_descriptor *pgio, int ds_idx; retry: - pnfs_generic_pg_check_layout(pgio); + ff_layout_pg_check_layout(pgio, req); /* Use full layout for now */ if (!pgio->pg_lseg) { ff_layout_pg_get_read(pgio, req, false); @@ -900,7 +908,7 @@ ff_layout_pg_init_write(struct nfs_pageio_descriptor *pgio, int i; retry: - pnfs_generic_pg_check_layout(pgio); + ff_layout_pg_check_layout(pgio, req); if (!pgio->pg_lseg) { pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode, nfs_req_openctx(req), diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index eba18f137fb0..6fcf26b16816 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -2611,7 +2611,7 @@ EXPORT_SYMBOL_GPL(pnfs_generic_pg_check_layout); * Check for any intersection between the request and the pgio->pg_lseg, * and if none, put this pgio->pg_lseg away. */ -static void +void pnfs_generic_pg_check_range(struct nfs_pageio_descriptor *pgio, struct nfs_page *req) { if (pgio->pg_lseg && !pnfs_lseg_request_intersecting(pgio->pg_lseg, req)) { @@ -2619,6 +2619,7 @@ pnfs_generic_pg_check_range(struct nfs_pageio_descriptor *pgio, struct nfs_page pgio->pg_lseg = NULL; } } +EXPORT_SYMBOL_GPL(pnfs_generic_pg_check_range); void pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req) diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h index b32025553f26..8e0ada581b92 100644 --- a/fs/nfs/pnfs.h +++ b/fs/nfs/pnfs.h @@ -252,6 +252,7 @@ void pnfs_put_lseg(struct pnfs_layout_segment *lseg); void set_pnfs_layoutdriver(struct nfs_server *, const struct nfs_fh *, struct nfs_fsinfo *); void unset_pnfs_layoutdriver(struct nfs_server *); void pnfs_generic_pg_check_layout(struct nfs_pageio_descriptor *pgio); +void pnfs_generic_pg_check_range(struct nfs_pageio_descriptor *pgio, struct nfs_page *req); void pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *, struct nfs_page *); int pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc); void pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio, From e70430d9398fc959d1392d416da78167087e1256 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 23 Mar 2020 14:33:11 -0400 Subject: [PATCH 288/635] pNFS/flexfiles: remove requirement for whole file layouts Remove the requirement that the server always sends whole file layouts. Signed-off-by: Trond Myklebust --- fs/nfs/flexfilelayout/flexfilelayout.c | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c index 2b45807a5221..42f581e213cc 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.c +++ b/fs/nfs/flexfilelayout/flexfilelayout.c @@ -257,24 +257,6 @@ static void ff_layout_free_mirror_array(struct nfs4_ff_layout_segment *fls) ff_layout_put_mirror(fls->mirror_array[i]); } -static int ff_layout_check_layout(struct nfs4_layoutget_res *lgr) -{ - int ret = 0; - - dprintk("--> %s\n", __func__); - - /* FIXME: remove this check when layout segment support is added */ - if (lgr->range.offset != 0 || - lgr->range.length != NFS4_MAX_UINT64) { - dprintk("%s Only whole file layouts supported. Use MDS i/o\n", - __func__); - ret = -EINVAL; - } - - dprintk("--> %s returns %d\n", __func__, ret); - return ret; -} - static void _ff_layout_free_lseg(struct nfs4_ff_layout_segment *fls) { if (fls) { @@ -556,9 +538,6 @@ ff_layout_alloc_lseg(struct pnfs_layout_hdr *lh, out_sort_mirrors: ff_layout_sort_mirrors(fls); - rc = ff_layout_check_layout(lgr); - if (rc) - goto out_err_free; ret = &fls->generic_hdr; dprintk("<-- %s (success)\n", __func__); out_free_page: From cbd7be43c4d40dbd1b33c8414d1bc019fa38849e Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 23 Mar 2020 15:18:12 -0400 Subject: [PATCH 289/635] pNFS/flexfiles: Specify the layout segment range in LAYOUTGET Move from requesting only full file layout segments, to requesting layout segments that match our I/O size. This means the server is still free to return a full file layout, but we will no longer error out if it does not. Signed-off-by: Trond Myklebust --- fs/nfs/flexfilelayout/flexfilelayout.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c index 42f581e213cc..7d399f72ebbb 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.c +++ b/fs/nfs/flexfilelayout/flexfilelayout.c @@ -798,8 +798,8 @@ ff_layout_pg_get_read(struct nfs_pageio_descriptor *pgio, pnfs_put_lseg(pgio->pg_lseg); pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode, nfs_req_openctx(req), - 0, - NFS4_MAX_UINT64, + req_offset(req), + req->wb_bytes, IOMODE_READ, strict_iomode, GFP_KERNEL); @@ -891,8 +891,8 @@ retry: if (!pgio->pg_lseg) { pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode, nfs_req_openctx(req), - 0, - NFS4_MAX_UINT64, + req_offset(req), + req->wb_bytes, IOMODE_RW, false, GFP_NOFS); From 1de3af9883fe2b689d1f61b205e9f5a0cedca8e7 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Sat, 28 Mar 2020 11:39:29 -0400 Subject: [PATCH 290/635] NFS: Remove unused FLUSH_SYNC support in nfs_initiate_pgio() If the FLUSH_SYNC flag is set, nfs_initiate_pgio() will currently wait for completion, and then return the status of the I/O operation. What we actually want to report in nfs_pageio_doio() is whether or not the RPC call was launched successfully, whereas actual I/O status is intended handled in the reply callbacks. Since FLUSH_SYNC is never set by any of the callers anyway, let's just remove that code altogether. Signed-off-by: Trond Myklebust --- fs/nfs/pagelist.c | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/fs/nfs/pagelist.c b/fs/nfs/pagelist.c index c9c3edefc5be..be5e209399ea 100644 --- a/fs/nfs/pagelist.c +++ b/fs/nfs/pagelist.c @@ -629,7 +629,6 @@ int nfs_initiate_pgio(struct rpc_clnt *clnt, struct nfs_pgio_header *hdr, .workqueue = nfsiod_workqueue, .flags = RPC_TASK_ASYNC | RPC_TASK_CRED_NOREF | flags, }; - int ret = 0; hdr->rw_ops->rw_initiate(hdr, &msg, rpc_ops, &task_setup_data, how); @@ -641,18 +640,10 @@ int nfs_initiate_pgio(struct rpc_clnt *clnt, struct nfs_pgio_header *hdr, (unsigned long long)hdr->args.offset); task = rpc_run_task(&task_setup_data); - if (IS_ERR(task)) { - ret = PTR_ERR(task); - goto out; - } - if (how & FLUSH_SYNC) { - ret = rpc_wait_for_completion_task(task); - if (ret == 0) - ret = task->tk_status; - } + if (IS_ERR(task)) + return PTR_ERR(task); rpc_put_task(task); -out: - return ret; + return 0; } EXPORT_SYMBOL_GPL(nfs_initiate_pgio); From cee416a347440628762db2257ff921ccf9f66923 Mon Sep 17 00:00:00 2001 From: Gwendal Grignou Date: Fri, 27 Mar 2020 15:34:32 -0700 Subject: [PATCH 291/635] platform/chrome: cros_ec_sensorhub: Add the number of sensors in sensorhub To better manage resources, store the number of sensors reported by the EC. Signed-off-by: Gwendal Grignou Signed-off-by: Enric Balletbo i Serra --- drivers/platform/chrome/cros_ec_sensorhub.c | 4 +++- include/linux/platform_data/cros_ec_sensorhub.h | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/platform/chrome/cros_ec_sensorhub.c b/drivers/platform/chrome/cros_ec_sensorhub.c index 79fefd3bb0fa..134ee5e46047 100644 --- a/drivers/platform/chrome/cros_ec_sensorhub.c +++ b/drivers/platform/chrome/cros_ec_sensorhub.c @@ -65,6 +65,7 @@ static int cros_ec_sensorhub_register(struct device *dev, return sensor_num; } + sensorhub->sensor_num = sensor_num; if (sensor_num == 0) { dev_err(dev, "Zero sensors reported.\n"); return -EINVAL; @@ -172,7 +173,8 @@ static int cros_ec_sensorhub_probe(struct platform_device *pdev) * If the device has sensors but does not claim to * be a sensor hub, we are in legacy mode. */ - for (i = 0; i < 2; i++) { + data->sensor_num = 2; + for (i = 0; i < data->sensor_num; i++) { ret = cros_ec_sensorhub_allocate_sensor(dev, "cros-ec-accel-legacy", i); if (ret) diff --git a/include/linux/platform_data/cros_ec_sensorhub.h b/include/linux/platform_data/cros_ec_sensorhub.h index bef7ffc7fce1..7e46a47fd642 100644 --- a/include/linux/platform_data/cros_ec_sensorhub.h +++ b/include/linux/platform_data/cros_ec_sensorhub.h @@ -22,9 +22,11 @@ struct cros_ec_sensor_platform { * struct cros_ec_sensorhub - Sensor Hub device data. * * @ec: Embedded Controller where the hub is located. + * @sensor_num: Number of MEMS sensors present in the EC. */ struct cros_ec_sensorhub { struct cros_ec_dev *ec; + int sensor_num; }; #endif /* __LINUX_PLATFORM_DATA_CROS_EC_SENSORHUB_H */ From 145d59baff5944b71551ac518d7fd7d377a9c820 Mon Sep 17 00:00:00 2001 From: Gwendal Grignou Date: Fri, 27 Mar 2020 15:34:33 -0700 Subject: [PATCH 292/635] platform/chrome: cros_ec_sensorhub: Add FIFO support cros_ec_sensorhub registers a listener and query motion sense FIFO, spread to iio sensors registers. To test, we can use libiio: iiod& iio_readdev -u ip:localhost -T 10000 -s 25 -b 16 cros-ec-gyro | od -x Signed-off-by: Gwendal Grignou Reviewed-by: Jonathan Cameron Acked-by: Andy Shevchenko Signed-off-by: Enric Balletbo i Serra --- drivers/platform/chrome/Makefile | 3 +- drivers/platform/chrome/cros_ec_sensorhub.c | 107 +++-- .../platform/chrome/cros_ec_sensorhub_ring.c | 439 ++++++++++++++++++ .../linux/platform_data/cros_ec_sensorhub.h | 88 +++- 4 files changed, 603 insertions(+), 34 deletions(-) create mode 100644 drivers/platform/chrome/cros_ec_sensorhub_ring.c diff --git a/drivers/platform/chrome/Makefile b/drivers/platform/chrome/Makefile index 198c155c7c4d..41baccba033f 100644 --- a/drivers/platform/chrome/Makefile +++ b/drivers/platform/chrome/Makefile @@ -20,7 +20,8 @@ obj-$(CONFIG_CROS_EC_CHARDEV) += cros_ec_chardev.o obj-$(CONFIG_CROS_EC_LIGHTBAR) += cros_ec_lightbar.o obj-$(CONFIG_CROS_EC_VBC) += cros_ec_vbc.o obj-$(CONFIG_CROS_EC_DEBUGFS) += cros_ec_debugfs.o -obj-$(CONFIG_CROS_EC_SENSORHUB) += cros_ec_sensorhub.o +cros-ec-sensorhub-objs := cros_ec_sensorhub.o cros_ec_sensorhub_ring.o +obj-$(CONFIG_CROS_EC_SENSORHUB) += cros-ec-sensorhub.o obj-$(CONFIG_CROS_EC_SYSFS) += cros_ec_sysfs.o obj-$(CONFIG_CROS_USBPD_LOGGER) += cros_usbpd_logger.o obj-$(CONFIG_CROS_USBPD_NOTIFY) += cros_usbpd_notify.o diff --git a/drivers/platform/chrome/cros_ec_sensorhub.c b/drivers/platform/chrome/cros_ec_sensorhub.c index 134ee5e46047..b7f2c00db5e1 100644 --- a/drivers/platform/chrome/cros_ec_sensorhub.c +++ b/drivers/platform/chrome/cros_ec_sensorhub.c @@ -50,10 +50,8 @@ static int cros_ec_sensorhub_register(struct device *dev, struct cros_ec_sensorhub *sensorhub) { int sensor_type[MOTIONSENSE_TYPE_MAX] = { 0 }; + struct cros_ec_command *msg = sensorhub->msg; struct cros_ec_dev *ec = sensorhub->ec; - struct ec_params_motion_sense *params; - struct ec_response_motion_sense *resp; - struct cros_ec_command *msg; int ret, i, sensor_num; char *name; @@ -71,22 +69,13 @@ static int cros_ec_sensorhub_register(struct device *dev, return -EINVAL; } - /* Prepare a message to send INFO command to each sensor. */ - msg = kzalloc(sizeof(*msg) + max(sizeof(*params), sizeof(*resp)), - GFP_KERNEL); - if (!msg) - return -ENOMEM; - msg->version = 1; - msg->command = EC_CMD_MOTION_SENSE_CMD + ec->cmd_offset; - msg->outsize = sizeof(*params); - msg->insize = sizeof(*resp); - params = (struct ec_params_motion_sense *)msg->data; - resp = (struct ec_response_motion_sense *)msg->data; + msg->insize = sizeof(struct ec_response_motion_sense); + msg->outsize = sizeof(struct ec_params_motion_sense); for (i = 0; i < sensor_num; i++) { - params->cmd = MOTIONSENSE_CMD_INFO; - params->info.sensor_num = i; + sensorhub->params->cmd = MOTIONSENSE_CMD_INFO; + sensorhub->params->info.sensor_num = i; ret = cros_ec_cmd_xfer_status(ec->ec_dev, msg); if (ret < 0) { @@ -95,7 +84,7 @@ static int cros_ec_sensorhub_register(struct device *dev, continue; } - switch (resp->info.type) { + switch (sensorhub->resp->info.type) { case MOTIONSENSE_TYPE_ACCEL: name = "cros-ec-accel"; break; @@ -118,15 +107,16 @@ static int cros_ec_sensorhub_register(struct device *dev, name = "cros-ec-activity"; break; default: - dev_warn(dev, "unknown type %d\n", resp->info.type); + dev_warn(dev, "unknown type %d\n", + sensorhub->resp->info.type); continue; } ret = cros_ec_sensorhub_allocate_sensor(dev, name, i); if (ret) - goto error; + return ret; - sensor_type[resp->info.type]++; + sensor_type[sensorhub->resp->info.type]++; } if (sensor_type[MOTIONSENSE_TYPE_ACCEL] >= 2) @@ -138,29 +128,41 @@ static int cros_ec_sensorhub_register(struct device *dev, "cros-ec-lid-angle", 0); if (ret) - goto error; + return ret; } - kfree(msg); return 0; - -error: - kfree(msg); - return ret; } static int cros_ec_sensorhub_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; + struct cros_ec_dev *ec = dev_get_drvdata(dev->parent); struct cros_ec_sensorhub *data; + struct cros_ec_command *msg; int ret; int i; + msg = devm_kzalloc(dev, sizeof(struct cros_ec_command) + + max((u16)sizeof(struct ec_params_motion_sense), + ec->ec_dev->max_response), GFP_KERNEL); + if (!msg) + return -ENOMEM; + + msg->command = EC_CMD_MOTION_SENSE_CMD + ec->cmd_offset; + data = devm_kzalloc(dev, sizeof(struct cros_ec_sensorhub), GFP_KERNEL); if (!data) return -ENOMEM; - data->ec = dev_get_drvdata(dev->parent); + mutex_init(&data->cmd_lock); + + data->dev = dev; + data->ec = ec; + data->msg = msg; + data->params = (struct ec_params_motion_sense *)msg->data; + data->resp = (struct ec_response_motion_sense *)msg->data; + dev_set_drvdata(dev, data); /* Check whether this EC is a sensor hub. */ @@ -182,12 +184,63 @@ static int cros_ec_sensorhub_probe(struct platform_device *pdev) } } + /* + * If the EC does not have a FIFO, the sensors will query their data + * themselves via sysfs or a software trigger. + */ + if (cros_ec_check_features(ec, EC_FEATURE_MOTION_SENSE_FIFO)) { + ret = cros_ec_sensorhub_ring_add(data); + if (ret) + return ret; + /* + * The msg and its data is not under the control of the ring + * handler. + */ + return devm_add_action_or_reset(dev, + cros_ec_sensorhub_ring_remove, + data); + } + return 0; } +#ifdef CONFIG_PM_SLEEP +/* + * When the EC is suspending, we must stop sending interrupt, + * we may use the same interrupt line for waking up the device. + * Tell the EC to stop sending non-interrupt event on the iio ring. + */ +static int cros_ec_sensorhub_suspend(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct cros_ec_sensorhub *sensorhub = platform_get_drvdata(pdev); + struct cros_ec_dev *ec = sensorhub->ec; + + if (cros_ec_check_features(ec, EC_FEATURE_MOTION_SENSE_FIFO)) + return cros_ec_sensorhub_ring_fifo_enable(sensorhub, false); + return 0; +} + +static int cros_ec_sensorhub_resume(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct cros_ec_sensorhub *sensorhub = platform_get_drvdata(pdev); + struct cros_ec_dev *ec = sensorhub->ec; + + if (cros_ec_check_features(ec, EC_FEATURE_MOTION_SENSE_FIFO)) + return cros_ec_sensorhub_ring_fifo_enable(sensorhub, true); + return 0; +} +#endif + +static SIMPLE_DEV_PM_OPS(cros_ec_sensorhub_pm_ops, + cros_ec_sensorhub_suspend, + cros_ec_sensorhub_resume); + static struct platform_driver cros_ec_sensorhub_driver = { .driver = { .name = DRV_NAME, + .pm = &cros_ec_sensorhub_pm_ops, }, .probe = cros_ec_sensorhub_probe, }; diff --git a/drivers/platform/chrome/cros_ec_sensorhub_ring.c b/drivers/platform/chrome/cros_ec_sensorhub_ring.c new file mode 100644 index 000000000000..da73757529ca --- /dev/null +++ b/drivers/platform/chrome/cros_ec_sensorhub_ring.c @@ -0,0 +1,439 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Driver for Chrome OS EC Sensor hub FIFO. + * + * Copyright 2020 Google LLC + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static inline int +cros_sensorhub_send_sample(struct cros_ec_sensorhub *sensorhub, + struct cros_ec_sensors_ring_sample *sample) +{ + cros_ec_sensorhub_push_data_cb_t cb; + int id = sample->sensor_id; + struct iio_dev *indio_dev; + + if (id > sensorhub->sensor_num) + return -EINVAL; + + cb = sensorhub->push_data[id].push_data_cb; + if (!cb) + return 0; + + indio_dev = sensorhub->push_data[id].indio_dev; + + if (sample->flag & MOTIONSENSE_SENSOR_FLAG_FLUSH) + return 0; + + return cb(indio_dev, sample->vector, sample->timestamp); +} + +/** + * cros_ec_sensorhub_register_push_data() - register the callback to the hub. + * + * @sensorhub : Sensor Hub object + * @sensor_num : The sensor the caller is interested in. + * @indio_dev : The iio device to use when a sample arrives. + * @cb : The callback to call when a sample arrives. + * + * The callback cb will be used by cros_ec_sensorhub_ring to distribute events + * from the EC. + * + * Return: 0 when callback is registered. + * EINVAL is the sensor number is invalid or the slot already used. + */ +int cros_ec_sensorhub_register_push_data(struct cros_ec_sensorhub *sensorhub, + u8 sensor_num, + struct iio_dev *indio_dev, + cros_ec_sensorhub_push_data_cb_t cb) +{ + if (sensor_num >= sensorhub->sensor_num) + return -EINVAL; + if (sensorhub->push_data[sensor_num].indio_dev) + return -EINVAL; + + sensorhub->push_data[sensor_num].indio_dev = indio_dev; + sensorhub->push_data[sensor_num].push_data_cb = cb; + + return 0; +} +EXPORT_SYMBOL_GPL(cros_ec_sensorhub_register_push_data); + +void cros_ec_sensorhub_unregister_push_data(struct cros_ec_sensorhub *sensorhub, + u8 sensor_num) +{ + sensorhub->push_data[sensor_num].indio_dev = NULL; + sensorhub->push_data[sensor_num].push_data_cb = NULL; +} +EXPORT_SYMBOL_GPL(cros_ec_sensorhub_unregister_push_data); + +/** + * cros_ec_sensorhub_ring_fifo_enable() - Enable or disable interrupt generation + * for FIFO events. + * @sensorhub: Sensor Hub object + * @on: true when events are requested. + * + * To be called before sleeping or when noone is listening. + * Return: 0 on success, or an error when we can not communicate with the EC. + * + */ +int cros_ec_sensorhub_ring_fifo_enable(struct cros_ec_sensorhub *sensorhub, + bool on) +{ + int ret; + + mutex_lock(&sensorhub->cmd_lock); + sensorhub->params->cmd = MOTIONSENSE_CMD_FIFO_INT_ENABLE; + sensorhub->params->fifo_int_enable.enable = on; + + sensorhub->msg->outsize = sizeof(struct ec_params_motion_sense); + sensorhub->msg->insize = sizeof(struct ec_response_motion_sense); + + ret = cros_ec_cmd_xfer_status(sensorhub->ec->ec_dev, sensorhub->msg); + mutex_unlock(&sensorhub->cmd_lock); + + /* We expect to receive a payload of 4 bytes, ignore. */ + if (ret > 0) + ret = 0; + + return ret; +} + +/** + * cros_ec_sensor_ring_process_event() - process one EC FIFO event + * + * @sensorhub: Sensor Hub object. + * @fifo_info: FIFO information from the EC (includes b point, EC timebase). + * @fifo_timestamp: EC IRQ, kernel timebase (aka c). + * @current_timestamp: calculated event timestamp, kernel timebase (aka a'). + * @in: incoming FIFO event from EC (includes a point, EC timebase). + * @out: outgoing event to user space (includes a'). + * + * Process one EC event, add it in the ring if necessary. + * + * Return: true if out event has been populated. + */ +static bool +cros_ec_sensor_ring_process_event(struct cros_ec_sensorhub *sensorhub, + const struct ec_response_motion_sense_fifo_info + *fifo_info, + const ktime_t fifo_timestamp, + ktime_t *current_timestamp, + struct ec_response_motion_sensor_data *in, + struct cros_ec_sensors_ring_sample *out) +{ + const s64 now = cros_ec_get_time_ns(); + int axis, async_flags; + + /* Do not populate the filter based on asynchronous events. */ + async_flags = in->flags & + (MOTIONSENSE_SENSOR_FLAG_ODR | MOTIONSENSE_SENSOR_FLAG_FLUSH); + + if (in->flags & MOTIONSENSE_SENSOR_FLAG_TIMESTAMP && !async_flags) { + s64 new_timestamp; + + /* + * Disable filtering since we might add more jitter + * if b is in a random point in time. + */ + new_timestamp = fifo_timestamp - + fifo_info->timestamp * 1000 + + in->timestamp * 1000; + + /* + * The timestamp can be stale if we had to use the fifo + * info timestamp. + */ + if (new_timestamp - *current_timestamp > 0) + *current_timestamp = new_timestamp; + } + + if (in->flags & MOTIONSENSE_SENSOR_FLAG_FLUSH) { + out->sensor_id = in->sensor_num; + out->timestamp = *current_timestamp; + out->flag = in->flags; + /* + * No other payload information provided with + * flush ack. + */ + return true; + } + + if (in->flags & MOTIONSENSE_SENSOR_FLAG_TIMESTAMP) + /* If we just have a timestamp, skip this entry. */ + return false; + + /* Regular sample */ + out->sensor_id = in->sensor_num; + if (*current_timestamp - now > 0) + /* If the timestamp is in the future. */ + out->timestamp = now; + else + out->timestamp = *current_timestamp; + + out->flag = in->flags; + for (axis = 0; axis < 3; axis++) + out->vector[axis] = in->data[axis]; + + return true; +} + +/** + * cros_ec_sensorhub_ring_handler() - The trigger handler function + * + * @sensorhub: Sensor Hub object. + * + * Called by the notifier, process the EC sensor FIFO queue. + */ +static void cros_ec_sensorhub_ring_handler(struct cros_ec_sensorhub *sensorhub) +{ + struct ec_response_motion_sense_fifo_info *fifo_info = + sensorhub->fifo_info; + struct cros_ec_dev *ec = sensorhub->ec; + ktime_t fifo_timestamp, current_timestamp; + int i, j, number_data, ret; + struct ec_response_motion_sensor_data *in; + struct cros_ec_sensors_ring_sample *out, *last_out; + + mutex_lock(&sensorhub->cmd_lock); + + /* Get FIFO information if there are lost vectors. */ + if (fifo_info->total_lost) { + int fifo_info_length = + sizeof(struct ec_response_motion_sense_fifo_info) + + sizeof(u16) * sensorhub->sensor_num; + + /* Need to retrieve the number of lost vectors per sensor */ + sensorhub->params->cmd = MOTIONSENSE_CMD_FIFO_INFO; + sensorhub->msg->outsize = 1; + sensorhub->msg->insize = fifo_info_length; + + if (cros_ec_cmd_xfer_status(ec->ec_dev, sensorhub->msg) < 0) + goto error; + + memcpy(fifo_info, &sensorhub->resp->fifo_info, + fifo_info_length); + + /* + * Update collection time, will not be as precise as the + * non-error case. + */ + fifo_timestamp = cros_ec_get_time_ns(); + } else { + fifo_timestamp = sensorhub->fifo_timestamp[ + CROS_EC_SENSOR_NEW_TS]; + } + + if (fifo_info->count > sensorhub->fifo_size || + fifo_info->size != sensorhub->fifo_size) { + dev_warn(sensorhub->dev, + "Mismatch EC data: count %d, size %d - expected %d", + fifo_info->count, fifo_info->size, + sensorhub->fifo_size); + goto error; + } + + /* Copy elements in the main fifo */ + current_timestamp = sensorhub->fifo_timestamp[CROS_EC_SENSOR_LAST_TS]; + out = sensorhub->ring; + for (i = 0; i < fifo_info->count; i += number_data) { + sensorhub->params->cmd = MOTIONSENSE_CMD_FIFO_READ; + sensorhub->params->fifo_read.max_data_vector = + fifo_info->count - i; + sensorhub->msg->outsize = + sizeof(struct ec_params_motion_sense); + sensorhub->msg->insize = + sizeof(sensorhub->resp->fifo_read) + + sensorhub->params->fifo_read.max_data_vector * + sizeof(struct ec_response_motion_sensor_data); + ret = cros_ec_cmd_xfer_status(ec->ec_dev, sensorhub->msg); + if (ret < 0) { + dev_warn(sensorhub->dev, "Fifo error: %d\n", ret); + break; + } + number_data = sensorhub->resp->fifo_read.number_data; + if (number_data == 0) { + dev_dbg(sensorhub->dev, "Unexpected empty FIFO\n"); + break; + } + if (number_data > fifo_info->count - i) { + dev_warn(sensorhub->dev, + "Invalid EC data: too many entry received: %d, expected %d", + number_data, fifo_info->count - i); + break; + } + if (out + number_data > + sensorhub->ring + fifo_info->count) { + dev_warn(sensorhub->dev, + "Too many samples: %d (%zd data) to %d entries for expected %d entries", + i, out - sensorhub->ring, i + number_data, + fifo_info->count); + break; + } + + for (in = sensorhub->resp->fifo_read.data, j = 0; + j < number_data; j++, in++) { + if (cros_ec_sensor_ring_process_event( + sensorhub, fifo_info, + fifo_timestamp, + ¤t_timestamp, + in, out)) + out++; + } + } + mutex_unlock(&sensorhub->cmd_lock); + last_out = out; + + if (out == sensorhub->ring) + /* Unexpected empty FIFO. */ + goto ring_handler_end; + + /* + * Check if current_timestamp is ahead of the last sample. + * Normally, the EC appends a timestamp after the last sample, but if + * the AP is slow to respond to the IRQ, the EC may have added new + * samples. Use the FIFO info timestamp as last timestamp then. + */ + if ((last_out - 1)->timestamp == current_timestamp) + current_timestamp = fifo_timestamp; + + /* Warn on lost samples. */ + if (fifo_info->total_lost) + for (i = 0; i < sensorhub->sensor_num; i++) { + if (fifo_info->lost[i]) + dev_warn_ratelimited(sensorhub->dev, + "Sensor %d: lost: %d out of %d\n", + i, fifo_info->lost[i], + fifo_info->total_lost); + } + + /* Push the event into the FIFO. */ + for (out = sensorhub->ring; out < last_out; out++) + cros_sensorhub_send_sample(sensorhub, out); + +ring_handler_end: + sensorhub->fifo_timestamp[CROS_EC_SENSOR_LAST_TS] = current_timestamp; + return; + +error: + mutex_unlock(&sensorhub->cmd_lock); +} + +static int cros_ec_sensorhub_event(struct notifier_block *nb, + unsigned long queued_during_suspend, + void *_notify) +{ + struct cros_ec_sensorhub *sensorhub; + struct cros_ec_device *ec_dev; + + sensorhub = container_of(nb, struct cros_ec_sensorhub, notifier); + ec_dev = sensorhub->ec->ec_dev; + + if (ec_dev->event_data.event_type != EC_MKBP_EVENT_SENSOR_FIFO) + return NOTIFY_DONE; + + if (ec_dev->event_size != sizeof(ec_dev->event_data.data.sensor_fifo)) { + dev_warn(ec_dev->dev, "Invalid fifo info size\n"); + return NOTIFY_DONE; + } + + if (queued_during_suspend) + return NOTIFY_OK; + + memcpy(sensorhub->fifo_info, &ec_dev->event_data.data.sensor_fifo.info, + sizeof(*sensorhub->fifo_info)); + sensorhub->fifo_timestamp[CROS_EC_SENSOR_NEW_TS] = + ec_dev->last_event_time; + cros_ec_sensorhub_ring_handler(sensorhub); + + return NOTIFY_OK; +} + +/** + * cros_ec_sensorhub_ring_add() - Add the FIFO functionality if the EC + * supports it. + * + * @sensorhub : Sensor Hub object. + * + * Return: 0 on success. + */ +int cros_ec_sensorhub_ring_add(struct cros_ec_sensorhub *sensorhub) +{ + struct cros_ec_dev *ec = sensorhub->ec; + int ret; + int fifo_info_length = + sizeof(struct ec_response_motion_sense_fifo_info) + + sizeof(u16) * sensorhub->sensor_num; + + /* Allocate the array for lost events. */ + sensorhub->fifo_info = devm_kzalloc(sensorhub->dev, fifo_info_length, + GFP_KERNEL); + if (!sensorhub->fifo_info) + return -ENOMEM; + + /* Retrieve FIFO information */ + sensorhub->msg->version = 2; + sensorhub->params->cmd = MOTIONSENSE_CMD_FIFO_INFO; + sensorhub->msg->outsize = 1; + sensorhub->msg->insize = fifo_info_length; + + ret = cros_ec_cmd_xfer_status(ec->ec_dev, sensorhub->msg); + if (ret < 0) + return ret; + + /* + * Allocate the full fifo. We need to copy the whole FIFO to set + * timestamps properly. + */ + sensorhub->fifo_size = sensorhub->resp->fifo_info.size; + sensorhub->ring = devm_kcalloc(sensorhub->dev, sensorhub->fifo_size, + sizeof(*sensorhub->ring), GFP_KERNEL); + if (!sensorhub->ring) + return -ENOMEM; + + /* + * Allocate the callback area based on the number of sensors. + */ + sensorhub->push_data = devm_kcalloc( + sensorhub->dev, sensorhub->sensor_num, + sizeof(*sensorhub->push_data), + GFP_KERNEL); + if (!sensorhub->push_data) + return -ENOMEM; + + sensorhub->fifo_timestamp[CROS_EC_SENSOR_LAST_TS] = + cros_ec_get_time_ns(); + + /* Register the notifier that will act as a top half interrupt. */ + sensorhub->notifier.notifier_call = cros_ec_sensorhub_event; + ret = blocking_notifier_chain_register(&ec->ec_dev->event_notifier, + &sensorhub->notifier); + if (ret < 0) + return ret; + + /* Start collection samples. */ + return cros_ec_sensorhub_ring_fifo_enable(sensorhub, true); +} + +void cros_ec_sensorhub_ring_remove(void *arg) +{ + struct cros_ec_sensorhub *sensorhub = arg; + struct cros_ec_device *ec_dev = sensorhub->ec->ec_dev; + + /* Disable the ring, prevent EC interrupt to the AP for nothing. */ + cros_ec_sensorhub_ring_fifo_enable(sensorhub, false); + blocking_notifier_chain_unregister(&ec_dev->event_notifier, + &sensorhub->notifier); +} diff --git a/include/linux/platform_data/cros_ec_sensorhub.h b/include/linux/platform_data/cros_ec_sensorhub.h index 7e46a47fd642..b0950814f820 100644 --- a/include/linux/platform_data/cros_ec_sensorhub.h +++ b/include/linux/platform_data/cros_ec_sensorhub.h @@ -8,8 +8,13 @@ #ifndef __LINUX_PLATFORM_DATA_CROS_EC_SENSORHUB_H #define __LINUX_PLATFORM_DATA_CROS_EC_SENSORHUB_H +#include +#include +#include #include +struct iio_dev; + /** * struct cros_ec_sensor_platform - ChromeOS EC sensor platform information. * @sensor_num: Id of the sensor, as reported by the EC. @@ -19,14 +24,85 @@ struct cros_ec_sensor_platform { }; /** - * struct cros_ec_sensorhub - Sensor Hub device data. + * typedef cros_ec_sensorhub_push_data_cb_t - Callback function to send datum + * to specific sensors. * - * @ec: Embedded Controller where the hub is located. - * @sensor_num: Number of MEMS sensors present in the EC. + * @indio_dev: The IIO device that will process the sample. + * @data: Vector array of the ring sample. + * @timestamp: Timestamp in host timespace when the sample was acquired by + * the EC. */ -struct cros_ec_sensorhub { - struct cros_ec_dev *ec; - int sensor_num; +typedef int (*cros_ec_sensorhub_push_data_cb_t)(struct iio_dev *indio_dev, + s16 *data, + s64 timestamp); + +struct cros_ec_sensorhub_sensor_push_data { + struct iio_dev *indio_dev; + cros_ec_sensorhub_push_data_cb_t push_data_cb; }; +enum { + CROS_EC_SENSOR_LAST_TS, + CROS_EC_SENSOR_NEW_TS, + CROS_EC_SENSOR_ALL_TS +}; + +struct cros_ec_sensors_ring_sample { + u8 sensor_id; + u8 flag; + s16 vector[3]; + s64 timestamp; +} __packed; + +/** + * struct cros_ec_sensorhub - Sensor Hub device data. + * + * @dev: Device object, mostly used for logging. + * @ec: Embedded Controller where the hub is located. + * @sensor_num: Number of MEMS sensors present in the EC. + * @msg: Structure to send FIFO requests. + * @params: Pointer to parameters in msg. + * @resp: Pointer to responses in msg. + * @cmd_lock : Lock for sending msg. + * @notifier: Notifier to kick the FIFO interrupt. + * @ring: Preprocessed ring to store events. + * @fifo_timestamp: array for event timestamp and spreading. + * @fifo_info: copy of FIFO information coming from the EC. + * @fifo_size: size of the ring. + * @push_data: array of callback to send datums to iio sensor object. + */ +struct cros_ec_sensorhub { + struct device *dev; + struct cros_ec_dev *ec; + int sensor_num; + + struct cros_ec_command *msg; + struct ec_params_motion_sense *params; + struct ec_response_motion_sense *resp; + struct mutex cmd_lock; /* Lock for protecting msg structure. */ + + struct notifier_block notifier; + + struct cros_ec_sensors_ring_sample *ring; + + ktime_t fifo_timestamp[CROS_EC_SENSOR_ALL_TS]; + struct ec_response_motion_sense_fifo_info *fifo_info; + int fifo_size; + + struct cros_ec_sensorhub_sensor_push_data *push_data; +}; + +int cros_ec_sensorhub_register_push_data(struct cros_ec_sensorhub *sensorhub, + u8 sensor_num, + struct iio_dev *indio_dev, + cros_ec_sensorhub_push_data_cb_t cb); + +void cros_ec_sensorhub_unregister_push_data(struct cros_ec_sensorhub *sensorhub, + u8 sensor_num); + +int cros_ec_sensorhub_ring_add(struct cros_ec_sensorhub *sensorhub); +void cros_ec_sensorhub_ring_remove(void *arg); +int cros_ec_sensorhub_ring_fifo_enable(struct cros_ec_sensorhub *sensorhub, + bool on); + #endif /* __LINUX_PLATFORM_DATA_CROS_EC_SENSORHUB_H */ From b9b05664ebf6cd58d2a7f9a7cffa119f331114e6 Mon Sep 17 00:00:00 2001 From: Gwendal Grignou Date: Fri, 27 Mar 2020 15:34:34 -0700 Subject: [PATCH 293/635] platform/chrome: cros_ec_sensorhub: Add code to spread timestmap EC FIFO can send sensor events in batch. Spread them based on previous (TSa) and currnet timestamp (TSb) EC FIFO iio events +-----------+ | TSa | +-----------+ +---------------------------------------+ | event 1 | | event 1 | TSb - (TSb - TSa)/n * (n-1) | +-----------+ +---------------------------------------+ | event 2 | | event 2 | TSb - (TSb - TSa)/n * (n-2) | +-----------+ +---------------------------------------+ | ... | ------> | .... | | +-----------+ +---------------------------------------+ | event n-1 | | event 2 | TSb - (TSb - TSa)/n | +-----------+ +---------------------------------------+ | event n | | event 2 | TSb | +-----------+ +---------------------------------------+ | TSb | +-----------+ Signed-off-by: Gwendal Grignou Acked-by: Jonathan Cameron Acked-by: Lee Jones Acked-by: Andy Shevchenko Signed-off-by: Enric Balletbo i Serra --- .../platform/chrome/cros_ec_sensorhub_ring.c | 117 ++++++++++++++++-- 1 file changed, 109 insertions(+), 8 deletions(-) diff --git a/drivers/platform/chrome/cros_ec_sensorhub_ring.c b/drivers/platform/chrome/cros_ec_sensorhub_ring.c index da73757529ca..443db8277d2b 100644 --- a/drivers/platform/chrome/cros_ec_sensorhub_ring.c +++ b/drivers/platform/chrome/cros_ec_sensorhub_ring.c @@ -190,6 +190,101 @@ cros_ec_sensor_ring_process_event(struct cros_ec_sensorhub *sensorhub, return true; } +/* + * cros_ec_sensor_ring_spread_add: Calculate proper timestamps then add to + * ringbuffer. + * + * If there is a sample with a proper timestamp + * + * timestamp | count + * ----------------- + * older_unprocess_out --> TS1 | 1 + * TS1 | 2 + * out --> TS1 | 3 + * next_out --> TS2 | + * + * We spread time for the samples [older_unprocess_out .. out] + * between TS1 and TS2: [TS1+1/4, TS1+2/4, TS1+3/4, TS2]. + * + * If we reach the end of the samples, we compare with the + * current timestamp: + * + * older_unprocess_out --> TS1 | 1 + * TS1 | 2 + * out --> TS1 | 3 + * + * We know have [TS1+1/3, TS1+2/3, current timestamp] + */ +static void cros_ec_sensor_ring_spread_add(struct cros_ec_sensorhub *sensorhub, + unsigned long sensor_mask, + s64 current_timestamp, + struct cros_ec_sensors_ring_sample + *last_out) +{ + struct cros_ec_sensors_ring_sample *out; + int i; + + for_each_set_bit(i, &sensor_mask, sensorhub->sensor_num) { + s64 older_timestamp; + s64 timestamp; + struct cros_ec_sensors_ring_sample *older_unprocess_out = + sensorhub->ring; + struct cros_ec_sensors_ring_sample *next_out; + int count = 1; + + for (out = sensorhub->ring; out < last_out; out = next_out) { + s64 time_period; + + next_out = out + 1; + if (out->sensor_id != i) + continue; + + /* Timestamp to start with */ + older_timestamp = out->timestamp; + + /* Find next sample. */ + while (next_out < last_out && next_out->sensor_id != i) + next_out++; + + if (next_out >= last_out) { + timestamp = current_timestamp; + } else { + timestamp = next_out->timestamp; + if (timestamp == older_timestamp) { + count++; + continue; + } + } + + /* + * The next sample has a new timestamp, spread the + * unprocessed samples. + */ + if (next_out < last_out) + count++; + time_period = div_s64(timestamp - older_timestamp, + count); + + for (; older_unprocess_out <= out; + older_unprocess_out++) { + if (older_unprocess_out->sensor_id != i) + continue; + older_timestamp += time_period; + older_unprocess_out->timestamp = + older_timestamp; + } + count = 1; + /* The next_out sample has a valid timestamp, skip. */ + next_out++; + older_unprocess_out = next_out; + } + } + + /* Push the event into the kfifo */ + for (out = sensorhub->ring; out < last_out; out++) + cros_sensorhub_send_sample(sensorhub, out); +} + /** * cros_ec_sensorhub_ring_handler() - The trigger handler function * @@ -204,6 +299,7 @@ static void cros_ec_sensorhub_ring_handler(struct cros_ec_sensorhub *sensorhub) struct cros_ec_dev *ec = sensorhub->ec; ktime_t fifo_timestamp, current_timestamp; int i, j, number_data, ret; + unsigned long sensor_mask = 0; struct ec_response_motion_sensor_data *in; struct cros_ec_sensors_ring_sample *out, *last_out; @@ -289,8 +385,10 @@ static void cros_ec_sensorhub_ring_handler(struct cros_ec_sensorhub *sensorhub) sensorhub, fifo_info, fifo_timestamp, ¤t_timestamp, - in, out)) + in, out)) { + sensor_mask |= BIT(in->sensor_num); out++; + } } } mutex_unlock(&sensorhub->cmd_lock); @@ -301,10 +399,10 @@ static void cros_ec_sensorhub_ring_handler(struct cros_ec_sensorhub *sensorhub) goto ring_handler_end; /* - * Check if current_timestamp is ahead of the last sample. - * Normally, the EC appends a timestamp after the last sample, but if - * the AP is slow to respond to the IRQ, the EC may have added new - * samples. Use the FIFO info timestamp as last timestamp then. + * Check if current_timestamp is ahead of the last sample. Normally, + * the EC appends a timestamp after the last sample, but if the AP + * is slow to respond to the IRQ, the EC may have added new samples. + * Use the FIFO info timestamp as last timestamp then. */ if ((last_out - 1)->timestamp == current_timestamp) current_timestamp = fifo_timestamp; @@ -319,9 +417,12 @@ static void cros_ec_sensorhub_ring_handler(struct cros_ec_sensorhub *sensorhub) fifo_info->total_lost); } - /* Push the event into the FIFO. */ - for (out = sensorhub->ring; out < last_out; out++) - cros_sensorhub_send_sample(sensorhub, out); + /* + * Spread samples in case of batching, then add them to the + * ringbuffer. + */ + cros_ec_sensor_ring_spread_add(sensorhub, sensor_mask, + current_timestamp, last_out); ring_handler_end: sensorhub->fifo_timestamp[CROS_EC_SENSOR_LAST_TS] = current_timestamp; From 93fe48a585905675719835f8269258736de0948f Mon Sep 17 00:00:00 2001 From: Gwendal Grignou Date: Fri, 27 Mar 2020 15:34:35 -0700 Subject: [PATCH 294/635] platform/chrome: cros_ec_sensorhub: Add median filter Events are timestamped in EC time space, their timestamps need to be converted in host time space. The assumption is the time delta between when the interrupt is sent by the EC and when it is receive by the host is a [small] constant. This is not always true, even with hard-wired interrupt. To mitigate worst offenders, add a median filter to weed out bigger than expected delays. Signed-off-by: Gwendal Grignou Acked-by: Jonathan Cameron Acked-by: Lee Jones Acked-by: Andy Shevchenko Signed-off-by: Enric Balletbo i Serra --- .../platform/chrome/cros_ec_sensorhub_ring.c | 560 +++++++++++++++++- .../linux/platform_data/cros_ec_sensorhub.h | 93 ++- 2 files changed, 622 insertions(+), 31 deletions(-) diff --git a/drivers/platform/chrome/cros_ec_sensorhub_ring.c b/drivers/platform/chrome/cros_ec_sensorhub_ring.c index 443db8277d2b..230e6cf3da2f 100644 --- a/drivers/platform/chrome/cros_ec_sensorhub_ring.c +++ b/drivers/platform/chrome/cros_ec_sensorhub_ring.c @@ -17,6 +17,21 @@ #include #include +/* Precision of fixed point for the m values from the filter */ +#define M_PRECISION BIT(23) + +/* Only activate the filter once we have at least this many elements. */ +#define TS_HISTORY_THRESHOLD 8 + +/* + * If we don't have any history entries for this long, empty the filter to + * make sure there are no big discontinuities. + */ +#define TS_HISTORY_BORED_US 500000 + +/* To measure by how much the filter is overshooting, if it happens. */ +#define FUTURE_TS_ANALYTICS_COUNT_MAX 100 + static inline int cros_sensorhub_send_sample(struct cros_ec_sensorhub *sensorhub, struct cros_ec_sensors_ring_sample *sample) @@ -92,9 +107,13 @@ EXPORT_SYMBOL_GPL(cros_ec_sensorhub_unregister_push_data); int cros_ec_sensorhub_ring_fifo_enable(struct cros_ec_sensorhub *sensorhub, bool on) { - int ret; + int ret, i; mutex_lock(&sensorhub->cmd_lock); + if (sensorhub->tight_timestamps) + for (i = 0; i < sensorhub->sensor_num; i++) + sensorhub->batch_state[i].last_len = 0; + sensorhub->params->cmd = MOTIONSENSE_CMD_FIFO_INT_ENABLE; sensorhub->params->fifo_int_enable.enable = on; @@ -111,8 +130,245 @@ int cros_ec_sensorhub_ring_fifo_enable(struct cros_ec_sensorhub *sensorhub, return ret; } +static int cros_ec_sensor_ring_median_cmp(const void *pv1, const void *pv2) +{ + s64 v1 = *(s64 *)pv1; + s64 v2 = *(s64 *)pv2; + + if (v1 > v2) + return 1; + else if (v1 < v2) + return -1; + else + return 0; +} + +/* + * cros_ec_sensor_ring_median: Gets median of an array of numbers + * + * For now it's implemented using an inefficient > O(n) sort then return + * the middle element. A more optimal method would be something like + * quickselect, but given that n = 64 we can probably live with it in the + * name of clarity. + * + * Warning: the input array gets modified (sorted)! + */ +static s64 cros_ec_sensor_ring_median(s64 *array, size_t length) +{ + sort(array, length, sizeof(s64), cros_ec_sensor_ring_median_cmp, NULL); + return array[length / 2]; +} + +/* + * IRQ Timestamp Filtering + * + * Lower down in cros_ec_sensor_ring_process_event(), for each sensor event + * we have to calculate it's timestamp in the AP timebase. There are 3 time + * points: + * a - EC timebase, sensor event + * b - EC timebase, IRQ + * c - AP timebase, IRQ + * a' - what we want: sensor even in AP timebase + * + * While a and b are recorded at accurate times (due to the EC real time + * nature); c is pretty untrustworthy, even though it's recorded the + * first thing in ec_irq_handler(). There is a very good change we'll get + * added lantency due to: + * other irqs + * ddrfreq + * cpuidle + * + * Normally a' = c - b + a, but if we do that naive math any jitter in c + * will get coupled in a', which we don't want. We want a function + * a' = cros_ec_sensor_ring_ts_filter(a) which will filter out outliers in c. + * + * Think of a graph of AP time(b) on the y axis vs EC time(c) on the x axis. + * The slope of the line won't be exactly 1, there will be some clock drift + * between the 2 chips for various reasons (mechanical stress, temperature, + * voltage). We need to extrapolate values for a future x, without trusting + * recent y values too much. + * + * We use a median filter for the slope, then another median filter for the + * y-intercept to calculate this function: + * dx[n] = x[n-1] - x[n] + * dy[n] = x[n-1] - x[n] + * m[n] = dy[n] / dx[n] + * median_m = median(m[n-k:n]) + * error[i] = y[n-i] - median_m * x[n-i] + * median_error = median(error[:k]) + * predicted_y = median_m * x + median_error + * + * Implementation differences from above: + * - Redefined y to be actually c - b, this gives us a lot more precision + * to do the math. (c-b)/b variations are more obvious than c/b variations. + * - Since we don't have floating point, any operations involving slope are + * done using fixed point math (*M_PRECISION) + * - Since x and y grow with time, we keep zeroing the graph (relative to + * the last sample), this way math involving *x[n-i] will not overflow + * - EC timestamps are kept in us, it improves the slope calculation precision + */ + /** - * cros_ec_sensor_ring_process_event() - process one EC FIFO event + * cros_ec_sensor_ring_ts_filter_update() - Update filter history. + * + * @state: Filter information. + * @b: IRQ timestamp, EC timebase (us) + * @c: IRQ timestamp, AP timebase (ns) + * + * Given a new IRQ timestamp pair (EC and AP timebases), add it to the filter + * history. + */ +static void +cros_ec_sensor_ring_ts_filter_update(struct cros_ec_sensors_ts_filter_state + *state, + s64 b, s64 c) +{ + s64 x, y; + s64 dx, dy; + s64 m; /* stored as *M_PRECISION */ + s64 *m_history_copy = state->temp_buf; + s64 *error = state->temp_buf; + int i; + + /* we trust b the most, that'll be our independent variable */ + x = b; + /* y is the offset between AP and EC times, in ns */ + y = c - b * 1000; + + dx = (state->x_history[0] + state->x_offset) - x; + if (dx == 0) + return; /* we already have this irq in the history */ + dy = (state->y_history[0] + state->y_offset) - y; + m = div64_s64(dy * M_PRECISION, dx); + + /* Empty filter if we haven't seen any action in a while. */ + if (-dx > TS_HISTORY_BORED_US) + state->history_len = 0; + + /* Move everything over, also update offset to all absolute coords .*/ + for (i = state->history_len - 1; i >= 1; i--) { + state->x_history[i] = state->x_history[i - 1] + dx; + state->y_history[i] = state->y_history[i - 1] + dy; + + state->m_history[i] = state->m_history[i - 1]; + /* + * Also use the same loop to copy m_history for future + * median extraction. + */ + m_history_copy[i] = state->m_history[i - 1]; + } + + /* Store the x and y, but remember offset is actually last sample. */ + state->x_offset = x; + state->y_offset = y; + state->x_history[0] = 0; + state->y_history[0] = 0; + + state->m_history[0] = m; + m_history_copy[0] = m; + + if (state->history_len < CROS_EC_SENSORHUB_TS_HISTORY_SIZE) + state->history_len++; + + /* Precalculate things for the filter. */ + if (state->history_len > TS_HISTORY_THRESHOLD) { + state->median_m = + cros_ec_sensor_ring_median(m_history_copy, + state->history_len - 1); + + /* + * Calculate y-intercepts as if m_median is the slope and + * points in the history are on the line. median_error will + * still be in the offset coordinate system. + */ + for (i = 0; i < state->history_len; i++) + error[i] = state->y_history[i] - + div_s64(state->median_m * state->x_history[i], + M_PRECISION); + state->median_error = + cros_ec_sensor_ring_median(error, state->history_len); + } else { + state->median_m = 0; + state->median_error = 0; + } +} + +/** + * cros_ec_sensor_ring_ts_filter() - Translate EC timebase timestamp to AP + * timebase + * + * @state: filter information. + * @x: any ec timestamp (us): + * + * cros_ec_sensor_ring_ts_filter(a) => a' event timestamp, AP timebase + * cros_ec_sensor_ring_ts_filter(b) => calculated timestamp when the EC IRQ + * should have happened on the AP, with low jitter + * + * Note: The filter will only activate once state->history_len goes + * over TS_HISTORY_THRESHOLD. Otherwise it'll just do the naive c - b + a + * transform. + * + * How to derive the formula, starting from: + * f(x) = median_m * x + median_error + * That's the calculated AP - EC offset (at the x point in time) + * Undo the coordinate system transform: + * f(x) = median_m * (x - x_offset) + median_error + y_offset + * Remember to undo the "y = c - b * 1000" modification: + * f(x) = median_m * (x - x_offset) + median_error + y_offset + x * 1000 + * + * Return: timestamp in AP timebase (ns) + */ +static s64 +cros_ec_sensor_ring_ts_filter(struct cros_ec_sensors_ts_filter_state *state, + s64 x) +{ + return div_s64(state->median_m * (x - state->x_offset), M_PRECISION) + + state->median_error + state->y_offset + x * 1000; +} + +/* + * Since a and b were originally 32 bit values from the EC, + * they overflow relatively often, casting is not enough, so we need to + * add an offset. + */ +static void +cros_ec_sensor_ring_fix_overflow(s64 *ts, + const s64 overflow_period, + struct cros_ec_sensors_ec_overflow_state + *state) +{ + s64 adjust; + + *ts += state->offset; + if (abs(state->last - *ts) > (overflow_period / 2)) { + adjust = state->last > *ts ? overflow_period : -overflow_period; + state->offset += adjust; + *ts += adjust; + } + state->last = *ts; +} + +static void +cros_ec_sensor_ring_check_for_past_timestamp(struct cros_ec_sensorhub + *sensorhub, + struct cros_ec_sensors_ring_sample + *sample) +{ + const u8 sensor_id = sample->sensor_id; + + /* If this event is earlier than one we saw before... */ + if (sensorhub->batch_state[sensor_id].newest_sensor_event > + sample->timestamp) + /* mark it for spreading. */ + sample->timestamp = + sensorhub->batch_state[sensor_id].last_ts; + else + sensorhub->batch_state[sensor_id].newest_sensor_event = + sample->timestamp; +} + +/** + * cros_ec_sensor_ring_process_event() - Process one EC FIFO event * * @sensorhub: Sensor Hub object. * @fifo_info: FIFO information from the EC (includes b point, EC timebase). @@ -142,28 +398,57 @@ cros_ec_sensor_ring_process_event(struct cros_ec_sensorhub *sensorhub, (MOTIONSENSE_SENSOR_FLAG_ODR | MOTIONSENSE_SENSOR_FLAG_FLUSH); if (in->flags & MOTIONSENSE_SENSOR_FLAG_TIMESTAMP && !async_flags) { - s64 new_timestamp; + s64 a = in->timestamp; + s64 b = fifo_info->timestamp; + s64 c = fifo_timestamp; - /* - * Disable filtering since we might add more jitter - * if b is in a random point in time. - */ - new_timestamp = fifo_timestamp - - fifo_info->timestamp * 1000 + - in->timestamp * 1000; + cros_ec_sensor_ring_fix_overflow(&a, 1LL << 32, + &sensorhub->overflow_a); + cros_ec_sensor_ring_fix_overflow(&b, 1LL << 32, + &sensorhub->overflow_b); + if (sensorhub->tight_timestamps) { + cros_ec_sensor_ring_ts_filter_update( + &sensorhub->filter, b, c); + *current_timestamp = cros_ec_sensor_ring_ts_filter( + &sensorhub->filter, a); + } else { + s64 new_timestamp; + + /* + * Disable filtering since we might add more jitter + * if b is in a random point in time. + */ + new_timestamp = fifo_timestamp - + fifo_info->timestamp * 1000 + + in->timestamp * 1000; + /* + * The timestamp can be stale if we had to use the fifo + * info timestamp. + */ + if (new_timestamp - *current_timestamp > 0) + *current_timestamp = new_timestamp; + } + } + + if (in->flags & MOTIONSENSE_SENSOR_FLAG_ODR) { + if (sensorhub->tight_timestamps) { + sensorhub->batch_state[in->sensor_num].last_len = 0; + sensorhub->batch_state[in->sensor_num].penul_len = 0; + } /* - * The timestamp can be stale if we had to use the fifo - * info timestamp. + * ODR change is only useful for the sensor_ring, it does not + * convey information to clients. */ - if (new_timestamp - *current_timestamp > 0) - *current_timestamp = new_timestamp; + return false; } if (in->flags & MOTIONSENSE_SENSOR_FLAG_FLUSH) { out->sensor_id = in->sensor_num; out->timestamp = *current_timestamp; out->flag = in->flags; + if (sensorhub->tight_timestamps) + sensorhub->batch_state[out->sensor_id].last_len = 0; /* * No other payload information provided with * flush ack. @@ -177,22 +462,221 @@ cros_ec_sensor_ring_process_event(struct cros_ec_sensorhub *sensorhub, /* Regular sample */ out->sensor_id = in->sensor_num; - if (*current_timestamp - now > 0) - /* If the timestamp is in the future. */ + if (*current_timestamp - now > 0) { + /* + * This fix is needed to overcome the timestamp filter putting + * events in the future. + */ + sensorhub->future_timestamp_total_ns += + *current_timestamp - now; + if (++sensorhub->future_timestamp_count == + FUTURE_TS_ANALYTICS_COUNT_MAX) { + s64 avg = div_s64(sensorhub->future_timestamp_total_ns, + sensorhub->future_timestamp_count); + dev_warn_ratelimited(sensorhub->dev, + "100 timestamps in the future, %lldns shaved on average\n", + avg); + sensorhub->future_timestamp_count = 0; + sensorhub->future_timestamp_total_ns = 0; + } out->timestamp = now; - else + } else { out->timestamp = *current_timestamp; + } out->flag = in->flags; for (axis = 0; axis < 3; axis++) out->vector[axis] = in->data[axis]; + if (sensorhub->tight_timestamps) + cros_ec_sensor_ring_check_for_past_timestamp(sensorhub, out); return true; } /* * cros_ec_sensor_ring_spread_add: Calculate proper timestamps then add to - * ringbuffer. + * ringbuffer. + * + * This is the new spreading code, assumes every sample's timestamp + * preceeds the sample. Run if tight_timestamps == true. + * + * Sometimes the EC receives only one interrupt (hence timestamp) for + * a batch of samples. Only the first sample will have the correct + * timestamp. So we must interpolate the other samples. + * We use the previous batch timestamp and our current batch timestamp + * as a way to calculate period, then spread the samples evenly. + * + * s0 int, 0ms + * s1 int, 10ms + * s2 int, 20ms + * 30ms point goes by, no interrupt, previous one is still asserted + * downloading s2 and s3 + * s3 sample, 20ms (incorrect timestamp) + * s4 int, 40ms + * + * The batches are [(s0), (s1), (s2, s3), (s4)]. Since the 3rd batch + * has 2 samples in them, we adjust the timestamp of s3. + * s2 - s1 = 10ms, so s3 must be s2 + 10ms => 20ms. If s1 would have + * been part of a bigger batch things would have gotten a little + * more complicated. + * + * Note: we also assume another sensor sample doesn't break up a batch + * in 2 or more partitions. Example, there can't ever be a sync sensor + * in between S2 and S3. This simplifies the following code. + */ +static void +cros_ec_sensor_ring_spread_add(struct cros_ec_sensorhub *sensorhub, + unsigned long sensor_mask, + struct cros_ec_sensors_ring_sample *last_out) +{ + struct cros_ec_sensors_ring_sample *batch_start, *next_batch_start; + int id; + + for_each_set_bit(id, &sensor_mask, sensorhub->sensor_num) { + for (batch_start = sensorhub->ring; batch_start < last_out; + batch_start = next_batch_start) { + /* + * For each batch (where all samples have the same + * timestamp). + */ + int batch_len, sample_idx; + struct cros_ec_sensors_ring_sample *batch_end = + batch_start; + struct cros_ec_sensors_ring_sample *s; + s64 batch_timestamp = batch_start->timestamp; + s64 sample_period; + + /* + * Skip over batches that start with the sensor types + * we're not looking at right now. + */ + if (batch_start->sensor_id != id) { + next_batch_start = batch_start + 1; + continue; + } + + /* + * Do not start a batch + * from a flush, as it happens asynchronously to the + * regular flow of events. + */ + if (batch_start->flag & MOTIONSENSE_SENSOR_FLAG_FLUSH) { + cros_sensorhub_send_sample(sensorhub, + batch_start); + next_batch_start = batch_start + 1; + continue; + } + + if (batch_start->timestamp <= + sensorhub->batch_state[id].last_ts) { + batch_timestamp = + sensorhub->batch_state[id].last_ts; + batch_len = sensorhub->batch_state[id].last_len; + + sample_idx = batch_len; + + sensorhub->batch_state[id].last_ts = + sensorhub->batch_state[id].penul_ts; + sensorhub->batch_state[id].last_len = + sensorhub->batch_state[id].penul_len; + } else { + /* + * Push first sample in the batch to the, + * kifo, it's guaranteed to be correct, the + * rest will follow later on. + */ + sample_idx = 1; + batch_len = 1; + cros_sensorhub_send_sample(sensorhub, + batch_start); + batch_start++; + } + + /* Find all samples have the same timestamp. */ + for (s = batch_start; s < last_out; s++) { + if (s->sensor_id != id) + /* + * Skip over other sensor types that + * are interleaved, don't count them. + */ + continue; + if (s->timestamp != batch_timestamp) + /* we discovered the next batch */ + break; + if (s->flag & MOTIONSENSE_SENSOR_FLAG_FLUSH) + /* break on flush packets */ + break; + batch_end = s; + batch_len++; + } + + if (batch_len == 1) + goto done_with_this_batch; + + /* Can we calculate period? */ + if (sensorhub->batch_state[id].last_len == 0) { + dev_warn(sensorhub->dev, "Sensor %d: lost %d samples when spreading\n", + id, batch_len - 1); + goto done_with_this_batch; + /* + * Note: we're dropping the rest of the samples + * in this batch since we have no idea where + * they're supposed to go without a period + * calculation. + */ + } + + sample_period = div_s64(batch_timestamp - + sensorhub->batch_state[id].last_ts, + sensorhub->batch_state[id].last_len); + dev_dbg(sensorhub->dev, + "Adjusting %d samples, sensor %d last_batch @%lld (%d samples) batch_timestamp=%lld => period=%lld\n", + batch_len, id, + sensorhub->batch_state[id].last_ts, + sensorhub->batch_state[id].last_len, + batch_timestamp, + sample_period); + + /* + * Adjust timestamps of the samples then push them to + * kfifo. + */ + for (s = batch_start; s <= batch_end; s++) { + if (s->sensor_id != id) + /* + * Skip over other sensor types that + * are interleaved, don't change them. + */ + continue; + + s->timestamp = batch_timestamp + + sample_period * sample_idx; + sample_idx++; + + cros_sensorhub_send_sample(sensorhub, s); + } + +done_with_this_batch: + sensorhub->batch_state[id].penul_ts = + sensorhub->batch_state[id].last_ts; + sensorhub->batch_state[id].penul_len = + sensorhub->batch_state[id].last_len; + + sensorhub->batch_state[id].last_ts = + batch_timestamp; + sensorhub->batch_state[id].last_len = batch_len; + + next_batch_start = batch_end + 1; + } + } +} + +/* + * cros_ec_sensor_ring_spread_add_legacy: Calculate proper timestamps then + * add to ringbuffer (legacy). + * + * Note: This assumes we're running old firmware, where every sample's timestamp + * is after the sample. Run if tight_timestamps == false. * * If there is a sample with a proper timestamp * @@ -215,11 +699,12 @@ cros_ec_sensor_ring_process_event(struct cros_ec_sensorhub *sensorhub, * * We know have [TS1+1/3, TS1+2/3, current timestamp] */ -static void cros_ec_sensor_ring_spread_add(struct cros_ec_sensorhub *sensorhub, - unsigned long sensor_mask, - s64 current_timestamp, - struct cros_ec_sensors_ring_sample - *last_out) +static void +cros_ec_sensor_ring_spread_add_legacy(struct cros_ec_sensorhub *sensorhub, + unsigned long sensor_mask, + s64 current_timestamp, + struct cros_ec_sensors_ring_sample + *last_out) { struct cros_ec_sensors_ring_sample *out; int i; @@ -404,25 +889,34 @@ static void cros_ec_sensorhub_ring_handler(struct cros_ec_sensorhub *sensorhub) * is slow to respond to the IRQ, the EC may have added new samples. * Use the FIFO info timestamp as last timestamp then. */ - if ((last_out - 1)->timestamp == current_timestamp) + if (!sensorhub->tight_timestamps && + (last_out - 1)->timestamp == current_timestamp) current_timestamp = fifo_timestamp; /* Warn on lost samples. */ if (fifo_info->total_lost) for (i = 0; i < sensorhub->sensor_num; i++) { - if (fifo_info->lost[i]) + if (fifo_info->lost[i]) { dev_warn_ratelimited(sensorhub->dev, "Sensor %d: lost: %d out of %d\n", i, fifo_info->lost[i], fifo_info->total_lost); + if (sensorhub->tight_timestamps) + sensorhub->batch_state[i].last_len = 0; + } } /* * Spread samples in case of batching, then add them to the * ringbuffer. */ - cros_ec_sensor_ring_spread_add(sensorhub, sensor_mask, - current_timestamp, last_out); + if (sensorhub->tight_timestamps) + cros_ec_sensor_ring_spread_add(sensorhub, sensor_mask, + last_out); + else + cros_ec_sensor_ring_spread_add_legacy(sensorhub, sensor_mask, + current_timestamp, + last_out); ring_handler_end: sensorhub->fifo_timestamp[CROS_EC_SENSOR_LAST_TS] = current_timestamp; @@ -517,6 +1011,18 @@ int cros_ec_sensorhub_ring_add(struct cros_ec_sensorhub *sensorhub) sensorhub->fifo_timestamp[CROS_EC_SENSOR_LAST_TS] = cros_ec_get_time_ns(); + sensorhub->tight_timestamps = cros_ec_check_features( + ec, EC_FEATURE_MOTION_SENSE_TIGHT_TIMESTAMPS); + + if (sensorhub->tight_timestamps) { + sensorhub->batch_state = devm_kcalloc(sensorhub->dev, + sensorhub->sensor_num, + sizeof(*sensorhub->batch_state), + GFP_KERNEL); + if (!sensorhub->batch_state) + return -ENOMEM; + } + /* Register the notifier that will act as a top half interrupt. */ sensorhub->notifier.notifier_call = cros_ec_sensorhub_event; ret = blocking_notifier_chain_register(&ec->ec_dev->event_notifier, diff --git a/include/linux/platform_data/cros_ec_sensorhub.h b/include/linux/platform_data/cros_ec_sensorhub.h index b0950814f820..c588be843f61 100644 --- a/include/linux/platform_data/cros_ec_sensorhub.h +++ b/include/linux/platform_data/cros_ec_sensorhub.h @@ -54,7 +54,64 @@ struct cros_ec_sensors_ring_sample { s64 timestamp; } __packed; +/* State used for cros_ec_ring_fix_overflow */ +struct cros_ec_sensors_ec_overflow_state { + s64 offset; + s64 last; +}; + +/* Length of the filter, how long to remember entries for */ +#define CROS_EC_SENSORHUB_TS_HISTORY_SIZE 64 + /** + * struct cros_ec_sensors_ts_filter_state - Timestamp filetr state. + * + * @x_offset: x is EC interrupt time. x_offset its last value. + * @y_offset: y is the difference between AP and EC time, y_offset its last + * value. + * @x_history: The past history of x, relative to x_offset. + * @y_history: The past history of y, relative to y_offset. + * @m_history: rate between y and x. + * @history_len: Amount of valid historic data in the arrays. + * @temp_buf: Temporary buffer used when updating the filter. + * @median_m: median value of m_history + * @median_error: final error to apply to AP interrupt timestamp to get the + * "true timestamp" the event occurred. + */ +struct cros_ec_sensors_ts_filter_state { + s64 x_offset, y_offset; + s64 x_history[CROS_EC_SENSORHUB_TS_HISTORY_SIZE]; + s64 y_history[CROS_EC_SENSORHUB_TS_HISTORY_SIZE]; + s64 m_history[CROS_EC_SENSORHUB_TS_HISTORY_SIZE]; + int history_len; + + s64 temp_buf[CROS_EC_SENSORHUB_TS_HISTORY_SIZE]; + + s64 median_m; + s64 median_error; +}; + +/* struct cros_ec_sensors_ts_batch_state - State of batch of a single sensor. + * + * Use to store information to batch data using median fileter information. + * + * @penul_ts: last but one batch timestamp (penultimate timestamp). + * Used for timestamp spreading calculations + * when a batch shows up. + * @penul_len: last but one batch length. + * @last_ts: Last batch timestam. + * @last_len: Last batch length. + * @newest_sensor_event: Last sensor timestamp. + */ +struct cros_ec_sensors_ts_batch_state { + s64 penul_ts; + int penul_len; + s64 last_ts; + int last_len; + s64 newest_sensor_event; +}; + +/* * struct cros_ec_sensorhub - Sensor Hub device data. * * @dev: Device object, mostly used for logging. @@ -66,10 +123,26 @@ struct cros_ec_sensors_ring_sample { * @cmd_lock : Lock for sending msg. * @notifier: Notifier to kick the FIFO interrupt. * @ring: Preprocessed ring to store events. - * @fifo_timestamp: array for event timestamp and spreading. - * @fifo_info: copy of FIFO information coming from the EC. - * @fifo_size: size of the ring. - * @push_data: array of callback to send datums to iio sensor object. + * @fifo_timestamp: Array for event timestamp and spreading. + * @fifo_info: Copy of FIFO information coming from the EC. + * @fifo_size: Size of the ring. + * @batch_state: Per sensor information of the last batches received. + * @overflow_a: For handling timestamp overflow for a time (sensor events) + * @overflow_b: For handling timestamp overflow for b time (ec interrupts) + * @filter: Medium fileter structure. + * @tight_timestamps: Set to truen when EC support tight timestamping: + * The timestamps reported from the EC have low jitter. + * Timestamps also come before every sample. Set either + * by feature bits coming from the EC or userspace. + * @future_timestamp_count: Statistics used to compute shaved time. + * This occurs when timestamp interpolation from EC + * time to AP time accidentally puts timestamps in + * the future. These timestamps are clamped to + * `now` and these count/total_ns maintain the + * statistics for how much time was removed in a + * given period. + * @future_timestamp_total_ns: Total amount of time shaved. + * @push_data: Array of callback to send datums to iio sensor object. */ struct cros_ec_sensorhub { struct device *dev; @@ -89,6 +162,18 @@ struct cros_ec_sensorhub { struct ec_response_motion_sense_fifo_info *fifo_info; int fifo_size; + struct cros_ec_sensors_ts_batch_state *batch_state; + + struct cros_ec_sensors_ec_overflow_state overflow_a; + struct cros_ec_sensors_ec_overflow_state overflow_b; + + struct cros_ec_sensors_ts_filter_state filter; + + int tight_timestamps; + + s32 future_timestamp_count; + s64 future_timestamp_total_ns; + struct cros_ec_sensorhub_sensor_push_data *push_data; }; From d9452adcc5b485ab1b50352d9356cde75ae6ac0e Mon Sep 17 00:00:00 2001 From: Gwendal Grignou Date: Fri, 27 Mar 2020 15:34:36 -0700 Subject: [PATCH 295/635] iio: cros_ec: Move function description to .c file To prevent comment rot, move function description to cros_ec_sensors_core.c. Signed-off-by: Gwendal Grignou Acked-by: Jonathan Cameron Signed-off-by: Enric Balletbo i Serra --- .../cros_ec_sensors/cros_ec_sensors_core.c | 69 ++++++++++++++++ .../linux/iio/common/cros_ec_sensors_core.h | 80 ------------------- 2 files changed, 69 insertions(+), 80 deletions(-) diff --git a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c index d3a3626c7cd8..f3c000448b90 100644 --- a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c +++ b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c @@ -82,6 +82,14 @@ static void get_default_min_max_freq(enum motionsensor_type type, } } +/** + * cros_ec_sensors_core_init() - basic initialization of the core structure + * @pdev: platform device created for the sensors + * @indio_dev: iio device structure of the device + * @physical_device: true if the device refers to a physical device + * + * Return: 0 on success, -errno on failure. + */ int cros_ec_sensors_core_init(struct platform_device *pdev, struct iio_dev *indio_dev, bool physical_device) @@ -159,6 +167,16 @@ int cros_ec_sensors_core_init(struct platform_device *pdev, } EXPORT_SYMBOL_GPL(cros_ec_sensors_core_init); +/** + * cros_ec_motion_send_host_cmd() - send motion sense host command + * @state: pointer to state information for device + * @opt_length: optional length to reduce the response size, useful on the data + * path. Otherwise, the maximal allowed response size is used + * + * When called, the sub-command is assumed to be set in param->cmd. + * + * Return: 0 on success, -errno on failure. + */ int cros_ec_motion_send_host_cmd(struct cros_ec_sensors_core_state *state, u16 opt_length) { @@ -421,6 +439,14 @@ int cros_ec_sensors_read_lpc(struct iio_dev *indio_dev, } EXPORT_SYMBOL_GPL(cros_ec_sensors_read_lpc); +/** + * cros_ec_sensors_read_cmd() - retrieve data using the EC command protocol + * @indio_dev: pointer to IIO device + * @scan_mask: bitmap of the sensor indices to scan + * @data: location to store data + * + * Return: 0 on success, -errno on failure. + */ int cros_ec_sensors_read_cmd(struct iio_dev *indio_dev, unsigned long scan_mask, s16 *data) { @@ -445,6 +471,18 @@ int cros_ec_sensors_read_cmd(struct iio_dev *indio_dev, } EXPORT_SYMBOL_GPL(cros_ec_sensors_read_cmd); +/** + * cros_ec_sensors_capture() - the trigger handler function + * @irq: the interrupt number. + * @p: a pointer to the poll function. + * + * On a trigger event occurring, if the pollfunc is attached then this + * handler is called as a threaded interrupt (and hence may sleep). It + * is responsible for grabbing data from the device and pushing it into + * the associated buffer. + * + * Return: IRQ_HANDLED + */ irqreturn_t cros_ec_sensors_capture(int irq, void *p) { struct iio_poll_func *pf = p; @@ -480,6 +518,16 @@ done: } EXPORT_SYMBOL_GPL(cros_ec_sensors_capture); +/** + * cros_ec_sensors_core_read() - function to request a value from the sensor + * @st: pointer to state information for device + * @chan: channel specification structure table + * @val: will contain one element making up the returned value + * @val2: will contain another element making up the returned value + * @mask: specifies which values to be requested + * + * Return: the type of value returned by the device + */ int cros_ec_sensors_core_read(struct cros_ec_sensors_core_state *st, struct iio_chan_spec const *chan, int *val, int *val2, long mask) @@ -520,6 +568,17 @@ int cros_ec_sensors_core_read(struct cros_ec_sensors_core_state *st, } EXPORT_SYMBOL_GPL(cros_ec_sensors_core_read); +/** + * cros_ec_sensors_core_read_avail() - get available values + * @indio_dev: pointer to state information for device + * @chan: channel specification structure table + * @vals: list of available values + * @type: type of data returned + * @length: number of data returned in the array + * @mask: specifies which values to be requested + * + * Return: an error code, IIO_AVAIL_RANGE or IIO_AVAIL_LIST + */ int cros_ec_sensors_core_read_avail(struct iio_dev *indio_dev, struct iio_chan_spec const *chan, const int **vals, @@ -541,6 +600,16 @@ int cros_ec_sensors_core_read_avail(struct iio_dev *indio_dev, } EXPORT_SYMBOL_GPL(cros_ec_sensors_core_read_avail); +/** + * cros_ec_sensors_core_write() - function to write a value to the sensor + * @st: pointer to state information for device + * @chan: channel specification structure table + * @val: first part of value to write + * @val2: second part of value to write + * @mask: specifies which values to write + * + * Return: the type of value returned by the device + */ int cros_ec_sensors_core_write(struct cros_ec_sensors_core_state *st, struct iio_chan_spec const *chan, int val, int val2, long mask) diff --git a/include/linux/iio/common/cros_ec_sensors_core.h b/include/linux/iio/common/cros_ec_sensors_core.h index bb331e6356a9..0af918978f97 100644 --- a/include/linux/iio/common/cros_ec_sensors_core.h +++ b/include/linux/iio/common/cros_ec_sensors_core.h @@ -79,95 +79,25 @@ struct cros_ec_sensors_core_state { int frequencies[3]; }; -/** - * cros_ec_sensors_read_lpc() - retrieve data from EC shared memory - * @indio_dev: pointer to IIO device - * @scan_mask: bitmap of the sensor indices to scan - * @data: location to store data - * - * This is the safe function for reading the EC data. It guarantees that the - * data sampled was not modified by the EC while being read. - * - * Return: 0 on success, -errno on failure. - */ int cros_ec_sensors_read_lpc(struct iio_dev *indio_dev, unsigned long scan_mask, s16 *data); -/** - * cros_ec_sensors_read_cmd() - retrieve data using the EC command protocol - * @indio_dev: pointer to IIO device - * @scan_mask: bitmap of the sensor indices to scan - * @data: location to store data - * - * Return: 0 on success, -errno on failure. - */ int cros_ec_sensors_read_cmd(struct iio_dev *indio_dev, unsigned long scan_mask, s16 *data); struct platform_device; -/** - * cros_ec_sensors_core_init() - basic initialization of the core structure - * @pdev: platform device created for the sensors - * @indio_dev: iio device structure of the device - * @physical_device: true if the device refers to a physical device - * - * Return: 0 on success, -errno on failure. - */ int cros_ec_sensors_core_init(struct platform_device *pdev, struct iio_dev *indio_dev, bool physical_device); -/** - * cros_ec_sensors_capture() - the trigger handler function - * @irq: the interrupt number. - * @p: a pointer to the poll function. - * - * On a trigger event occurring, if the pollfunc is attached then this - * handler is called as a threaded interrupt (and hence may sleep). It - * is responsible for grabbing data from the device and pushing it into - * the associated buffer. - * - * Return: IRQ_HANDLED - */ irqreturn_t cros_ec_sensors_capture(int irq, void *p); -/** - * cros_ec_motion_send_host_cmd() - send motion sense host command - * @st: pointer to state information for device - * @opt_length: optional length to reduce the response size, useful on the data - * path. Otherwise, the maximal allowed response size is used - * - * When called, the sub-command is assumed to be set in param->cmd. - * - * Return: 0 on success, -errno on failure. - */ int cros_ec_motion_send_host_cmd(struct cros_ec_sensors_core_state *st, u16 opt_length); -/** - * cros_ec_sensors_core_read() - function to request a value from the sensor - * @st: pointer to state information for device - * @chan: channel specification structure table - * @val: will contain one element making up the returned value - * @val2: will contain another element making up the returned value - * @mask: specifies which values to be requested - * - * Return: the type of value returned by the device - */ int cros_ec_sensors_core_read(struct cros_ec_sensors_core_state *st, struct iio_chan_spec const *chan, int *val, int *val2, long mask); -/** - * cros_ec_sensors_core_read_avail() - get available values - * @indio_dev: pointer to state information for device - * @chan: channel specification structure table - * @vals: list of available values - * @type: type of data returned - * @length: number of data returned in the array - * @mask: specifies which values to be requested - * - * Return: an error code, IIO_AVAIL_RANGE or IIO_AVAIL_LIST - */ int cros_ec_sensors_core_read_avail(struct iio_dev *indio_dev, struct iio_chan_spec const *chan, const int **vals, @@ -175,16 +105,6 @@ int cros_ec_sensors_core_read_avail(struct iio_dev *indio_dev, int *length, long mask); -/** - * cros_ec_sensors_core_write() - function to write a value to the sensor - * @st: pointer to state information for device - * @chan: channel specification structure table - * @val: first part of value to write - * @val2: second part of value to write - * @mask: specifies which values to write - * - * Return: the type of value returned by the device - */ int cros_ec_sensors_core_write(struct cros_ec_sensors_core_state *st, struct iio_chan_spec const *chan, int val, int val2, long mask); From 69f0793eb60dacd153388974bbaaa1d3184d171d Mon Sep 17 00:00:00 2001 From: Gwendal Grignou Date: Fri, 27 Mar 2020 15:34:37 -0700 Subject: [PATCH 296/635] iio: expose iio_device_set_clock Some IIO devices may want to override the default (realtime) to another clock source by default. It can beneficial when timestamps coming from the hardware or underlying drivers are already in that format. It can always be overridden by attribute current_timestamp_clock. Signed-off-by: Gwendal Grignou Reviewed-by: Jonathan Cameron Signed-off-by: Enric Balletbo i Serra --- drivers/iio/industrialio-core.c | 8 +++++++- include/linux/iio/iio.h | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/iio/industrialio-core.c b/drivers/iio/industrialio-core.c index 65ff0d067018..26e963483bab 100644 --- a/drivers/iio/industrialio-core.c +++ b/drivers/iio/industrialio-core.c @@ -189,7 +189,12 @@ ssize_t iio_read_const_attr(struct device *dev, } EXPORT_SYMBOL(iio_read_const_attr); -static int iio_device_set_clock(struct iio_dev *indio_dev, clockid_t clock_id) +/** + * iio_device_set_clock() - Set current timestamping clock for the device + * @indio_dev: IIO device structure containing the device + * @clock_id: timestamping clock posix identifier to set. + */ +int iio_device_set_clock(struct iio_dev *indio_dev, clockid_t clock_id) { int ret; const struct iio_event_interface *ev_int = indio_dev->event_interface; @@ -207,6 +212,7 @@ static int iio_device_set_clock(struct iio_dev *indio_dev, clockid_t clock_id) return 0; } +EXPORT_SYMBOL(iio_device_set_clock); /** * iio_get_time_ns() - utility function to get a time stamp for events etc diff --git a/include/linux/iio/iio.h b/include/linux/iio/iio.h index 862ce0019eba..b18f34a8901f 100644 --- a/include/linux/iio/iio.h +++ b/include/linux/iio/iio.h @@ -627,6 +627,8 @@ static inline clockid_t iio_device_get_clock(const struct iio_dev *indio_dev) return indio_dev->clock_id; } +int iio_device_set_clock(struct iio_dev *indio_dev, clockid_t clock_id); + /** * dev_to_iio_dev() - Get IIO device struct from a device struct * @dev: The device embedded in the IIO device From aa984f1ba4a477c8ea39d2fa975a4f8de8a126e9 Mon Sep 17 00:00:00 2001 From: Gwendal Grignou Date: Fri, 27 Mar 2020 15:34:38 -0700 Subject: [PATCH 297/635] iio: cros_ec: Register to cros_ec_sensorhub when EC supports FIFO When EC supports FIFO, each IIO device registers a callback, to put samples in the buffer when they arrives from the FIFO. When no FIFO, the user space app needs to call trigger_new, or better register a high precision timer. Signed-off-by: Gwendal Grignou Reviewed-by: Jonathan Cameron Signed-off-by: Enric Balletbo i Serra --- drivers/iio/accel/cros_ec_accel_legacy.c | 8 +- .../cros_ec_sensors/cros_ec_lid_angle.c | 2 +- .../common/cros_ec_sensors/cros_ec_sensors.c | 9 +- .../cros_ec_sensors/cros_ec_sensors_core.c | 101 +++++++++++++++++- drivers/iio/light/cros_ec_light_prox.c | 9 +- drivers/iio/pressure/cros_ec_baro.c | 9 +- .../linux/iio/common/cros_ec_sensors_core.h | 10 +- 7 files changed, 119 insertions(+), 29 deletions(-) diff --git a/drivers/iio/accel/cros_ec_accel_legacy.c b/drivers/iio/accel/cros_ec_accel_legacy.c index 68e847c6255e..2532b9ad3384 100644 --- a/drivers/iio/accel/cros_ec_accel_legacy.c +++ b/drivers/iio/accel/cros_ec_accel_legacy.c @@ -170,7 +170,8 @@ static int cros_ec_accel_legacy_probe(struct platform_device *pdev) if (!indio_dev) return -ENOMEM; - ret = cros_ec_sensors_core_init(pdev, indio_dev, true); + ret = cros_ec_sensors_core_init(pdev, indio_dev, true, + cros_ec_sensors_capture, NULL); if (ret) return ret; @@ -190,11 +191,6 @@ static int cros_ec_accel_legacy_probe(struct platform_device *pdev) state->sign[CROS_EC_SENSOR_Z] = -1; } - ret = devm_iio_triggered_buffer_setup(dev, indio_dev, NULL, - cros_ec_sensors_capture, NULL); - if (ret) - return ret; - return devm_iio_device_register(dev, indio_dev); } diff --git a/drivers/iio/common/cros_ec_sensors/cros_ec_lid_angle.c b/drivers/iio/common/cros_ec_sensors/cros_ec_lid_angle.c index 1dcc2a16ab2d..e30a59fcf0f9 100644 --- a/drivers/iio/common/cros_ec_sensors/cros_ec_lid_angle.c +++ b/drivers/iio/common/cros_ec_sensors/cros_ec_lid_angle.c @@ -97,7 +97,7 @@ static int cros_ec_lid_angle_probe(struct platform_device *pdev) if (!indio_dev) return -ENOMEM; - ret = cros_ec_sensors_core_init(pdev, indio_dev, false); + ret = cros_ec_sensors_core_init(pdev, indio_dev, false, NULL, NULL); if (ret) return ret; diff --git a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c index 576e45faafaf..711134d67dde 100644 --- a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c +++ b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c @@ -230,7 +230,9 @@ static int cros_ec_sensors_probe(struct platform_device *pdev) if (!indio_dev) return -ENOMEM; - ret = cros_ec_sensors_core_init(pdev, indio_dev, true); + ret = cros_ec_sensors_core_init(pdev, indio_dev, true, + cros_ec_sensors_capture, + cros_ec_sensors_push_data); if (ret) return ret; @@ -292,11 +294,6 @@ static int cros_ec_sensors_probe(struct platform_device *pdev) else state->core.read_ec_sensors_data = cros_ec_sensors_read_cmd; - ret = devm_iio_triggered_buffer_setup(dev, indio_dev, NULL, - cros_ec_sensors_capture, NULL); - if (ret) - return ret; - return devm_iio_device_register(dev, indio_dev); } diff --git a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c index f3c000448b90..01513cb93365 100644 --- a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c +++ b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -82,17 +83,71 @@ static void get_default_min_max_freq(enum motionsensor_type type, } } +int cros_ec_sensors_push_data(struct iio_dev *indio_dev, + s16 *data, + s64 timestamp) +{ + struct cros_ec_sensors_core_state *st = iio_priv(indio_dev); + s16 *out; + s64 delta; + unsigned int i; + + /* + * Ignore samples if the buffer is not set: it is needed if the ODR is + * set but the buffer is not enabled yet. + */ + if (!iio_buffer_enabled(indio_dev)) + return 0; + + out = (s16 *)st->samples; + for_each_set_bit(i, + indio_dev->active_scan_mask, + indio_dev->masklength) { + *out = data[i]; + out++; + } + + if (iio_device_get_clock(indio_dev) != CLOCK_BOOTTIME) + delta = iio_get_time_ns(indio_dev) - cros_ec_get_time_ns(); + else + delta = 0; + + iio_push_to_buffers_with_timestamp(indio_dev, st->samples, + timestamp + delta); + + return 0; +} +EXPORT_SYMBOL_GPL(cros_ec_sensors_push_data); + +static void cros_ec_sensors_core_clean(void *arg) +{ + struct platform_device *pdev = (struct platform_device *)arg; + struct cros_ec_sensorhub *sensor_hub = + dev_get_drvdata(pdev->dev.parent); + struct iio_dev *indio_dev = platform_get_drvdata(pdev); + struct cros_ec_sensors_core_state *st = iio_priv(indio_dev); + u8 sensor_num = st->param.info.sensor_num; + + cros_ec_sensorhub_unregister_push_data(sensor_hub, sensor_num); +} + /** * cros_ec_sensors_core_init() - basic initialization of the core structure * @pdev: platform device created for the sensors * @indio_dev: iio device structure of the device * @physical_device: true if the device refers to a physical device + * @trigger_capture: function pointer to call buffer is triggered, + * for backward compatibility. + * @push_data: function to call when cros_ec_sensorhub receives + * a sample for that sensor. * * Return: 0 on success, -errno on failure. */ int cros_ec_sensors_core_init(struct platform_device *pdev, struct iio_dev *indio_dev, - bool physical_device) + bool physical_device, + cros_ec_sensors_capture_t trigger_capture, + cros_ec_sensorhub_push_data_cb_t push_data) { struct device *dev = &pdev->dev; struct cros_ec_sensors_core_state *state = iio_priv(indio_dev); @@ -131,8 +186,6 @@ int cros_ec_sensors_core_init(struct platform_device *pdev, indio_dev->name = pdev->name; if (physical_device) { - indio_dev->modes = INDIO_DIRECT_MODE; - state->param.cmd = MOTIONSENSE_CMD_INFO; state->param.info.sensor_num = sensor_platform->sensor_num; ret = cros_ec_motion_send_host_cmd(state, 0); @@ -161,6 +214,48 @@ int cros_ec_sensors_core_init(struct platform_device *pdev, state->frequencies[2] = state->resp->info_3.max_frequency; } + + if (cros_ec_check_features(ec, EC_FEATURE_MOTION_SENSE_FIFO)) { + /* + * Create a software buffer, feed by the EC FIFO. + * We can not use trigger here, as events are generated + * as soon as sample_frequency is set. + */ + struct iio_buffer *buffer; + + buffer = devm_iio_kfifo_allocate(dev); + if (!buffer) + return -ENOMEM; + + iio_device_attach_buffer(indio_dev, buffer); + indio_dev->modes = INDIO_BUFFER_SOFTWARE; + + ret = cros_ec_sensorhub_register_push_data( + sensor_hub, sensor_platform->sensor_num, + indio_dev, push_data); + if (ret) + return ret; + + ret = devm_add_action_or_reset( + dev, cros_ec_sensors_core_clean, pdev); + if (ret) + return ret; + + /* Timestamp coming from FIFO are in ns since boot. */ + ret = iio_device_set_clock(indio_dev, CLOCK_BOOTTIME); + if (ret) + return ret; + } else { + /* + * The only way to get samples in buffer is to set a + * software tigger (systrig, hrtimer). + */ + ret = devm_iio_triggered_buffer_setup( + dev, indio_dev, NULL, trigger_capture, + NULL); + if (ret) + return ret; + } } return 0; diff --git a/drivers/iio/light/cros_ec_light_prox.c b/drivers/iio/light/cros_ec_light_prox.c index 7a838e2956f4..03c951ff4a3c 100644 --- a/drivers/iio/light/cros_ec_light_prox.c +++ b/drivers/iio/light/cros_ec_light_prox.c @@ -177,7 +177,9 @@ static int cros_ec_light_prox_probe(struct platform_device *pdev) if (!indio_dev) return -ENOMEM; - ret = cros_ec_sensors_core_init(pdev, indio_dev, true); + ret = cros_ec_sensors_core_init(pdev, indio_dev, true, + cros_ec_sensors_capture, + cros_ec_sensors_push_data); if (ret) return ret; @@ -236,11 +238,6 @@ static int cros_ec_light_prox_probe(struct platform_device *pdev) state->core.read_ec_sensors_data = cros_ec_sensors_read_cmd; - ret = devm_iio_triggered_buffer_setup(dev, indio_dev, NULL, - cros_ec_sensors_capture, NULL); - if (ret) - return ret; - return devm_iio_device_register(dev, indio_dev); } diff --git a/drivers/iio/pressure/cros_ec_baro.c b/drivers/iio/pressure/cros_ec_baro.c index b521bebd551c..6add499f11aa 100644 --- a/drivers/iio/pressure/cros_ec_baro.c +++ b/drivers/iio/pressure/cros_ec_baro.c @@ -134,7 +134,9 @@ static int cros_ec_baro_probe(struct platform_device *pdev) if (!indio_dev) return -ENOMEM; - ret = cros_ec_sensors_core_init(pdev, indio_dev, true); + ret = cros_ec_sensors_core_init(pdev, indio_dev, true, + cros_ec_sensors_capture, + cros_ec_sensors_push_data); if (ret) return ret; @@ -182,11 +184,6 @@ static int cros_ec_baro_probe(struct platform_device *pdev) state->core.read_ec_sensors_data = cros_ec_sensors_read_cmd; - ret = devm_iio_triggered_buffer_setup(dev, indio_dev, NULL, - cros_ec_sensors_capture, NULL); - if (ret) - return ret; - return devm_iio_device_register(dev, indio_dev); } diff --git a/include/linux/iio/common/cros_ec_sensors_core.h b/include/linux/iio/common/cros_ec_sensors_core.h index 0af918978f97..b8f573ca9dcc 100644 --- a/include/linux/iio/common/cros_ec_sensors_core.h +++ b/include/linux/iio/common/cros_ec_sensors_core.h @@ -12,6 +12,7 @@ #include #include #include +#include enum { CROS_EC_SENSOR_X, @@ -32,6 +33,8 @@ enum { /* Minimum sampling period to use when device is suspending */ #define CROS_EC_MIN_SUSPEND_SAMPLING_FREQUENCY 1000 /* 1 second */ +typedef irqreturn_t (*cros_ec_sensors_capture_t)(int irq, void *p); + /** * struct cros_ec_sensors_core_state - state data for EC sensors IIO driver * @ec: cros EC device structure @@ -87,9 +90,14 @@ int cros_ec_sensors_read_cmd(struct iio_dev *indio_dev, unsigned long scan_mask, struct platform_device; int cros_ec_sensors_core_init(struct platform_device *pdev, - struct iio_dev *indio_dev, bool physical_device); + struct iio_dev *indio_dev, bool physical_device, + cros_ec_sensors_capture_t trigger_capture, + cros_ec_sensorhub_push_data_cb_t push_data); irqreturn_t cros_ec_sensors_capture(int irq, void *p); +int cros_ec_sensors_push_data(struct iio_dev *indio_dev, + s16 *data, + s64 timestamp); int cros_ec_motion_send_host_cmd(struct cros_ec_sensors_core_state *st, u16 opt_length); From 2861be4ca9125ee1b7c49895948ca4236449a7fe Mon Sep 17 00:00:00 2001 From: Gwendal Grignou Date: Fri, 27 Mar 2020 15:34:39 -0700 Subject: [PATCH 298/635] iio: cros_ec: Remove pm function Since cros_ec_sensorhub is shutting down the FIFO when the device suspends, no need to slow down the EC sampling period rate. It was necesseary to do that before command CMD_FIFO_INT_ENABLE was introduced, but now all supported chromebooks have it. Signed-off-by: Gwendal Grignou Acked-by: Jonathan Cameron Acked-by: Lee Jones Signed-off-by: Enric Balletbo i Serra --- .../cros_ec_sensors/cros_ec_lid_angle.c | 1 - .../common/cros_ec_sensors/cros_ec_sensors.c | 1 - .../cros_ec_sensors/cros_ec_sensors_core.c | 47 ------------------- drivers/iio/light/cros_ec_light_prox.c | 1 - .../linux/iio/common/cros_ec_sensors_core.h | 5 -- 5 files changed, 55 deletions(-) diff --git a/drivers/iio/common/cros_ec_sensors/cros_ec_lid_angle.c b/drivers/iio/common/cros_ec_sensors/cros_ec_lid_angle.c index e30a59fcf0f9..af801e203623 100644 --- a/drivers/iio/common/cros_ec_sensors/cros_ec_lid_angle.c +++ b/drivers/iio/common/cros_ec_sensors/cros_ec_lid_angle.c @@ -127,7 +127,6 @@ MODULE_DEVICE_TABLE(platform, cros_ec_lid_angle_ids); static struct platform_driver cros_ec_lid_angle_platform_driver = { .driver = { .name = DRV_NAME, - .pm = &cros_ec_sensors_pm_ops, }, .probe = cros_ec_lid_angle_probe, .id_table = cros_ec_lid_angle_ids, diff --git a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c index 711134d67dde..fad21a90bc7e 100644 --- a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c +++ b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c @@ -314,7 +314,6 @@ MODULE_DEVICE_TABLE(platform, cros_ec_sensors_ids); static struct platform_driver cros_ec_sensors_platform_driver = { .driver = { .name = "cros-ec-sensors", - .pm = &cros_ec_sensors_pm_ops, }, .probe = cros_ec_sensors_probe, .id_table = cros_ec_sensors_ids, diff --git a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c index 01513cb93365..a1ecbd55ea76 100644 --- a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c +++ b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c @@ -738,52 +738,5 @@ int cros_ec_sensors_core_write(struct cros_ec_sensors_core_state *st, } EXPORT_SYMBOL_GPL(cros_ec_sensors_core_write); -static int __maybe_unused cros_ec_sensors_prepare(struct device *dev) -{ - struct iio_dev *indio_dev = dev_get_drvdata(dev); - struct cros_ec_sensors_core_state *st = iio_priv(indio_dev); - - if (st->curr_sampl_freq == 0) - return 0; - - /* - * If the sensors are sampled at high frequency, we will not be able to - * sleep. Set sampling to a long period if necessary. - */ - if (st->curr_sampl_freq < CROS_EC_MIN_SUSPEND_SAMPLING_FREQUENCY) { - mutex_lock(&st->cmd_lock); - st->param.cmd = MOTIONSENSE_CMD_EC_RATE; - st->param.ec_rate.data = CROS_EC_MIN_SUSPEND_SAMPLING_FREQUENCY; - cros_ec_motion_send_host_cmd(st, 0); - mutex_unlock(&st->cmd_lock); - } - return 0; -} - -static void __maybe_unused cros_ec_sensors_complete(struct device *dev) -{ - struct iio_dev *indio_dev = dev_get_drvdata(dev); - struct cros_ec_sensors_core_state *st = iio_priv(indio_dev); - - if (st->curr_sampl_freq == 0) - return; - - if (st->curr_sampl_freq < CROS_EC_MIN_SUSPEND_SAMPLING_FREQUENCY) { - mutex_lock(&st->cmd_lock); - st->param.cmd = MOTIONSENSE_CMD_EC_RATE; - st->param.ec_rate.data = st->curr_sampl_freq; - cros_ec_motion_send_host_cmd(st, 0); - mutex_unlock(&st->cmd_lock); - } -} - -const struct dev_pm_ops cros_ec_sensors_pm_ops = { -#ifdef CONFIG_PM_SLEEP - .prepare = cros_ec_sensors_prepare, - .complete = cros_ec_sensors_complete -#endif -}; -EXPORT_SYMBOL_GPL(cros_ec_sensors_pm_ops); - MODULE_DESCRIPTION("ChromeOS EC sensor hub core functions"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/iio/light/cros_ec_light_prox.c b/drivers/iio/light/cros_ec_light_prox.c index 03c951ff4a3c..e10b35de4c2f 100644 --- a/drivers/iio/light/cros_ec_light_prox.c +++ b/drivers/iio/light/cros_ec_light_prox.c @@ -255,7 +255,6 @@ MODULE_DEVICE_TABLE(platform, cros_ec_light_prox_ids); static struct platform_driver cros_ec_light_prox_platform_driver = { .driver = { .name = "cros-ec-light-prox", - .pm = &cros_ec_sensors_pm_ops, }, .probe = cros_ec_light_prox_probe, .id_table = cros_ec_light_prox_ids, diff --git a/include/linux/iio/common/cros_ec_sensors_core.h b/include/linux/iio/common/cros_ec_sensors_core.h index b8f573ca9dcc..96ea4551945e 100644 --- a/include/linux/iio/common/cros_ec_sensors_core.h +++ b/include/linux/iio/common/cros_ec_sensors_core.h @@ -30,9 +30,6 @@ enum { */ #define CROS_EC_SAMPLE_SIZE (sizeof(s64) * 2) -/* Minimum sampling period to use when device is suspending */ -#define CROS_EC_MIN_SUSPEND_SAMPLING_FREQUENCY 1000 /* 1 second */ - typedef irqreturn_t (*cros_ec_sensors_capture_t)(int irq, void *p); /** @@ -117,8 +114,6 @@ int cros_ec_sensors_core_write(struct cros_ec_sensors_core_state *st, struct iio_chan_spec const *chan, int val, int val2, long mask); -extern const struct dev_pm_ops cros_ec_sensors_pm_ops; - /* List of extended channel specification for all sensors */ extern const struct iio_chan_spec_ext_info cros_ec_sensors_ext_info[]; From 6562793b55c58b6b1dcb9cd581c7905afc25e89f Mon Sep 17 00:00:00 2001 From: Gwendal Grignou Date: Fri, 27 Mar 2020 15:34:40 -0700 Subject: [PATCH 299/635] iio: cros_ec: Expose hwfifo_timeout Expose EC minimal interrupt period through buffer/hwfifo_timeout: - Maximal timeout is limited to 65s. - When timeout for all sensors is set to 0, EC will not send events, even if the sensor sampling rate is greater than 0. Rename frequency to sampling_frequency to match IIO ABI. Signed-off-by: Gwendal Grignou Reviewed-by: Jonathan Cameron Signed-off-by: Enric Balletbo i Serra --- .../common/cros_ec_sensors/cros_ec_sensors.c | 3 +- .../cros_ec_sensors/cros_ec_sensors_core.c | 95 ++++++++++++++----- drivers/iio/light/cros_ec_light_prox.c | 5 +- drivers/iio/pressure/cros_ec_baro.c | 5 +- .../linux/iio/common/cros_ec_sensors_core.h | 4 +- 5 files changed, 82 insertions(+), 30 deletions(-) diff --git a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c index fad21a90bc7e..a66941fdb385 100644 --- a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c +++ b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c @@ -236,6 +236,8 @@ static int cros_ec_sensors_probe(struct platform_device *pdev) if (ret) return ret; + iio_buffer_set_attrs(indio_dev->buffer, cros_ec_sensor_fifo_attributes); + indio_dev->info = &ec_sensors_info; state = iio_priv(indio_dev); for (channel = state->channels, i = CROS_EC_SENSOR_X; @@ -247,7 +249,6 @@ static int cros_ec_sensors_probe(struct platform_device *pdev) BIT(IIO_CHAN_INFO_CALIBSCALE); channel->info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SCALE) | - BIT(IIO_CHAN_INFO_FREQUENCY) | BIT(IIO_CHAN_INFO_SAMP_FREQ); channel->info_mask_shared_by_all_available = BIT(IIO_CHAN_INFO_SAMP_FREQ); diff --git a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c index a1ecbd55ea76..b8eac7e5d5e5 100644 --- a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c +++ b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -83,6 +84,77 @@ static void get_default_min_max_freq(enum motionsensor_type type, } } +static int cros_ec_sensor_set_ec_rate(struct cros_ec_sensors_core_state *st, + int rate) +{ + int ret; + + if (rate > U16_MAX) + rate = U16_MAX; + + mutex_lock(&st->cmd_lock); + st->param.cmd = MOTIONSENSE_CMD_EC_RATE; + st->param.ec_rate.data = rate; + ret = cros_ec_motion_send_host_cmd(st, 0); + mutex_unlock(&st->cmd_lock); + return ret; +} + +static ssize_t cros_ec_sensor_set_report_latency(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t len) +{ + struct iio_dev *indio_dev = dev_to_iio_dev(dev); + struct cros_ec_sensors_core_state *st = iio_priv(indio_dev); + int integer, fract, ret; + int latency; + + ret = iio_str_to_fixpoint(buf, 100000, &integer, &fract); + if (ret) + return ret; + + /* EC rate is in ms. */ + latency = integer * 1000 + fract / 1000; + ret = cros_ec_sensor_set_ec_rate(st, latency); + if (ret < 0) + return ret; + + return len; +} + +static ssize_t cros_ec_sensor_get_report_latency(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct iio_dev *indio_dev = dev_to_iio_dev(dev); + struct cros_ec_sensors_core_state *st = iio_priv(indio_dev); + int latency, ret; + + mutex_lock(&st->cmd_lock); + st->param.cmd = MOTIONSENSE_CMD_EC_RATE; + st->param.ec_rate.data = EC_MOTION_SENSE_NO_VALUE; + + ret = cros_ec_motion_send_host_cmd(st, 0); + latency = st->resp->ec_rate.ret; + mutex_unlock(&st->cmd_lock); + if (ret < 0) + return ret; + + return sprintf(buf, "%d.%06u\n", + latency / 1000, + (latency % 1000) * 1000); +} + +static IIO_DEVICE_ATTR(hwfifo_timeout, 0644, + cros_ec_sensor_get_report_latency, + cros_ec_sensor_set_report_latency, 0); + +const struct attribute *cros_ec_sensor_fifo_attributes[] = { + &iio_dev_attr_hwfifo_timeout.dev_attr.attr, + NULL, +}; +EXPORT_SYMBOL_GPL(cros_ec_sensor_fifo_attributes); + int cros_ec_sensors_push_data(struct iio_dev *indio_dev, s16 *data, s64 timestamp) @@ -631,18 +703,6 @@ int cros_ec_sensors_core_read(struct cros_ec_sensors_core_state *st, switch (mask) { case IIO_CHAN_INFO_SAMP_FREQ: - st->param.cmd = MOTIONSENSE_CMD_EC_RATE; - st->param.ec_rate.data = - EC_MOTION_SENSE_NO_VALUE; - - ret = cros_ec_motion_send_host_cmd(st, 0); - if (ret) - break; - - *val = st->resp->ec_rate.ret; - ret = IIO_VAL_INT; - break; - case IIO_CHAN_INFO_FREQUENCY: st->param.cmd = MOTIONSENSE_CMD_SENSOR_ODR; st->param.sensor_odr.data = EC_MOTION_SENSE_NO_VALUE; @@ -712,7 +772,7 @@ int cros_ec_sensors_core_write(struct cros_ec_sensors_core_state *st, int ret; switch (mask) { - case IIO_CHAN_INFO_FREQUENCY: + case IIO_CHAN_INFO_SAMP_FREQ: st->param.cmd = MOTIONSENSE_CMD_SENSOR_ODR; st->param.sensor_odr.data = val; @@ -721,15 +781,6 @@ int cros_ec_sensors_core_write(struct cros_ec_sensors_core_state *st, ret = cros_ec_motion_send_host_cmd(st, 0); break; - case IIO_CHAN_INFO_SAMP_FREQ: - st->param.cmd = MOTIONSENSE_CMD_EC_RATE; - st->param.ec_rate.data = val; - - ret = cros_ec_motion_send_host_cmd(st, 0); - if (ret) - break; - st->curr_sampl_freq = val; - break; default: ret = -EINVAL; break; diff --git a/drivers/iio/light/cros_ec_light_prox.c b/drivers/iio/light/cros_ec_light_prox.c index e10b35de4c2f..2198b50909ed 100644 --- a/drivers/iio/light/cros_ec_light_prox.c +++ b/drivers/iio/light/cros_ec_light_prox.c @@ -183,6 +183,8 @@ static int cros_ec_light_prox_probe(struct platform_device *pdev) if (ret) return ret; + iio_buffer_set_attrs(indio_dev->buffer, cros_ec_sensor_fifo_attributes); + indio_dev->info = &cros_ec_light_prox_info; state = iio_priv(indio_dev); state->core.type = state->core.resp->info.type; @@ -191,8 +193,7 @@ static int cros_ec_light_prox_probe(struct platform_device *pdev) /* Common part */ channel->info_mask_shared_by_all = - BIT(IIO_CHAN_INFO_SAMP_FREQ) | - BIT(IIO_CHAN_INFO_FREQUENCY); + BIT(IIO_CHAN_INFO_SAMP_FREQ); channel->info_mask_shared_by_all_available = BIT(IIO_CHAN_INFO_SAMP_FREQ); channel->scan_type.realbits = CROS_EC_SENSOR_BITS; diff --git a/drivers/iio/pressure/cros_ec_baro.c b/drivers/iio/pressure/cros_ec_baro.c index 6add499f11aa..c079b8960082 100644 --- a/drivers/iio/pressure/cros_ec_baro.c +++ b/drivers/iio/pressure/cros_ec_baro.c @@ -140,6 +140,8 @@ static int cros_ec_baro_probe(struct platform_device *pdev) if (ret) return ret; + iio_buffer_set_attrs(indio_dev->buffer, cros_ec_sensor_fifo_attributes); + indio_dev->info = &cros_ec_baro_info; state = iio_priv(indio_dev); state->core.type = state->core.resp->info.type; @@ -149,8 +151,7 @@ static int cros_ec_baro_probe(struct platform_device *pdev) channel->info_mask_separate = BIT(IIO_CHAN_INFO_RAW); channel->info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SCALE) | - BIT(IIO_CHAN_INFO_SAMP_FREQ) | - BIT(IIO_CHAN_INFO_FREQUENCY); + BIT(IIO_CHAN_INFO_SAMP_FREQ); channel->info_mask_shared_by_all_available = BIT(IIO_CHAN_INFO_SAMP_FREQ); channel->scan_type.realbits = CROS_EC_SENSOR_BITS; diff --git a/include/linux/iio/common/cros_ec_sensors_core.h b/include/linux/iio/common/cros_ec_sensors_core.h index 96ea4551945e..5b0acc14c891 100644 --- a/include/linux/iio/common/cros_ec_sensors_core.h +++ b/include/linux/iio/common/cros_ec_sensors_core.h @@ -50,7 +50,6 @@ typedef irqreturn_t (*cros_ec_sensors_capture_t)(int irq, void *p); * the timestamp. The timestamp is always last and * is always 8-byte aligned. * @read_ec_sensors_data: function used for accessing sensors values - * @cuur_sampl_freq: current sampling period */ struct cros_ec_sensors_core_state { struct cros_ec_device *ec; @@ -73,8 +72,6 @@ struct cros_ec_sensors_core_state { int (*read_ec_sensors_data)(struct iio_dev *indio_dev, unsigned long scan_mask, s16 *data); - int curr_sampl_freq; - /* Table of known available frequencies : 0, Min and Max in mHz */ int frequencies[3]; }; @@ -116,5 +113,6 @@ int cros_ec_sensors_core_write(struct cros_ec_sensors_core_state *st, /* List of extended channel specification for all sensors */ extern const struct iio_chan_spec_ext_info cros_ec_sensors_ext_info[]; +extern const struct attribute *cros_ec_sensor_fifo_attributes[]; #endif /* __CROS_EC_SENSORS_CORE_H */ From cb87556068146de5c9933397706d3bde88b4a14d Mon Sep 17 00:00:00 2001 From: Gwendal Grignou Date: Fri, 27 Mar 2020 15:34:41 -0700 Subject: [PATCH 300/635] iio: cros_ec: Report hwfifo_watermark_max Report the maximum amount of sample the EC can hold. This is not tunable, but can be useful for application to find out the maximum amount of time it can sleep when hwfifo_timeout is set to a large number. Signed-off-by: Gwendal Grignou Reviewed-by: Jonathan Cameron Signed-off-by: Enric Balletbo i Serra --- .../cros_ec_sensors/cros_ec_sensors_core.c | 33 +++++++++++++++++-- .../linux/iio/common/cros_ec_sensors_core.h | 3 ++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c index b8eac7e5d5e5..67e8eff038cf 100644 --- a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c +++ b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c @@ -22,6 +22,12 @@ #include #include +/* + * Hard coded to the first device to support sensor fifo. The EC has a 2048 + * byte fifo and will trigger an interrupt when fifo is 2/3 full. + */ +#define CROS_EC_FIFO_SIZE (2048 * 2 / 3) + static char *cros_ec_loc[] = { [MOTIONSENSE_LOC_BASE] = "base", [MOTIONSENSE_LOC_LID] = "lid", @@ -55,8 +61,15 @@ static int cros_ec_get_host_cmd_version_mask(struct cros_ec_device *ec_dev, static void get_default_min_max_freq(enum motionsensor_type type, u32 *min_freq, - u32 *max_freq) + u32 *max_freq, + u32 *max_fifo_events) { + /* + * We don't know fifo size, set to size previously used by older + * hardware. + */ + *max_fifo_events = CROS_EC_FIFO_SIZE; + switch (type) { case MOTIONSENSE_TYPE_ACCEL: case MOTIONSENSE_TYPE_GYRO: @@ -149,8 +162,21 @@ static IIO_DEVICE_ATTR(hwfifo_timeout, 0644, cros_ec_sensor_get_report_latency, cros_ec_sensor_set_report_latency, 0); +static ssize_t hwfifo_watermark_max_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct iio_dev *indio_dev = dev_to_iio_dev(dev); + struct cros_ec_sensors_core_state *st = iio_priv(indio_dev); + + return sprintf(buf, "%d\n", st->fifo_max_event_count); +} + +static IIO_DEVICE_ATTR_RO(hwfifo_watermark_max, 0); + const struct attribute *cros_ec_sensor_fifo_attributes[] = { &iio_dev_attr_hwfifo_timeout.dev_attr.attr, + &iio_dev_attr_hwfifo_watermark_max.dev_attr.attr, NULL, }; EXPORT_SYMBOL_GPL(cros_ec_sensor_fifo_attributes); @@ -279,12 +305,15 @@ int cros_ec_sensors_core_init(struct platform_device *pdev, if (state->msg->version < 3) { get_default_min_max_freq(state->resp->info.type, &state->frequencies[1], - &state->frequencies[2]); + &state->frequencies[2], + &state->fifo_max_event_count); } else { state->frequencies[1] = state->resp->info_3.min_frequency; state->frequencies[2] = state->resp->info_3.max_frequency; + state->fifo_max_event_count = + state->resp->info_3.fifo_max_event_count; } if (cros_ec_check_features(ec, EC_FEATURE_MOTION_SENSE_FIFO)) { diff --git a/include/linux/iio/common/cros_ec_sensors_core.h b/include/linux/iio/common/cros_ec_sensors_core.h index 5b0acc14c891..bc26ae2e3272 100644 --- a/include/linux/iio/common/cros_ec_sensors_core.h +++ b/include/linux/iio/common/cros_ec_sensors_core.h @@ -50,6 +50,7 @@ typedef irqreturn_t (*cros_ec_sensors_capture_t)(int irq, void *p); * the timestamp. The timestamp is always last and * is always 8-byte aligned. * @read_ec_sensors_data: function used for accessing sensors values + * @fifo_max_event_count: Size of the EC sensor FIFO */ struct cros_ec_sensors_core_state { struct cros_ec_device *ec; @@ -72,6 +73,8 @@ struct cros_ec_sensors_core_state { int (*read_ec_sensors_data)(struct iio_dev *indio_dev, unsigned long scan_mask, s16 *data); + u32 fifo_max_event_count; + /* Table of known available frequencies : 0, Min and Max in mHz */ int frequencies[3]; }; From 317a0ebe53f46527aed912f7c3df963cd9a41536 Mon Sep 17 00:00:00 2001 From: Gwendal Grignou Date: Fri, 27 Mar 2020 15:34:42 -0700 Subject: [PATCH 301/635] iio: cros_ec: Use Hertz as unit for sampling frequency To be compliant with other sensors, set and get sensor sampling frequency in Hz, not mHz. Fixes: ae7b02ad2f32 ("iio: common: cros_ec_sensors: Expose cros_ec_sensors frequency range via iio sysfs") Signed-off-by: Gwendal Grignou Acked-by: Jonathan Cameron Signed-off-by: Enric Balletbo i Serra --- .../cros_ec_sensors/cros_ec_sensors_core.c | 32 +++++++++++-------- .../linux/iio/common/cros_ec_sensors_core.h | 6 ++-- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c index 67e8eff038cf..c831915ca7e5 100644 --- a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c +++ b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c @@ -253,6 +253,7 @@ int cros_ec_sensors_core_init(struct platform_device *pdev, struct cros_ec_dev *ec = sensor_hub->ec; struct cros_ec_sensor_platform *sensor_platform = dev_get_platdata(dev); u32 ver_mask; + int frequencies[ARRAY_SIZE(state->frequencies) / 2] = { 0 }; int ret, i; platform_set_drvdata(pdev, indio_dev); @@ -301,20 +302,22 @@ int cros_ec_sensors_core_init(struct platform_device *pdev, state->calib[i].scale = MOTION_SENSE_DEFAULT_SCALE; /* 0 is a correct value used to stop the device */ - state->frequencies[0] = 0; if (state->msg->version < 3) { get_default_min_max_freq(state->resp->info.type, - &state->frequencies[1], - &state->frequencies[2], + &frequencies[1], + &frequencies[2], &state->fifo_max_event_count); } else { - state->frequencies[1] = - state->resp->info_3.min_frequency; - state->frequencies[2] = - state->resp->info_3.max_frequency; + frequencies[1] = state->resp->info_3.min_frequency; + frequencies[2] = state->resp->info_3.max_frequency; state->fifo_max_event_count = state->resp->info_3.fifo_max_event_count; } + for (i = 0; i < ARRAY_SIZE(frequencies); i++) { + state->frequencies[2 * i] = frequencies[i] / 1000; + state->frequencies[2 * i + 1] = + (frequencies[i] % 1000) * 1000; + } if (cros_ec_check_features(ec, EC_FEATURE_MOTION_SENSE_FIFO)) { /* @@ -728,7 +731,7 @@ int cros_ec_sensors_core_read(struct cros_ec_sensors_core_state *st, struct iio_chan_spec const *chan, int *val, int *val2, long mask) { - int ret; + int ret, frequency; switch (mask) { case IIO_CHAN_INFO_SAMP_FREQ: @@ -740,8 +743,10 @@ int cros_ec_sensors_core_read(struct cros_ec_sensors_core_state *st, if (ret) break; - *val = st->resp->sensor_odr.ret; - ret = IIO_VAL_INT; + frequency = st->resp->sensor_odr.ret; + *val = frequency / 1000; + *val2 = (frequency % 1000) * 1000; + ret = IIO_VAL_INT_PLUS_MICRO; break; default: ret = -EINVAL; @@ -776,7 +781,7 @@ int cros_ec_sensors_core_read_avail(struct iio_dev *indio_dev, case IIO_CHAN_INFO_SAMP_FREQ: *length = ARRAY_SIZE(state->frequencies); *vals = (const int *)&state->frequencies; - *type = IIO_VAL_INT; + *type = IIO_VAL_INT_PLUS_MICRO; return IIO_AVAIL_LIST; } @@ -798,12 +803,13 @@ int cros_ec_sensors_core_write(struct cros_ec_sensors_core_state *st, struct iio_chan_spec const *chan, int val, int val2, long mask) { - int ret; + int ret, frequency; switch (mask) { case IIO_CHAN_INFO_SAMP_FREQ: + frequency = val * 1000 + val2 / 1000; st->param.cmd = MOTIONSENSE_CMD_SENSOR_ODR; - st->param.sensor_odr.data = val; + st->param.sensor_odr.data = frequency; /* Always roundup, so caller gets at least what it asks for. */ st->param.sensor_odr.roundup = 1; diff --git a/include/linux/iio/common/cros_ec_sensors_core.h b/include/linux/iio/common/cros_ec_sensors_core.h index bc26ae2e3272..7bc961defa87 100644 --- a/include/linux/iio/common/cros_ec_sensors_core.h +++ b/include/linux/iio/common/cros_ec_sensors_core.h @@ -51,6 +51,8 @@ typedef irqreturn_t (*cros_ec_sensors_capture_t)(int irq, void *p); * is always 8-byte aligned. * @read_ec_sensors_data: function used for accessing sensors values * @fifo_max_event_count: Size of the EC sensor FIFO + * @frequencies: Table of known available frequencies: + * 0, Min and Max in mHz */ struct cros_ec_sensors_core_state { struct cros_ec_device *ec; @@ -74,9 +76,7 @@ struct cros_ec_sensors_core_state { unsigned long scan_mask, s16 *data); u32 fifo_max_event_count; - - /* Table of known available frequencies : 0, Min and Max in mHz */ - int frequencies[3]; + int frequencies[6]; }; int cros_ec_sensors_read_lpc(struct iio_dev *indio_dev, unsigned long scan_mask, From f8db89d14efb770dd59aa0ca74386e5de68310d5 Mon Sep 17 00:00:00 2001 From: Prashant Malani Date: Mon, 10 Feb 2020 11:06:24 -0800 Subject: [PATCH 302/635] mfd: cros_ec: Check DT node for usbpd-notify add Add a check to ensure there is indeed an EC device tree entry before adding the cros-usbpd-notify device. This covers configs where both CONFIG_ACPI and CONFIG_OF are defined, but the EC device is defined using device tree and not in ACPI. Fixes: 4602dce0361e ("mfd: cros_ec: Add cros-usbpd-notify subdevice") Signed-off-by: Prashant Malani Tested-by: Enric Balletbo i Serra Signed-off-by: Lee Jones --- drivers/mfd/cros_ec_dev.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/mfd/cros_ec_dev.c b/drivers/mfd/cros_ec_dev.c index 39e611695053..32c2b912b58b 100644 --- a/drivers/mfd/cros_ec_dev.c +++ b/drivers/mfd/cros_ec_dev.c @@ -211,7 +211,7 @@ static int ec_device_probe(struct platform_device *pdev) * explicitly added on platforms that don't have the PD notifier ACPI * device entry defined. */ - if (IS_ENABLED(CONFIG_OF)) { + if (IS_ENABLED(CONFIG_OF) && ec->ec_dev->dev->of_node) { if (cros_ec_check_features(ec, EC_FEATURE_USB_PD)) { retval = mfd_add_hotplug_devices(ec->dev, cros_usbpd_notify_cells, From c703797c1d5466f9af70bc7692fa49399107e66c Mon Sep 17 00:00:00 2001 From: Tony Lindgren Date: Tue, 11 Feb 2020 09:15:02 -0800 Subject: [PATCH 303/635] mfd: cpcap: Fix compile if MFD_CORE is not selected If only cpcap mfd driver is selected we will get: ERROR: "devm_mfd_add_devices" [drivers/mfd/motorola-cpcap.ko] undefined! This is because Kconfig is missing select for MFD_CORE. Signed-off-by: Tony Lindgren Signed-off-by: Lee Jones --- drivers/mfd/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig index da4a9a0c5ca2..611fe602f213 100644 --- a/drivers/mfd/Kconfig +++ b/drivers/mfd/Kconfig @@ -906,6 +906,7 @@ config MFD_CPCAP tristate "Support for Motorola CPCAP" depends on SPI depends on OF || COMPILE_TEST + select MFD_CORE select REGMAP_SPI select REGMAP_IRQ help From a0c8498c076d0338860df7e45e5e6f48158a02d8 Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Wed, 12 Feb 2020 17:56:42 -0600 Subject: [PATCH 304/635] mfd: omap-usb-tll: Replace zero-length array with flexible-array member The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertently introduced[3] to the codebase from now on. Also, notice that, dynamic memory allocations won't be affected by this change: "Flexible array members have incomplete type, and so the sizeof operator may not be applied. As a quirk of the original implementation of zero-length arrays, sizeof evaluates to zero."[1] This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Signed-off-by: Lee Jones --- drivers/mfd/omap-usb-tll.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/mfd/omap-usb-tll.c b/drivers/mfd/omap-usb-tll.c index 265f5e350e1c..1cf259a48966 100644 --- a/drivers/mfd/omap-usb-tll.c +++ b/drivers/mfd/omap-usb-tll.c @@ -99,7 +99,7 @@ struct usbtll_omap { void __iomem *base; int nch; /* num. of channels */ - struct clk *ch_clk[0]; /* must be the last member */ + struct clk *ch_clk[]; /* must be the last member */ }; /*-------------------------------------------------------------------------*/ From 7235d9e48fda6902018e04669eda6f5d1878599f Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Wed, 12 Feb 2020 17:59:11 -0600 Subject: [PATCH 305/635] mfd: pm8xxx: Replace zero-length array with flexible-array member The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertently introduced[3] to the codebase from now on. Also, notice that, dynamic memory allocations won't be affected by this change: "Flexible array members have incomplete type, and so the sizeof operator may not be applied. As a quirk of the original implementation of zero-length arrays, sizeof evaluates to zero."[1] This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Signed-off-by: Lee Jones --- drivers/mfd/qcom-pm8xxx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/mfd/qcom-pm8xxx.c b/drivers/mfd/qcom-pm8xxx.c index 29133326c6fd..acd172ddcbd6 100644 --- a/drivers/mfd/qcom-pm8xxx.c +++ b/drivers/mfd/qcom-pm8xxx.c @@ -76,7 +76,7 @@ struct pm_irq_chip { unsigned int num_masters; const struct pm_irq_data *pm_irq_data; /* MUST BE AT THE END OF THIS STRUCT */ - u8 config[0]; + u8 config[]; }; static int pm8xxx_read_block_irq(struct pm_irq_chip *chip, unsigned int bp, From 74391043a42fd58a3cf5c93f6dcf9ddbdce55332 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sun, 16 Feb 2020 12:32:42 +0100 Subject: [PATCH 306/635] mfd: Kconfig: Fix some misspelling of the word functionality Fix several variations of typo around functionali{ty,es}. Signed-off-by: Christophe JAILLET Signed-off-by: Lee Jones --- drivers/mfd/Kconfig | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig index 611fe602f213..0a59249198d3 100644 --- a/drivers/mfd/Kconfig +++ b/drivers/mfd/Kconfig @@ -1216,7 +1216,7 @@ config AB8500_CORE chip. This connects to U8500 either on the SSP/SPI bus (deprecated since hardware version v1.0) or the I2C bus via PRCMU. It also adds the irq_chip parts for handling the Mixed Signal chip events. - This chip embeds various other multimedia funtionalities as well. + This chip embeds various other multimedia functionalities as well. config AB8500_DEBUG bool "Enable debug info via debugfs" @@ -1866,7 +1866,7 @@ config MFD_WM8994 has on board GPIO and regulator functionality which is supported via the relevant subsystems. This driver provides core support for the WM8994, in order to use the actual - functionaltiy of the device other drivers must be enabled. + functionality of the device other drivers must be enabled. config MFD_WM97xx tristate "Wolfson Microelectronics WM97xx" @@ -1879,7 +1879,7 @@ config MFD_WM97xx designed for smartphone applications. As well as audio functionality it has on board GPIO and a touchscreen functionality which is supported via the relevant subsystems. This driver provides core - support for the WM97xx, in order to use the actual functionaltiy of + support for the WM97xx, in order to use the actual functionality of the device other drivers must be enabled. config MFD_STW481X @@ -1972,7 +1972,7 @@ config MFD_STPMIC1 Support for ST Microelectronics STPMIC1 PMIC. STPMIC1 has power on key, watchdog and regulator functionalities which are supported via the relevant subsystems. This driver provides core support for the - STPMIC1. In order to use the actual functionaltiy of the device other + STPMIC1. In order to use the actual functionality of the device other drivers must be enabled. To compile this driver as a module, choose M here: the From 9a153b0ed196cc6052ea6a32f517cbf5015c8d29 Mon Sep 17 00:00:00 2001 From: Corentin Labbe Date: Tue, 18 Feb 2020 20:09:01 +0000 Subject: [PATCH 307/635] mfd: omap: Remove useless cast for driver.name device_driver name is const char pointer, so it not useful to cast xx_driver_name (which is already const char). Signed-off-by: Corentin Labbe Signed-off-by: Lee Jones --- drivers/mfd/omap-usb-host.c | 2 +- drivers/mfd/omap-usb-tll.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/mfd/omap-usb-host.c b/drivers/mfd/omap-usb-host.c index 4798d9f3f9d5..1f4f01b02d98 100644 --- a/drivers/mfd/omap-usb-host.c +++ b/drivers/mfd/omap-usb-host.c @@ -840,7 +840,7 @@ MODULE_DEVICE_TABLE(of, usbhs_omap_dt_ids); static struct platform_driver usbhs_omap_driver = { .driver = { - .name = (char *)usbhs_driver_name, + .name = usbhs_driver_name, .pm = &usbhsomap_dev_pm_ops, .of_match_table = usbhs_omap_dt_ids, }, diff --git a/drivers/mfd/omap-usb-tll.c b/drivers/mfd/omap-usb-tll.c index 1cf259a48966..4b7f73c317e8 100644 --- a/drivers/mfd/omap-usb-tll.c +++ b/drivers/mfd/omap-usb-tll.c @@ -304,7 +304,7 @@ MODULE_DEVICE_TABLE(of, usbtll_omap_dt_ids); static struct platform_driver usbtll_omap_driver = { .driver = { - .name = (char *)usbtll_driver_name, + .name = usbtll_driver_name, .of_match_table = usbtll_omap_dt_ids, }, .probe = usbtll_omap_probe, From d8f083a302f7f69eecd513ce1aa828cfff41f0cf Mon Sep 17 00:00:00 2001 From: Soeren Moch Date: Sun, 12 Jan 2020 01:55:00 +0000 Subject: [PATCH 308/635] mfd: rk808: Always use poweroff when requested With the device tree property "rockchip,system-power-controller" we explicitly request to use this PMIC to power off the system. So always register our poweroff function, even if some other handler (probably PSCI poweroff) was registered before. This does tend to reveal a warning on shutdown due to the Rockchip I2C driver not implementing an atomic transfer method, however since the write to DEV_OFF takes effect immediately the I2C completion interrupt is moot anyway, and as the very last thing written to the console it is only visible to users going out of their way to capture serial output. Signed-off-by: Soeren Moch Reviewed-by: Heiko Stuebner [ rm: note potential warning in commit message ] Signed-off-by: Robin Murphy Signed-off-by: Lee Jones --- drivers/mfd/rk808.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/drivers/mfd/rk808.c b/drivers/mfd/rk808.c index a69a6742ecdc..616e44e7ef98 100644 --- a/drivers/mfd/rk808.c +++ b/drivers/mfd/rk808.c @@ -550,7 +550,7 @@ static int rk808_probe(struct i2c_client *client, const struct mfd_cell *cells; int nr_pre_init_regs; int nr_cells; - int pm_off = 0, msb, lsb; + int msb, lsb; unsigned char pmic_id_msb, pmic_id_lsb; int ret; int i; @@ -674,16 +674,9 @@ static int rk808_probe(struct i2c_client *client, goto err_irq; } - pm_off = of_property_read_bool(np, - "rockchip,system-power-controller"); - if (pm_off && !pm_power_off) { + if (of_property_read_bool(np, "rockchip,system-power-controller")) { rk808_i2c_client = client; pm_power_off = rk808->pm_pwroff_fn; - } - - if (pm_off && !pm_power_off_prepare) { - if (!rk808_i2c_client) - rk808_i2c_client = client; pm_power_off_prepare = rk808->pm_pwroff_prep_fn; } From 08e8c0d9e9fa78e62a51d5d7af608a70d29ca473 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Sun, 12 Jan 2020 01:55:01 +0000 Subject: [PATCH 309/635] mfd: rk808: Ensure suspend/resume hooks always work The RK809/RK817 suspend/resume hooks should not have to depend on whether this driver owns the pm_power_off hook, and thus the global rk808_i2c_client is set - indeed, the GPIO-based control is really only relevant when PSCI firmware is in charge of power rather than the kernel. As driver model callbacks, they have an appropriate device argument to hand, so can just always use that. Signed-off-by: Robin Murphy Signed-off-by: Lee Jones --- drivers/mfd/rk808.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/mfd/rk808.c b/drivers/mfd/rk808.c index 616e44e7ef98..ac798053c26a 100644 --- a/drivers/mfd/rk808.c +++ b/drivers/mfd/rk808.c @@ -712,7 +712,7 @@ static int rk808_remove(struct i2c_client *client) static int __maybe_unused rk8xx_suspend(struct device *dev) { - struct rk808 *rk808 = i2c_get_clientdata(rk808_i2c_client); + struct rk808 *rk808 = i2c_get_clientdata(to_i2c_client(dev)); int ret = 0; switch (rk808->variant) { @@ -732,7 +732,7 @@ static int __maybe_unused rk8xx_suspend(struct device *dev) static int __maybe_unused rk8xx_resume(struct device *dev) { - struct rk808 *rk808 = i2c_get_clientdata(rk808_i2c_client); + struct rk808 *rk808 = i2c_get_clientdata(to_i2c_client(dev)); int ret = 0; switch (rk808->variant) { From 90df3a8230ef8b02370bb2eb0354fba57ecb2b9d Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Sun, 12 Jan 2020 01:55:02 +0000 Subject: [PATCH 310/635] mfd: rk808: Stop using syscore ops Setting the SLEEP pin to its shutdown function for appropriate PMICs doesn't need to happen in single-CPU context, so there's really no point involving the syscore machinery. Hook it up to the standard driver model shutdown method instead. This also obviates the issue that the syscore ops weren't being unregistered on probe failure or module removal. Signed-off-by: Robin Murphy Signed-off-by: Lee Jones --- drivers/mfd/rk808.c | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/drivers/mfd/rk808.c b/drivers/mfd/rk808.c index ac798053c26a..8116ed6cf2e7 100644 --- a/drivers/mfd/rk808.c +++ b/drivers/mfd/rk808.c @@ -19,7 +19,6 @@ #include #include #include -#include struct rk808_reg_data { int addr; @@ -509,28 +508,27 @@ static void rk818_device_shutdown(void) dev_err(&rk808_i2c_client->dev, "Failed to shutdown device!\n"); } -static void rk8xx_syscore_shutdown(void) +static void rk8xx_shutdown(struct i2c_client *client) { - struct rk808 *rk808 = i2c_get_clientdata(rk808_i2c_client); + struct rk808 *rk808 = i2c_get_clientdata(client); int ret; - if (system_state == SYSTEM_POWER_OFF && - (rk808->variant == RK809_ID || rk808->variant == RK817_ID)) { + switch (rk808->variant) { + case RK809_ID: + case RK817_ID: ret = regmap_update_bits(rk808->regmap, RK817_SYS_CFG(3), RK817_SLPPIN_FUNC_MSK, SLPPIN_DN_FUN); - if (ret) { - dev_warn(&rk808_i2c_client->dev, - "Cannot switch to power down function\n"); - } + break; + default: + return; } + if (ret) + dev_warn(&client->dev, + "Cannot switch to power down function\n"); } -static struct syscore_ops rk808_syscore_ops = { - .shutdown = rk8xx_syscore_shutdown, -}; - static const struct of_device_id rk808_of_match[] = { { .compatible = "rockchip,rk805" }, { .compatible = "rockchip,rk808" }, @@ -623,7 +621,6 @@ static int rk808_probe(struct i2c_client *client, nr_pre_init_regs = ARRAY_SIZE(rk817_pre_init_reg); cells = rk817s; nr_cells = ARRAY_SIZE(rk817s); - register_syscore_ops(&rk808_syscore_ops); break; default: dev_err(&client->dev, "Unsupported RK8XX ID %lu\n", @@ -759,6 +756,7 @@ static struct i2c_driver rk808_i2c_driver = { }, .probe = rk808_probe, .remove = rk808_remove, + .shutdown = rk8xx_shutdown, }; module_i2c_driver(rk808_i2c_driver); From 7a52cbccee8df0edfee30b81fdbb7d4f9d27ffd5 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Sun, 12 Jan 2020 01:55:03 +0000 Subject: [PATCH 311/635] mfd: rk808: Reduce shutdown duplication Rather than having 3 almost-identical functions plus the machinery to keep track of them, it's far simpler to just dynamically select the appropriate register field per variant. Signed-off-by: Robin Murphy Signed-off-by: Lee Jones --- drivers/mfd/rk808.c | 61 +++++++++++++-------------------------- include/linux/mfd/rk808.h | 1 - 2 files changed, 20 insertions(+), 42 deletions(-) diff --git a/drivers/mfd/rk808.c b/drivers/mfd/rk808.c index 8116ed6cf2e7..b2265c6e94ae 100644 --- a/drivers/mfd/rk808.c +++ b/drivers/mfd/rk808.c @@ -448,21 +448,6 @@ static const struct regmap_irq_chip rk818_irq_chip = { static struct i2c_client *rk808_i2c_client; -static void rk805_device_shutdown(void) -{ - int ret; - struct rk808 *rk808 = i2c_get_clientdata(rk808_i2c_client); - - if (!rk808) - return; - - ret = regmap_update_bits(rk808->regmap, - RK805_DEV_CTRL_REG, - DEV_OFF, DEV_OFF); - if (ret) - dev_err(&rk808_i2c_client->dev, "Failed to shutdown device!\n"); -} - static void rk805_device_shutdown_prepare(void) { int ret; @@ -478,32 +463,29 @@ static void rk805_device_shutdown_prepare(void) dev_err(&rk808_i2c_client->dev, "Failed to shutdown device!\n"); } -static void rk808_device_shutdown(void) +static void rk808_pm_power_off(void) { int ret; + unsigned int reg, bit; struct rk808 *rk808 = i2c_get_clientdata(rk808_i2c_client); - if (!rk808) + switch (rk808->variant) { + case RK805_ID: + reg = RK805_DEV_CTRL_REG; + bit = DEV_OFF; + break; + case RK808_ID: + reg = RK808_DEVCTRL_REG, + bit = DEV_OFF_RST; + break; + case RK818_ID: + reg = RK818_DEVCTRL_REG; + bit = DEV_OFF; + break; + default: return; - - ret = regmap_update_bits(rk808->regmap, - RK808_DEVCTRL_REG, - DEV_OFF_RST, DEV_OFF_RST); - if (ret) - dev_err(&rk808_i2c_client->dev, "Failed to shutdown device!\n"); -} - -static void rk818_device_shutdown(void) -{ - int ret; - struct rk808 *rk808 = i2c_get_clientdata(rk808_i2c_client); - - if (!rk808) - return; - - ret = regmap_update_bits(rk808->regmap, - RK818_DEVCTRL_REG, - DEV_OFF, DEV_OFF); + } + ret = regmap_update_bits(rk808->regmap, reg, bit, bit); if (ret) dev_err(&rk808_i2c_client->dev, "Failed to shutdown device!\n"); } @@ -592,7 +574,6 @@ static int rk808_probe(struct i2c_client *client, nr_pre_init_regs = ARRAY_SIZE(rk805_pre_init_reg); cells = rk805s; nr_cells = ARRAY_SIZE(rk805s); - rk808->pm_pwroff_fn = rk805_device_shutdown; rk808->pm_pwroff_prep_fn = rk805_device_shutdown_prepare; break; case RK808_ID: @@ -602,7 +583,6 @@ static int rk808_probe(struct i2c_client *client, nr_pre_init_regs = ARRAY_SIZE(rk808_pre_init_reg); cells = rk808s; nr_cells = ARRAY_SIZE(rk808s); - rk808->pm_pwroff_fn = rk808_device_shutdown; break; case RK818_ID: rk808->regmap_cfg = &rk818_regmap_config; @@ -611,7 +591,6 @@ static int rk808_probe(struct i2c_client *client, nr_pre_init_regs = ARRAY_SIZE(rk818_pre_init_reg); cells = rk818s; nr_cells = ARRAY_SIZE(rk818s); - rk808->pm_pwroff_fn = rk818_device_shutdown; break; case RK809_ID: case RK817_ID: @@ -673,7 +652,7 @@ static int rk808_probe(struct i2c_client *client, if (of_property_read_bool(np, "rockchip,system-power-controller")) { rk808_i2c_client = client; - pm_power_off = rk808->pm_pwroff_fn; + pm_power_off = rk808_pm_power_off; pm_power_off_prepare = rk808->pm_pwroff_prep_fn; } @@ -694,7 +673,7 @@ static int rk808_remove(struct i2c_client *client) * pm_power_off may points to a function from another module. * Check if the pointer is set by us and only then overwrite it. */ - if (rk808->pm_pwroff_fn && pm_power_off == rk808->pm_pwroff_fn) + if (pm_power_off == rk808_pm_power_off) pm_power_off = NULL; /** diff --git a/include/linux/mfd/rk808.h b/include/linux/mfd/rk808.h index a59bf323f713..b038653fa87e 100644 --- a/include/linux/mfd/rk808.h +++ b/include/linux/mfd/rk808.h @@ -620,7 +620,6 @@ struct rk808 { long variant; const struct regmap_config *regmap_cfg; const struct regmap_irq_chip *regmap_irq_chip; - void (*pm_pwroff_fn)(void); void (*pm_pwroff_prep_fn)(void); }; #endif /* __LINUX_REGULATOR_RK808_H */ From 42679765faf286259b16acf284eb52d68877ff32 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Sun, 12 Jan 2020 01:55:04 +0000 Subject: [PATCH 312/635] mfd: rk808: Convert RK805 to shutdown/suspend hooks RK805 has the same kind of dual-role sleep/shutdown pin as RK809/RK817, so it makes little sense for the driver to have to have two completely different mechanisms to handle essentially the same thing. Move RK805 over to the shutdown/suspend flow to clean things up. Signed-off-by: Robin Murphy Signed-off-by: Lee Jones --- drivers/mfd/rk808.c | 37 ++++++++++++------------------------- include/linux/mfd/rk808.h | 1 - 2 files changed, 12 insertions(+), 26 deletions(-) diff --git a/drivers/mfd/rk808.c b/drivers/mfd/rk808.c index b2265c6e94ae..d109b9f14407 100644 --- a/drivers/mfd/rk808.c +++ b/drivers/mfd/rk808.c @@ -185,7 +185,6 @@ static const struct rk808_reg_data rk805_pre_init_reg[] = { {RK805_BUCK4_CONFIG_REG, RK805_BUCK3_4_ILMAX_MASK, RK805_BUCK4_ILMAX_3500MA}, {RK805_BUCK4_CONFIG_REG, BUCK_ILMIN_MASK, BUCK_ILMIN_400MA}, - {RK805_GPIO_IO_POL_REG, SLP_SD_MSK, SLEEP_FUN}, {RK805_THERMAL_REG, TEMP_HOTDIE_MSK, TEMP115C}, }; @@ -448,21 +447,6 @@ static const struct regmap_irq_chip rk818_irq_chip = { static struct i2c_client *rk808_i2c_client; -static void rk805_device_shutdown_prepare(void) -{ - int ret; - struct rk808 *rk808 = i2c_get_clientdata(rk808_i2c_client); - - if (!rk808) - return; - - ret = regmap_update_bits(rk808->regmap, - RK805_GPIO_IO_POL_REG, - SLP_SD_MSK, SHUTDOWN_FUN); - if (ret) - dev_err(&rk808_i2c_client->dev, "Failed to shutdown device!\n"); -} - static void rk808_pm_power_off(void) { int ret; @@ -496,6 +480,12 @@ static void rk8xx_shutdown(struct i2c_client *client) int ret; switch (rk808->variant) { + case RK805_ID: + ret = regmap_update_bits(rk808->regmap, + RK805_GPIO_IO_POL_REG, + SLP_SD_MSK, + SHUTDOWN_FUN); + break; case RK809_ID: case RK817_ID: ret = regmap_update_bits(rk808->regmap, @@ -574,7 +564,6 @@ static int rk808_probe(struct i2c_client *client, nr_pre_init_regs = ARRAY_SIZE(rk805_pre_init_reg); cells = rk805s; nr_cells = ARRAY_SIZE(rk805s); - rk808->pm_pwroff_prep_fn = rk805_device_shutdown_prepare; break; case RK808_ID: rk808->regmap_cfg = &rk808_regmap_config; @@ -653,7 +642,6 @@ static int rk808_probe(struct i2c_client *client, if (of_property_read_bool(np, "rockchip,system-power-controller")) { rk808_i2c_client = client; pm_power_off = rk808_pm_power_off; - pm_power_off_prepare = rk808->pm_pwroff_prep_fn; } return 0; @@ -676,13 +664,6 @@ static int rk808_remove(struct i2c_client *client) if (pm_power_off == rk808_pm_power_off) pm_power_off = NULL; - /** - * As above, check if the pointer is set by us before overwrite. - */ - if (rk808->pm_pwroff_prep_fn && - pm_power_off_prepare == rk808->pm_pwroff_prep_fn) - pm_power_off_prepare = NULL; - return 0; } @@ -692,6 +673,12 @@ static int __maybe_unused rk8xx_suspend(struct device *dev) int ret = 0; switch (rk808->variant) { + case RK805_ID: + ret = regmap_update_bits(rk808->regmap, + RK805_GPIO_IO_POL_REG, + SLP_SD_MSK, + SLEEP_FUN); + break; case RK809_ID: case RK817_ID: ret = regmap_update_bits(rk808->regmap, diff --git a/include/linux/mfd/rk808.h b/include/linux/mfd/rk808.h index b038653fa87e..e07f6e61cd38 100644 --- a/include/linux/mfd/rk808.h +++ b/include/linux/mfd/rk808.h @@ -620,6 +620,5 @@ struct rk808 { long variant; const struct regmap_config *regmap_cfg; const struct regmap_irq_chip *regmap_irq_chip; - void (*pm_pwroff_prep_fn)(void); }; #endif /* __LINUX_REGULATOR_RK808_H */ From ba5836933a71160c40356dfbd493135a7134b069 Mon Sep 17 00:00:00 2001 From: Benjamin Gaignard Date: Mon, 17 Feb 2020 14:45:44 +0100 Subject: [PATCH 313/635] dt-bindings: mfd: Document STM32 low power timer bindings Add a subnode to STM low power timer bindings to support timer driver Signed-off-by: Benjamin Gaignard Acked-by: Rob Herring Signed-off-by: Lee Jones --- .../bindings/mfd/st,stm32-lptimer.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Documentation/devicetree/bindings/mfd/st,stm32-lptimer.yaml b/Documentation/devicetree/bindings/mfd/st,stm32-lptimer.yaml index 1a4cc5f3fb33..ddf190cb800b 100644 --- a/Documentation/devicetree/bindings/mfd/st,stm32-lptimer.yaml +++ b/Documentation/devicetree/bindings/mfd/st,stm32-lptimer.yaml @@ -39,6 +39,8 @@ properties: "#size-cells": const: 0 + wakeup-source: true + pwm: type: object @@ -81,6 +83,16 @@ patternProperties: required: - compatible + timer: + type: object + + properties: + compatible: + const: st,stm32-lptimer-timer + + required: + - compatible + required: - "#address-cells" - "#size-cells" @@ -115,6 +127,10 @@ examples: counter { compatible = "st,stm32-lptimer-counter"; }; + + timer { + compatible = "st,stm32-lptimer-timer"; + }; }; ... From 2a7e7274f3d43d2a072cab25c0035dc994903bb9 Mon Sep 17 00:00:00 2001 From: Baolin Wang Date: Mon, 17 Feb 2020 10:26:16 +0800 Subject: [PATCH 314/635] mfd: sc27xx: Add USB charger type detection support The Spreadtrum SC27XX series PMICs supply the USB charger type detection function, and related registers are located on the PMIC global registers region, thus we implement and export this function in the MFD driver for users to get the USB charger type. Signed-off-by: Baolin Wang Signed-off-by: Lee Jones --- drivers/mfd/sprd-sc27xx-spi.c | 52 +++++++++++++++++++++++++++++++++ include/linux/mfd/sc27xx-pmic.h | 7 +++++ 2 files changed, 59 insertions(+) create mode 100644 include/linux/mfd/sc27xx-pmic.h diff --git a/drivers/mfd/sprd-sc27xx-spi.c b/drivers/mfd/sprd-sc27xx-spi.c index c0529a1cd5ea..ebdf2f11ae28 100644 --- a/drivers/mfd/sprd-sc27xx-spi.c +++ b/drivers/mfd/sprd-sc27xx-spi.c @@ -10,6 +10,7 @@ #include #include #include +#include #define SPRD_PMIC_INT_MASK_STATUS 0x0 #define SPRD_PMIC_INT_RAW_STATUS 0x4 @@ -17,6 +18,16 @@ #define SPRD_SC2731_IRQ_BASE 0x140 #define SPRD_SC2731_IRQ_NUMS 16 +#define SPRD_SC2731_CHG_DET 0xedc + +/* PMIC charger detection definition */ +#define SPRD_PMIC_CHG_DET_DELAY_US 200000 +#define SPRD_PMIC_CHG_DET_TIMEOUT 2000000 +#define SPRD_PMIC_CHG_DET_DONE BIT(11) +#define SPRD_PMIC_SDP_TYPE BIT(7) +#define SPRD_PMIC_DCP_TYPE BIT(6) +#define SPRD_PMIC_CDP_TYPE BIT(5) +#define SPRD_PMIC_CHG_TYPE_MASK GENMASK(7, 5) struct sprd_pmic { struct regmap *regmap; @@ -24,12 +35,14 @@ struct sprd_pmic { struct regmap_irq *irqs; struct regmap_irq_chip irq_chip; struct regmap_irq_chip_data *irq_data; + const struct sprd_pmic_data *pdata; int irq; }; struct sprd_pmic_data { u32 irq_base; u32 num_irqs; + u32 charger_det; }; /* @@ -40,8 +53,46 @@ struct sprd_pmic_data { static const struct sprd_pmic_data sc2731_data = { .irq_base = SPRD_SC2731_IRQ_BASE, .num_irqs = SPRD_SC2731_IRQ_NUMS, + .charger_det = SPRD_SC2731_CHG_DET, }; +enum usb_charger_type sprd_pmic_detect_charger_type(struct device *dev) +{ + struct spi_device *spi = to_spi_device(dev); + struct sprd_pmic *ddata = spi_get_drvdata(spi); + const struct sprd_pmic_data *pdata = ddata->pdata; + enum usb_charger_type type; + u32 val; + int ret; + + ret = regmap_read_poll_timeout(ddata->regmap, pdata->charger_det, val, + (val & SPRD_PMIC_CHG_DET_DONE), + SPRD_PMIC_CHG_DET_DELAY_US, + SPRD_PMIC_CHG_DET_TIMEOUT); + if (ret) { + dev_err(&spi->dev, "failed to detect charger type\n"); + return UNKNOWN_TYPE; + } + + switch (val & SPRD_PMIC_CHG_TYPE_MASK) { + case SPRD_PMIC_CDP_TYPE: + type = CDP_TYPE; + break; + case SPRD_PMIC_DCP_TYPE: + type = DCP_TYPE; + break; + case SPRD_PMIC_SDP_TYPE: + type = SDP_TYPE; + break; + default: + type = UNKNOWN_TYPE; + break; + } + + return type; +} +EXPORT_SYMBOL_GPL(sprd_pmic_detect_charger_type); + static const struct mfd_cell sprd_pmic_devs[] = { { .name = "sc27xx-wdt", @@ -181,6 +232,7 @@ static int sprd_pmic_probe(struct spi_device *spi) spi_set_drvdata(spi, ddata); ddata->dev = &spi->dev; ddata->irq = spi->irq; + ddata->pdata = pdata; ddata->irq_chip.name = dev_name(&spi->dev); ddata->irq_chip.status_base = diff --git a/include/linux/mfd/sc27xx-pmic.h b/include/linux/mfd/sc27xx-pmic.h new file mode 100644 index 000000000000..57e45c0b3ae2 --- /dev/null +++ b/include/linux/mfd/sc27xx-pmic.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LINUX_MFD_SC27XX_PMIC_H +#define __LINUX_MFD_SC27XX_PMIC_H + +extern enum usb_charger_type sprd_pmic_detect_charger_type(struct device *dev); + +#endif /* __LINUX_MFD_SC27XX_PMIC_H */ From 4e213b45d2b50f0254fd5a840db0f8c11be99766 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 13 Jan 2020 14:57:29 +0200 Subject: [PATCH 315/635] mfd: intel-lpss: Add Intel Comet Lake PCH-V PCI IDs Intel Comet Lake PCH-V has the same LPSS than Intel Kaby Lake. Add the new IDs to the list of supported devices. Signed-off-by: Andy Shevchenko Signed-off-by: Lee Jones --- drivers/mfd/intel-lpss-pci.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/mfd/intel-lpss-pci.c b/drivers/mfd/intel-lpss-pci.c index c40a6c7d0cf8..378024a3ba28 100644 --- a/drivers/mfd/intel-lpss-pci.c +++ b/drivers/mfd/intel-lpss-pci.c @@ -347,6 +347,16 @@ static const struct pci_device_id intel_lpss_pci_ids[] = { { PCI_VDEVICE(INTEL, 0xa36a), (kernel_ulong_t)&cnl_i2c_info }, { PCI_VDEVICE(INTEL, 0xa36b), (kernel_ulong_t)&cnl_i2c_info }, { PCI_VDEVICE(INTEL, 0xa37b), (kernel_ulong_t)&spt_info }, + /* CML-V */ + { PCI_VDEVICE(INTEL, 0xa3a7), (kernel_ulong_t)&spt_uart_info }, + { PCI_VDEVICE(INTEL, 0xa3a8), (kernel_ulong_t)&spt_uart_info }, + { PCI_VDEVICE(INTEL, 0xa3a9), (kernel_ulong_t)&spt_info }, + { PCI_VDEVICE(INTEL, 0xa3aa), (kernel_ulong_t)&spt_info }, + { PCI_VDEVICE(INTEL, 0xa3e0), (kernel_ulong_t)&spt_i2c_info }, + { PCI_VDEVICE(INTEL, 0xa3e1), (kernel_ulong_t)&spt_i2c_info }, + { PCI_VDEVICE(INTEL, 0xa3e2), (kernel_ulong_t)&spt_i2c_info }, + { PCI_VDEVICE(INTEL, 0xa3e3), (kernel_ulong_t)&spt_i2c_info }, + { PCI_VDEVICE(INTEL, 0xa3e6), (kernel_ulong_t)&spt_uart_info }, { } }; MODULE_DEVICE_TABLE(pci, intel_lpss_pci_ids); From fb945c95a482200876993977008b67ea658bd938 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 26 Feb 2020 16:51:58 +0200 Subject: [PATCH 316/635] mfd: dln2: Fix sanity checking for endpoints While the commit 2b8bd606b1e6 ("mfd: dln2: More sanity checking for endpoints") tries to harden the sanity checks it made at the same time a regression, i.e. mixed in and out endpoints. Obviously it should have been not tested on real hardware at that time, but unluckily it didn't happen. So, fix above mentioned typo and make device being enumerated again. While here, introduce an enumerator for magic values to prevent similar issue to happen in the future. Fixes: 2b8bd606b1e6 ("mfd: dln2: More sanity checking for endpoints") Cc: Oliver Neukum Cc: Greg Kroah-Hartman Signed-off-by: Andy Shevchenko Signed-off-by: Lee Jones --- drivers/mfd/dln2.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/mfd/dln2.c b/drivers/mfd/dln2.c index 7841c11411d0..4faa8d2e5d04 100644 --- a/drivers/mfd/dln2.c +++ b/drivers/mfd/dln2.c @@ -90,6 +90,11 @@ struct dln2_mod_rx_slots { spinlock_t lock; }; +enum dln2_endpoint { + DLN2_EP_OUT = 0, + DLN2_EP_IN = 1, +}; + struct dln2_dev { struct usb_device *usb_dev; struct usb_interface *interface; @@ -733,10 +738,10 @@ static int dln2_probe(struct usb_interface *interface, hostif->desc.bNumEndpoints < 2) return -ENODEV; - epin = &hostif->endpoint[0].desc; - epout = &hostif->endpoint[1].desc; + epout = &hostif->endpoint[DLN2_EP_OUT].desc; if (!usb_endpoint_is_bulk_out(epout)) return -ENODEV; + epin = &hostif->endpoint[DLN2_EP_IN].desc; if (!usb_endpoint_is_bulk_in(epin)) return -ENODEV; From a045f3578e0c14edced021a8d38887008bc000b1 Mon Sep 17 00:00:00 2001 From: Matti Vaittinen Date: Fri, 20 Mar 2020 11:42:33 +0200 Subject: [PATCH 317/635] dt-bindings: bd718x7: Yamlify and add BD71850 Convert ROHM bd71837 and bd71847 PMIC binding text docs to yaml. Split the binding document to two separate documents (own documents for BD71837 and BD71847) as they have different amount of regulators. This way we can better enforce the node name check for regulators. ROHM is also providing BD71850 - which is almost identical to BD71847 - main difference is some initial regulator states. The BD71850 can be driven by same driver and it has same buck/LDO setup as BD71847 - add it to BD71847 binding document and introduce compatible for it. Signed-off-by: Matti Vaittinen Reviewed-by: Rob Herring Acked-by: Mark Brown Signed-off-by: Lee Jones --- .../bindings/mfd/rohm,bd71837-pmic.txt | 90 ------- .../bindings/mfd/rohm,bd71837-pmic.yaml | 236 ++++++++++++++++++ .../bindings/mfd/rohm,bd71847-pmic.yaml | 222 ++++++++++++++++ .../regulator/rohm,bd71837-regulator.txt | 162 ------------ .../regulator/rohm,bd71837-regulator.yaml | 103 ++++++++ .../regulator/rohm,bd71847-regulator.yaml | 97 +++++++ 6 files changed, 658 insertions(+), 252 deletions(-) delete mode 100644 Documentation/devicetree/bindings/mfd/rohm,bd71837-pmic.txt create mode 100644 Documentation/devicetree/bindings/mfd/rohm,bd71837-pmic.yaml create mode 100644 Documentation/devicetree/bindings/mfd/rohm,bd71847-pmic.yaml delete mode 100644 Documentation/devicetree/bindings/regulator/rohm,bd71837-regulator.txt create mode 100644 Documentation/devicetree/bindings/regulator/rohm,bd71837-regulator.yaml create mode 100644 Documentation/devicetree/bindings/regulator/rohm,bd71847-regulator.yaml diff --git a/Documentation/devicetree/bindings/mfd/rohm,bd71837-pmic.txt b/Documentation/devicetree/bindings/mfd/rohm,bd71837-pmic.txt deleted file mode 100644 index f22d74c7a8db..000000000000 --- a/Documentation/devicetree/bindings/mfd/rohm,bd71837-pmic.txt +++ /dev/null @@ -1,90 +0,0 @@ -* ROHM BD71837 and BD71847 Power Management Integrated Circuit bindings - -BD71837MWV and BD71847MWV are programmable Power Management ICs for powering -single-core, dual-core, and quad-core SoCs such as NXP-i.MX 8M. They are -optimized for low BOM cost and compact solution footprint. BD71837MWV -integrates 8 Buck regulators and 7 LDOs. BD71847MWV contains 6 Buck regulators -and 6 LDOs. - -Datasheet for BD71837 is available at: -https://www.rohm.com/datasheet/BD71837MWV/bd71837mwv-e -Datasheet for BD71847 is available at: -https://www.rohm.com/datasheet/BD71847AMWV/bd71847amwv-e - -Required properties: - - compatible : Should be "rohm,bd71837" for bd71837 - "rohm,bd71847" for bd71847. - - reg : I2C slave address. - - interrupt-parent : Phandle to the parent interrupt controller. - - interrupts : The interrupt line the device is connected to. - - clocks : The parent clock connected to PMIC. If this is missing - 32768 KHz clock is assumed. - - #clock-cells : Should be 0. - - regulators: : List of child nodes that specify the regulators. - Please see ../regulator/rohm,bd71837-regulator.txt - -Optional properties: -- clock-output-names : Should contain name for output clock. -- rohm,reset-snvs-powered : Transfer BD718x7 to SNVS state at reset. - -The BD718x7 supports two different HW states as reset target states. States -are called as SNVS and READY. At READY state all the PMIC power outputs go -down and OTP is reload. At the SNVS state all other logic and external -devices apart from the SNVS power domain are shut off. Please refer to NXP -i.MX8 documentation for further information regarding SNVS state. When a -reset is done via SNVS state the PMIC OTP data is not reload. This causes -power outputs that have been under SW control to stay down when reset has -switched power state to SNVS. If reset is done via READY state the power -outputs will be returned to HW control by OTP loading. Thus the reset -target state is set to READY by default. If SNVS state is used the boot -crucial regulators must have the regulator-always-on and regulator-boot-on -properties set in regulator node. - -- rohm,short-press-ms : Short press duration in milliseconds -- rohm,long-press-ms : Long press duration in milliseconds - -Configure the "short press" and "long press" timers for the power button. -Values are rounded to what hardware supports (500ms multiple for short and -1000ms multiple for long). If these properties are not present the existing -configuration (from bootloader or OTP) is not touched. - -Example: - - /* external oscillator node */ - osc: oscillator { - compatible = "fixed-clock"; - #clock-cells = <1>; - clock-frequency = <32768>; - clock-output-names = "osc"; - }; - - pmic: pmic@4b { - compatible = "rohm,bd71837"; - reg = <0x4b>; - interrupt-parent = <&gpio1>; - interrupts = <29 GPIO_ACTIVE_LOW>; - interrupt-names = "irq"; - #clock-cells = <0>; - clocks = <&osc 0>; - clock-output-names = "bd71837-32k-out"; - rohm,reset-snvs-powered; - - regulators { - buck1: BUCK1 { - regulator-name = "buck1"; - regulator-min-microvolt = <700000>; - regulator-max-microvolt = <1300000>; - regulator-boot-on; - regulator-always-on; - regulator-ramp-delay = <1250>; - }; - // [...] - }; - }; - - /* Clock consumer node */ - rtc@0 { - compatible = "company,my-rtc"; - clock-names = "my-clock"; - clocks = <&pmic>; - }; diff --git a/Documentation/devicetree/bindings/mfd/rohm,bd71837-pmic.yaml b/Documentation/devicetree/bindings/mfd/rohm,bd71837-pmic.yaml new file mode 100644 index 000000000000..aa922c560fcc --- /dev/null +++ b/Documentation/devicetree/bindings/mfd/rohm,bd71837-pmic.yaml @@ -0,0 +1,236 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mfd/rohm,bd71837-pmic.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: ROHM BD71837 Power Management Integrated Circuit bindings + +maintainers: + - Matti Vaittinen + +description: | + BD71837MWV is programmable Power Management ICs for powering single-core, + dual-core, and quad-core SoCs such as NXP-i.MX 8M. It is optimized for low + BOM cost and compact solution footprint. BD71837MWV integrates 8 Buck + regulators and 7 LDOs. + Datasheet for BD71837 is available at + https://www.rohm.com/products/power-management/power-management-ic-for-system/industrial-consumer-applications/nxp-imx/bd71837amwv-product + +properties: + compatible: + const: rohm,bd71837 + + reg: + description: + I2C slave address. + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 1 + + "#clock-cells": + const: 0 + +# The BD718x7 supports two different HW states as reset target states. States +# are called as SNVS and READY. At READY state all the PMIC power outputs go +# down and OTP is reload. At the SNVS state all other logic and external +# devices apart from the SNVS power domain are shut off. Please refer to NXP +# i.MX8 documentation for further information regarding SNVS state. When a +# reset is done via SNVS state the PMIC OTP data is not reload. This causes +# power outputs that have been under SW control to stay down when reset has +# switched power state to SNVS. If reset is done via READY state the power +# outputs will be returned to HW control by OTP loading. Thus the reset +# target state is set to READY by default. If SNVS state is used the boot +# crucial regulators must have the regulator-always-on and regulator-boot-on +# properties set in regulator node. + + rohm,reset-snvs-powered: + description: | + Transfer PMIC to SNVS state at reset + type: boolean + +# Configure the "short press" and "long press" timers for the power button. +# Values are rounded to what hardware supports +# Short-press: +# Shortest being 10ms, next 500ms and then multiple of 500ms up to 7,5s +# Long-press: +# Shortest being 10ms, next 1000ms and then multiple of 1000ms up to 15s +# If these properties are not present the existing configuration (from +# bootloader or OTP) is not touched. + + rohm,short-press-ms: + description: + Short press duration in milliseconds + enum: + - 10 + - 500 + - 1000 + - 1500 + - 2000 + - 2500 + - 3000 + - 3500 + - 4000 + - 4500 + - 5000 + - 5500 + - 6000 + - 6500 + - 7000 + + rohm,long-press-ms: + description: + Long press duration in milliseconds + enum: + - 10 + - 1000 + - 2000 + - 3000 + - 4000 + - 5000 + - 6000 + - 7000 + - 8000 + - 9000 + - 10000 + - 11000 + - 12000 + - 13000 + - 14000 + + regulators: + $ref: ../regulator/rohm,bd71837-regulator.yaml + description: + List of child nodes that specify the regulators. + +required: + - compatible + - reg + - interrupts + - clocks + - "#clock-cells" + - regulators + +additionalProperties: false + +examples: + - | + #include + #include + + i2c { + pmic: pmic@4b { + compatible = "rohm,bd71837"; + reg = <0x4b>; + interrupt-parent = <&gpio1>; + interrupts = <29 IRQ_TYPE_LEVEL_LOW>; + #clock-cells = <0>; + clocks = <&osc 0>; + rohm,reset-snvs-powered; + rohm,short-press-ms = <10>; + rohm,long-press-ms = <2000>; + + regulators { + buck1: BUCK1 { + regulator-name = "buck1"; + regulator-min-microvolt = <700000>; + regulator-max-microvolt = <1300000>; + regulator-boot-on; + regulator-always-on; + regulator-ramp-delay = <1250>; + rohm,dvs-run-voltage = <900000>; + rohm,dvs-idle-voltage = <850000>; + rohm,dvs-suspend-voltage = <800000>; + }; + buck2: BUCK2 { + regulator-name = "buck2"; + regulator-min-microvolt = <700000>; + regulator-max-microvolt = <1300000>; + regulator-boot-on; + regulator-always-on; + regulator-ramp-delay = <1250>; + rohm,dvs-run-voltage = <1000000>; + rohm,dvs-idle-voltage = <900000>; + }; + buck3: BUCK3 { + regulator-name = "buck3"; + regulator-min-microvolt = <700000>; + regulator-max-microvolt = <1300000>; + regulator-boot-on; + rohm,dvs-run-voltage = <1000000>; + }; + buck4: BUCK4 { + regulator-name = "buck4"; + regulator-min-microvolt = <700000>; + regulator-max-microvolt = <1300000>; + regulator-boot-on; + rohm,dvs-run-voltage = <1000000>; + }; + buck5: BUCK5 { + regulator-name = "buck5"; + regulator-min-microvolt = <700000>; + regulator-max-microvolt = <1350000>; + regulator-boot-on; + }; + buck6: BUCK6 { + regulator-name = "buck6"; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + }; + buck7: BUCK7 { + regulator-name = "buck7"; + regulator-min-microvolt = <1605000>; + regulator-max-microvolt = <1995000>; + regulator-boot-on; + }; + buck8: BUCK8 { + regulator-name = "buck8"; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <1400000>; + }; + + ldo1: LDO1 { + regulator-name = "ldo1"; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + }; + ldo2: LDO2 { + regulator-name = "ldo2"; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-boot-on; + }; + ldo3: LDO3 { + regulator-name = "ldo3"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + }; + ldo4: LDO4 { + regulator-name = "ldo4"; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <1800000>; + }; + ldo5: LDO5 { + regulator-name = "ldo5"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + }; + ldo6: LDO6 { + regulator-name = "ldo6"; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <1800000>; + }; + ldo7_reg: LDO7 { + regulator-name = "ldo7"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + }; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/mfd/rohm,bd71847-pmic.yaml b/Documentation/devicetree/bindings/mfd/rohm,bd71847-pmic.yaml new file mode 100644 index 000000000000..402e40dfe0b8 --- /dev/null +++ b/Documentation/devicetree/bindings/mfd/rohm,bd71847-pmic.yaml @@ -0,0 +1,222 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mfd/rohm,bd71847-pmic.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: ROHM BD71847 and BD71850 Power Management Integrated Circuit bindings + +maintainers: + - Matti Vaittinen + +description: | + BD71847AMWV and BD71850MWV are programmable Power Management ICs for powering + single-core, dual-core, and quad-core SoCs such as NXP-i.MX 8M. It is + optimized for low BOM cost and compact solution footprint. BD71847MWV and + BD71850MWV integrate 6 Buck regulators and 6 LDOs. + Datasheets are available at + https://www.rohm.com/products/power-management/power-management-ic-for-system/industrial-consumer-applications/nxp-imx/bd71847amwv-product + https://www.rohm.com/products/power-management/power-management-ic-for-system/industrial-consumer-applications/nxp-imx/bd71850mwv-product + +properties: + compatible: + enum: + - rohm,bd71847 + - rohm,bd71850 + + reg: + description: + I2C slave address. + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 1 + + "#clock-cells": + const: 0 + +# The BD71847 abd BD71850 support two different HW states as reset target +# states. States are called as SNVS and READY. At READY state all the PMIC +# power outputs go down and OTP is reload. At the SNVS state all other logic +# and external devices apart from the SNVS power domain are shut off. Please +# refer to NXP i.MX8 documentation for further information regarding SNVS +# state. When a reset is done via SNVS state the PMIC OTP data is not reload. +# This causes power outputs that have been under SW control to stay down when +# reset has switched power state to SNVS. If reset is done via READY state the +# power outputs will be returned to HW control by OTP loading. Thus the reset +# target state is set to READY by default. If SNVS state is used the boot +# crucial regulators must have the regulator-always-on and regulator-boot-on +# properties set in regulator node. + + rohm,reset-snvs-powered: + description: + Transfer PMIC to SNVS state at reset. + type: boolean + +# Configure the "short press" and "long press" timers for the power button. +# Values are rounded to what hardware supports +# Short-press: +# Shortest being 10ms, next 500ms and then multiple of 500ms up to 7,5s +# Long-press: +# Shortest being 10ms, next 1000ms and then multiple of 1000ms up to 15s +# If these properties are not present the existing # configuration (from +# bootloader or OTP) is not touched. + + rohm,short-press-ms: + description: + Short press duration in milliseconds + enum: + - 10 + - 500 + - 1000 + - 1500 + - 2000 + - 2500 + - 3000 + - 3500 + - 4000 + - 4500 + - 5000 + - 5500 + - 6000 + - 6500 + - 7000 + - 7500 + + rohm,long-press-ms: + description: + Long press duration in milliseconds + enum: + - 10 + - 1000 + - 2000 + - 3000 + - 4000 + - 5000 + - 6000 + - 7000 + - 8000 + - 9000 + - 10000 + - 11000 + - 12000 + - 13000 + - 14000 + - 15000 + + regulators: + $ref: ../regulator/rohm,bd71847-regulator.yaml + description: + List of child nodes that specify the regulators. + +required: + - compatible + - reg + - interrupts + - clocks + - "#clock-cells" + - regulators + +additionalProperties: false + +examples: + - | + #include + #include + + i2c { + pmic: pmic@4b { + compatible = "rohm,bd71847"; + reg = <0x4b>; + interrupt-parent = <&gpio1>; + interrupts = <29 IRQ_TYPE_LEVEL_LOW>; + #clock-cells = <0>; + clocks = <&osc 0>; + rohm,reset-snvs-powered; + rohm,short-press-ms = <10>; + rohm,long-press-ms = <2000>; + + regulators { + buck1: BUCK1 { + regulator-name = "buck1"; + regulator-min-microvolt = <700000>; + regulator-max-microvolt = <1300000>; + regulator-boot-on; + regulator-always-on; + regulator-ramp-delay = <1250>; + rohm,dvs-run-voltage = <900000>; + rohm,dvs-idle-voltage = <850000>; + rohm,dvs-suspend-voltage = <800000>; + }; + buck2: BUCK2 { + regulator-name = "buck2"; + regulator-min-microvolt = <700000>; + regulator-max-microvolt = <1300000>; + regulator-boot-on; + regulator-always-on; + regulator-ramp-delay = <1250>; + rohm,dvs-run-voltage = <1000000>; + rohm,dvs-idle-voltage = <900000>; + }; + buck3: BUCK3 { + regulator-name = "buck3"; + regulator-min-microvolt = <550000>; + regulator-max-microvolt = <1350000>; + regulator-boot-on; + }; + buck4: BUCK4 { + regulator-name = "buck4"; + regulator-min-microvolt = <2600000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + }; + buck5: BUCK5 { + regulator-name = "buck5"; + regulator-min-microvolt = <1605000>; + regulator-max-microvolt = <1995000>; + regulator-boot-on; + }; + buck8: BUCK6 { + regulator-name = "buck6"; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <1400000>; + }; + + ldo1: LDO1 { + regulator-name = "ldo1"; + regulator-min-microvolt = <1600000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + }; + ldo2: LDO2 { + regulator-name = "ldo2"; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <900000>; + regulator-boot-on; + }; + ldo3: LDO3 { + regulator-name = "ldo3"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + }; + ldo4: LDO4 { + regulator-name = "ldo4"; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <1800000>; + }; + ldo5: LDO5 { + regulator-name = "ldo5"; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <3300000>; + }; + ldo6: LDO6 { + regulator-name = "ldo6"; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <1800000>; + }; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/regulator/rohm,bd71837-regulator.txt b/Documentation/devicetree/bindings/regulator/rohm,bd71837-regulator.txt deleted file mode 100644 index cbce62c22b60..000000000000 --- a/Documentation/devicetree/bindings/regulator/rohm,bd71837-regulator.txt +++ /dev/null @@ -1,162 +0,0 @@ -ROHM BD71837 and BD71847 Power Management Integrated Circuit regulator bindings - -Required properties: - - regulator-name: should be "buck1", ..., "buck8" and "ldo1", ..., "ldo7" for - BD71837. For BD71847 names should be "buck1", ..., "buck6" - and "ldo1", ..., "ldo6" - -List of regulators provided by this controller. BD71837 regulators node -should be sub node of the BD71837 MFD node. See BD71837 MFD bindings at -Documentation/devicetree/bindings/mfd/rohm,bd71837-pmic.txt -Regulator nodes should be named to BUCK_ and LDO_. The -definition for each of these nodes is defined using the standard -binding for regulators at -Documentation/devicetree/bindings/regulator/regulator.txt. -Note that if BD71837 starts at RUN state you probably want to use -regulator-boot-on at least for BUCK6 and BUCK7 so that those are not -disabled by driver at startup. LDO5 and LDO6 are supplied by those and -if they are disabled at startup the voltage monitoring for LDO5/LDO6 will -cause PMIC to reset. - -The valid names for BD71837 regulator nodes are: -BUCK1, BUCK2, BUCK3, BUCK4, BUCK5, BUCK6, BUCK7, BUCK8 -LDO1, LDO2, LDO3, LDO4, LDO5, LDO6, LDO7 - -The valid names for BD71847 regulator nodes are: -BUCK1, BUCK2, BUCK3, BUCK4, BUCK5, BUCK6 -LDO1, LDO2, LDO3, LDO4, LDO5, LDO6 - -Optional properties: -- rohm,dvs-run-voltage : PMIC default "RUN" state voltage in uV. - See below table for bucks which support this. -- rohm,dvs-idle-voltage : PMIC default "IDLE" state voltage in uV. - See below table for bucks which support this. -- rohm,dvs-suspend-voltage : PMIC default "SUSPEND" state voltage in uV. - See below table for bucks which support this. -- Any optional property defined in bindings/regulator/regulator.txt - -Supported default DVS states: - -BD71837: -buck | dvs-run-voltage | dvs-idle-voltage | dvs-suspend-voltage ------------------------------------------------------------------------------ -1 | supported | supported | supported ----------------------------------------------------------------------------- -2 | supported | supported | not supported ----------------------------------------------------------------------------- -3 | supported | not supported | not supported ----------------------------------------------------------------------------- -4 | supported | not supported | not supported ----------------------------------------------------------------------------- -rest | not supported | not supported | not supported - -BD71847: -buck | dvs-run-voltage | dvs-idle-voltage | dvs-suspend-voltage ------------------------------------------------------------------------------ -1 | supported | supported | supported ----------------------------------------------------------------------------- -2 | supported | supported | not supported ----------------------------------------------------------------------------- -rest | not supported | not supported | not supported - -Example: -regulators { - buck1: BUCK1 { - regulator-name = "buck1"; - regulator-min-microvolt = <700000>; - regulator-max-microvolt = <1300000>; - regulator-boot-on; - regulator-always-on; - regulator-ramp-delay = <1250>; - rohm,dvs-run-voltage = <900000>; - rohm,dvs-idle-voltage = <850000>; - rohm,dvs-suspend-voltage = <800000>; - }; - buck2: BUCK2 { - regulator-name = "buck2"; - regulator-min-microvolt = <700000>; - regulator-max-microvolt = <1300000>; - regulator-boot-on; - regulator-always-on; - regulator-ramp-delay = <1250>; - rohm,dvs-run-voltage = <1000000>; - rohm,dvs-idle-voltage = <900000>; - }; - buck3: BUCK3 { - regulator-name = "buck3"; - regulator-min-microvolt = <700000>; - regulator-max-microvolt = <1300000>; - regulator-boot-on; - rohm,dvs-run-voltage = <1000000>; - }; - buck4: BUCK4 { - regulator-name = "buck4"; - regulator-min-microvolt = <700000>; - regulator-max-microvolt = <1300000>; - regulator-boot-on; - rohm,dvs-run-voltage = <1000000>; - }; - buck5: BUCK5 { - regulator-name = "buck5"; - regulator-min-microvolt = <700000>; - regulator-max-microvolt = <1350000>; - regulator-boot-on; - }; - buck6: BUCK6 { - regulator-name = "buck6"; - regulator-min-microvolt = <3000000>; - regulator-max-microvolt = <3300000>; - regulator-boot-on; - }; - buck7: BUCK7 { - regulator-name = "buck7"; - regulator-min-microvolt = <1605000>; - regulator-max-microvolt = <1995000>; - regulator-boot-on; - }; - buck8: BUCK8 { - regulator-name = "buck8"; - regulator-min-microvolt = <800000>; - regulator-max-microvolt = <1400000>; - }; - - ldo1: LDO1 { - regulator-name = "ldo1"; - regulator-min-microvolt = <3000000>; - regulator-max-microvolt = <3300000>; - regulator-boot-on; - }; - ldo2: LDO2 { - regulator-name = "ldo2"; - regulator-min-microvolt = <900000>; - regulator-max-microvolt = <900000>; - regulator-boot-on; - }; - ldo3: LDO3 { - regulator-name = "ldo3"; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <3300000>; - }; - ldo4: LDO4 { - regulator-name = "ldo4"; - regulator-min-microvolt = <900000>; - regulator-max-microvolt = <1800000>; - }; - ldo5: LDO5 { - regulator-name = "ldo5"; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <3300000>; - }; - ldo6: LDO6 { - regulator-name = "ldo6"; - regulator-min-microvolt = <900000>; - regulator-max-microvolt = <1800000>; - }; - ldo7_reg: LDO7 { - regulator-name = "ldo7"; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <3300000>; - }; -}; - - diff --git a/Documentation/devicetree/bindings/regulator/rohm,bd71837-regulator.yaml b/Documentation/devicetree/bindings/regulator/rohm,bd71837-regulator.yaml new file mode 100644 index 000000000000..a323b1696eee --- /dev/null +++ b/Documentation/devicetree/bindings/regulator/rohm,bd71837-regulator.yaml @@ -0,0 +1,103 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/regulator/rohm,bd71837-regulator.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: ROHM BD71837 Power Management Integrated Circuit regulators + +maintainers: + - Matti Vaittinen + +description: | + List of regulators provided by this controller. BD71837 regulators node + should be sub node of the BD71837 MFD node. See BD71837 MFD bindings at + Documentation/devicetree/bindings/mfd/rohm,bd71837-pmic.yaml + Regulator nodes should be named to BUCK_ and LDO_. The + definition for each of these nodes is defined using the standard + binding for regulators at + Documentation/devicetree/bindings/regulator/regulator.txt. + Note that if BD71837 starts at RUN state you probably want to use + regulator-boot-on at least for BUCK6 and BUCK7 so that those are not + disabled by driver at startup. LDO5 and LDO6 are supplied by those and + if they are disabled at startup the voltage monitoring for LDO5/LDO6 will + cause PMIC to reset. + +#The valid names for BD71837 regulator nodes are: +#BUCK1, BUCK2, BUCK3, BUCK4, BUCK5, BUCK6, BUCK7, BUCK8 +#LDO1, LDO2, LDO3, LDO4, LDO5, LDO6, LDO7 + +patternProperties: + "^LDO[1-7]$": + type: object + allOf: + - $ref: regulator.yaml# + description: + Properties for single LDO regulator. + + properties: + regulator-name: + pattern: "^ldo[1-7]$" + description: + should be "ldo1", ..., "ldo7" + + "^BUCK[1-8]$": + type: object + allOf: + - $ref: regulator.yaml# + description: + Properties for single BUCK regulator. + + properties: + regulator-name: + pattern: "^buck[1-8]$" + description: + should be "buck1", ..., "buck8" + + rohm,dvs-run-voltage: + allOf: + - $ref: "/schemas/types.yaml#/definitions/uint32" + - minimum: 0 + maximum: 1300000 + description: + PMIC default "RUN" state voltage in uV. See below table for + bucks which support this. 0 means disabled. + + rohm,dvs-idle-voltage: + allOf: + - $ref: "/schemas/types.yaml#/definitions/uint32" + - minimum: 0 + maximum: 1300000 + description: + PMIC default "IDLE" state voltage in uV. See below table for + bucks which support this. 0 means disabled. + + rohm,dvs-suspend-voltage: + allOf: + - $ref: "/schemas/types.yaml#/definitions/uint32" + - minimum: 0 + maximum: 1300000 + description: + PMIC default "SUSPEND" state voltage in uV. See below table for + bucks which support this. 0 means disabled. + + # Supported default DVS states: + # + # BD71837: + # buck | dvs-run-voltage | dvs-idle-voltage | dvs-suspend-voltage + # ---------------------------------------------------------------- + # 1 | supported | supported | supported + # ---------------------------------------------------------------- + # 2 | supported | supported | not supported + # ---------------------------------------------------------------- + # 3 | supported | not supported | not supported + # ---------------------------------------------------------------- + # 4 | supported | not supported | not supported + # ---------------------------------------------------------------- + # rest | not supported | not supported | not supported + + + required: + - regulator-name + additionalProperties: false +additionalProperties: false diff --git a/Documentation/devicetree/bindings/regulator/rohm,bd71847-regulator.yaml b/Documentation/devicetree/bindings/regulator/rohm,bd71847-regulator.yaml new file mode 100644 index 000000000000..526fd00bcb16 --- /dev/null +++ b/Documentation/devicetree/bindings/regulator/rohm,bd71847-regulator.yaml @@ -0,0 +1,97 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/regulator/rohm,bd71847-regulator.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: ROHM BD71847 and BD71850 Power Management Integrated Circuit regulators + +maintainers: + - Matti Vaittinen + +description: | + List of regulators provided by this controller. BD71847 regulators node + should be sub node of the BD71847 MFD node. See BD71847 MFD bindings at + Documentation/devicetree/bindings/mfd/rohm,bd71847-pmic.yaml + Regulator nodes should be named to BUCK_ and LDO_. The + definition for each of these nodes is defined using the standard + binding for regulators at + Documentation/devicetree/bindings/regulator/regulator.txt. + Note that if BD71847 starts at RUN state you probably want to use + regulator-boot-on at least for BUCK5. LDO6 is supplied by it and it must + not be disabled by driver at startup. If BUCK5 is disabled at startup the + voltage monitoring for LDO5/LDO6 can cause PMIC to reset. + +#The valid names for BD71847 regulator nodes are: +#BUCK1, BUCK2, BUCK3, BUCK4, BUCK5, BUCK6 +#LDO1, LDO2, LDO3, LDO4, LDO5, LDO6 + +patternProperties: + "^LDO[1-6]$": + type: object + allOf: + - $ref: regulator.yaml# + description: + Properties for single LDO regulator. + + properties: + regulator-name: + pattern: "^ldo[1-6]$" + description: + should be "ldo1", ..., "ldo6" + + "^BUCK[1-6]$": + type: object + allOf: + - $ref: regulator.yaml# + description: + Properties for single BUCK regulator. + + properties: + regulator-name: + pattern: "^buck[1-6]$" + description: + should be "buck1", ..., "buck6" + + rohm,dvs-run-voltage: + allOf: + - $ref: "/schemas/types.yaml#/definitions/uint32" + - minimum: 0 + maximum: 1300000 + description: + PMIC default "RUN" state voltage in uV. See below table for + bucks which support this. 0 means disabled. + + rohm,dvs-idle-voltage: + allOf: + - $ref: "/schemas/types.yaml#/definitions/uint32" + - minimum: 0 + maximum: 1300000 + description: + PMIC default "IDLE" state voltage in uV. See below table for + bucks which support this. 0 means disabled. + + rohm,dvs-suspend-voltage: + allOf: + - $ref: "/schemas/types.yaml#/definitions/uint32" + - minimum: 0 + maximum: 1300000 + description: + PMIC default "SUSPEND" state voltage in uV. See below table for + bucks which support this. 0 means disabled. + + # Supported default DVS states: + # + # BD71847: + # buck | dvs-run-voltage | dvs-idle-voltage | dvs-suspend-voltage + # ---------------------------------------------------------------- + # 1 | supported | supported | supported + # ---------------------------------------------------------------- + # 2 | supported | supported | not supported + # ---------------------------------------------------------------- + # rest | not supported | not supported | not supported + + required: + - regulator-name + additionalProperties: false +additionalProperties: false From b1cc5409f08ef548db9cfc9e7b4fcbd07e32fae0 Mon Sep 17 00:00:00 2001 From: Shreyas Joshi Date: Wed, 26 Feb 2020 11:07:22 +1000 Subject: [PATCH 318/635] mfd: da9062: Add support for interrupt polarity defined in device tree The da9062 interrupt handler cannot necessarily be low active. Add a function to configure the interrupt type based on what is defined in the device tree. The allowable interrupt type is either low or high level trigger. Signed-off-by: Shreyas Joshi Reviewed-by: Linus Walleij Reviewed-by: Adam Thomson Signed-off-by: Lee Jones --- drivers/mfd/da9062-core.c | 44 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/drivers/mfd/da9062-core.c b/drivers/mfd/da9062-core.c index 419c73533401..fc30726e2e27 100644 --- a/drivers/mfd/da9062-core.c +++ b/drivers/mfd/da9062-core.c @@ -21,6 +21,9 @@ #define DA9062_REG_EVENT_B_OFFSET 1 #define DA9062_REG_EVENT_C_OFFSET 2 +#define DA9062_IRQ_LOW 0 +#define DA9062_IRQ_HIGH 1 + static struct regmap_irq da9061_irqs[] = { /* EVENT A */ [DA9061_IRQ_ONKEY] = { @@ -369,6 +372,33 @@ static int da9062_get_device_type(struct da9062 *chip) return ret; } +static u32 da9062_configure_irq_type(struct da9062 *chip, int irq, u32 *trigger) +{ + u32 irq_type = 0; + struct irq_data *irq_data = irq_get_irq_data(irq); + + if (!irq_data) { + dev_err(chip->dev, "Invalid IRQ: %d\n", irq); + return -EINVAL; + } + *trigger = irqd_get_trigger_type(irq_data); + + switch (*trigger) { + case IRQ_TYPE_LEVEL_HIGH: + irq_type = DA9062_IRQ_HIGH; + break; + case IRQ_TYPE_LEVEL_LOW: + irq_type = DA9062_IRQ_LOW; + break; + default: + dev_warn(chip->dev, "Unsupported IRQ type: %d\n", *trigger); + return -EINVAL; + } + return regmap_update_bits(chip->regmap, DA9062AA_CONFIG_A, + DA9062AA_IRQ_TYPE_MASK, + irq_type << DA9062AA_IRQ_TYPE_SHIFT); +} + static const struct regmap_range da9061_aa_readable_ranges[] = { regmap_reg_range(DA9062AA_PAGE_CON, DA9062AA_STATUS_B), regmap_reg_range(DA9062AA_STATUS_D, DA9062AA_EVENT_C), @@ -388,6 +418,7 @@ static const struct regmap_range da9061_aa_readable_ranges[] = { regmap_reg_range(DA9062AA_VBUCK1_A, DA9062AA_VBUCK4_A), regmap_reg_range(DA9062AA_VBUCK3_A, DA9062AA_VBUCK3_A), regmap_reg_range(DA9062AA_VLDO1_A, DA9062AA_VLDO4_A), + regmap_reg_range(DA9062AA_CONFIG_A, DA9062AA_CONFIG_A), regmap_reg_range(DA9062AA_VBUCK1_B, DA9062AA_VBUCK4_B), regmap_reg_range(DA9062AA_VBUCK3_B, DA9062AA_VBUCK3_B), regmap_reg_range(DA9062AA_VLDO1_B, DA9062AA_VLDO4_B), @@ -417,6 +448,7 @@ static const struct regmap_range da9061_aa_writeable_ranges[] = { regmap_reg_range(DA9062AA_VBUCK1_A, DA9062AA_VBUCK4_A), regmap_reg_range(DA9062AA_VBUCK3_A, DA9062AA_VBUCK3_A), regmap_reg_range(DA9062AA_VLDO1_A, DA9062AA_VLDO4_A), + regmap_reg_range(DA9062AA_CONFIG_A, DA9062AA_CONFIG_A), regmap_reg_range(DA9062AA_VBUCK1_B, DA9062AA_VBUCK4_B), regmap_reg_range(DA9062AA_VBUCK3_B, DA9062AA_VBUCK3_B), regmap_reg_range(DA9062AA_VLDO1_B, DA9062AA_VLDO4_B), @@ -596,6 +628,7 @@ static int da9062_i2c_probe(struct i2c_client *i2c, const struct regmap_irq_chip *irq_chip; const struct regmap_config *config; int cell_num; + u32 trigger_type = 0; int ret; chip = devm_kzalloc(&i2c->dev, sizeof(*chip), GFP_KERNEL); @@ -654,10 +687,15 @@ static int da9062_i2c_probe(struct i2c_client *i2c, if (ret) return ret; + ret = da9062_configure_irq_type(chip, i2c->irq, &trigger_type); + if (ret < 0) { + dev_err(chip->dev, "Failed to configure IRQ type\n"); + return ret; + } + ret = regmap_add_irq_chip(chip->regmap, i2c->irq, - IRQF_TRIGGER_LOW | IRQF_ONESHOT | IRQF_SHARED, - -1, irq_chip, - &chip->regmap_irq); + trigger_type | IRQF_SHARED | IRQF_ONESHOT, + -1, irq_chip, &chip->regmap_irq); if (ret) { dev_err(chip->dev, "Failed to request IRQ %d: %d\n", i2c->irq, ret); From e3fadb35bc1be0078e9ff5f9a55811f7eb1a5d05 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 23 Mar 2020 22:02:37 +0200 Subject: [PATCH 319/635] mfd: dln2: Allow to be enumerated via ACPI On some platforms user may want to enumerate DLN2 device, its children, to be enumerated via ACPI. In order to achieve this, let's distinguish children by _ADR value. Signed-off-by: Andy Shevchenko Signed-off-by: Lee Jones --- drivers/mfd/dln2.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/drivers/mfd/dln2.c b/drivers/mfd/dln2.c index 4faa8d2e5d04..39276fa626d2 100644 --- a/drivers/mfd/dln2.c +++ b/drivers/mfd/dln2.c @@ -645,35 +645,56 @@ static int dln2_start_rx_urbs(struct dln2_dev *dln2, gfp_t gfp) return 0; } +enum { + DLN2_ACPI_MATCH_GPIO = 0, + DLN2_ACPI_MATCH_I2C = 1, + DLN2_ACPI_MATCH_SPI = 2, +}; + static struct dln2_platform_data dln2_pdata_gpio = { .handle = DLN2_HANDLE_GPIO, }; +static struct mfd_cell_acpi_match dln2_acpi_match_gpio = { + .adr = DLN2_ACPI_MATCH_GPIO, +}; + /* Only one I2C port seems to be supported on current hardware */ static struct dln2_platform_data dln2_pdata_i2c = { .handle = DLN2_HANDLE_I2C, .port = 0, }; +static struct mfd_cell_acpi_match dln2_acpi_match_i2c = { + .adr = DLN2_ACPI_MATCH_I2C, +}; + /* Only one SPI port supported */ static struct dln2_platform_data dln2_pdata_spi = { .handle = DLN2_HANDLE_SPI, .port = 0, }; +static struct mfd_cell_acpi_match dln2_acpi_match_spi = { + .adr = DLN2_ACPI_MATCH_SPI, +}; + static const struct mfd_cell dln2_devs[] = { { .name = "dln2-gpio", + .acpi_match = &dln2_acpi_match_gpio, .platform_data = &dln2_pdata_gpio, .pdata_size = sizeof(struct dln2_platform_data), }, { .name = "dln2-i2c", + .acpi_match = &dln2_acpi_match_i2c, .platform_data = &dln2_pdata_i2c, .pdata_size = sizeof(struct dln2_platform_data), }, { .name = "dln2-spi", + .acpi_match = &dln2_acpi_match_spi, .platform_data = &dln2_pdata_spi, .pdata_size = sizeof(struct dln2_platform_data), }, From c2b5fdfba2a915b5001d3ec63f4b3fc651c22da4 Mon Sep 17 00:00:00 2001 From: Takashi Iwai Date: Wed, 11 Mar 2020 08:47:38 +0100 Subject: [PATCH 320/635] mfd: aat2870: Use scnprintf() for avoiding potential buffer overflow There is still one call of sprintf() without checking the proper buffer overflow in aat2870_dump_reg(). Replace it with scnprintf() call for covering that. Signed-off-by: Takashi Iwai Signed-off-by: Lee Jones --- drivers/mfd/aat2870-core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/mfd/aat2870-core.c b/drivers/mfd/aat2870-core.c index 78ee4b28fca2..a17cf759739d 100644 --- a/drivers/mfd/aat2870-core.c +++ b/drivers/mfd/aat2870-core.c @@ -221,7 +221,7 @@ static ssize_t aat2870_dump_reg(struct aat2870_data *aat2870, char *buf) count += sprintf(buf, "aat2870 registers\n"); for (addr = 0; addr < AAT2870_REG_NUM; addr++) { - count += sprintf(buf + count, "0x%02x: ", addr); + count += snprintf(buf + count, PAGE_SIZE - count, "0x%02x: ", addr); if (count >= PAGE_SIZE - 1) break; From d2923aa4535664ae5c46c3a093985afd18fec118 Mon Sep 17 00:00:00 2001 From: Jarkko Nikula Date: Mon, 16 Mar 2020 16:32:24 +0200 Subject: [PATCH 321/635] mfd: intel-lpss: Fix Intel Elkhart Lake LPSS I2C input clock Intel Elkhart Lake LPSS I2C has 100 MHz input clock instead of 133 MHz that was our preliminary information. This will result slower I2C bus clock when driver calculates its timing parameters in case ACPI tables don't provide them. Slower I2C bus clock is allowed but let's fix this to match with reality. While at it, keep the same default I2C device properties as Intel Broxton since it is not known do they need any update. Signed-off-by: Jarkko Nikula Reviewed-by: Andy Shevchenko Signed-off-by: Lee Jones --- drivers/mfd/intel-lpss-pci.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/drivers/mfd/intel-lpss-pci.c b/drivers/mfd/intel-lpss-pci.c index 378024a3ba28..7fc0c5d4edff 100644 --- a/drivers/mfd/intel-lpss-pci.c +++ b/drivers/mfd/intel-lpss-pci.c @@ -139,6 +139,11 @@ static const struct intel_lpss_platform_info cnl_i2c_info = { .properties = spt_i2c_properties, }; +static const struct intel_lpss_platform_info ehl_i2c_info = { + .clk_rate = 100000000, + .properties = bxt_i2c_properties, +}; + static const struct pci_device_id intel_lpss_pci_ids[] = { /* CML-LP */ { PCI_VDEVICE(INTEL, 0x02a8), (kernel_ulong_t)&spt_uart_info }, @@ -231,15 +236,15 @@ static const struct pci_device_id intel_lpss_pci_ids[] = { { PCI_VDEVICE(INTEL, 0x4b2a), (kernel_ulong_t)&bxt_info }, { PCI_VDEVICE(INTEL, 0x4b2b), (kernel_ulong_t)&bxt_info }, { PCI_VDEVICE(INTEL, 0x4b37), (kernel_ulong_t)&bxt_info }, - { PCI_VDEVICE(INTEL, 0x4b44), (kernel_ulong_t)&bxt_i2c_info }, - { PCI_VDEVICE(INTEL, 0x4b45), (kernel_ulong_t)&bxt_i2c_info }, - { PCI_VDEVICE(INTEL, 0x4b4b), (kernel_ulong_t)&bxt_i2c_info }, - { PCI_VDEVICE(INTEL, 0x4b4c), (kernel_ulong_t)&bxt_i2c_info }, + { PCI_VDEVICE(INTEL, 0x4b44), (kernel_ulong_t)&ehl_i2c_info }, + { PCI_VDEVICE(INTEL, 0x4b45), (kernel_ulong_t)&ehl_i2c_info }, + { PCI_VDEVICE(INTEL, 0x4b4b), (kernel_ulong_t)&ehl_i2c_info }, + { PCI_VDEVICE(INTEL, 0x4b4c), (kernel_ulong_t)&ehl_i2c_info }, { PCI_VDEVICE(INTEL, 0x4b4d), (kernel_ulong_t)&bxt_uart_info }, - { PCI_VDEVICE(INTEL, 0x4b78), (kernel_ulong_t)&bxt_i2c_info }, - { PCI_VDEVICE(INTEL, 0x4b79), (kernel_ulong_t)&bxt_i2c_info }, - { PCI_VDEVICE(INTEL, 0x4b7a), (kernel_ulong_t)&bxt_i2c_info }, - { PCI_VDEVICE(INTEL, 0x4b7b), (kernel_ulong_t)&bxt_i2c_info }, + { PCI_VDEVICE(INTEL, 0x4b78), (kernel_ulong_t)&ehl_i2c_info }, + { PCI_VDEVICE(INTEL, 0x4b79), (kernel_ulong_t)&ehl_i2c_info }, + { PCI_VDEVICE(INTEL, 0x4b7a), (kernel_ulong_t)&ehl_i2c_info }, + { PCI_VDEVICE(INTEL, 0x4b7b), (kernel_ulong_t)&ehl_i2c_info }, /* JSL */ { PCI_VDEVICE(INTEL, 0x4da8), (kernel_ulong_t)&spt_uart_info }, { PCI_VDEVICE(INTEL, 0x4da9), (kernel_ulong_t)&spt_uart_info }, From 3db0a2fc5668c6dc60218e0f55d2443bcd726aa0 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Thu, 4 Apr 2019 08:05:38 -0400 Subject: [PATCH 322/635] ceph: register MDS request with dir inode from the start When the unsafe reply to a request comes in, the request is put on the r_unsafe_dir inode's list. In future patches, we're going to need to wait on requests that may not have gotten an unsafe reply yet. Change __register_request to put the entry on the dir inode's list when the pointer is set in the request, and don't check the CEPH_MDS_R_GOT_UNSAFE flag when unregistering it. The only place that uses this list today is fsync codepath, and with the coming changes, we'll want to wait on all operations whether it has gotten an unsafe reply or not. Signed-off-by: Jeff Layton Reviewed-by: "Yan, Zheng" Signed-off-by: Ilya Dryomov --- fs/ceph/mds_client.c | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index bbbbddf71326..ea5244896646 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -793,8 +793,13 @@ static void __register_request(struct ceph_mds_client *mdsc, mdsc->oldest_tid = req->r_tid; if (dir) { + struct ceph_inode_info *ci = ceph_inode(dir); + ihold(dir); req->r_unsafe_dir = dir; + spin_lock(&ci->i_unsafe_lock); + list_add_tail(&req->r_unsafe_dir_item, &ci->i_unsafe_dirops); + spin_unlock(&ci->i_unsafe_lock); } } @@ -822,8 +827,7 @@ static void __unregister_request(struct ceph_mds_client *mdsc, erase_request(&mdsc->request_tree, req); - if (req->r_unsafe_dir && - test_bit(CEPH_MDS_R_GOT_UNSAFE, &req->r_req_flags)) { + if (req->r_unsafe_dir) { struct ceph_inode_info *ci = ceph_inode(req->r_unsafe_dir); spin_lock(&ci->i_unsafe_lock); list_del_init(&req->r_unsafe_dir_item); @@ -2935,14 +2939,6 @@ static void handle_reply(struct ceph_mds_session *session, struct ceph_msg *msg) } else { set_bit(CEPH_MDS_R_GOT_UNSAFE, &req->r_req_flags); list_add_tail(&req->r_unsafe_item, &req->r_session->s_unsafe); - if (req->r_unsafe_dir) { - struct ceph_inode_info *ci = - ceph_inode(req->r_unsafe_dir); - spin_lock(&ci->i_unsafe_lock); - list_add_tail(&req->r_unsafe_dir_item, - &ci->i_unsafe_dirops); - spin_unlock(&ci->i_unsafe_lock); - } } dout("handle_reply tid %lld result %d\n", tid, result); From f85122afeb230b4ad0b90ac40aba0fe6532baeea Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Tue, 2 Apr 2019 08:04:30 -0400 Subject: [PATCH 323/635] ceph: add refcounting for Fx caps In future patches we'll be taking and relying on Fx caps. Add proper refcounting for them. Signed-off-by: Jeff Layton Reviewed-by: "Yan, Zheng" Signed-off-by: Ilya Dryomov --- fs/ceph/caps.c | 7 +++++++ fs/ceph/inode.c | 1 + fs/ceph/super.h | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index 28ae0c134700..1a17f19fd8ad 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -965,6 +965,8 @@ int __ceph_caps_used(struct ceph_inode_info *ci) used |= CEPH_CAP_FILE_WR; if (ci->i_wb_ref || ci->i_wrbuffer_ref) used |= CEPH_CAP_FILE_BUFFER; + if (ci->i_fx_ref) + used |= CEPH_CAP_FILE_EXCL; return used; } @@ -2500,6 +2502,8 @@ static void __take_cap_refs(struct ceph_inode_info *ci, int got, ci->i_rd_ref++; if (got & CEPH_CAP_FILE_CACHE) ci->i_rdcache_ref++; + if (got & CEPH_CAP_FILE_EXCL) + ci->i_fx_ref++; if (got & CEPH_CAP_FILE_WR) { if (ci->i_wr_ref == 0 && !ci->i_head_snapc) { BUG_ON(!snap_rwsem_locked); @@ -2911,6 +2915,9 @@ void ceph_put_cap_refs(struct ceph_inode_info *ci, int had) if (had & CEPH_CAP_FILE_CACHE) if (--ci->i_rdcache_ref == 0) last++; + if (had & CEPH_CAP_FILE_EXCL) + if (--ci->i_fx_ref == 0) + last++; if (had & CEPH_CAP_FILE_BUFFER) { if (--ci->i_wb_ref == 0) { last++; diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c index d01710a16a4a..094b8fc37787 100644 --- a/fs/ceph/inode.c +++ b/fs/ceph/inode.c @@ -496,6 +496,7 @@ struct inode *ceph_alloc_inode(struct super_block *sb) ci->i_rdcache_ref = 0; ci->i_wr_ref = 0; ci->i_wb_ref = 0; + ci->i_fx_ref = 0; ci->i_wrbuffer_ref = 0; ci->i_wrbuffer_ref_head = 0; atomic_set(&ci->i_filelock_ref, 0); diff --git a/fs/ceph/super.h b/fs/ceph/super.h index 037cdfb2ad4f..2eee34b9ac71 100644 --- a/fs/ceph/super.h +++ b/fs/ceph/super.h @@ -375,7 +375,7 @@ struct ceph_inode_info { /* held references to caps */ int i_pin_ref; - int i_rd_ref, i_rdcache_ref, i_wr_ref, i_wb_ref; + int i_rd_ref, i_rdcache_ref, i_wr_ref, i_wb_ref, i_fx_ref; int i_wrbuffer_ref, i_wrbuffer_ref_head; atomic_t i_filelock_ref; atomic_t i_shared_gen; /* increment each time we get FILE_SHARED */ From 525d15e8e5ad770d17681d6f9f1513faa361017c Mon Sep 17 00:00:00 2001 From: "Yan, Zheng" Date: Sat, 11 May 2019 17:27:59 +0800 Subject: [PATCH 324/635] ceph: check inode type for CEPH_CAP_FILE_{CACHE,RD,REXTEND,LAZYIO} These bits will have new meaning for directory inodes. Signed-off-by: "Yan, Zheng" Reviewed-by: Jeff Layton Signed-off-by: Ilya Dryomov --- fs/ceph/caps.c | 44 ++++++++++++++++++++++++++++++++------------ fs/ceph/mds_client.c | 3 ++- fs/ceph/super.h | 12 +----------- 3 files changed, 35 insertions(+), 24 deletions(-) diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index 1a17f19fd8ad..d05717397c2a 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -573,7 +573,8 @@ static void __check_cap_issue(struct ceph_inode_info *ci, struct ceph_cap *cap, * Each time we receive FILE_CACHE anew, we increment * i_rdcache_gen. */ - if ((issued & (CEPH_CAP_FILE_CACHE|CEPH_CAP_FILE_LAZYIO)) && + if (S_ISREG(ci->vfs_inode.i_mode) && + (issued & (CEPH_CAP_FILE_CACHE|CEPH_CAP_FILE_LAZYIO)) && (had & (CEPH_CAP_FILE_CACHE|CEPH_CAP_FILE_LAZYIO)) == 0) { ci->i_rdcache_gen++; } @@ -958,7 +959,7 @@ int __ceph_caps_used(struct ceph_inode_info *ci) if (ci->i_rd_ref) used |= CEPH_CAP_FILE_RD; if (ci->i_rdcache_ref || - (!S_ISDIR(ci->vfs_inode.i_mode) && /* ignore readdir cache */ + (S_ISREG(ci->vfs_inode.i_mode) && ci->vfs_inode.i_data.nrpages)) used |= CEPH_CAP_FILE_CACHE; if (ci->i_wr_ref) @@ -985,6 +986,20 @@ int __ceph_caps_file_wanted(struct ceph_inode_info *ci) return ceph_caps_for_mode(bits >> 1); } +/* + * wanted, by virtue of open file modes AND cap refs (buffered/cached data) + */ +int __ceph_caps_wanted(struct ceph_inode_info *ci) +{ + int w = __ceph_caps_file_wanted(ci) | __ceph_caps_used(ci); + if (!S_ISDIR(ci->vfs_inode.i_mode)) { + /* we want EXCL if dirty data */ + if (w & CEPH_CAP_FILE_BUFFER) + w |= CEPH_CAP_FILE_EXCL; + } + return w; +} + /* * Return caps we have registered with the MDS(s) as 'wanted'. */ @@ -1903,7 +1918,7 @@ retry_locked: * If we fail, it's because pages are locked.... try again later. */ if ((!no_delay || mdsc->stopping) && - !S_ISDIR(inode->i_mode) && /* ignore readdir cache */ + S_ISREG(inode->i_mode) && !(ci->i_wb_ref || ci->i_wrbuffer_ref) && /* no dirty pages... */ inode->i_data.nrpages && /* have cached pages */ (revoking & (CEPH_CAP_FILE_CACHE| @@ -2635,7 +2650,8 @@ again: snap_rwsem_locked = true; } *got = need | (have & want); - if ((need & CEPH_CAP_FILE_RD) && + if (S_ISREG(inode->i_mode) && + (need & CEPH_CAP_FILE_RD) && !(*got & CEPH_CAP_FILE_CACHE)) ceph_disable_fscache_readpage(ci); __take_cap_refs(ci, *got, true); @@ -2643,7 +2659,8 @@ again: } } else { int session_readonly = false; - if ((need & CEPH_CAP_FILE_WR) && ci->i_auth_cap) { + if (ci->i_auth_cap && + (need & (CEPH_CAP_FILE_WR | CEPH_CAP_FILE_EXCL))) { struct ceph_mds_session *s = ci->i_auth_cap->session; spin_lock(&s->s_cap_lock); session_readonly = s->s_readonly; @@ -2817,7 +2834,8 @@ int ceph_get_caps(struct file *filp, int need, int want, return ret; } - if (ci->i_inline_version != CEPH_INLINE_NONE && + if (S_ISREG(ci->vfs_inode.i_mode) && + ci->i_inline_version != CEPH_INLINE_NONE && (_got & (CEPH_CAP_FILE_CACHE|CEPH_CAP_FILE_LAZYIO)) && i_size_read(inode) > 0) { struct page *page = @@ -2850,7 +2868,8 @@ int ceph_get_caps(struct file *filp, int need, int want, break; } - if ((_got & CEPH_CAP_FILE_RD) && (_got & CEPH_CAP_FILE_CACHE)) + if (S_ISREG(ci->vfs_inode.i_mode) && + (_got & CEPH_CAP_FILE_RD) && (_got & CEPH_CAP_FILE_CACHE)) ceph_fscache_revalidate_cookie(ci); *got = _got; @@ -3140,7 +3159,7 @@ static void handle_cap_grant(struct inode *inode, * try to invalidate (once). (If there are dirty buffers, we * will invalidate _after_ writeback.) */ - if (!S_ISDIR(inode->i_mode) && /* don't invalidate readdir cache */ + if (S_ISREG(inode->i_mode) && /* don't invalidate readdir cache */ ((cap->issued & ~newcaps) & CEPH_CAP_FILE_CACHE) && (newcaps & CEPH_CAP_FILE_LAZYIO) == 0 && !(ci->i_wrbuffer_ref || ci->i_wb_ref)) { @@ -3304,11 +3323,12 @@ static void handle_cap_grant(struct inode *inode, ceph_cap_string(cap->issued), ceph_cap_string(newcaps), ceph_cap_string(revoking)); - if (revoking & used & CEPH_CAP_FILE_BUFFER) + if (S_ISREG(inode->i_mode) && + (revoking & used & CEPH_CAP_FILE_BUFFER)) writeback = true; /* initiate writeback; will delay ack */ - else if (revoking == CEPH_CAP_FILE_CACHE && - (newcaps & CEPH_CAP_FILE_LAZYIO) == 0 && - queue_invalidate) + else if (queue_invalidate && + revoking == CEPH_CAP_FILE_CACHE && + (newcaps & CEPH_CAP_FILE_LAZYIO) == 0) ; /* do nothing yet, invalidation will be queued */ else if (cap == ci->i_auth_cap) check_caps = 1; /* check auth cap only */ diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index ea5244896646..2980e57ca7b9 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -1776,7 +1776,8 @@ static int trim_caps_cb(struct inode *inode, struct ceph_cap *cap, void *arg) } /* The inode has cached pages, but it's no longer used. * we can safely drop it */ - if (wanted == 0 && used == CEPH_CAP_FILE_CACHE && + if (S_ISREG(inode->i_mode) && + wanted == 0 && used == CEPH_CAP_FILE_CACHE && !(oissued & CEPH_CAP_FILE_CACHE)) { used = 0; oissued = 0; diff --git a/fs/ceph/super.h b/fs/ceph/super.h index 2eee34b9ac71..37dc1ac8f6c3 100644 --- a/fs/ceph/super.h +++ b/fs/ceph/super.h @@ -675,17 +675,7 @@ extern int ceph_caps_revoking(struct ceph_inode_info *ci, int mask); extern int __ceph_caps_used(struct ceph_inode_info *ci); extern int __ceph_caps_file_wanted(struct ceph_inode_info *ci); - -/* - * wanted, by virtue of open file modes AND cap refs (buffered/cached data) - */ -static inline int __ceph_caps_wanted(struct ceph_inode_info *ci) -{ - int w = __ceph_caps_file_wanted(ci) | __ceph_caps_used(ci); - if (w & CEPH_CAP_FILE_BUFFER) - w |= CEPH_CAP_FILE_EXCL; /* we want EXCL if dirty data */ - return w; -} +extern int __ceph_caps_wanted(struct ceph_inode_info *ci); /* what the mds thinks we want */ extern int __ceph_caps_mds_wanted(struct ceph_inode_info *ci, bool check); From 072eaf3c0f0fd2bd8f53799c8dee3ab907db1242 Mon Sep 17 00:00:00 2001 From: Ilya Dryomov Date: Tue, 28 Jan 2020 20:12:22 +0100 Subject: [PATCH 325/635] libceph: drop CEPH_DEFINE_SHOW_FUNC Although CEPH_DEFINE_SHOW_FUNC is much older, it now duplicates DEFINE_SHOW_ATTRIBUTE from linux/seq_file.h. Signed-off-by: Ilya Dryomov Reviewed-by: Jeff Layton --- fs/ceph/debugfs.c | 16 ++++++++-------- include/linux/ceph/debugfs.h | 14 -------------- net/ceph/debugfs.c | 20 ++++++++++---------- 3 files changed, 18 insertions(+), 32 deletions(-) diff --git a/fs/ceph/debugfs.c b/fs/ceph/debugfs.c index fb7cabd98e7b..481ac97b4d25 100644 --- a/fs/ceph/debugfs.c +++ b/fs/ceph/debugfs.c @@ -218,10 +218,10 @@ static int mds_sessions_show(struct seq_file *s, void *ptr) return 0; } -CEPH_DEFINE_SHOW_FUNC(mdsmap_show) -CEPH_DEFINE_SHOW_FUNC(mdsc_show) -CEPH_DEFINE_SHOW_FUNC(caps_show) -CEPH_DEFINE_SHOW_FUNC(mds_sessions_show) +DEFINE_SHOW_ATTRIBUTE(mdsmap); +DEFINE_SHOW_ATTRIBUTE(mdsc); +DEFINE_SHOW_ATTRIBUTE(caps); +DEFINE_SHOW_ATTRIBUTE(mds_sessions); /* @@ -281,25 +281,25 @@ void ceph_fs_debugfs_init(struct ceph_fs_client *fsc) 0400, fsc->client->debugfs_dir, fsc, - &mdsmap_show_fops); + &mdsmap_fops); fsc->debugfs_mds_sessions = debugfs_create_file("mds_sessions", 0400, fsc->client->debugfs_dir, fsc, - &mds_sessions_show_fops); + &mds_sessions_fops); fsc->debugfs_mdsc = debugfs_create_file("mdsc", 0400, fsc->client->debugfs_dir, fsc, - &mdsc_show_fops); + &mdsc_fops); fsc->debugfs_caps = debugfs_create_file("caps", 0400, fsc->client->debugfs_dir, fsc, - &caps_show_fops); + &caps_fops); } diff --git a/include/linux/ceph/debugfs.h b/include/linux/ceph/debugfs.h index cf5e840eec71..8b3a1a7a953a 100644 --- a/include/linux/ceph/debugfs.h +++ b/include/linux/ceph/debugfs.h @@ -2,22 +2,8 @@ #ifndef _FS_CEPH_DEBUGFS_H #define _FS_CEPH_DEBUGFS_H -#include #include -#define CEPH_DEFINE_SHOW_FUNC(name) \ -static int name##_open(struct inode *inode, struct file *file) \ -{ \ - return single_open(file, name, inode->i_private); \ -} \ - \ -static const struct file_operations name##_fops = { \ - .open = name##_open, \ - .read = seq_read, \ - .llseek = seq_lseek, \ - .release = single_release, \ -}; - /* debugfs.c */ extern void ceph_debugfs_init(void); extern void ceph_debugfs_cleanup(void); diff --git a/net/ceph/debugfs.c b/net/ceph/debugfs.c index 7cb992e55475..1344f232ecc5 100644 --- a/net/ceph/debugfs.c +++ b/net/ceph/debugfs.c @@ -383,11 +383,11 @@ static int client_options_show(struct seq_file *s, void *p) return 0; } -CEPH_DEFINE_SHOW_FUNC(monmap_show) -CEPH_DEFINE_SHOW_FUNC(osdmap_show) -CEPH_DEFINE_SHOW_FUNC(monc_show) -CEPH_DEFINE_SHOW_FUNC(osdc_show) -CEPH_DEFINE_SHOW_FUNC(client_options_show) +DEFINE_SHOW_ATTRIBUTE(monmap); +DEFINE_SHOW_ATTRIBUTE(osdmap); +DEFINE_SHOW_ATTRIBUTE(monc); +DEFINE_SHOW_ATTRIBUTE(osdc); +DEFINE_SHOW_ATTRIBUTE(client_options); void __init ceph_debugfs_init(void) { @@ -414,31 +414,31 @@ void ceph_debugfs_client_init(struct ceph_client *client) 0400, client->debugfs_dir, client, - &monc_show_fops); + &monc_fops); client->osdc.debugfs_file = debugfs_create_file("osdc", 0400, client->debugfs_dir, client, - &osdc_show_fops); + &osdc_fops); client->debugfs_monmap = debugfs_create_file("monmap", 0400, client->debugfs_dir, client, - &monmap_show_fops); + &monmap_fops); client->debugfs_osdmap = debugfs_create_file("osdmap", 0400, client->debugfs_dir, client, - &osdmap_show_fops); + &osdmap_fops); client->debugfs_options = debugfs_create_file("client_options", 0400, client->debugfs_dir, client, - &client_options_show_fops); + &client_options_fops); } void ceph_debugfs_client_cleanup(struct ceph_client *client) From 94f4857f4ba21aad4cf11dde961ea23a07b5161c Mon Sep 17 00:00:00 2001 From: Ilya Dryomov Date: Thu, 30 Jan 2020 13:54:59 +0100 Subject: [PATCH 326/635] rbd: remove barriers from img_request_layered_{set,clear,test}() IMG_REQ_LAYERED is set in rbd_img_request_create(), and tested and cleared in rbd_img_request_destroy() when the image request is about to be destroyed. The barriers are unnecessary. Signed-off-by: Ilya Dryomov --- drivers/block/rbd.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c index 6343402c09e6..162fd1df06dd 100644 --- a/drivers/block/rbd.c +++ b/drivers/block/rbd.c @@ -1366,18 +1366,15 @@ static void rbd_osd_submit(struct ceph_osd_request *osd_req) static void img_request_layered_set(struct rbd_img_request *img_request) { set_bit(IMG_REQ_LAYERED, &img_request->flags); - smp_mb(); } static void img_request_layered_clear(struct rbd_img_request *img_request) { clear_bit(IMG_REQ_LAYERED, &img_request->flags); - smp_mb(); } static bool img_request_layered_test(struct rbd_img_request *img_request) { - smp_mb(); return test_bit(IMG_REQ_LAYERED, &img_request->flags) != 0; } From 70837470b41e5d0e8d655348067219b915def0c7 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Tue, 4 Feb 2020 09:37:48 -0500 Subject: [PATCH 327/635] ceph: don't ClearPageChecked in ceph_invalidatepage() CephFS doesn't set this bit to begin with, so there should be no need to clear it. Reported-by: David Howells Signed-off-by: Jeff Layton Reviewed-by: Ilya Dryomov Signed-off-by: Ilya Dryomov --- fs/ceph/addr.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c index 7ab616601141..6067847bc03b 100644 --- a/fs/ceph/addr.c +++ b/fs/ceph/addr.c @@ -159,8 +159,6 @@ static void ceph_invalidatepage(struct page *page, unsigned int offset, if (!PagePrivate(page)) return; - ClearPageChecked(page); - dout("%p invalidatepage %p idx %lu full dirty page\n", inode, page, page->index); From 5107d7d505cb32fc5e74b792bce14b03f5beac7f Mon Sep 17 00:00:00 2001 From: Xiubo Li Date: Wed, 29 Jan 2020 03:27:07 -0500 Subject: [PATCH 328/635] ceph: move ceph_osdc_{read,write}pages to ceph.ko Since these helpers are only used by ceph.ko, move them there and rename them with _sync_ qualifiers. Signed-off-by: Xiubo Li Reviewed-by: Jeff Layton Signed-off-by: Ilya Dryomov --- fs/ceph/addr.c | 86 ++++++++++++++++++++++++++++++++- include/linux/ceph/osd_client.h | 17 ------- net/ceph/osd_client.c | 79 ------------------------------ 3 files changed, 84 insertions(+), 98 deletions(-) diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c index 6067847bc03b..7136f9947354 100644 --- a/fs/ceph/addr.c +++ b/fs/ceph/addr.c @@ -180,6 +180,47 @@ static int ceph_releasepage(struct page *page, gfp_t g) return !PagePrivate(page); } +/* + * Read some contiguous pages. If we cross a stripe boundary, shorten + * *plen. Return number of bytes read, or error. + */ +static int ceph_sync_readpages(struct ceph_fs_client *fsc, + struct ceph_vino vino, + struct ceph_file_layout *layout, + u64 off, u64 *plen, + u32 truncate_seq, u64 truncate_size, + struct page **pages, int num_pages, + int page_align) +{ + struct ceph_osd_client *osdc = &fsc->client->osdc; + struct ceph_osd_request *req; + int rc = 0; + + dout("readpages on ino %llx.%llx on %llu~%llu\n", vino.ino, + vino.snap, off, *plen); + req = ceph_osdc_new_request(osdc, layout, vino, off, plen, 0, 1, + CEPH_OSD_OP_READ, CEPH_OSD_FLAG_READ, + NULL, truncate_seq, truncate_size, + false); + if (IS_ERR(req)) + return PTR_ERR(req); + + /* it may be a short read due to an object boundary */ + osd_req_op_extent_osd_data_pages(req, 0, + pages, *plen, page_align, false, false); + + dout("readpages final extent is %llu~%llu (%llu bytes align %d)\n", + off, *plen, *plen, page_align); + + rc = ceph_osdc_start_request(osdc, req, false); + if (!rc) + rc = ceph_osdc_wait_request(osdc, req); + + ceph_osdc_put_request(req); + dout("readpages result %d\n", rc); + return rc; +} + /* * read a single page, without unlocking it. */ @@ -216,7 +257,7 @@ static int ceph_do_readpage(struct file *filp, struct page *page) dout("readpage inode %p file %p page %p index %lu\n", inode, filp, page, page->index); - err = ceph_osdc_readpages(&fsc->client->osdc, ceph_vino(inode), + err = ceph_sync_readpages(fsc, ceph_vino(inode), &ci->i_layout, off, &len, ci->i_truncate_seq, ci->i_truncate_size, &page, 1, 0); @@ -568,6 +609,47 @@ static u64 get_writepages_data_length(struct inode *inode, return end > start ? end - start : 0; } +/* + * do a synchronous write on N pages + */ +static int ceph_sync_writepages(struct ceph_fs_client *fsc, + struct ceph_vino vino, + struct ceph_file_layout *layout, + struct ceph_snap_context *snapc, + u64 off, u64 len, + u32 truncate_seq, u64 truncate_size, + struct timespec64 *mtime, + struct page **pages, int num_pages) +{ + struct ceph_osd_client *osdc = &fsc->client->osdc; + struct ceph_osd_request *req; + int rc = 0; + int page_align = off & ~PAGE_MASK; + + req = ceph_osdc_new_request(osdc, layout, vino, off, &len, 0, 1, + CEPH_OSD_OP_WRITE, CEPH_OSD_FLAG_WRITE, + snapc, truncate_seq, truncate_size, + true); + if (IS_ERR(req)) + return PTR_ERR(req); + + /* it may be a short write due to an object boundary */ + osd_req_op_extent_osd_data_pages(req, 0, pages, len, page_align, + false, false); + dout("writepages %llu~%llu (%llu bytes)\n", off, len, len); + + req->r_mtime = *mtime; + rc = ceph_osdc_start_request(osdc, req, true); + if (!rc) + rc = ceph_osdc_wait_request(osdc, req); + + ceph_osdc_put_request(req); + if (rc == 0) + rc = len; + dout("writepages result %d\n", rc); + return rc; +} + /* * Write a single page, but leave the page locked. * @@ -626,7 +708,7 @@ static int writepage_nounlock(struct page *page, struct writeback_control *wbc) set_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC); set_page_writeback(page); - err = ceph_osdc_writepages(&fsc->client->osdc, ceph_vino(inode), + err = ceph_sync_writepages(fsc, ceph_vino(inode), &ci->i_layout, snapc, page_off, len, ceph_wbc.truncate_seq, ceph_wbc.truncate_size, diff --git a/include/linux/ceph/osd_client.h b/include/linux/ceph/osd_client.h index 5a62dbd3f4c2..9d9f745b98a1 100644 --- a/include/linux/ceph/osd_client.h +++ b/include/linux/ceph/osd_client.h @@ -509,23 +509,6 @@ int ceph_osdc_call(struct ceph_osd_client *osdc, struct page *req_page, size_t req_len, struct page **resp_pages, size_t *resp_len); -extern int ceph_osdc_readpages(struct ceph_osd_client *osdc, - struct ceph_vino vino, - struct ceph_file_layout *layout, - u64 off, u64 *plen, - u32 truncate_seq, u64 truncate_size, - struct page **pages, int nr_pages, - int page_align); - -extern int ceph_osdc_writepages(struct ceph_osd_client *osdc, - struct ceph_vino vino, - struct ceph_file_layout *layout, - struct ceph_snap_context *sc, - u64 off, u64 len, - u32 truncate_seq, u64 truncate_size, - struct timespec64 *mtime, - struct page **pages, int nr_pages); - int ceph_osdc_copy_from(struct ceph_osd_client *osdc, u64 src_snapid, u64 src_version, struct ceph_object_id *src_oid, diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c index af868d3923b9..f9ec6a6568bd 100644 --- a/net/ceph/osd_client.c +++ b/net/ceph/osd_client.c @@ -5228,85 +5228,6 @@ void ceph_osdc_stop(struct ceph_osd_client *osdc) ceph_msgpool_destroy(&osdc->msgpool_op_reply); } -/* - * Read some contiguous pages. If we cross a stripe boundary, shorten - * *plen. Return number of bytes read, or error. - */ -int ceph_osdc_readpages(struct ceph_osd_client *osdc, - struct ceph_vino vino, struct ceph_file_layout *layout, - u64 off, u64 *plen, - u32 truncate_seq, u64 truncate_size, - struct page **pages, int num_pages, int page_align) -{ - struct ceph_osd_request *req; - int rc = 0; - - dout("readpages on ino %llx.%llx on %llu~%llu\n", vino.ino, - vino.snap, off, *plen); - req = ceph_osdc_new_request(osdc, layout, vino, off, plen, 0, 1, - CEPH_OSD_OP_READ, CEPH_OSD_FLAG_READ, - NULL, truncate_seq, truncate_size, - false); - if (IS_ERR(req)) - return PTR_ERR(req); - - /* it may be a short read due to an object boundary */ - osd_req_op_extent_osd_data_pages(req, 0, - pages, *plen, page_align, false, false); - - dout("readpages final extent is %llu~%llu (%llu bytes align %d)\n", - off, *plen, *plen, page_align); - - rc = ceph_osdc_start_request(osdc, req, false); - if (!rc) - rc = ceph_osdc_wait_request(osdc, req); - - ceph_osdc_put_request(req); - dout("readpages result %d\n", rc); - return rc; -} -EXPORT_SYMBOL(ceph_osdc_readpages); - -/* - * do a synchronous write on N pages - */ -int ceph_osdc_writepages(struct ceph_osd_client *osdc, struct ceph_vino vino, - struct ceph_file_layout *layout, - struct ceph_snap_context *snapc, - u64 off, u64 len, - u32 truncate_seq, u64 truncate_size, - struct timespec64 *mtime, - struct page **pages, int num_pages) -{ - struct ceph_osd_request *req; - int rc = 0; - int page_align = off & ~PAGE_MASK; - - req = ceph_osdc_new_request(osdc, layout, vino, off, &len, 0, 1, - CEPH_OSD_OP_WRITE, CEPH_OSD_FLAG_WRITE, - snapc, truncate_seq, truncate_size, - true); - if (IS_ERR(req)) - return PTR_ERR(req); - - /* it may be a short write due to an object boundary */ - osd_req_op_extent_osd_data_pages(req, 0, pages, len, page_align, - false, false); - dout("writepages %llu~%llu (%llu bytes)\n", off, len, len); - - req->r_mtime = *mtime; - rc = ceph_osdc_start_request(osdc, req, true); - if (!rc) - rc = ceph_osdc_wait_request(osdc, req); - - ceph_osdc_put_request(req); - if (rc == 0) - rc = len; - dout("writepages result %d\n", rc); - return rc; -} -EXPORT_SYMBOL(ceph_osdc_writepages); - static int osd_req_op_copy_from_init(struct ceph_osd_request *req, u64 src_snapid, u64 src_version, struct ceph_object_id *src_oid, From cb63483ad06d5ae57c76cf15ba0f51fad0900417 Mon Sep 17 00:00:00 2001 From: Xiubo Li Date: Tue, 11 Feb 2020 10:31:20 -0500 Subject: [PATCH 329/635] ceph: fix description of some mount options Based on the latest code, the default value for wsize/rsize is 64MB and the default value for the mount_timeout is 60 seconds. Signed-off-by: Xiubo Li Reviewed-by: Jeff Layton Signed-off-by: Ilya Dryomov --- Documentation/filesystems/ceph.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation/filesystems/ceph.txt b/Documentation/filesystems/ceph.txt index b19b6a03f91c..92ffc9b3b018 100644 --- a/Documentation/filesystems/ceph.txt +++ b/Documentation/filesystems/ceph.txt @@ -103,17 +103,17 @@ Mount Options address its connection to the monitor originates from. wsize=X - Specify the maximum write size in bytes. Default: 16 MB. + Specify the maximum write size in bytes. Default: 64 MB. rsize=X - Specify the maximum read size in bytes. Default: 16 MB. + Specify the maximum read size in bytes. Default: 64 MB. rasize=X Specify the maximum readahead size in bytes. Default: 8 MB. mount_timeout=X Specify the timeout value for mount (in seconds), in the case - of a non-responsive Ceph file system. The default is 30 + of a non-responsive Ceph file system. The default is 60 seconds. caps_max=X From bbb480ab05b2c148d8f8378660bfddd46761e86e Mon Sep 17 00:00:00 2001 From: "Yan, Zheng" Date: Tue, 11 Feb 2020 22:54:43 +0800 Subject: [PATCH 330/635] ceph: check if file lock exists before sending unlock request When a process exits, kernel closes its files. locks_remove_file() is called to remove file locks on these files. locks_remove_file() tries unlocking files even there is no file lock. Signed-off-by: "Yan, Zheng" Reviewed-by: Jeff Layton Signed-off-by: Ilya Dryomov --- fs/ceph/locks.c | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/fs/ceph/locks.c b/fs/ceph/locks.c index 544e9e85b120..d6b9166e71e4 100644 --- a/fs/ceph/locks.c +++ b/fs/ceph/locks.c @@ -210,6 +210,21 @@ static int ceph_lock_wait_for_completion(struct ceph_mds_client *mdsc, return 0; } +static int try_unlock_file(struct file *file, struct file_lock *fl) +{ + int err; + unsigned int orig_flags = fl->fl_flags; + fl->fl_flags |= FL_EXISTS; + err = locks_lock_file_wait(file, fl); + fl->fl_flags = orig_flags; + if (err == -ENOENT) { + if (!(orig_flags & FL_EXISTS)) + err = 0; + return err; + } + return 1; +} + /** * Attempt to set an fcntl lock. * For now, this just goes away to the server. Later it may be more awesome. @@ -255,9 +270,15 @@ int ceph_lock(struct file *file, int cmd, struct file_lock *fl) else lock_cmd = CEPH_LOCK_UNLOCK; + if (op == CEPH_MDS_OP_SETFILELOCK && F_UNLCK == fl->fl_type) { + err = try_unlock_file(file, fl); + if (err <= 0) + return err; + } + err = ceph_lock_message(CEPH_LOCK_FCNTL, op, inode, lock_cmd, wait, fl); if (!err) { - if (op == CEPH_MDS_OP_SETFILELOCK) { + if (op == CEPH_MDS_OP_SETFILELOCK && F_UNLCK != fl->fl_type) { dout("mds locked, locking locally\n"); err = posix_lock_file(file, fl, NULL); if (err) { @@ -311,9 +332,15 @@ int ceph_flock(struct file *file, int cmd, struct file_lock *fl) else lock_cmd = CEPH_LOCK_UNLOCK; + if (F_UNLCK == fl->fl_type) { + err = try_unlock_file(file, fl); + if (err <= 0) + return err; + } + err = ceph_lock_message(CEPH_LOCK_FLOCK, CEPH_MDS_OP_SETFILELOCK, inode, lock_cmd, wait, fl); - if (!err) { + if (!err && F_UNLCK != fl->fl_type) { err = locks_lock_file_wait(file, fl); if (err) { ceph_lock_message(CEPH_LOCK_FLOCK, From 679a97d28627647e5a68dd684537e499cd741e2a Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Fri, 31 Jan 2020 11:37:36 +0100 Subject: [PATCH 331/635] rbd: kill img_request kref The reference counter is never increased, so we can as well call rbd_img_request_destroy() directly and drop the kref. Signed-off-by: Hannes Reinecke Reviewed-by: Ilya Dryomov Signed-off-by: Ilya Dryomov --- drivers/block/rbd.c | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c index 162fd1df06dd..b077c0fb9f70 100644 --- a/drivers/block/rbd.c +++ b/drivers/block/rbd.c @@ -349,7 +349,6 @@ struct rbd_img_request { struct pending_result pending; struct work_struct work; int work_result; - struct kref kref; }; #define for_each_obj_request(ireq, oreq) \ @@ -1320,15 +1319,6 @@ static void rbd_obj_request_put(struct rbd_obj_request *obj_request) kref_put(&obj_request->kref, rbd_obj_request_destroy); } -static void rbd_img_request_destroy(struct kref *kref); -static void rbd_img_request_put(struct rbd_img_request *img_request) -{ - rbd_assert(img_request != NULL); - dout("%s: img %p (was %d)\n", __func__, img_request, - kref_read(&img_request->kref)); - kref_put(&img_request->kref, rbd_img_request_destroy); -} - static inline void rbd_img_obj_request_add(struct rbd_img_request *img_request, struct rbd_obj_request *obj_request) { @@ -1656,19 +1646,15 @@ static struct rbd_img_request *rbd_img_request_create( INIT_LIST_HEAD(&img_request->lock_item); INIT_LIST_HEAD(&img_request->object_extents); mutex_init(&img_request->state_mutex); - kref_init(&img_request->kref); return img_request; } -static void rbd_img_request_destroy(struct kref *kref) +static void rbd_img_request_destroy(struct rbd_img_request *img_request) { - struct rbd_img_request *img_request; struct rbd_obj_request *obj_request; struct rbd_obj_request *next_obj_request; - img_request = container_of(kref, struct rbd_img_request, kref); - dout("%s: img %p\n", __func__, img_request); WARN_ON(!list_empty(&img_request->lock_item)); @@ -2885,7 +2871,7 @@ static int rbd_obj_read_from_parent(struct rbd_obj_request *obj_req) obj_req->copyup_bvecs); } if (ret) { - rbd_img_request_put(child_img_req); + rbd_img_request_destroy(child_img_req); return ret; } @@ -3644,7 +3630,7 @@ again: if (test_bit(IMG_REQ_CHILD, &img_req->flags)) { struct rbd_obj_request *obj_req = img_req->obj_request; - rbd_img_request_put(img_req); + rbd_img_request_destroy(img_req); if (__rbd_obj_handle_request(obj_req, &result)) { img_req = obj_req->img_request; goto again; @@ -3652,7 +3638,7 @@ again: } else { struct request *rq = img_req->rq; - rbd_img_request_put(img_req); + rbd_img_request_destroy(img_req); blk_mq_end_request(rq, errno_to_blk_status(result)); } } @@ -4798,7 +4784,7 @@ static void rbd_queue_workfn(struct work_struct *work) return; err_img_request: - rbd_img_request_put(img_request); + rbd_img_request_destroy(img_request); err_rq: if (result) rbd_warn(rbd_dev, "%s %llx at %llx result %d", From 78b42a871a654face984c844b43c777d66adb1fe Mon Sep 17 00:00:00 2001 From: Ilya Dryomov Date: Wed, 12 Feb 2020 14:34:03 +0100 Subject: [PATCH 332/635] rbd: get rid of img_request_layered_clear() No need to clear IMG_REQ_LAYERED before destroying the request. Signed-off-by: Ilya Dryomov --- drivers/block/rbd.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c index b077c0fb9f70..c61c5dd424fa 100644 --- a/drivers/block/rbd.c +++ b/drivers/block/rbd.c @@ -1358,11 +1358,6 @@ static void img_request_layered_set(struct rbd_img_request *img_request) set_bit(IMG_REQ_LAYERED, &img_request->flags); } -static void img_request_layered_clear(struct rbd_img_request *img_request) -{ - clear_bit(IMG_REQ_LAYERED, &img_request->flags); -} - static bool img_request_layered_test(struct rbd_img_request *img_request) { return test_bit(IMG_REQ_LAYERED, &img_request->flags) != 0; @@ -1661,10 +1656,8 @@ static void rbd_img_request_destroy(struct rbd_img_request *img_request) for_each_obj_request_safe(img_request, obj_request, next_obj_request) rbd_img_obj_request_del(img_request, obj_request); - if (img_request_layered_test(img_request)) { - img_request_layered_clear(img_request); + if (img_request_layered_test(img_request)) rbd_dev_parent_put(img_request->rbd_dev); - } if (rbd_img_is_write(img_request)) ceph_put_snap_context(img_request->snapc); From a52cc685753568e5bcbe762586c2bfbe7175255e Mon Sep 17 00:00:00 2001 From: Ilya Dryomov Date: Wed, 12 Feb 2020 15:08:39 +0100 Subject: [PATCH 333/635] rbd: acquire header_rwsem just once in rbd_queue_workfn() Currently header_rwsem is acquired twice: once in rbd_dev_parent_get() when the image request is being created and then in rbd_queue_workfn() to capture mapping_size and snapc. Introduce rbd_img_capture_header() and move image request allocation so that header_rwsem can be acquired just once. Signed-off-by: Ilya Dryomov --- drivers/block/rbd.c | 59 ++++++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c index c61c5dd424fa..acda61f5be03 100644 --- a/drivers/block/rbd.c +++ b/drivers/block/rbd.c @@ -1601,10 +1601,8 @@ static bool rbd_dev_parent_get(struct rbd_device *rbd_dev) if (!rbd_dev->parent_spec) return false; - down_read(&rbd_dev->header_rwsem); if (rbd_dev->parent_overlap) counter = atomic_inc_return_safe(&rbd_dev->parent_ref); - up_read(&rbd_dev->header_rwsem); if (counter < 0) rbd_warn(rbd_dev, "parent reference overflow"); @@ -1619,8 +1617,7 @@ static bool rbd_dev_parent_get(struct rbd_device *rbd_dev) */ static struct rbd_img_request *rbd_img_request_create( struct rbd_device *rbd_dev, - enum obj_operation_type op_type, - struct ceph_snap_context *snapc) + enum obj_operation_type op_type) { struct rbd_img_request *img_request; @@ -1630,13 +1627,6 @@ static struct rbd_img_request *rbd_img_request_create( img_request->rbd_dev = rbd_dev; img_request->op_type = op_type; - if (!rbd_img_is_write(img_request)) - img_request->snap_id = rbd_dev->spec->snap_id; - else - img_request->snapc = snapc; - - if (rbd_dev_parent_get(rbd_dev)) - img_request_layered_set(img_request); INIT_LIST_HEAD(&img_request->lock_item); INIT_LIST_HEAD(&img_request->object_extents); @@ -1645,6 +1635,21 @@ static struct rbd_img_request *rbd_img_request_create( return img_request; } +static void rbd_img_capture_header(struct rbd_img_request *img_req) +{ + struct rbd_device *rbd_dev = img_req->rbd_dev; + + lockdep_assert_held(&rbd_dev->header_rwsem); + + if (rbd_img_is_write(img_req)) + img_req->snapc = ceph_get_snap_context(rbd_dev->header.snapc); + else + img_req->snap_id = rbd_dev->spec->snap_id; + + if (rbd_dev_parent_get(rbd_dev)) + img_request_layered_set(img_req); +} + static void rbd_img_request_destroy(struct rbd_img_request *img_request) { struct rbd_obj_request *obj_request; @@ -2825,17 +2830,21 @@ static int rbd_obj_read_object(struct rbd_obj_request *obj_req) static int rbd_obj_read_from_parent(struct rbd_obj_request *obj_req) { struct rbd_img_request *img_req = obj_req->img_request; + struct rbd_device *parent = img_req->rbd_dev->parent; struct rbd_img_request *child_img_req; int ret; - child_img_req = rbd_img_request_create(img_req->rbd_dev->parent, - OBJ_OP_READ, NULL); + child_img_req = rbd_img_request_create(parent, OBJ_OP_READ); if (!child_img_req) return -ENOMEM; __set_bit(IMG_REQ_CHILD, &child_img_req->flags); child_img_req->obj_request = obj_req; + down_read(&parent->header_rwsem); + rbd_img_capture_header(child_img_req); + up_read(&parent->header_rwsem); + dout("%s child_img_req %p for obj_req %p\n", __func__, child_img_req, obj_req); @@ -4686,7 +4695,6 @@ static void rbd_queue_workfn(struct work_struct *work) struct request *rq = blk_mq_rq_from_pdu(work); struct rbd_device *rbd_dev = rq->q->queuedata; struct rbd_img_request *img_request; - struct ceph_snap_context *snapc = NULL; u64 offset = (u64)blk_rq_pos(rq) << SECTOR_SHIFT; u64 length = blk_rq_bytes(rq); enum obj_operation_type op_type; @@ -4739,29 +4747,25 @@ static void rbd_queue_workfn(struct work_struct *work) blk_mq_start_request(rq); + img_request = rbd_img_request_create(rbd_dev, op_type); + if (!img_request) { + result = -ENOMEM; + goto err_rq; + } + img_request->rq = rq; + down_read(&rbd_dev->header_rwsem); mapping_size = rbd_dev->mapping.size; - if (op_type != OBJ_OP_READ) { - snapc = rbd_dev->header.snapc; - ceph_get_snap_context(snapc); - } + rbd_img_capture_header(img_request); up_read(&rbd_dev->header_rwsem); if (offset + length > mapping_size) { rbd_warn(rbd_dev, "beyond EOD (%llu~%llu > %llu)", offset, length, mapping_size); result = -EIO; - goto err_rq; + goto err_img_request; } - img_request = rbd_img_request_create(rbd_dev, op_type, snapc); - if (!img_request) { - result = -ENOMEM; - goto err_rq; - } - img_request->rq = rq; - snapc = NULL; /* img_request consumes a ref */ - dout("%s rbd_dev %p img_req %p %s %llu~%llu\n", __func__, rbd_dev, img_request, obj_op_name(op_type), offset, length); @@ -4782,7 +4786,6 @@ err_rq: if (result) rbd_warn(rbd_dev, "%s %llx at %llx result %d", obj_op_name(op_type), length, offset, result); - ceph_put_snap_context(snapc); err: blk_mq_end_request(rq, errno_to_blk_status(result)); } From 59e542c869895fb37005b60058a342187bb63c61 Mon Sep 17 00:00:00 2001 From: Ilya Dryomov Date: Wed, 12 Feb 2020 15:23:58 +0100 Subject: [PATCH 334/635] rbd: embed image request in blk-mq pdu Avoid making allocations for !IMG_REQ_CHILD image requests. Only IMG_REQ_CHILD image requests need to be freed now. Move the initial request checks to rbd_queue_rq(). Unfortunately we can't fill the image request and kick the state machine directly from rbd_queue_rq() because ->queue_rq() isn't allowed to block. Signed-off-by: Ilya Dryomov --- drivers/block/rbd.c | 138 ++++++++++++++++---------------------------- 1 file changed, 51 insertions(+), 87 deletions(-) diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c index acda61f5be03..3af09a0f208b 100644 --- a/drivers/block/rbd.c +++ b/drivers/block/rbd.c @@ -337,10 +337,7 @@ struct rbd_img_request { u64 snap_id; /* for reads */ struct ceph_snap_context *snapc; /* for writes */ }; - union { - struct request *rq; /* block request */ - struct rbd_obj_request *obj_request; /* obj req initiator */ - }; + struct rbd_obj_request *obj_request; /* obj req initiator */ struct list_head lock_item; struct list_head object_extents; /* obj_req.ex structs */ @@ -1610,20 +1607,11 @@ static bool rbd_dev_parent_get(struct rbd_device *rbd_dev) return counter > 0; } -/* - * Caller is responsible for filling in the list of object requests - * that comprises the image request, and the Linux request pointer - * (if there is one). - */ -static struct rbd_img_request *rbd_img_request_create( - struct rbd_device *rbd_dev, - enum obj_operation_type op_type) +static void rbd_img_request_init(struct rbd_img_request *img_request, + struct rbd_device *rbd_dev, + enum obj_operation_type op_type) { - struct rbd_img_request *img_request; - - img_request = kmem_cache_zalloc(rbd_img_request_cache, GFP_NOIO); - if (!img_request) - return NULL; + memset(img_request, 0, sizeof(*img_request)); img_request->rbd_dev = rbd_dev; img_request->op_type = op_type; @@ -1631,8 +1619,6 @@ static struct rbd_img_request *rbd_img_request_create( INIT_LIST_HEAD(&img_request->lock_item); INIT_LIST_HEAD(&img_request->object_extents); mutex_init(&img_request->state_mutex); - - return img_request; } static void rbd_img_capture_header(struct rbd_img_request *img_req) @@ -1667,7 +1653,8 @@ static void rbd_img_request_destroy(struct rbd_img_request *img_request) if (rbd_img_is_write(img_request)) ceph_put_snap_context(img_request->snapc); - kmem_cache_free(rbd_img_request_cache, img_request); + if (test_bit(IMG_REQ_CHILD, &img_request->flags)) + kmem_cache_free(rbd_img_request_cache, img_request); } #define BITS_PER_OBJ 2 @@ -2834,10 +2821,11 @@ static int rbd_obj_read_from_parent(struct rbd_obj_request *obj_req) struct rbd_img_request *child_img_req; int ret; - child_img_req = rbd_img_request_create(parent, OBJ_OP_READ); + child_img_req = kmem_cache_alloc(rbd_img_request_cache, GFP_NOIO); if (!child_img_req) return -ENOMEM; + rbd_img_request_init(child_img_req, parent, OBJ_OP_READ); __set_bit(IMG_REQ_CHILD, &child_img_req->flags); child_img_req->obj_request = obj_req; @@ -3638,7 +3626,7 @@ again: goto again; } } else { - struct request *rq = img_req->rq; + struct request *rq = blk_mq_rq_from_pdu(img_req); rbd_img_request_destroy(img_req); blk_mq_end_request(rq, errno_to_blk_status(result)); @@ -4692,68 +4680,25 @@ static int rbd_obj_method_sync(struct rbd_device *rbd_dev, static void rbd_queue_workfn(struct work_struct *work) { - struct request *rq = blk_mq_rq_from_pdu(work); - struct rbd_device *rbd_dev = rq->q->queuedata; - struct rbd_img_request *img_request; + struct rbd_img_request *img_request = + container_of(work, struct rbd_img_request, work); + struct rbd_device *rbd_dev = img_request->rbd_dev; + enum obj_operation_type op_type = img_request->op_type; + struct request *rq = blk_mq_rq_from_pdu(img_request); u64 offset = (u64)blk_rq_pos(rq) << SECTOR_SHIFT; u64 length = blk_rq_bytes(rq); - enum obj_operation_type op_type; u64 mapping_size; int result; - switch (req_op(rq)) { - case REQ_OP_DISCARD: - op_type = OBJ_OP_DISCARD; - break; - case REQ_OP_WRITE_ZEROES: - op_type = OBJ_OP_ZEROOUT; - break; - case REQ_OP_WRITE: - op_type = OBJ_OP_WRITE; - break; - case REQ_OP_READ: - op_type = OBJ_OP_READ; - break; - default: - dout("%s: non-fs request type %d\n", __func__, req_op(rq)); - result = -EIO; - goto err; - } - /* Ignore/skip any zero-length requests */ - if (!length) { dout("%s: zero-length request\n", __func__); result = 0; - goto err_rq; - } - - if (op_type != OBJ_OP_READ) { - if (rbd_is_ro(rbd_dev)) { - rbd_warn(rbd_dev, "%s on read-only mapping", - obj_op_name(op_type)); - result = -EIO; - goto err; - } - rbd_assert(!rbd_is_snap(rbd_dev)); - } - - if (offset && length > U64_MAX - offset + 1) { - rbd_warn(rbd_dev, "bad request range (%llu~%llu)", offset, - length); - result = -EINVAL; - goto err_rq; /* Shouldn't happen */ + goto err_img_request; } blk_mq_start_request(rq); - img_request = rbd_img_request_create(rbd_dev, op_type); - if (!img_request) { - result = -ENOMEM; - goto err_rq; - } - img_request->rq = rq; - down_read(&rbd_dev->header_rwsem); mapping_size = rbd_dev->mapping.size; rbd_img_capture_header(img_request); @@ -4782,21 +4727,50 @@ static void rbd_queue_workfn(struct work_struct *work) err_img_request: rbd_img_request_destroy(img_request); -err_rq: if (result) rbd_warn(rbd_dev, "%s %llx at %llx result %d", obj_op_name(op_type), length, offset, result); -err: blk_mq_end_request(rq, errno_to_blk_status(result)); } static blk_status_t rbd_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) { - struct request *rq = bd->rq; - struct work_struct *work = blk_mq_rq_to_pdu(rq); + struct rbd_device *rbd_dev = hctx->queue->queuedata; + struct rbd_img_request *img_req = blk_mq_rq_to_pdu(bd->rq); + enum obj_operation_type op_type; - queue_work(rbd_wq, work); + switch (req_op(bd->rq)) { + case REQ_OP_DISCARD: + op_type = OBJ_OP_DISCARD; + break; + case REQ_OP_WRITE_ZEROES: + op_type = OBJ_OP_ZEROOUT; + break; + case REQ_OP_WRITE: + op_type = OBJ_OP_WRITE; + break; + case REQ_OP_READ: + op_type = OBJ_OP_READ; + break; + default: + rbd_warn(rbd_dev, "unknown req_op %d", req_op(bd->rq)); + return BLK_STS_IOERR; + } + + rbd_img_request_init(img_req, rbd_dev, op_type); + + if (rbd_img_is_write(img_req)) { + if (rbd_is_ro(rbd_dev)) { + rbd_warn(rbd_dev, "%s on read-only mapping", + obj_op_name(img_req->op_type)); + return BLK_STS_IOERR; + } + rbd_assert(!rbd_is_snap(rbd_dev)); + } + + INIT_WORK(&img_req->work, rbd_queue_workfn); + queue_work(rbd_wq, &img_req->work); return BLK_STS_OK; } @@ -4963,18 +4937,8 @@ out: return ret; } -static int rbd_init_request(struct blk_mq_tag_set *set, struct request *rq, - unsigned int hctx_idx, unsigned int numa_node) -{ - struct work_struct *work = blk_mq_rq_to_pdu(rq); - - INIT_WORK(work, rbd_queue_workfn); - return 0; -} - static const struct blk_mq_ops rbd_mq_ops = { .queue_rq = rbd_queue_rq, - .init_request = rbd_init_request, }; static int rbd_init_disk(struct rbd_device *rbd_dev) @@ -5007,7 +4971,7 @@ static int rbd_init_disk(struct rbd_device *rbd_dev) rbd_dev->tag_set.numa_node = NUMA_NO_NODE; rbd_dev->tag_set.flags = BLK_MQ_F_SHOULD_MERGE; rbd_dev->tag_set.nr_hw_queues = 1; - rbd_dev->tag_set.cmd_size = sizeof(struct work_struct); + rbd_dev->tag_set.cmd_size = sizeof(struct rbd_img_request); err = blk_mq_alloc_tag_set(&rbd_dev->tag_set); if (err) From f9b6b98d24f7cec5b8269217f9d4fdec1ca43218 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Fri, 31 Jan 2020 11:37:39 +0100 Subject: [PATCH 335/635] rbd: enable multiple blk-mq queues Allocate one queue per CPU and get a performance boost from higher parallelism. Signed-off-by: Hannes Reinecke Reviewed-by: Ilya Dryomov Signed-off-by: Ilya Dryomov --- drivers/block/rbd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c index 3af09a0f208b..1e0a6b19ae0d 100644 --- a/drivers/block/rbd.c +++ b/drivers/block/rbd.c @@ -4970,7 +4970,7 @@ static int rbd_init_disk(struct rbd_device *rbd_dev) rbd_dev->tag_set.queue_depth = rbd_dev->opts->queue_depth; rbd_dev->tag_set.numa_node = NUMA_NO_NODE; rbd_dev->tag_set.flags = BLK_MQ_F_SHOULD_MERGE; - rbd_dev->tag_set.nr_hw_queues = 1; + rbd_dev->tag_set.nr_hw_queues = num_present_cpus(); rbd_dev->tag_set.cmd_size = sizeof(struct rbd_img_request); err = blk_mq_alloc_tag_set(&rbd_dev->tag_set); From f682dc713c0af79effde49209f3895072579791b Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Thu, 13 Feb 2020 10:00:04 -0600 Subject: [PATCH 336/635] ceph: replace zero-length array with flexible-array member The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertently introduced[3] to the codebase from now on. Also, notice that, dynamic memory allocations won't be affected by this change: "Flexible array members have incomplete type, and so the sizeof operator may not be applied. As a quirk of the original implementation of zero-length arrays, sizeof evaluates to zero."[1] This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Reviewed-by: Jeff Layton Signed-off-by: Ilya Dryomov --- fs/ceph/cache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/ceph/cache.c b/fs/ceph/cache.c index 270b769607a2..2f5cb6bc78e1 100644 --- a/fs/ceph/cache.c +++ b/fs/ceph/cache.c @@ -32,7 +32,7 @@ struct ceph_fscache_entry { size_t uniq_len; /* The following members must be last */ struct ceph_fsid fsid; - char uniquifier[0]; + char uniquifier[]; }; static const struct fscache_cookie_def ceph_fscache_fsid_object_def = { From cb03c14390fb0d345f098730b9ef67ca6058c439 Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Thu, 13 Feb 2020 21:24:22 +0100 Subject: [PATCH 337/635] ceph: switch to page_mkwrite_check_truncate in ceph_page_mkwrite Use the "page has been truncated" logic in page_mkwrite_check_truncate instead of reimplementing it here. Other than with the existing code, fail with -EFAULT / VM_FAULT_NOPAGE when page_offset(page) == size here as well, as should be expected. Signed-off-by: Andreas Gruenbacher Reviewed-by: Jeff Layton Signed-off-by: Ilya Dryomov --- fs/ceph/addr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c index 7136f9947354..6f4678d98df7 100644 --- a/fs/ceph/addr.c +++ b/fs/ceph/addr.c @@ -1655,7 +1655,7 @@ static vm_fault_t ceph_page_mkwrite(struct vm_fault *vmf) do { lock_page(page); - if ((off > size) || (page->mapping != inode->i_mapping)) { + if (page_mkwrite_check_truncate(page, inode) < 0) { unlock_page(page); ret = VM_FAULT_NOPAGE; break; From c36d641493c9d170adf895b7650c26c944d37636 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Mon, 17 Feb 2020 10:19:14 -0500 Subject: [PATCH 338/635] ceph: reorganize fields in ceph_mds_request This shrinks the struct size by 16 bytes. Signed-off-by: Jeff Layton Signed-off-by: Ilya Dryomov --- fs/ceph/mds_client.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fs/ceph/mds_client.h b/fs/ceph/mds_client.h index 27a7446e10d3..a0918d00117c 100644 --- a/fs/ceph/mds_client.h +++ b/fs/ceph/mds_client.h @@ -263,6 +263,7 @@ struct ceph_mds_request { int r_fmode; /* file mode, if expecting cap */ kuid_t r_uid; kgid_t r_gid; + int r_request_release_offset; struct timespec64 r_stamp; /* for choosing which mds to send this request to */ @@ -280,11 +281,12 @@ struct ceph_mds_request { int r_old_inode_drop, r_old_inode_unless; struct ceph_msg *r_request; /* original request */ - int r_request_release_offset; struct ceph_msg *r_reply; struct ceph_mds_reply_info_parsed r_reply_info; struct page *r_locked_page; int r_err; + int r_num_caps; + u32 r_readdir_offset; unsigned long r_timeout; /* optional. jiffies, 0 is "wait forever" */ unsigned long r_started; /* start time to measure timeout against */ @@ -315,10 +317,8 @@ struct ceph_mds_request { long long r_dir_release_cnt; long long r_dir_ordered_cnt; int r_readdir_cache_idx; - u32 r_readdir_offset; struct ceph_cap_reservation r_caps_reservation; - int r_num_caps; }; struct ceph_pool_perm { From 058daab79d6b597a20fd49b5e445b1b2929c2c1c Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Mon, 17 Feb 2020 18:38:37 -0500 Subject: [PATCH 339/635] ceph: move to a dedicated slabcache for mds requests On my machine (x86_64) this struct is 952 bytes, which gets rounded up to 1024 by kmalloc. Move this to a dedicated slabcache, so we can allocate them without the extra 72 bytes of overhead per. Signed-off-by: Jeff Layton Reviewed-by: Ilya Dryomov Signed-off-by: Ilya Dryomov --- fs/ceph/mds_client.c | 5 +++-- fs/ceph/super.c | 8 ++++++++ include/linux/ceph/libceph.h | 1 + 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index 2980e57ca7b9..fab9d6461a65 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -736,7 +736,7 @@ void ceph_mdsc_release_request(struct kref *kref) put_request_session(req); ceph_unreserve_caps(req->r_mdsc, &req->r_caps_reservation); WARN_ON_ONCE(!list_empty(&req->r_wait)); - kfree(req); + kmem_cache_free(ceph_mds_request_cachep, req); } DEFINE_RB_FUNCS(request, struct ceph_mds_request, r_tid, r_node) @@ -2094,8 +2094,9 @@ int ceph_alloc_readdir_reply_buffer(struct ceph_mds_request *req, struct ceph_mds_request * ceph_mdsc_create_request(struct ceph_mds_client *mdsc, int op, int mode) { - struct ceph_mds_request *req = kzalloc(sizeof(*req), GFP_NOFS); + struct ceph_mds_request *req; + req = kmem_cache_zalloc(ceph_mds_request_cachep, GFP_NOFS); if (!req) return ERR_PTR(-ENOMEM); diff --git a/fs/ceph/super.c b/fs/ceph/super.c index c7f150686a53..b1329cd5388a 100644 --- a/fs/ceph/super.c +++ b/fs/ceph/super.c @@ -729,6 +729,7 @@ struct kmem_cache *ceph_cap_flush_cachep; struct kmem_cache *ceph_dentry_cachep; struct kmem_cache *ceph_file_cachep; struct kmem_cache *ceph_dir_file_cachep; +struct kmem_cache *ceph_mds_request_cachep; static void ceph_inode_init_once(void *foo) { @@ -769,6 +770,10 @@ static int __init init_caches(void) if (!ceph_dir_file_cachep) goto bad_dir_file; + ceph_mds_request_cachep = KMEM_CACHE(ceph_mds_request, SLAB_MEM_SPREAD); + if (!ceph_mds_request_cachep) + goto bad_mds_req; + error = ceph_fscache_register(); if (error) goto bad_fscache; @@ -776,6 +781,8 @@ static int __init init_caches(void) return 0; bad_fscache: + kmem_cache_destroy(ceph_mds_request_cachep); +bad_mds_req: kmem_cache_destroy(ceph_dir_file_cachep); bad_dir_file: kmem_cache_destroy(ceph_file_cachep); @@ -804,6 +811,7 @@ static void destroy_caches(void) kmem_cache_destroy(ceph_dentry_cachep); kmem_cache_destroy(ceph_file_cachep); kmem_cache_destroy(ceph_dir_file_cachep); + kmem_cache_destroy(ceph_mds_request_cachep); ceph_fscache_unregister(); } diff --git a/include/linux/ceph/libceph.h b/include/linux/ceph/libceph.h index ec73ebc4827d..525b7c3f1c81 100644 --- a/include/linux/ceph/libceph.h +++ b/include/linux/ceph/libceph.h @@ -272,6 +272,7 @@ extern struct kmem_cache *ceph_cap_flush_cachep; extern struct kmem_cache *ceph_dentry_cachep; extern struct kmem_cache *ceph_file_cachep; extern struct kmem_cache *ceph_dir_file_cachep; +extern struct kmem_cache *ceph_mds_request_cachep; /* ceph_common.c */ extern bool libceph_compatible(void *data); From 1b0c3b9f91f0df03088d293fc9e62743fd789ad2 Mon Sep 17 00:00:00 2001 From: Luis Henriques Date: Mon, 24 Feb 2020 13:44:32 +0000 Subject: [PATCH 340/635] ceph: re-org copy_file_range and fix some error paths This patch re-organizes copy_file_range, trying to fix a few issues in the error handling. Here's the summary: - Abort copy if initial do_splice_direct() returns fewer bytes than requested. - Move the 'size' initialization (with i_size_read()) further down in the code, after the initial call to do_splice_direct(). This avoids issues with a possibly stale value if a manual copy is done. - Move the object copy loop into a separate function. This makes it easier to handle errors (e.g, dirtying caps and updating the MDS metadata if only some objects have been copied before an error has occurred). - Added calls to ceph_oloc_destroy() to avoid leaking memory with src_oloc and dst_oloc - After the object copy loop, the new file size to be reported to the MDS (if there's file size change) is now the actual file size, and not the size after an eventual extra manual copy. - Added a few dout() to show the number of bytes copied in the two manual copies and in the object copy loop. Signed-off-by: Luis Henriques Reviewed-by: Jeff Layton Signed-off-by: Ilya Dryomov --- fs/ceph/file.c | 173 ++++++++++++++++++++++++++++--------------------- 1 file changed, 100 insertions(+), 73 deletions(-) diff --git a/fs/ceph/file.c b/fs/ceph/file.c index 5a478cd06e11..7f8c4e308301 100644 --- a/fs/ceph/file.c +++ b/fs/ceph/file.c @@ -1944,6 +1944,71 @@ static int is_file_size_ok(struct inode *src_inode, struct inode *dst_inode, return 0; } +static ssize_t ceph_do_objects_copy(struct ceph_inode_info *src_ci, u64 *src_off, + struct ceph_inode_info *dst_ci, u64 *dst_off, + struct ceph_fs_client *fsc, + size_t len, unsigned int flags) +{ + struct ceph_object_locator src_oloc, dst_oloc; + struct ceph_object_id src_oid, dst_oid; + size_t bytes = 0; + u64 src_objnum, src_objoff, dst_objnum, dst_objoff; + u32 src_objlen, dst_objlen; + u32 object_size = src_ci->i_layout.object_size; + int ret; + + src_oloc.pool = src_ci->i_layout.pool_id; + src_oloc.pool_ns = ceph_try_get_string(src_ci->i_layout.pool_ns); + dst_oloc.pool = dst_ci->i_layout.pool_id; + dst_oloc.pool_ns = ceph_try_get_string(dst_ci->i_layout.pool_ns); + + while (len >= object_size) { + ceph_calc_file_object_mapping(&src_ci->i_layout, *src_off, + object_size, &src_objnum, + &src_objoff, &src_objlen); + ceph_calc_file_object_mapping(&dst_ci->i_layout, *dst_off, + object_size, &dst_objnum, + &dst_objoff, &dst_objlen); + ceph_oid_init(&src_oid); + ceph_oid_printf(&src_oid, "%llx.%08llx", + src_ci->i_vino.ino, src_objnum); + ceph_oid_init(&dst_oid); + ceph_oid_printf(&dst_oid, "%llx.%08llx", + dst_ci->i_vino.ino, dst_objnum); + /* Do an object remote copy */ + ret = ceph_osdc_copy_from(&fsc->client->osdc, + src_ci->i_vino.snap, 0, + &src_oid, &src_oloc, + CEPH_OSD_OP_FLAG_FADVISE_SEQUENTIAL | + CEPH_OSD_OP_FLAG_FADVISE_NOCACHE, + &dst_oid, &dst_oloc, + CEPH_OSD_OP_FLAG_FADVISE_SEQUENTIAL | + CEPH_OSD_OP_FLAG_FADVISE_DONTNEED, + dst_ci->i_truncate_seq, + dst_ci->i_truncate_size, + CEPH_OSD_COPY_FROM_FLAG_TRUNCATE_SEQ); + if (ret) { + if (ret == -EOPNOTSUPP) { + fsc->have_copy_from2 = false; + pr_notice("OSDs don't support copy-from2; disabling copy offload\n"); + } + dout("ceph_osdc_copy_from returned %d\n", ret); + if (!bytes) + bytes = ret; + goto out; + } + len -= object_size; + bytes += object_size; + *src_off += object_size; + *dst_off += object_size; + } + +out: + ceph_oloc_destroy(&src_oloc); + ceph_oloc_destroy(&dst_oloc); + return bytes; +} + static ssize_t __ceph_copy_file_range(struct file *src_file, loff_t src_off, struct file *dst_file, loff_t dst_off, size_t len, unsigned int flags) @@ -1954,14 +2019,11 @@ static ssize_t __ceph_copy_file_range(struct file *src_file, loff_t src_off, struct ceph_inode_info *dst_ci = ceph_inode(dst_inode); struct ceph_cap_flush *prealloc_cf; struct ceph_fs_client *src_fsc = ceph_inode_to_client(src_inode); - struct ceph_object_locator src_oloc, dst_oloc; - struct ceph_object_id src_oid, dst_oid; - loff_t endoff = 0, size; - ssize_t ret = -EIO; + loff_t size; + ssize_t ret = -EIO, bytes; u64 src_objnum, dst_objnum, src_objoff, dst_objoff; - u32 src_objlen, dst_objlen, object_size; + u32 src_objlen, dst_objlen; int src_got = 0, dst_got = 0, err, dirty; - bool do_final_copy = false; if (src_inode->i_sb != dst_inode->i_sb) { struct ceph_fs_client *dst_fsc = ceph_inode_to_client(dst_inode); @@ -2039,22 +2101,14 @@ static ssize_t __ceph_copy_file_range(struct file *src_file, loff_t src_off, if (ret < 0) goto out_caps; - size = i_size_read(dst_inode); - endoff = dst_off + len; - /* Drop dst file cached pages */ ret = invalidate_inode_pages2_range(dst_inode->i_mapping, dst_off >> PAGE_SHIFT, - endoff >> PAGE_SHIFT); + (dst_off + len) >> PAGE_SHIFT); if (ret < 0) { dout("Failed to invalidate inode pages (%zd)\n", ret); ret = 0; /* XXX */ } - src_oloc.pool = src_ci->i_layout.pool_id; - src_oloc.pool_ns = ceph_try_get_string(src_ci->i_layout.pool_ns); - dst_oloc.pool = dst_ci->i_layout.pool_id; - dst_oloc.pool_ns = ceph_try_get_string(dst_ci->i_layout.pool_ns); - ceph_calc_file_object_mapping(&src_ci->i_layout, src_off, src_ci->i_layout.object_size, &src_objnum, &src_objoff, &src_objlen); @@ -2073,6 +2127,8 @@ static ssize_t __ceph_copy_file_range(struct file *src_file, loff_t src_off, * starting at the src_off */ if (src_objoff) { + dout("Initial partial copy of %u bytes\n", src_objlen); + /* * we need to temporarily drop all caps as we'll be calling * {read,write}_iter, which will get caps again. @@ -2080,8 +2136,9 @@ static ssize_t __ceph_copy_file_range(struct file *src_file, loff_t src_off, put_rd_wr_caps(src_ci, src_got, dst_ci, dst_got); ret = do_splice_direct(src_file, &src_off, dst_file, &dst_off, src_objlen, flags); - if (ret < 0) { - dout("do_splice_direct returned %d\n", err); + /* Abort on short copies or on error */ + if (ret < src_objlen) { + dout("Failed partial copy (%zd)\n", ret); goto out; } len -= ret; @@ -2094,62 +2151,29 @@ static ssize_t __ceph_copy_file_range(struct file *src_file, loff_t src_off, if (err < 0) goto out_caps; } - object_size = src_ci->i_layout.object_size; - while (len >= object_size) { - ceph_calc_file_object_mapping(&src_ci->i_layout, src_off, - object_size, &src_objnum, - &src_objoff, &src_objlen); - ceph_calc_file_object_mapping(&dst_ci->i_layout, dst_off, - object_size, &dst_objnum, - &dst_objoff, &dst_objlen); - ceph_oid_init(&src_oid); - ceph_oid_printf(&src_oid, "%llx.%08llx", - src_ci->i_vino.ino, src_objnum); - ceph_oid_init(&dst_oid); - ceph_oid_printf(&dst_oid, "%llx.%08llx", - dst_ci->i_vino.ino, dst_objnum); - /* Do an object remote copy */ - err = ceph_osdc_copy_from( - &src_fsc->client->osdc, - src_ci->i_vino.snap, 0, - &src_oid, &src_oloc, - CEPH_OSD_OP_FLAG_FADVISE_SEQUENTIAL | - CEPH_OSD_OP_FLAG_FADVISE_NOCACHE, - &dst_oid, &dst_oloc, - CEPH_OSD_OP_FLAG_FADVISE_SEQUENTIAL | - CEPH_OSD_OP_FLAG_FADVISE_DONTNEED, - dst_ci->i_truncate_seq, dst_ci->i_truncate_size, - CEPH_OSD_COPY_FROM_FLAG_TRUNCATE_SEQ); - if (err) { - if (err == -EOPNOTSUPP) { - src_fsc->have_copy_from2 = false; - pr_notice("OSDs don't support copy-from2; disabling copy offload\n"); - } - dout("ceph_osdc_copy_from returned %d\n", err); - if (!ret) - ret = err; - goto out_caps; - } - len -= object_size; - src_off += object_size; - dst_off += object_size; - ret += object_size; - } - if (len) - /* We still need one final local copy */ - do_final_copy = true; + size = i_size_read(dst_inode); + bytes = ceph_do_objects_copy(src_ci, &src_off, dst_ci, &dst_off, + src_fsc, len, flags); + if (bytes <= 0) { + if (!ret) + ret = bytes; + goto out_caps; + } + dout("Copied %zu bytes out of %zu\n", bytes, len); + len -= bytes; + ret += bytes; file_update_time(dst_file); inode_inc_iversion_raw(dst_inode); - if (endoff > size) { + if (dst_off > size) { int caps_flags = 0; /* Let the MDS know about dst file size change */ - if (ceph_quota_is_max_bytes_approaching(dst_inode, endoff)) + if (ceph_quota_is_max_bytes_approaching(dst_inode, dst_off)) caps_flags |= CHECK_CAPS_NODELAY; - if (ceph_inode_set_size(dst_inode, endoff)) + if (ceph_inode_set_size(dst_inode, dst_off)) caps_flags |= CHECK_CAPS_AUTHONLY; if (caps_flags) ceph_check_caps(dst_ci, caps_flags, NULL); @@ -2165,15 +2189,18 @@ static ssize_t __ceph_copy_file_range(struct file *src_file, loff_t src_off, out_caps: put_rd_wr_caps(src_ci, src_got, dst_ci, dst_got); - if (do_final_copy) { - err = do_splice_direct(src_file, &src_off, dst_file, - &dst_off, len, flags); - if (err < 0) { - dout("do_splice_direct returned %d\n", err); - goto out; - } - len -= err; - ret += err; + /* + * Do the final manual copy if we still have some bytes left, unless + * there were errors in remote object copies (len >= object_size). + */ + if (len && (len < src_ci->i_layout.object_size)) { + dout("Final partial copy of %zu bytes\n", len); + bytes = do_splice_direct(src_file, &src_off, dst_file, + &dst_off, len, flags); + if (bytes > 0) + ret += bytes; + else + dout("Failed partial copy (%zd)\n", bytes); } out: From 8ccf7fcce191c11a2d85ef58ad1a82dd6c7e2f7c Mon Sep 17 00:00:00 2001 From: Xiubo Li Date: Sun, 23 Feb 2020 22:23:11 -0500 Subject: [PATCH 341/635] ceph: return ETIMEDOUT errno to userland when request timed out req->r_timeout is only used during mounting, so this error will be more accurate. URL: https://tracker.ceph.com/issues/44215 Signed-off-by: Xiubo Li Reviewed-by: Jeff Layton Signed-off-by: Ilya Dryomov --- fs/ceph/mds_client.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index fab9d6461a65..2da98b6cc064 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -2579,7 +2579,7 @@ static void __do_request(struct ceph_mds_client *mdsc, if (req->r_timeout && time_after_eq(jiffies, req->r_started + req->r_timeout)) { dout("do_request timed out\n"); - err = -EIO; + err = -ETIMEDOUT; goto finish; } if (READ_ONCE(mdsc->fsc->mount_state) == CEPH_MOUNT_SHUTDOWN) { @@ -2753,7 +2753,7 @@ static int ceph_mdsc_wait_request(struct ceph_mds_client *mdsc, if (timeleft > 0) err = 0; else if (!timeleft) - err = -EIO; /* timed out */ + err = -ETIMEDOUT; /* timed out */ else err = timeleft; /* killed */ } From 4d8b8fb4940a46f8d64bf1f9d116e1d2ae32b01c Mon Sep 17 00:00:00 2001 From: Ilya Dryomov Date: Wed, 26 Feb 2020 16:37:55 +0100 Subject: [PATCH 342/635] libceph: simplify ceph_monc_handle_map() ceph_monc_handle_map() confuses static checkers which report a false use-after-free on monc->monmap, missing that monc->monmap and client->monc.monmap is the same pointer. Use monc->monmap consistently and get rid of "old", which is redundant. Signed-off-by: Ilya Dryomov --- net/ceph/mon_client.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/net/ceph/mon_client.c b/net/ceph/mon_client.c index 9d9e4e4ea600..3d8c8015e976 100644 --- a/net/ceph/mon_client.c +++ b/net/ceph/mon_client.c @@ -467,7 +467,7 @@ static void ceph_monc_handle_map(struct ceph_mon_client *monc, struct ceph_msg *msg) { struct ceph_client *client = monc->client; - struct ceph_monmap *monmap = NULL, *old = monc->monmap; + struct ceph_monmap *monmap; void *p, *end; mutex_lock(&monc->mutex); @@ -484,13 +484,13 @@ static void ceph_monc_handle_map(struct ceph_mon_client *monc, goto out; } - if (ceph_check_fsid(monc->client, &monmap->fsid) < 0) { + if (ceph_check_fsid(client, &monmap->fsid) < 0) { kfree(monmap); goto out; } - client->monc.monmap = monmap; - kfree(old); + kfree(monc->monmap); + monc->monmap = monmap; __ceph_monc_got_map(monc, CEPH_SUB_MONMAP, monc->monmap->epoch); client->have_fsid = true; From bb0e681dda33adac516dd74723b757c5c2feafe6 Mon Sep 17 00:00:00 2001 From: Ilya Dryomov Date: Fri, 30 Aug 2019 17:38:31 +0200 Subject: [PATCH 343/635] libceph: directly skip to the end of redirect reply Coverity complains about a double write to *p. Don't bother with osd_instructions and directly skip to the end of redirect reply. Reported-by: Colin Ian King Signed-off-by: Ilya Dryomov --- net/ceph/osd_client.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c index f9ec6a6568bd..998e26b75a78 100644 --- a/net/ceph/osd_client.c +++ b/net/ceph/osd_client.c @@ -3483,9 +3483,6 @@ static int ceph_redirect_decode(void **p, void *end, goto e_inval; } - len = ceph_decode_32(p); - *p += len; /* skip osd_instructions */ - /* skip the rest */ *p = struct_end; out: From e8a4d26771547cd690616bb204e0f350c4fa0e09 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Tue, 25 Feb 2020 11:49:53 -0800 Subject: [PATCH 344/635] ceph: clean up kick_flushing_inode_caps() The last thing that this function does is release i_ceph_lock, so have the caller do that instead. Add a lockdep assertion to ensure that the function is always called with i_ceph_lock held. Change the prototype to take a ceph_inode_info pointer and drop the separate mdsc argument as we can get that from the session. While at it, make it non-static. We'll need this to kick any flushing caps once the create reply comes in. Signed-off-by: Jeff Layton Reviewed-by: Ilya Dryomov Signed-off-by: Ilya Dryomov --- fs/ceph/caps.c | 21 +++++++++------------ fs/ceph/super.h | 2 ++ 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index d05717397c2a..9fa2f8777aa4 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -2474,16 +2474,15 @@ void ceph_kick_flushing_caps(struct ceph_mds_client *mdsc, } } -static void kick_flushing_inode_caps(struct ceph_mds_client *mdsc, - struct ceph_mds_session *session, - struct inode *inode) - __releases(ci->i_ceph_lock) +void ceph_kick_flushing_inode_caps(struct ceph_mds_session *session, + struct ceph_inode_info *ci) { - struct ceph_inode_info *ci = ceph_inode(inode); - struct ceph_cap *cap; + struct ceph_mds_client *mdsc = session->s_mdsc; + struct ceph_cap *cap = ci->i_auth_cap; - cap = ci->i_auth_cap; - dout("kick_flushing_inode_caps %p flushing %s\n", inode, + lockdep_assert_held(&ci->i_ceph_lock); + + dout("%s %p flushing %s\n", __func__, &ci->vfs_inode, ceph_cap_string(ci->i_flushing_caps)); if (!list_empty(&ci->i_cap_flush_list)) { @@ -2495,9 +2494,6 @@ static void kick_flushing_inode_caps(struct ceph_mds_client *mdsc, spin_unlock(&mdsc->cap_dirty_lock); __kick_flushing_caps(mdsc, session, ci, oldest_flush_tid); - spin_unlock(&ci->i_ceph_lock); - } else { - spin_unlock(&ci->i_ceph_lock); } } @@ -3366,7 +3362,8 @@ static void handle_cap_grant(struct inode *inode, if (le32_to_cpu(grant->op) == CEPH_CAP_OP_IMPORT) { if (newcaps & ~extra_info->issued) wake = true; - kick_flushing_inode_caps(session->s_mdsc, session, inode); + ceph_kick_flushing_inode_caps(session, ci); + spin_unlock(&ci->i_ceph_lock); up_read(&session->s_mdsc->snap_rwsem); } else { spin_unlock(&ci->i_ceph_lock); diff --git a/fs/ceph/super.h b/fs/ceph/super.h index 37dc1ac8f6c3..b2711ee56b09 100644 --- a/fs/ceph/super.h +++ b/fs/ceph/super.h @@ -1048,6 +1048,8 @@ extern void ceph_early_kick_flushing_caps(struct ceph_mds_client *mdsc, struct ceph_mds_session *session); extern void ceph_kick_flushing_caps(struct ceph_mds_client *mdsc, struct ceph_mds_session *session); +void ceph_kick_flushing_inode_caps(struct ceph_mds_session *session, + struct ceph_inode_info *ci); extern struct ceph_cap *ceph_get_cap_for_mds(struct ceph_inode_info *ci, int mds); extern void ceph_get_cap_refs(struct ceph_inode_info *ci, int caps); From c7e4f85ce9672470adee5c2f9ade1aa0d15d9334 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Tue, 25 Feb 2020 11:08:33 -0800 Subject: [PATCH 345/635] ceph: more caps.c lockdep assertions Signed-off-by: Jeff Layton Reviewed-by: "Yan, Zheng" Signed-off-by: Ilya Dryomov --- fs/ceph/caps.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index 9fa2f8777aa4..739c06611522 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -1634,6 +1634,8 @@ int __ceph_mark_dirty_caps(struct ceph_inode_info *ci, int mask, int was = ci->i_dirty_caps; int dirty = 0; + lockdep_assert_held(&ci->i_ceph_lock); + if (!ci->i_auth_cap) { pr_warn("__mark_dirty_caps %p %llx mask %s, " "but no auth cap (session was closed?)\n", @@ -1743,6 +1745,7 @@ static u64 __mark_caps_flushing(struct inode *inode, struct ceph_cap_flush *cf = NULL; int flushing; + lockdep_assert_held(&ci->i_ceph_lock); BUG_ON(ci->i_dirty_caps == 0); BUG_ON(list_empty(&ci->i_dirty_item)); BUG_ON(!ci->i_prealloc_cap_flush); From 3bb48b4142bbf72045af5ebe72e65ccff6d02680 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Mon, 2 Dec 2019 13:47:57 -0500 Subject: [PATCH 346/635] ceph: add flag to designate that a request is asynchronous ...and ensure that such requests are never queued. The MDS has need to know that a request is asynchronous so add flags and proper infrastructure for that. Also, delegated inode numbers and directory caps are associated with the session, so ensure that async requests are always transmitted on the first attempt and are never queued to wait for session reestablishment. If it does end up looking like we'll need to queue the request, then have it return -EJUKEBOX so the caller can reattempt with a synchronous request. Signed-off-by: Jeff Layton Reviewed-by: "Yan, Zheng" Signed-off-by: Ilya Dryomov --- fs/ceph/inode.c | 1 + fs/ceph/mds_client.c | 15 +++++++++++++++ fs/ceph/mds_client.h | 1 + include/linux/ceph/ceph_fs.h | 5 +++-- 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c index 094b8fc37787..9869ec101e88 100644 --- a/fs/ceph/inode.c +++ b/fs/ceph/inode.c @@ -1311,6 +1311,7 @@ retry_lookup: err = fill_inode(in, req->r_locked_page, &rinfo->targeti, NULL, session, (!test_bit(CEPH_MDS_R_ABORTED, &req->r_req_flags) && + !test_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags) && rinfo->head->result == 0) ? req->r_fmode : -1, &req->r_caps_reservation); if (err < 0) { diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index 2da98b6cc064..9e8b2099e63f 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -2528,6 +2528,8 @@ static int __prepare_send_request(struct ceph_mds_client *mdsc, rhead->oldest_client_tid = cpu_to_le64(__get_oldest_tid(mdsc)); if (test_bit(CEPH_MDS_R_GOT_UNSAFE, &req->r_req_flags)) flags |= CEPH_MDS_FLAG_REPLAY; + if (test_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags)) + flags |= CEPH_MDS_FLAG_ASYNC; if (req->r_parent) flags |= CEPH_MDS_FLAG_WANT_DENTRY; rhead->flags = cpu_to_le32(flags); @@ -2611,6 +2613,10 @@ static void __do_request(struct ceph_mds_client *mdsc, mds = __choose_mds(mdsc, req, &random); if (mds < 0 || ceph_mdsmap_get_state(mdsc->mdsmap, mds) < CEPH_MDS_STATE_ACTIVE) { + if (test_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags)) { + err = -EJUKEBOX; + goto finish; + } dout("do_request no mds or not active, waiting for map\n"); list_add(&req->r_wait, &mdsc->waiting_for_map); return; @@ -2635,6 +2641,15 @@ static void __do_request(struct ceph_mds_client *mdsc, err = -EACCES; goto out_session; } + /* + * We cannot queue async requests since the caps and delegated + * inodes are bound to the session. Just return -EJUKEBOX and + * let the caller retry a sync request in that case. + */ + if (test_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags)) { + err = -EJUKEBOX; + goto out_session; + } if (session->s_state == CEPH_MDS_SESSION_NEW || session->s_state == CEPH_MDS_SESSION_CLOSING) { __open_session(mdsc, session); diff --git a/fs/ceph/mds_client.h b/fs/ceph/mds_client.h index a0918d00117c..95ac00e59e66 100644 --- a/fs/ceph/mds_client.h +++ b/fs/ceph/mds_client.h @@ -255,6 +255,7 @@ struct ceph_mds_request { #define CEPH_MDS_R_GOT_RESULT (5) /* got a result */ #define CEPH_MDS_R_DID_PREPOPULATE (6) /* prepopulated readdir */ #define CEPH_MDS_R_PARENT_LOCKED (7) /* is r_parent->i_rwsem wlocked? */ +#define CEPH_MDS_R_ASYNC (8) /* async request */ unsigned long r_req_flags; struct mutex r_fill_mutex; diff --git a/include/linux/ceph/ceph_fs.h b/include/linux/ceph/ceph_fs.h index cb21c5cf12c3..9f747a1b8788 100644 --- a/include/linux/ceph/ceph_fs.h +++ b/include/linux/ceph/ceph_fs.h @@ -444,8 +444,9 @@ union ceph_mds_request_args { } __attribute__ ((packed)) lookupino; } __attribute__ ((packed)); -#define CEPH_MDS_FLAG_REPLAY 1 /* this is a replayed op */ -#define CEPH_MDS_FLAG_WANT_DENTRY 2 /* want dentry in reply */ +#define CEPH_MDS_FLAG_REPLAY 1 /* this is a replayed op */ +#define CEPH_MDS_FLAG_WANT_DENTRY 2 /* want dentry in reply */ +#define CEPH_MDS_FLAG_ASYNC 4 /* request is asynchronous */ struct ceph_mds_request_head { __le64 oldest_client_tid; From f5e17aed3accb406f51ae528d657c275efc1edfc Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Tue, 18 Feb 2020 14:12:32 -0500 Subject: [PATCH 347/635] ceph: track primary dentry link Newer versions of the MDS will flag a dentry as "primary". In later patches, we'll need to consult this info, so track it in di->flags. Signed-off-by: Jeff Layton Reviewed-by: "Yan, Zheng" Signed-off-by: Ilya Dryomov --- fs/ceph/dir.c | 1 + fs/ceph/inode.c | 8 +++++++- fs/ceph/super.h | 1 + include/linux/ceph/ceph_fs.h | 3 +++ 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/fs/ceph/dir.c b/fs/ceph/dir.c index d0cd0aba5843..a87274935a09 100644 --- a/fs/ceph/dir.c +++ b/fs/ceph/dir.c @@ -1411,6 +1411,7 @@ void ceph_invalidate_dentry_lease(struct dentry *dentry) spin_lock(&dentry->d_lock); di->time = jiffies; di->lease_shared_gen = 0; + di->flags &= ~CEPH_DENTRY_PRIMARY_LINK; __dentry_lease_unlist(di); spin_unlock(&dentry->d_lock); } diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c index 9869ec101e88..7478bd0283c1 100644 --- a/fs/ceph/inode.c +++ b/fs/ceph/inode.c @@ -1051,6 +1051,7 @@ static void __update_dentry_lease(struct inode *dir, struct dentry *dentry, struct ceph_mds_session **old_lease_session) { struct ceph_dentry_info *di = ceph_dentry(dentry); + unsigned mask = le16_to_cpu(lease->mask); long unsigned duration = le32_to_cpu(lease->duration_ms); long unsigned ttl = from_time + (duration * HZ) / 1000; long unsigned half_ttl = from_time + (duration * HZ / 2) / 1000; @@ -1062,8 +1063,13 @@ static void __update_dentry_lease(struct inode *dir, struct dentry *dentry, if (ceph_snap(dir) != CEPH_NOSNAP) return; + if (mask & CEPH_LEASE_PRIMARY_LINK) + di->flags |= CEPH_DENTRY_PRIMARY_LINK; + else + di->flags &= ~CEPH_DENTRY_PRIMARY_LINK; + di->lease_shared_gen = atomic_read(&ceph_inode(dir)->i_shared_gen); - if (duration == 0) { + if (!(mask & CEPH_LEASE_VALID)) { __ceph_dentry_dir_lease_touch(di); return; } diff --git a/fs/ceph/super.h b/fs/ceph/super.h index b2711ee56b09..7a93f37e7708 100644 --- a/fs/ceph/super.h +++ b/fs/ceph/super.h @@ -284,6 +284,7 @@ struct ceph_dentry_info { #define CEPH_DENTRY_REFERENCED 1 #define CEPH_DENTRY_LEASE_LIST 2 #define CEPH_DENTRY_SHRINK_LIST 4 +#define CEPH_DENTRY_PRIMARY_LINK 8 struct ceph_inode_xattrs_info { /* diff --git a/include/linux/ceph/ceph_fs.h b/include/linux/ceph/ceph_fs.h index 9f747a1b8788..94cc4b047987 100644 --- a/include/linux/ceph/ceph_fs.h +++ b/include/linux/ceph/ceph_fs.h @@ -531,6 +531,9 @@ struct ceph_mds_reply_lease { __le32 seq; } __attribute__ ((packed)); +#define CEPH_LEASE_VALID (1 | 2) /* old and new bit values */ +#define CEPH_LEASE_PRIMARY_LINK 4 /* primary linkage */ + struct ceph_mds_reply_dirfrag { __le32 frag; /* fragment */ __le32 auth; /* auth mds, if this is a delegation point */ From 891f3f5a6a0615a2ed93cc495b54d1a8121d0968 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Tue, 14 Jan 2020 15:06:40 -0500 Subject: [PATCH 348/635] ceph: add infrastructure for waiting for async create to complete When we issue an async create, we must ensure that any later on-the-wire requests involving it wait for the create reply. Expand i_ceph_flags to be an unsigned long, and add a new bit that MDS requests can wait on. If the bit is set in the inode when sending caps, then don't send it and just return that it has been delayed. Signed-off-by: Jeff Layton Reviewed-by: "Yan, Zheng" Signed-off-by: Ilya Dryomov --- fs/ceph/caps.c | 13 ++++++++++++- fs/ceph/dir.c | 2 +- fs/ceph/mds_client.c | 20 +++++++++++++++++++- fs/ceph/mds_client.h | 7 +++++++ fs/ceph/super.h | 4 +++- 5 files changed, 42 insertions(+), 4 deletions(-) diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index 739c06611522..fcad5d8ebcfa 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -511,7 +511,7 @@ static void __cap_delay_requeue(struct ceph_mds_client *mdsc, struct ceph_inode_info *ci, bool set_timeout) { - dout("__cap_delay_requeue %p flags %d at %lu\n", &ci->vfs_inode, + dout("__cap_delay_requeue %p flags 0x%lx at %lu\n", &ci->vfs_inode, ci->i_ceph_flags, ci->i_hold_caps_max); if (!mdsc->stopping) { spin_lock(&mdsc->cap_delay_lock); @@ -1294,6 +1294,13 @@ static int __send_cap(struct ceph_mds_client *mdsc, struct ceph_cap *cap, int delayed = 0; int ret; + /* Don't send anything if it's still being created. Return delayed */ + if (ci->i_ceph_flags & CEPH_I_ASYNC_CREATE) { + spin_unlock(&ci->i_ceph_lock); + dout("%s async create in flight for %p\n", __func__, inode); + return 1; + } + held = cap->issued | cap->implemented; revoking = cap->implemented & ~cap->issued; retain &= ~revoking; @@ -2253,6 +2260,10 @@ int ceph_fsync(struct file *file, loff_t start, loff_t end, int datasync) if (datasync) goto out; + ret = ceph_wait_on_async_create(inode); + if (ret) + goto out; + dirty = try_flush_caps(inode, &flush_tid); dout("fsync dirty caps are %s\n", ceph_cap_string(dirty)); diff --git a/fs/ceph/dir.c b/fs/ceph/dir.c index a87274935a09..5b83bda57056 100644 --- a/fs/ceph/dir.c +++ b/fs/ceph/dir.c @@ -752,7 +752,7 @@ static struct dentry *ceph_lookup(struct inode *dir, struct dentry *dentry, struct ceph_dentry_info *di = ceph_dentry(dentry); spin_lock(&ci->i_ceph_lock); - dout(" dir %p flags are %d\n", dir, ci->i_ceph_flags); + dout(" dir %p flags are 0x%lx\n", dir, ci->i_ceph_flags); if (strncmp(dentry->d_name.name, fsc->mount_options->snapdir_name, dentry->d_name.len) && diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index 9e8b2099e63f..c433655eba62 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -2730,7 +2730,7 @@ static void kick_requests(struct ceph_mds_client *mdsc, int mds) int ceph_mdsc_submit_request(struct ceph_mds_client *mdsc, struct inode *dir, struct ceph_mds_request *req) { - int err; + int err = 0; /* take CAP_PIN refs for r_inode, r_parent, r_old_dentry */ if (req->r_inode) @@ -2743,6 +2743,24 @@ int ceph_mdsc_submit_request(struct ceph_mds_client *mdsc, struct inode *dir, ceph_get_cap_refs(ceph_inode(req->r_old_dentry_dir), CEPH_CAP_PIN); + if (req->r_inode) { + err = ceph_wait_on_async_create(req->r_inode); + if (err) { + dout("%s: wait for async create returned: %d\n", + __func__, err); + return err; + } + } + + if (!err && req->r_old_inode) { + err = ceph_wait_on_async_create(req->r_old_inode); + if (err) { + dout("%s: wait for async create returned: %d\n", + __func__, err); + return err; + } + } + dout("submit_request on %p for inode %p\n", req, dir); mutex_lock(&mdsc->mutex); __register_request(mdsc, req, dir); diff --git a/fs/ceph/mds_client.h b/fs/ceph/mds_client.h index 95ac00e59e66..8043f2b439b1 100644 --- a/fs/ceph/mds_client.h +++ b/fs/ceph/mds_client.h @@ -538,4 +538,11 @@ extern void ceph_mdsc_open_export_target_sessions(struct ceph_mds_client *mdsc, extern int ceph_trim_caps(struct ceph_mds_client *mdsc, struct ceph_mds_session *session, int max_caps); +static inline int ceph_wait_on_async_create(struct inode *inode) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + + return wait_on_bit(&ci->i_ceph_flags, CEPH_ASYNC_CREATE_BIT, + TASK_INTERRUPTIBLE); +} #endif diff --git a/fs/ceph/super.h b/fs/ceph/super.h index 7a93f37e7708..00bf846e0014 100644 --- a/fs/ceph/super.h +++ b/fs/ceph/super.h @@ -316,7 +316,7 @@ struct ceph_inode_info { u64 i_inline_version; u32 i_time_warp_seq; - unsigned i_ceph_flags; + unsigned long i_ceph_flags; atomic64_t i_release_count; atomic64_t i_ordered_count; atomic64_t i_complete_seq[2]; @@ -524,6 +524,8 @@ static inline struct inode *ceph_find_inode(struct super_block *sb, #define CEPH_I_ERROR_WRITE (1 << 10) /* have seen write errors */ #define CEPH_I_ERROR_FILELOCK (1 << 11) /* have seen file lock errors */ #define CEPH_I_ODIRECT (1 << 12) /* inode in direct I/O mode */ +#define CEPH_ASYNC_CREATE_BIT (13) /* async create in flight for this */ +#define CEPH_I_ASYNC_CREATE (1 << CEPH_ASYNC_CREATE_BIT) /* * Masks of ceph inode work. From 40dcf75e82f875fd95abb99d08ef3eeb025e16a9 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Tue, 14 Jan 2020 09:23:49 -0500 Subject: [PATCH 349/635] ceph: make __take_cap_refs non-static Rename it to ceph_take_cap_refs and make it available to other files. Also replace a comment with a lockdep assertion. Signed-off-by: Jeff Layton Reviewed-by: "Yan, Zheng" Signed-off-by: Ilya Dryomov --- fs/ceph/caps.c | 12 ++++++------ fs/ceph/super.h | 2 ++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index fcad5d8ebcfa..569484a57111 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -2515,12 +2515,12 @@ void ceph_kick_flushing_inode_caps(struct ceph_mds_session *session, /* * Take references to capabilities we hold, so that we don't release * them to the MDS prematurely. - * - * Protected by i_ceph_lock. */ -static void __take_cap_refs(struct ceph_inode_info *ci, int got, +void ceph_take_cap_refs(struct ceph_inode_info *ci, int got, bool snap_rwsem_locked) { + lockdep_assert_held(&ci->i_ceph_lock); + if (got & CEPH_CAP_PIN) ci->i_pin_ref++; if (got & CEPH_CAP_FILE_RD) @@ -2541,7 +2541,7 @@ static void __take_cap_refs(struct ceph_inode_info *ci, int got, if (ci->i_wb_ref == 0) ihold(&ci->vfs_inode); ci->i_wb_ref++; - dout("__take_cap_refs %p wb %d -> %d (?)\n", + dout("%s %p wb %d -> %d (?)\n", __func__, &ci->vfs_inode, ci->i_wb_ref-1, ci->i_wb_ref); } } @@ -2664,7 +2664,7 @@ again: (need & CEPH_CAP_FILE_RD) && !(*got & CEPH_CAP_FILE_CACHE)) ceph_disable_fscache_readpage(ci); - __take_cap_refs(ci, *got, true); + ceph_take_cap_refs(ci, *got, true); ret = 1; } } else { @@ -2893,7 +2893,7 @@ int ceph_get_caps(struct file *filp, int need, int want, void ceph_get_cap_refs(struct ceph_inode_info *ci, int caps) { spin_lock(&ci->i_ceph_lock); - __take_cap_refs(ci, caps, false); + ceph_take_cap_refs(ci, caps, false); spin_unlock(&ci->i_ceph_lock); } diff --git a/fs/ceph/super.h b/fs/ceph/super.h index 00bf846e0014..4cc00c4012c2 100644 --- a/fs/ceph/super.h +++ b/fs/ceph/super.h @@ -1055,6 +1055,8 @@ void ceph_kick_flushing_inode_caps(struct ceph_mds_session *session, struct ceph_inode_info *ci); extern struct ceph_cap *ceph_get_cap_for_mds(struct ceph_inode_info *ci, int mds); +extern void ceph_take_cap_refs(struct ceph_inode_info *ci, int caps, + bool snap_rwsem_locked); extern void ceph_get_cap_refs(struct ceph_inode_info *ci, int caps); extern void ceph_put_cap_refs(struct ceph_inode_info *ci, int had); extern void ceph_put_wrbuffer_cap_refs(struct ceph_inode_info *ci, int nr, From a25949b99003b7e6c2604a3fc8b8d62385508477 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Tue, 18 Feb 2020 14:12:45 -0500 Subject: [PATCH 350/635] ceph: cap tracking for async directory operations Track and correctly handle directory caps for asynchronous operations. Add aliases for Frc caps that we now designate at Dcu caps (when dealing with directories). Unlike file caps, we don't reclaim these when the session goes away, and instead preemptively release them. In-flight async dirops are instead handled during reconnect phase. The client needs to re-do a synchronous operation in order to re-get directory caps. Signed-off-by: Jeff Layton Reviewed-by: "Yan, Zheng" Signed-off-by: Ilya Dryomov --- fs/ceph/caps.c | 27 +++++++++++++++++++-------- fs/ceph/mds_client.c | 31 ++++++++++++++++++++++++++----- fs/ceph/mds_client.h | 6 +++++- include/linux/ceph/ceph_fs.h | 6 ++++++ 4 files changed, 56 insertions(+), 14 deletions(-) diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index 569484a57111..d3274120f738 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -992,7 +992,11 @@ int __ceph_caps_file_wanted(struct ceph_inode_info *ci) int __ceph_caps_wanted(struct ceph_inode_info *ci) { int w = __ceph_caps_file_wanted(ci) | __ceph_caps_used(ci); - if (!S_ISDIR(ci->vfs_inode.i_mode)) { + if (S_ISDIR(ci->vfs_inode.i_mode)) { + /* we want EXCL if holding caps of dir ops */ + if (w & CEPH_CAP_ANY_DIR_OPS) + w |= CEPH_CAP_FILE_EXCL; + } else { /* we want EXCL if dirty data */ if (w & CEPH_CAP_FILE_BUFFER) w |= CEPH_CAP_FILE_EXCL; @@ -1893,10 +1897,13 @@ retry_locked: * revoking the shared cap on every create/unlink * operation. */ - if (IS_RDONLY(inode)) + if (IS_RDONLY(inode)) { want = CEPH_CAP_ANY_SHARED; - else - want = CEPH_CAP_ANY_SHARED | CEPH_CAP_FILE_EXCL; + } else { + want = CEPH_CAP_ANY_SHARED | + CEPH_CAP_FILE_EXCL | + CEPH_CAP_ANY_DIR_OPS; + } retain |= want; } else { @@ -2749,10 +2756,14 @@ int ceph_try_get_caps(struct inode *inode, int need, int want, int ret; BUG_ON(need & ~CEPH_CAP_FILE_RD); - BUG_ON(want & ~(CEPH_CAP_FILE_CACHE|CEPH_CAP_FILE_LAZYIO|CEPH_CAP_FILE_SHARED)); - ret = ceph_pool_perm_check(inode, need); - if (ret < 0) - return ret; + BUG_ON(want & ~(CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_LAZYIO | + CEPH_CAP_FILE_SHARED | CEPH_CAP_FILE_EXCL | + CEPH_CAP_ANY_DIR_OPS)); + if (need) { + ret = ceph_pool_perm_check(inode, need); + if (ret < 0) + return ret; + } ret = try_get_cap_refs(inode, need, want, 0, (nonblock ? NON_BLOCKING : 0), got); diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index c433655eba62..9ffb2ee623af 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -699,6 +699,7 @@ void ceph_mdsc_release_request(struct kref *kref) struct ceph_mds_request *req = container_of(kref, struct ceph_mds_request, r_kref); + ceph_mdsc_release_dir_caps(req); destroy_reply_info(&req->r_reply_info); if (req->r_request) ceph_msg_put(req->r_request); @@ -3280,6 +3281,17 @@ bad: return; } +void ceph_mdsc_release_dir_caps(struct ceph_mds_request *req) +{ + int dcaps; + + dcaps = xchg(&req->r_dir_caps, 0); + if (dcaps) { + dout("releasing r_dir_caps=%s\n", ceph_cap_string(dcaps)); + ceph_put_cap_refs(ceph_inode(req->r_parent), dcaps); + } +} + /* * called under session->mutex. */ @@ -3307,9 +3319,14 @@ static void replay_unsafe_requests(struct ceph_mds_client *mdsc, continue; if (req->r_attempts == 0) continue; /* only old requests */ - if (req->r_session && - req->r_session->s_mds == session->s_mds) - __send_request(mdsc, session, req, true); + if (!req->r_session) + continue; + if (req->r_session->s_mds != session->s_mds) + continue; + + ceph_mdsc_release_dir_caps(req); + + __send_request(mdsc, session, req, true); } mutex_unlock(&mdsc->mutex); } @@ -3393,7 +3410,7 @@ fail_msg: /* * Encode information about a cap for a reconnect with the MDS. */ -static int encode_caps_cb(struct inode *inode, struct ceph_cap *cap, +static int reconnect_caps_cb(struct inode *inode, struct ceph_cap *cap, void *arg) { union { @@ -3416,6 +3433,10 @@ static int encode_caps_cb(struct inode *inode, struct ceph_cap *cap, cap->mseq = 0; /* and migrate_seq */ cap->cap_gen = cap->session->s_cap_gen; + /* These are lost when the session goes away */ + if (S_ISDIR(inode->i_mode)) + cap->issued &= ~CEPH_CAP_ANY_DIR_OPS; + if (recon_state->msg_version >= 2) { rec.v2.cap_id = cpu_to_le64(cap->cap_id); rec.v2.wanted = cpu_to_le32(__ceph_caps_wanted(ci)); @@ -3712,7 +3733,7 @@ static void send_mds_reconnect(struct ceph_mds_client *mdsc, recon_state.msg_version = 2; } /* trsaverse this session's caps */ - err = ceph_iterate_session_caps(session, encode_caps_cb, &recon_state); + err = ceph_iterate_session_caps(session, reconnect_caps_cb, &recon_state); spin_lock(&session->s_cap_lock); session->s_cap_reconnect = 0; diff --git a/fs/ceph/mds_client.h b/fs/ceph/mds_client.h index 8043f2b439b1..f10d342ea585 100644 --- a/fs/ceph/mds_client.h +++ b/fs/ceph/mds_client.h @@ -284,8 +284,11 @@ struct ceph_mds_request { struct ceph_msg *r_request; /* original request */ struct ceph_msg *r_reply; struct ceph_mds_reply_info_parsed r_reply_info; - struct page *r_locked_page; int r_err; + + + struct page *r_locked_page; + int r_dir_caps; int r_num_caps; u32 r_readdir_offset; @@ -489,6 +492,7 @@ extern int ceph_mdsc_submit_request(struct ceph_mds_client *mdsc, extern int ceph_mdsc_do_request(struct ceph_mds_client *mdsc, struct inode *dir, struct ceph_mds_request *req); +extern void ceph_mdsc_release_dir_caps(struct ceph_mds_request *req); static inline void ceph_mdsc_get_request(struct ceph_mds_request *req) { kref_get(&req->r_kref); diff --git a/include/linux/ceph/ceph_fs.h b/include/linux/ceph/ceph_fs.h index 94cc4b047987..91d09cf37649 100644 --- a/include/linux/ceph/ceph_fs.h +++ b/include/linux/ceph/ceph_fs.h @@ -663,6 +663,12 @@ int ceph_flags_to_mode(int flags); #define CEPH_CAP_LOCKS (CEPH_LOCK_IFILE | CEPH_LOCK_IAUTH | CEPH_LOCK_ILINK | \ CEPH_LOCK_IXATTR) +/* cap masks async dir operations */ +#define CEPH_CAP_DIR_CREATE CEPH_CAP_FILE_CACHE +#define CEPH_CAP_DIR_UNLINK CEPH_CAP_FILE_RD +#define CEPH_CAP_ANY_DIR_OPS (CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_RD | \ + CEPH_CAP_FILE_WREXTEND | CEPH_CAP_FILE_LAZYIO) + int ceph_caps_for_mode(int mode); enum { From 173e70e8ac0f1e7e3eb98f1ffb953e80a5cc8a5f Mon Sep 17 00:00:00 2001 From: "Yan, Zheng" Date: Tue, 18 Feb 2020 08:17:08 -0500 Subject: [PATCH 351/635] ceph: don't take refs to want mask unless we have all bits If we don't have all of the cap bits for the want mask in try_get_cap_refs, then just take refs on the need bits. Signed-off-by: "Yan, Zheng" Reviewed-by: Jeff Layton Signed-off-by: Ilya Dryomov --- fs/ceph/caps.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index d3274120f738..d09b9d91cf4f 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -2666,7 +2666,10 @@ again: } snap_rwsem_locked = true; } - *got = need | (have & want); + if ((have & want) == want) + *got = need | want; + else + *got = need; if (S_ISREG(inode->i_mode) && (need & CEPH_CAP_FILE_RD) && !(*got & CEPH_CAP_FILE_CACHE)) From 2ccb45462aeaf0831397b90d31d3d50a7704fa1f Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Tue, 2 Apr 2019 15:35:56 -0400 Subject: [PATCH 352/635] ceph: perform asynchronous unlink if we have sufficient caps The MDS is getting a new lock-caching facility that will allow it to cache the necessary locks to allow asynchronous directory operations. Since the CEPH_CAP_FILE_* caps are currently unused on directories, we can repurpose those bits for this purpose. When performing an unlink, if we have Fx on the parent directory, and CEPH_CAP_DIR_UNLINK (aka Fr), and we know that the dentry being removed is the primary link, then then we can fire off an unlink request immediately and don't need to wait on reply before returning. In that situation, just fix up the dcache and link count and return immediately after issuing the call to the MDS. This does mean that we need to hold an extra reference to the inode being unlinked, and extra references to the caps to avoid races. Those references are put and error handling is done in the r_callback routine. If the operation ends up failing, then set a writeback error on the directory inode, and the inode itself that can be fetched later by an fsync on the dir. The behavior of dir caps is slightly different from caps on normal files. Because these are just considered an optimization, if the session is reconnected, we will not automatically reclaim them. They are instead considered lost until we do another synchronous op in the parent directory. Async dirops are enabled via the "nowsync" mount option, which is patterned after the xfs "wsync" mount option. For now, the default is "wsync", but eventually we may flip that. Signed-off-by: Jeff Layton Reviewed-by: "Yan, Zheng" Signed-off-by: Ilya Dryomov --- fs/ceph/dir.c | 108 ++++++++++++++++++++++++++++++++++++++++++++++-- fs/ceph/super.c | 20 +++++++++ fs/ceph/super.h | 5 ++- 3 files changed, 128 insertions(+), 5 deletions(-) diff --git a/fs/ceph/dir.c b/fs/ceph/dir.c index 5b83bda57056..ee6b319e5481 100644 --- a/fs/ceph/dir.c +++ b/fs/ceph/dir.c @@ -1036,6 +1036,78 @@ static int ceph_link(struct dentry *old_dentry, struct inode *dir, return err; } +static void ceph_async_unlink_cb(struct ceph_mds_client *mdsc, + struct ceph_mds_request *req) +{ + int result = req->r_err ? req->r_err : + le32_to_cpu(req->r_reply_info.head->result); + + if (result == -EJUKEBOX) + goto out; + + /* If op failed, mark everyone involved for errors */ + if (result) { + int pathlen; + u64 base; + char *path = ceph_mdsc_build_path(req->r_dentry, &pathlen, + &base, 0); + + /* mark error on parent + clear complete */ + mapping_set_error(req->r_parent->i_mapping, result); + ceph_dir_clear_complete(req->r_parent); + + /* drop the dentry -- we don't know its status */ + if (!d_unhashed(req->r_dentry)) + d_drop(req->r_dentry); + + /* mark inode itself for an error (since metadata is bogus) */ + mapping_set_error(req->r_old_inode->i_mapping, result); + + pr_warn("ceph: async unlink failure path=(%llx)%s result=%d!\n", + base, IS_ERR(path) ? "<>" : path, result); + ceph_mdsc_free_path(path, pathlen); + } +out: + iput(req->r_old_inode); + ceph_mdsc_release_dir_caps(req); +} + +static int get_caps_for_async_unlink(struct inode *dir, struct dentry *dentry) +{ + struct ceph_inode_info *ci = ceph_inode(dir); + struct ceph_dentry_info *di; + int got = 0, want = CEPH_CAP_FILE_EXCL | CEPH_CAP_DIR_UNLINK; + + spin_lock(&ci->i_ceph_lock); + if ((__ceph_caps_issued(ci, NULL) & want) == want) { + ceph_take_cap_refs(ci, want, false); + got = want; + } + spin_unlock(&ci->i_ceph_lock); + + /* If we didn't get anything, return 0 */ + if (!got) + return 0; + + spin_lock(&dentry->d_lock); + di = ceph_dentry(dentry); + /* + * - We are holding Fx, which implies Fs caps. + * - Only support async unlink for primary linkage + */ + if (atomic_read(&ci->i_shared_gen) != di->lease_shared_gen || + !(di->flags & CEPH_DENTRY_PRIMARY_LINK)) + want = 0; + spin_unlock(&dentry->d_lock); + + /* Do we still want what we've got? */ + if (want == got) + return got; + + ceph_put_cap_refs(ci, got); + return 0; +} + /* * rmdir and unlink are differ only by the metadata op code */ @@ -1045,6 +1117,7 @@ static int ceph_unlink(struct inode *dir, struct dentry *dentry) struct ceph_mds_client *mdsc = fsc->mdsc; struct inode *inode = d_inode(dentry); struct ceph_mds_request *req; + bool try_async = ceph_test_mount_opt(fsc, ASYNC_DIROPS); int err = -EROFS; int op; @@ -1059,6 +1132,7 @@ static int ceph_unlink(struct inode *dir, struct dentry *dentry) CEPH_MDS_OP_RMDIR : CEPH_MDS_OP_UNLINK; } else goto out; +retry: req = ceph_mdsc_create_request(mdsc, op, USE_AUTH_MDS); if (IS_ERR(req)) { err = PTR_ERR(req); @@ -1067,13 +1141,39 @@ static int ceph_unlink(struct inode *dir, struct dentry *dentry) req->r_dentry = dget(dentry); req->r_num_caps = 2; req->r_parent = dir; - set_bit(CEPH_MDS_R_PARENT_LOCKED, &req->r_req_flags); req->r_dentry_drop = CEPH_CAP_FILE_SHARED; req->r_dentry_unless = CEPH_CAP_FILE_EXCL; req->r_inode_drop = ceph_drop_caps_for_unlink(inode); - err = ceph_mdsc_do_request(mdsc, dir, req); - if (!err && !req->r_reply_info.head->is_dentry) - d_delete(dentry); + + if (try_async && op == CEPH_MDS_OP_UNLINK && + (req->r_dir_caps = get_caps_for_async_unlink(dir, dentry))) { + dout("async unlink on %lu/%.*s caps=%s", dir->i_ino, + dentry->d_name.len, dentry->d_name.name, + ceph_cap_string(req->r_dir_caps)); + set_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags); + req->r_callback = ceph_async_unlink_cb; + req->r_old_inode = d_inode(dentry); + ihold(req->r_old_inode); + err = ceph_mdsc_submit_request(mdsc, dir, req); + if (!err) { + /* + * We have enough caps, so we assume that the unlink + * will succeed. Fix up the target inode and dcache. + */ + drop_nlink(inode); + d_delete(dentry); + } else if (err == -EJUKEBOX) { + try_async = false; + ceph_mdsc_put_request(req); + goto retry; + } + } else { + set_bit(CEPH_MDS_R_PARENT_LOCKED, &req->r_req_flags); + err = ceph_mdsc_do_request(mdsc, dir, req); + if (!err && !req->r_reply_info.head->is_dentry) + d_delete(dentry); + } + ceph_mdsc_put_request(req); out: return err; diff --git a/fs/ceph/super.c b/fs/ceph/super.c index b1329cd5388a..c9784eb1159a 100644 --- a/fs/ceph/super.c +++ b/fs/ceph/super.c @@ -155,6 +155,7 @@ enum { Opt_acl, Opt_quotadf, Opt_copyfrom, + Opt_wsync, }; enum ceph_recover_session_mode { @@ -194,6 +195,7 @@ static const struct fs_parameter_spec ceph_mount_parameters[] = { fsparam_string ("snapdirname", Opt_snapdirname), fsparam_string ("source", Opt_source), fsparam_u32 ("wsize", Opt_wsize), + fsparam_flag_no ("wsync", Opt_wsync), {} }; @@ -444,6 +446,12 @@ static int ceph_parse_mount_param(struct fs_context *fc, fc->sb_flags &= ~SB_POSIXACL; } break; + case Opt_wsync: + if (!result.negated) + fsopt->flags &= ~CEPH_MOUNT_OPT_ASYNC_DIROPS; + else + fsopt->flags |= CEPH_MOUNT_OPT_ASYNC_DIROPS; + break; default: BUG(); } @@ -567,6 +575,9 @@ static int ceph_show_options(struct seq_file *m, struct dentry *root) if (fsopt->flags & CEPH_MOUNT_OPT_CLEANRECOVER) seq_show_option(m, "recover_session", "clean"); + if (fsopt->flags & CEPH_MOUNT_OPT_ASYNC_DIROPS) + seq_puts(m, ",nowsync"); + if (fsopt->wsize != CEPH_MAX_WRITE_SIZE) seq_printf(m, ",wsize=%u", fsopt->wsize); if (fsopt->rsize != CEPH_MAX_READ_SIZE) @@ -1115,6 +1126,15 @@ static void ceph_free_fc(struct fs_context *fc) static int ceph_reconfigure_fc(struct fs_context *fc) { + struct ceph_parse_opts_ctx *pctx = fc->fs_private; + struct ceph_mount_options *fsopt = pctx->opts; + struct ceph_fs_client *fsc = ceph_sb_to_client(fc->root->d_sb); + + if (fsopt->flags & CEPH_MOUNT_OPT_ASYNC_DIROPS) + ceph_set_mount_opt(fsc, ASYNC_DIROPS); + else + ceph_clear_mount_opt(fsc, ASYNC_DIROPS); + sync_filesystem(fc->root->d_sb); return 0; } diff --git a/fs/ceph/super.h b/fs/ceph/super.h index 4cc00c4012c2..f5313d79ae27 100644 --- a/fs/ceph/super.h +++ b/fs/ceph/super.h @@ -43,13 +43,16 @@ #define CEPH_MOUNT_OPT_MOUNTWAIT (1<<12) /* mount waits if no mds is up */ #define CEPH_MOUNT_OPT_NOQUOTADF (1<<13) /* no root dir quota in statfs */ #define CEPH_MOUNT_OPT_NOCOPYFROM (1<<14) /* don't use RADOS 'copy-from' op */ +#define CEPH_MOUNT_OPT_ASYNC_DIROPS (1<<15) /* allow async directory ops */ #define CEPH_MOUNT_OPT_DEFAULT \ (CEPH_MOUNT_OPT_DCACHE | \ CEPH_MOUNT_OPT_NOCOPYFROM) #define ceph_set_mount_opt(fsc, opt) \ - (fsc)->mount_options->flags |= CEPH_MOUNT_OPT_##opt; + (fsc)->mount_options->flags |= CEPH_MOUNT_OPT_##opt +#define ceph_clear_mount_opt(fsc, opt) \ + (fsc)->mount_options->flags &= ~CEPH_MOUNT_OPT_##opt #define ceph_test_mount_opt(fsc, opt) \ (!!((fsc)->mount_options->flags & CEPH_MOUNT_OPT_##opt)) From 966c716018de1c0860b3cfa65e316394d84723d2 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Thu, 5 Dec 2019 09:09:25 -0500 Subject: [PATCH 353/635] ceph: make ceph_fill_inode non-static Signed-off-by: Jeff Layton Reviewed-by: "Yan, Zheng" Signed-off-by: Ilya Dryomov --- fs/ceph/inode.c | 47 ++++++++++++++++++++++++----------------------- fs/ceph/super.h | 8 ++++++++ 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c index 7478bd0283c1..4056c7968b86 100644 --- a/fs/ceph/inode.c +++ b/fs/ceph/inode.c @@ -728,11 +728,11 @@ void ceph_fill_file_time(struct inode *inode, int issued, * Populate an inode based on info from mds. May be called on new or * existing inodes. */ -static int fill_inode(struct inode *inode, struct page *locked_page, - struct ceph_mds_reply_info_in *iinfo, - struct ceph_mds_reply_dirfrag *dirinfo, - struct ceph_mds_session *session, int cap_fmode, - struct ceph_cap_reservation *caps_reservation) +int ceph_fill_inode(struct inode *inode, struct page *locked_page, + struct ceph_mds_reply_info_in *iinfo, + struct ceph_mds_reply_dirfrag *dirinfo, + struct ceph_mds_session *session, int cap_fmode, + struct ceph_cap_reservation *caps_reservation) { struct ceph_mds_client *mdsc = ceph_inode_to_client(inode)->mdsc; struct ceph_mds_reply_inode *info = iinfo->in; @@ -749,7 +749,7 @@ static int fill_inode(struct inode *inode, struct page *locked_page, bool new_version = false; bool fill_inline = false; - dout("fill_inode %p ino %llx.%llx v %llu had %llu\n", + dout("%s %p ino %llx.%llx v %llu had %llu\n", __func__, inode, ceph_vinop(inode), le64_to_cpu(info->version), ci->i_version); @@ -770,7 +770,7 @@ static int fill_inode(struct inode *inode, struct page *locked_page, if (iinfo->xattr_len > 4) { xattr_blob = ceph_buffer_new(iinfo->xattr_len, GFP_NOFS); if (!xattr_blob) - pr_err("fill_inode ENOMEM xattr blob %d bytes\n", + pr_err("%s ENOMEM xattr blob %d bytes\n", __func__, iinfo->xattr_len); } @@ -933,8 +933,9 @@ static int fill_inode(struct inode *inode, struct page *locked_page, spin_unlock(&ci->i_ceph_lock); if (symlen != i_size_read(inode)) { - pr_err("fill_inode %llx.%llx BAD symlink " - "size %lld\n", ceph_vinop(inode), + pr_err("%s %llx.%llx BAD symlink " + "size %lld\n", __func__, + ceph_vinop(inode), i_size_read(inode)); i_size_write(inode, symlen); inode->i_blocks = calc_inode_blocks(symlen); @@ -958,7 +959,7 @@ static int fill_inode(struct inode *inode, struct page *locked_page, inode->i_fop = &ceph_dir_fops; break; default: - pr_err("fill_inode %llx.%llx BAD mode 0%o\n", + pr_err("%s %llx.%llx BAD mode 0%o\n", __func__, ceph_vinop(inode), inode->i_mode); } @@ -1246,10 +1247,9 @@ int ceph_fill_trace(struct super_block *sb, struct ceph_mds_request *req) struct inode *dir = req->r_parent; if (dir) { - err = fill_inode(dir, NULL, - &rinfo->diri, rinfo->dirfrag, - session, -1, - &req->r_caps_reservation); + err = ceph_fill_inode(dir, NULL, &rinfo->diri, + rinfo->dirfrag, session, -1, + &req->r_caps_reservation); if (err < 0) goto done; } else { @@ -1314,14 +1314,14 @@ retry_lookup: goto done; } - err = fill_inode(in, req->r_locked_page, &rinfo->targeti, NULL, - session, + err = ceph_fill_inode(in, req->r_locked_page, &rinfo->targeti, + NULL, session, (!test_bit(CEPH_MDS_R_ABORTED, &req->r_req_flags) && !test_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags) && rinfo->head->result == 0) ? req->r_fmode : -1, &req->r_caps_reservation); if (err < 0) { - pr_err("fill_inode badness %p %llx.%llx\n", + pr_err("ceph_fill_inode badness %p %llx.%llx\n", in, ceph_vinop(in)); if (in->i_state & I_NEW) discard_new_inode(in); @@ -1508,10 +1508,11 @@ static int readdir_prepopulate_inodes_only(struct ceph_mds_request *req, dout("new_inode badness got %d\n", err); continue; } - rc = fill_inode(in, NULL, &rde->inode, NULL, session, - -1, &req->r_caps_reservation); + rc = ceph_fill_inode(in, NULL, &rde->inode, NULL, session, + -1, &req->r_caps_reservation); if (rc < 0) { - pr_err("fill_inode badness on %p got %d\n", in, rc); + pr_err("ceph_fill_inode badness on %p got %d\n", + in, rc); err = rc; if (in->i_state & I_NEW) { ihold(in); @@ -1715,10 +1716,10 @@ retry_lookup: } } - ret = fill_inode(in, NULL, &rde->inode, NULL, session, - -1, &req->r_caps_reservation); + ret = ceph_fill_inode(in, NULL, &rde->inode, NULL, session, + -1, &req->r_caps_reservation); if (ret < 0) { - pr_err("fill_inode badness on %p\n", in); + pr_err("ceph_fill_inode badness on %p\n", in); if (d_really_is_negative(dn)) { /* avoid calling iput_final() in mds * dispatch threads */ diff --git a/fs/ceph/super.h b/fs/ceph/super.h index f5313d79ae27..2c479d761d32 100644 --- a/fs/ceph/super.h +++ b/fs/ceph/super.h @@ -895,6 +895,9 @@ static inline bool __ceph_have_pending_cap_snap(struct ceph_inode_info *ci) } /* inode.c */ +struct ceph_mds_reply_info_in; +struct ceph_mds_reply_dirfrag; + extern const struct inode_operations ceph_file_iops; extern struct inode *ceph_alloc_inode(struct super_block *sb); @@ -910,6 +913,11 @@ extern void ceph_fill_file_time(struct inode *inode, int issued, u64 time_warp_seq, struct timespec64 *ctime, struct timespec64 *mtime, struct timespec64 *atime); +extern int ceph_fill_inode(struct inode *inode, struct page *locked_page, + struct ceph_mds_reply_info_in *iinfo, + struct ceph_mds_reply_dirfrag *dirinfo, + struct ceph_mds_session *session, int cap_fmode, + struct ceph_cap_reservation *caps_reservation); extern int ceph_fill_trace(struct super_block *sb, struct ceph_mds_request *req); extern int ceph_readdir_prepopulate(struct ceph_mds_request *req, From d4846487870897a5a149a3220c95bfd5728f9247 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Fri, 15 Nov 2019 11:51:55 -0500 Subject: [PATCH 354/635] ceph: decode interval_sets for delegated inos Starting in Octopus, the MDS will hand out caps that allow the client to do asynchronous file creates under certain conditions. As part of that, the MDS will delegate ranges of inode numbers to the client. Add the infrastructure to decode these ranges, and stuff them into an xarray for later consumption by the async creation code. Because the xarray code currently only handles unsigned long indexes, and those are 32-bits on 32-bit arches, we only enable the decoding when running on a 64-bit arch. Signed-off-by: Jeff Layton Reviewed-by: "Yan, Zheng" Signed-off-by: Ilya Dryomov --- fs/ceph/mds_client.c | 122 +++++++++++++++++++++++++++++++++++++++---- fs/ceph/mds_client.h | 9 +++- 2 files changed, 121 insertions(+), 10 deletions(-) diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index 9ffb2ee623af..1e738d67586d 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -415,21 +415,121 @@ bad: return -EIO; } + +#if BITS_PER_LONG == 64 + +#define DELEGATED_INO_AVAILABLE xa_mk_value(1) + +static int ceph_parse_deleg_inos(void **p, void *end, + struct ceph_mds_session *s) +{ + u32 sets; + + ceph_decode_32_safe(p, end, sets, bad); + dout("got %u sets of delegated inodes\n", sets); + while (sets--) { + u64 start, len, ino; + + ceph_decode_64_safe(p, end, start, bad); + ceph_decode_64_safe(p, end, len, bad); + while (len--) { + int err = xa_insert(&s->s_delegated_inos, ino = start++, + DELEGATED_INO_AVAILABLE, + GFP_KERNEL); + if (!err) { + dout("added delegated inode 0x%llx\n", + start - 1); + } else if (err == -EBUSY) { + pr_warn("ceph: MDS delegated inode 0x%llx more than once.\n", + start - 1); + } else { + return err; + } + } + } + return 0; +bad: + return -EIO; +} + +u64 ceph_get_deleg_ino(struct ceph_mds_session *s) +{ + unsigned long ino; + void *val; + + xa_for_each(&s->s_delegated_inos, ino, val) { + val = xa_erase(&s->s_delegated_inos, ino); + if (val == DELEGATED_INO_AVAILABLE) + return ino; + } + return 0; +} + +int ceph_restore_deleg_ino(struct ceph_mds_session *s, u64 ino) +{ + return xa_insert(&s->s_delegated_inos, ino, DELEGATED_INO_AVAILABLE, + GFP_KERNEL); +} +#else /* BITS_PER_LONG == 64 */ +/* + * FIXME: xarrays can't handle 64-bit indexes on a 32-bit arch. For now, just + * ignore delegated_inos on 32 bit arch. Maybe eventually add xarrays for top + * and bottom words? + */ +static int ceph_parse_deleg_inos(void **p, void *end, + struct ceph_mds_session *s) +{ + u32 sets; + + ceph_decode_32_safe(p, end, sets, bad); + if (sets) + ceph_decode_skip_n(p, end, sets * 2 * sizeof(__le64), bad); + return 0; +bad: + return -EIO; +} + +u64 ceph_get_deleg_ino(struct ceph_mds_session *s) +{ + return 0; +} + +int ceph_restore_deleg_ino(struct ceph_mds_session *s, u64 ino) +{ + return 0; +} +#endif /* BITS_PER_LONG == 64 */ + /* * parse create results */ static int parse_reply_info_create(void **p, void *end, struct ceph_mds_reply_info_parsed *info, - u64 features) + u64 features, struct ceph_mds_session *s) { + int ret; + if (features == (u64)-1 || (features & CEPH_FEATURE_REPLY_CREATE_INODE)) { - /* Malformed reply? */ if (*p == end) { + /* Malformed reply? */ info->has_create_ino = false; - } else { + } else if (test_bit(CEPHFS_FEATURE_DELEG_INO, &s->s_features)) { + u8 struct_v, struct_compat; + u32 len; + info->has_create_ino = true; + ceph_decode_8_safe(p, end, struct_v, bad); + ceph_decode_8_safe(p, end, struct_compat, bad); + ceph_decode_32_safe(p, end, len, bad); ceph_decode_64_safe(p, end, info->ino, bad); + ret = ceph_parse_deleg_inos(p, end, s); + if (ret) + return ret; + } else { + /* legacy */ + ceph_decode_64_safe(p, end, info->ino, bad); + info->has_create_ino = true; } } else { if (*p != end) @@ -448,7 +548,7 @@ bad: */ static int parse_reply_info_extra(void **p, void *end, struct ceph_mds_reply_info_parsed *info, - u64 features) + u64 features, struct ceph_mds_session *s) { u32 op = le32_to_cpu(info->head->op); @@ -457,7 +557,7 @@ static int parse_reply_info_extra(void **p, void *end, else if (op == CEPH_MDS_OP_READDIR || op == CEPH_MDS_OP_LSSNAP) return parse_reply_info_readdir(p, end, info, features); else if (op == CEPH_MDS_OP_CREATE) - return parse_reply_info_create(p, end, info, features); + return parse_reply_info_create(p, end, info, features, s); else return -EIO; } @@ -465,7 +565,7 @@ static int parse_reply_info_extra(void **p, void *end, /* * parse entire mds reply */ -static int parse_reply_info(struct ceph_msg *msg, +static int parse_reply_info(struct ceph_mds_session *s, struct ceph_msg *msg, struct ceph_mds_reply_info_parsed *info, u64 features) { @@ -490,7 +590,7 @@ static int parse_reply_info(struct ceph_msg *msg, ceph_decode_32_safe(&p, end, len, bad); if (len > 0) { ceph_decode_need(&p, end, len, bad); - err = parse_reply_info_extra(&p, p+len, info, features); + err = parse_reply_info_extra(&p, p+len, info, features, s); if (err < 0) goto out_bad; } @@ -558,6 +658,7 @@ void ceph_put_mds_session(struct ceph_mds_session *s) if (refcount_dec_and_test(&s->s_ref)) { if (s->s_auth.authorizer) ceph_auth_destroy_authorizer(s->s_auth.authorizer); + xa_destroy(&s->s_delegated_inos); kfree(s); } } @@ -645,6 +746,7 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc, refcount_set(&s->s_ref, 1); INIT_LIST_HEAD(&s->s_waiting); INIT_LIST_HEAD(&s->s_unsafe); + xa_init(&s->s_delegated_inos); s->s_num_cap_releases = 0; s->s_cap_reconnect = 0; s->s_cap_iterator = NULL; @@ -2980,9 +3082,9 @@ static void handle_reply(struct ceph_mds_session *session, struct ceph_msg *msg) dout("handle_reply tid %lld result %d\n", tid, result); rinfo = &req->r_reply_info; if (test_bit(CEPHFS_FEATURE_REPLY_ENCODING, &session->s_features)) - err = parse_reply_info(msg, rinfo, (u64)-1); + err = parse_reply_info(session, msg, rinfo, (u64)-1); else - err = parse_reply_info(msg, rinfo, session->s_con.peer_features); + err = parse_reply_info(session, msg, rinfo, session->s_con.peer_features); mutex_unlock(&mdsc->mutex); mutex_lock(&session->s_mutex); @@ -3678,6 +3780,8 @@ static void send_mds_reconnect(struct ceph_mds_client *mdsc, if (!reply) goto fail_nomsg; + xa_destroy(&session->s_delegated_inos); + mutex_lock(&session->s_mutex); session->s_state = CEPH_MDS_SESSION_RECONNECTING; session->s_seq = 0; diff --git a/fs/ceph/mds_client.h b/fs/ceph/mds_client.h index f10d342ea585..4c3b71707470 100644 --- a/fs/ceph/mds_client.h +++ b/fs/ceph/mds_client.h @@ -23,8 +23,9 @@ enum ceph_feature_type { CEPHFS_FEATURE_RECLAIM_CLIENT, CEPHFS_FEATURE_LAZY_CAP_WANTED, CEPHFS_FEATURE_MULTI_RECONNECT, + CEPHFS_FEATURE_DELEG_INO, - CEPHFS_FEATURE_MAX = CEPHFS_FEATURE_MULTI_RECONNECT, + CEPHFS_FEATURE_MAX = CEPHFS_FEATURE_DELEG_INO, }; /* @@ -37,6 +38,7 @@ enum ceph_feature_type { CEPHFS_FEATURE_REPLY_ENCODING, \ CEPHFS_FEATURE_LAZY_CAP_WANTED, \ CEPHFS_FEATURE_MULTI_RECONNECT, \ + CEPHFS_FEATURE_DELEG_INO, \ \ CEPHFS_FEATURE_MAX, \ } @@ -201,6 +203,7 @@ struct ceph_mds_session { struct list_head s_waiting; /* waiting requests */ struct list_head s_unsafe; /* unsafe requests */ + struct xarray s_delegated_inos; }; /* @@ -542,6 +545,7 @@ extern void ceph_mdsc_open_export_target_sessions(struct ceph_mds_client *mdsc, extern int ceph_trim_caps(struct ceph_mds_client *mdsc, struct ceph_mds_session *session, int max_caps); + static inline int ceph_wait_on_async_create(struct inode *inode) { struct ceph_inode_info *ci = ceph_inode(inode); @@ -549,4 +553,7 @@ static inline int ceph_wait_on_async_create(struct inode *inode) return wait_on_bit(&ci->i_ceph_flags, CEPH_ASYNC_CREATE_BIT, TASK_INTERRUPTIBLE); } + +extern u64 ceph_get_deleg_ino(struct ceph_mds_session *session); +extern int ceph_restore_deleg_ino(struct ceph_mds_session *session, u64 ino); #endif From 6deb8008a8e64eec3b1e06cc4286905ee6b14e42 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Mon, 13 Jan 2020 13:04:08 -0500 Subject: [PATCH 355/635] ceph: add new MDS req field to hold delegated inode number Add new request field to hold the delegated inode number. Encode that into the message when it's set. Signed-off-by: Jeff Layton Reviewed-by: "Yan, Zheng" Signed-off-by: Ilya Dryomov --- fs/ceph/mds_client.c | 3 +-- fs/ceph/mds_client.h | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index 1e738d67586d..a0ad9bd56b7f 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -2477,7 +2477,7 @@ static struct ceph_msg *create_request_message(struct ceph_mds_client *mdsc, head->op = cpu_to_le32(req->r_op); head->caller_uid = cpu_to_le32(from_kuid(&init_user_ns, req->r_uid)); head->caller_gid = cpu_to_le32(from_kgid(&init_user_ns, req->r_gid)); - head->ino = 0; + head->ino = cpu_to_le64(req->r_deleg_ino); head->args = req->r_args; ceph_encode_filepath(&p, end, ino1, path1); @@ -2638,7 +2638,6 @@ static int __prepare_send_request(struct ceph_mds_client *mdsc, rhead->flags = cpu_to_le32(flags); rhead->num_fwd = req->r_num_fwd; rhead->num_retry = req->r_attempts - 1; - rhead->ino = 0; dout(" r_parent = %p\n", req->r_parent); return 0; diff --git a/fs/ceph/mds_client.h b/fs/ceph/mds_client.h index 4c3b71707470..4e5be79bf080 100644 --- a/fs/ceph/mds_client.h +++ b/fs/ceph/mds_client.h @@ -313,6 +313,7 @@ struct ceph_mds_request { int r_num_fwd; /* number of forward attempts */ int r_resend_mds; /* mds to resend to next, if any*/ u32 r_sent_on_mseq; /* cap mseq request was sent at*/ + u64 r_deleg_ino; struct list_head r_wait; struct completion r_completion; From 785892fe88f693028a996d1cb363d60609848e5e Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Thu, 2 Jan 2020 07:11:38 -0500 Subject: [PATCH 356/635] ceph: cache layout in parent dir on first sync create If a create is done, then typically we'll end up writing to the file soon afterward. We don't want to wait for the reply before doing that when doing an async create, so that means we need the layout for the new file before we've gotten the response from the MDS. All files created in a directory will initially inherit the same layout, so copy off the requisite info from the first synchronous create in the directory, and save it in a new i_cached_layout field. Zero out the layout when we lose Dc caps in the dir. Signed-off-by: Jeff Layton Reviewed-by: "Yan, Zheng" Signed-off-by: Ilya Dryomov --- fs/ceph/caps.c | 13 ++++++++++--- fs/ceph/file.c | 22 +++++++++++++++++++++- fs/ceph/inode.c | 2 ++ fs/ceph/mds_client.c | 7 ++++++- fs/ceph/super.h | 1 + 5 files changed, 40 insertions(+), 5 deletions(-) diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index d09b9d91cf4f..be85225b962c 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -561,14 +561,14 @@ static void __cap_delay_cancel(struct ceph_mds_client *mdsc, spin_unlock(&mdsc->cap_delay_lock); } -/* - * Common issue checks for add_cap, handle_cap_grant. - */ +/* Common issue checks for add_cap, handle_cap_grant. */ static void __check_cap_issue(struct ceph_inode_info *ci, struct ceph_cap *cap, unsigned issued) { unsigned had = __ceph_caps_issued(ci, NULL); + lockdep_assert_held(&ci->i_ceph_lock); + /* * Each time we receive FILE_CACHE anew, we increment * i_rdcache_gen. @@ -593,6 +593,13 @@ static void __check_cap_issue(struct ceph_inode_info *ci, struct ceph_cap *cap, __ceph_dir_clear_complete(ci); } } + + /* Wipe saved layout if we're losing DIR_CREATE caps */ + if (S_ISDIR(ci->vfs_inode.i_mode) && (had & CEPH_CAP_DIR_CREATE) && + !(issued & CEPH_CAP_DIR_CREATE)) { + ceph_put_string(rcu_dereference_raw(ci->i_cached_layout.pool_ns)); + memset(&ci->i_cached_layout, 0, sizeof(ci->i_cached_layout)); + } } /* diff --git a/fs/ceph/file.c b/fs/ceph/file.c index 7f8c4e308301..b6ad64b8863a 100644 --- a/fs/ceph/file.c +++ b/fs/ceph/file.c @@ -430,6 +430,23 @@ out: return err; } +/* Clone the layout from a synchronous create, if the dir now has Dc caps */ +static void +cache_file_layout(struct inode *dst, struct inode *src) +{ + struct ceph_inode_info *cdst = ceph_inode(dst); + struct ceph_inode_info *csrc = ceph_inode(src); + + spin_lock(&cdst->i_ceph_lock); + if ((__ceph_caps_issued(cdst, NULL) & CEPH_CAP_DIR_CREATE) && + !ceph_file_layout_is_valid(&cdst->i_cached_layout)) { + memcpy(&cdst->i_cached_layout, &csrc->i_layout, + sizeof(cdst->i_cached_layout)); + rcu_assign_pointer(cdst->i_cached_layout.pool_ns, + ceph_try_get_string(csrc->i_layout.pool_ns)); + } + spin_unlock(&cdst->i_ceph_lock); +} /* * Do a lookup + open with a single request. If we get a non-existent @@ -518,7 +535,10 @@ int ceph_atomic_open(struct inode *dir, struct dentry *dentry, } else { dout("atomic_open finish_open on dn %p\n", dn); if (req->r_op == CEPH_MDS_OP_CREATE && req->r_reply_info.has_create_ino) { - ceph_init_inode_acls(d_inode(dentry), &as_ctx); + struct inode *newino = d_inode(dentry); + + cache_file_layout(dir, newino); + ceph_init_inode_acls(newino, &as_ctx); file->f_mode |= FMODE_CREATED; } err = finish_open(file, dentry, ceph_open); diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c index 4056c7968b86..73f986efb1fd 100644 --- a/fs/ceph/inode.c +++ b/fs/ceph/inode.c @@ -447,6 +447,7 @@ struct inode *ceph_alloc_inode(struct super_block *sb) ci->i_max_files = 0; memset(&ci->i_dir_layout, 0, sizeof(ci->i_dir_layout)); + memset(&ci->i_cached_layout, 0, sizeof(ci->i_cached_layout)); RCU_INIT_POINTER(ci->i_layout.pool_ns, NULL); ci->i_fragtree = RB_ROOT; @@ -587,6 +588,7 @@ void ceph_evict_inode(struct inode *inode) ceph_buffer_put(ci->i_xattrs.prealloc_blob); ceph_put_string(rcu_dereference_raw(ci->i_layout.pool_ns)); + ceph_put_string(rcu_dereference_raw(ci->i_cached_layout.pool_ns)); } static inline blkcnt_t calc_inode_blocks(u64 size) diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index a0ad9bd56b7f..177acb9f456b 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -3535,8 +3535,13 @@ static int reconnect_caps_cb(struct inode *inode, struct ceph_cap *cap, cap->cap_gen = cap->session->s_cap_gen; /* These are lost when the session goes away */ - if (S_ISDIR(inode->i_mode)) + if (S_ISDIR(inode->i_mode)) { + if (cap->issued & CEPH_CAP_DIR_CREATE) { + ceph_put_string(rcu_dereference_raw(ci->i_cached_layout.pool_ns)); + memset(&ci->i_cached_layout, 0, sizeof(ci->i_cached_layout)); + } cap->issued &= ~CEPH_CAP_ANY_DIR_OPS; + } if (recon_state->msg_version >= 2) { rec.v2.cap_id = cpu_to_le64(cap->cap_id); diff --git a/fs/ceph/super.h b/fs/ceph/super.h index 2c479d761d32..22b89eaeaad4 100644 --- a/fs/ceph/super.h +++ b/fs/ceph/super.h @@ -326,6 +326,7 @@ struct ceph_inode_info { struct ceph_dir_layout i_dir_layout; struct ceph_file_layout i_layout; + struct ceph_file_layout i_cached_layout; // for async creates char *i_symlink; /* for dirs */ From 9a8d03ca2e2c334d08ee91a3e07dcce31a02fdc6 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Wed, 27 Nov 2019 12:06:14 -0500 Subject: [PATCH 357/635] ceph: attempt to do async create when possible With the Octopus release, the MDS will hand out directory create caps. If we have Fxc caps on the directory, and complete directory information or a known negative dentry, then we can return without waiting on the reply, allowing the open() call to return very quickly to userland. We use the normal ceph_fill_inode() routine to fill in the inode, so we have to gin up some reply inode information with what we'd expect the newly-created inode to have. The client assumes that it has a full set of caps on the new inode, and that the MDS will revoke them when there is conflicting access. This functionality is gated on the wsync/nowsync mount options. Signed-off-by: Jeff Layton Reviewed-by: "Yan, Zheng" Signed-off-by: Ilya Dryomov --- fs/ceph/file.c | 247 ++++++++++++++++++++++++++++++++++- include/linux/ceph/ceph_fs.h | 3 + 2 files changed, 243 insertions(+), 7 deletions(-) diff --git a/fs/ceph/file.c b/fs/ceph/file.c index b6ad64b8863a..67080721cec8 100644 --- a/fs/ceph/file.c +++ b/fs/ceph/file.c @@ -448,6 +448,216 @@ cache_file_layout(struct inode *dst, struct inode *src) spin_unlock(&cdst->i_ceph_lock); } +/* + * Try to set up an async create. We need caps, a file layout, and inode number, + * and either a lease on the dentry or complete dir info. If any of those + * criteria are not satisfied, then return false and the caller can go + * synchronous. + */ +static int try_prep_async_create(struct inode *dir, struct dentry *dentry, + struct ceph_file_layout *lo, u64 *pino) +{ + struct ceph_inode_info *ci = ceph_inode(dir); + struct ceph_dentry_info *di = ceph_dentry(dentry); + int got = 0, want = CEPH_CAP_FILE_EXCL | CEPH_CAP_DIR_CREATE; + u64 ino; + + spin_lock(&ci->i_ceph_lock); + /* No auth cap means no chance for Dc caps */ + if (!ci->i_auth_cap) + goto no_async; + + /* Any delegated inos? */ + if (xa_empty(&ci->i_auth_cap->session->s_delegated_inos)) + goto no_async; + + if (!ceph_file_layout_is_valid(&ci->i_cached_layout)) + goto no_async; + + if ((__ceph_caps_issued(ci, NULL) & want) != want) + goto no_async; + + if (d_in_lookup(dentry)) { + if (!__ceph_dir_is_complete(ci)) + goto no_async; + } else if (atomic_read(&ci->i_shared_gen) != + READ_ONCE(di->lease_shared_gen)) { + goto no_async; + } + + ino = ceph_get_deleg_ino(ci->i_auth_cap->session); + if (!ino) + goto no_async; + + *pino = ino; + ceph_take_cap_refs(ci, want, false); + memcpy(lo, &ci->i_cached_layout, sizeof(*lo)); + rcu_assign_pointer(lo->pool_ns, + ceph_try_get_string(ci->i_cached_layout.pool_ns)); + got = want; +no_async: + spin_unlock(&ci->i_ceph_lock); + return got; +} + +static void restore_deleg_ino(struct inode *dir, u64 ino) +{ + struct ceph_inode_info *ci = ceph_inode(dir); + struct ceph_mds_session *s = NULL; + + spin_lock(&ci->i_ceph_lock); + if (ci->i_auth_cap) + s = ceph_get_mds_session(ci->i_auth_cap->session); + spin_unlock(&ci->i_ceph_lock); + if (s) { + int err = ceph_restore_deleg_ino(s, ino); + if (err) + pr_warn("ceph: unable to restore delegated ino 0x%llx to session: %d\n", + ino, err); + ceph_put_mds_session(s); + } +} + +static void ceph_async_create_cb(struct ceph_mds_client *mdsc, + struct ceph_mds_request *req) +{ + int result = req->r_err ? req->r_err : + le32_to_cpu(req->r_reply_info.head->result); + + if (result == -EJUKEBOX) + goto out; + + mapping_set_error(req->r_parent->i_mapping, result); + + if (result) { + struct dentry *dentry = req->r_dentry; + int pathlen; + u64 base; + char *path = ceph_mdsc_build_path(req->r_dentry, &pathlen, + &base, 0); + + ceph_dir_clear_complete(req->r_parent); + if (!d_unhashed(dentry)) + d_drop(dentry); + + /* FIXME: start returning I/O errors on all accesses? */ + pr_warn("ceph: async create failure path=(%llx)%s result=%d!\n", + base, IS_ERR(path) ? "<>" : path, result); + ceph_mdsc_free_path(path, pathlen); + } + + if (req->r_target_inode) { + struct ceph_inode_info *ci = ceph_inode(req->r_target_inode); + u64 ino = ceph_vino(req->r_target_inode).ino; + + if (req->r_deleg_ino != ino) + pr_warn("%s: inode number mismatch! err=%d deleg_ino=0x%llx target=0x%llx\n", + __func__, req->r_err, req->r_deleg_ino, ino); + mapping_set_error(req->r_target_inode->i_mapping, result); + + spin_lock(&ci->i_ceph_lock); + if (ci->i_ceph_flags & CEPH_I_ASYNC_CREATE) { + ci->i_ceph_flags &= ~CEPH_I_ASYNC_CREATE; + wake_up_bit(&ci->i_ceph_flags, CEPH_ASYNC_CREATE_BIT); + } + ceph_kick_flushing_inode_caps(req->r_session, ci); + spin_unlock(&ci->i_ceph_lock); + } else { + pr_warn("%s: no req->r_target_inode for 0x%llx\n", __func__, + req->r_deleg_ino); + } +out: + ceph_mdsc_release_dir_caps(req); +} + +static int ceph_finish_async_create(struct inode *dir, struct dentry *dentry, + struct file *file, umode_t mode, + struct ceph_mds_request *req, + struct ceph_acl_sec_ctx *as_ctx, + struct ceph_file_layout *lo) +{ + int ret; + char xattr_buf[4]; + struct ceph_mds_reply_inode in = { }; + struct ceph_mds_reply_info_in iinfo = { .in = &in }; + struct ceph_inode_info *ci = ceph_inode(dir); + struct inode *inode; + struct timespec64 now; + struct ceph_vino vino = { .ino = req->r_deleg_ino, + .snap = CEPH_NOSNAP }; + + ktime_get_real_ts64(&now); + + inode = ceph_get_inode(dentry->d_sb, vino); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + iinfo.inline_version = CEPH_INLINE_NONE; + iinfo.change_attr = 1; + ceph_encode_timespec64(&iinfo.btime, &now); + + iinfo.xattr_len = ARRAY_SIZE(xattr_buf); + iinfo.xattr_data = xattr_buf; + memset(iinfo.xattr_data, 0, iinfo.xattr_len); + + in.ino = cpu_to_le64(vino.ino); + in.snapid = cpu_to_le64(CEPH_NOSNAP); + in.version = cpu_to_le64(1); // ??? + in.cap.caps = in.cap.wanted = cpu_to_le32(CEPH_CAP_ALL_FILE); + in.cap.cap_id = cpu_to_le64(1); + in.cap.realm = cpu_to_le64(ci->i_snap_realm->ino); + in.cap.flags = CEPH_CAP_FLAG_AUTH; + in.ctime = in.mtime = in.atime = iinfo.btime; + in.mode = cpu_to_le32((u32)mode); + in.truncate_seq = cpu_to_le32(1); + in.truncate_size = cpu_to_le64(-1ULL); + in.xattr_version = cpu_to_le64(1); + in.uid = cpu_to_le32(from_kuid(&init_user_ns, current_fsuid())); + in.gid = cpu_to_le32(from_kgid(&init_user_ns, dir->i_mode & S_ISGID ? + dir->i_gid : current_fsgid())); + in.nlink = cpu_to_le32(1); + in.max_size = cpu_to_le64(lo->stripe_unit); + + ceph_file_layout_to_legacy(lo, &in.layout); + + ret = ceph_fill_inode(inode, NULL, &iinfo, NULL, req->r_session, + req->r_fmode, NULL); + if (ret) { + dout("%s failed to fill inode: %d\n", __func__, ret); + ceph_dir_clear_complete(dir); + if (!d_unhashed(dentry)) + d_drop(dentry); + if (inode->i_state & I_NEW) + discard_new_inode(inode); + } else { + struct dentry *dn; + + dout("%s d_adding new inode 0x%llx to 0x%lx/%s\n", __func__, + vino.ino, dir->i_ino, dentry->d_name.name); + ceph_dir_clear_ordered(dir); + ceph_init_inode_acls(inode, as_ctx); + if (inode->i_state & I_NEW) { + /* + * If it's not I_NEW, then someone created this before + * we got here. Assume the server is aware of it at + * that point and don't worry about setting + * CEPH_I_ASYNC_CREATE. + */ + ceph_inode(inode)->i_ceph_flags = CEPH_I_ASYNC_CREATE; + unlock_new_inode(inode); + } + if (d_in_lookup(dentry) || d_really_is_negative(dentry)) { + if (!d_unhashed(dentry)) + d_drop(dentry); + dn = d_splice_alias(inode, dentry); + WARN_ON_ONCE(dn && dn != dentry); + } + file->f_mode |= FMODE_CREATED; + ret = finish_open(file, dentry, ceph_open); + } + return ret; +} + /* * Do a lookup + open with a single request. If we get a non-existent * file or symlink, return 1 so the VFS can retry. @@ -460,6 +670,7 @@ int ceph_atomic_open(struct inode *dir, struct dentry *dentry, struct ceph_mds_request *req; struct dentry *dn; struct ceph_acl_sec_ctx as_ctx = {}; + bool try_async = ceph_test_mount_opt(fsc, ASYNC_DIROPS); int mask; int err; @@ -483,7 +694,7 @@ int ceph_atomic_open(struct inode *dir, struct dentry *dentry, /* If it's not being looked up, it's negative */ return -ENOENT; } - +retry: /* do the open */ req = prepare_open_request(dir->i_sb, flags, mode); if (IS_ERR(req)) { @@ -492,21 +703,43 @@ int ceph_atomic_open(struct inode *dir, struct dentry *dentry, } req->r_dentry = dget(dentry); req->r_num_caps = 2; + mask = CEPH_STAT_CAP_INODE | CEPH_CAP_AUTH_SHARED; + if (ceph_security_xattr_wanted(dir)) + mask |= CEPH_CAP_XATTR_SHARED; + req->r_args.open.mask = cpu_to_le32(mask); + req->r_parent = dir; + if (flags & O_CREAT) { + struct ceph_file_layout lo; + req->r_dentry_drop = CEPH_CAP_FILE_SHARED | CEPH_CAP_AUTH_EXCL; req->r_dentry_unless = CEPH_CAP_FILE_EXCL; if (as_ctx.pagelist) { req->r_pagelist = as_ctx.pagelist; as_ctx.pagelist = NULL; } + if (try_async && + (req->r_dir_caps = + try_prep_async_create(dir, dentry, &lo, + &req->r_deleg_ino))) { + set_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags); + req->r_args.open.flags |= cpu_to_le32(CEPH_O_EXCL); + req->r_callback = ceph_async_create_cb; + err = ceph_mdsc_submit_request(mdsc, dir, req); + if (!err) { + err = ceph_finish_async_create(dir, dentry, + file, mode, req, + &as_ctx, &lo); + } else if (err == -EJUKEBOX) { + restore_deleg_ino(dir, req->r_deleg_ino); + ceph_mdsc_put_request(req); + try_async = false; + goto retry; + } + goto out_req; + } } - mask = CEPH_STAT_CAP_INODE | CEPH_CAP_AUTH_SHARED; - if (ceph_security_xattr_wanted(dir)) - mask |= CEPH_CAP_XATTR_SHARED; - req->r_args.open.mask = cpu_to_le32(mask); - - req->r_parent = dir; set_bit(CEPH_MDS_R_PARENT_LOCKED, &req->r_req_flags); err = ceph_mdsc_do_request(mdsc, (flags & (O_CREAT|O_TRUNC)) ? dir : NULL, diff --git a/include/linux/ceph/ceph_fs.h b/include/linux/ceph/ceph_fs.h index 91d09cf37649..e035c5194005 100644 --- a/include/linux/ceph/ceph_fs.h +++ b/include/linux/ceph/ceph_fs.h @@ -659,6 +659,9 @@ int ceph_flags_to_mode(int flags); #define CEPH_CAP_ANY (CEPH_CAP_ANY_RD | CEPH_CAP_ANY_EXCL | \ CEPH_CAP_ANY_FILE_WR | CEPH_CAP_FILE_LAZYIO | \ CEPH_CAP_PIN) +#define CEPH_CAP_ALL_FILE (CEPH_CAP_PIN | CEPH_CAP_ANY_SHARED | \ + CEPH_CAP_AUTH_EXCL | CEPH_CAP_XATTR_EXCL | \ + CEPH_CAP_ANY_FILE_RD | CEPH_CAP_ANY_FILE_WR) #define CEPH_CAP_LOCKS (CEPH_LOCK_IFILE | CEPH_LOCK_IAUTH | CEPH_LOCK_ILINK | \ CEPH_LOCK_IXATTR) From 3313f66a57ed73a47162af642e2c8d3995cae058 Mon Sep 17 00:00:00 2001 From: "Yan, Zheng" Date: Wed, 4 Mar 2020 21:22:20 +0800 Subject: [PATCH 358/635] ceph: update dentry lease for async create Otherwise ceph_d_delete() may return 1 for the dentry, which makes dput() prune the dentry and clear parent dir's complete flag. Signed-off-by: "Yan, Zheng" Reviewed-by: Jeff Layton Signed-off-by: Ilya Dryomov --- fs/ceph/file.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fs/ceph/file.c b/fs/ceph/file.c index 67080721cec8..a6c484123d98 100644 --- a/fs/ceph/file.c +++ b/fs/ceph/file.c @@ -480,6 +480,9 @@ static int try_prep_async_create(struct inode *dir, struct dentry *dentry, if (d_in_lookup(dentry)) { if (!__ceph_dir_is_complete(ci)) goto no_async; + spin_lock(&dentry->d_lock); + di->lease_shared_gen = atomic_read(&ci->i_shared_gen); + spin_unlock(&dentry->d_lock); } else if (atomic_read(&ci->i_shared_gen) != READ_ONCE(di->lease_shared_gen)) { goto no_async; From c0e385b1067a9d1d6558d58a2f4d56b4db787177 Mon Sep 17 00:00:00 2001 From: "Yan, Zheng" Date: Thu, 5 Mar 2020 20:20:59 +0800 Subject: [PATCH 359/635] ceph: always renew caps if mds_wanted is insufficient Original code only renews caps for inodes with CEPH_I_CAP_DROPPED flag, which indicates that mds has closed the session and caps were dropped. Remove this flag in preparation for not requesting caps for idle open files. Signed-off-by: "Yan, Zheng" Reviewed-by: Jeff Layton Signed-off-by: Ilya Dryomov --- fs/ceph/caps.c | 36 +++++++++++++++--------------------- fs/ceph/mds_client.c | 5 ----- fs/ceph/super.h | 13 ++++++------- 3 files changed, 21 insertions(+), 33 deletions(-) diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index be85225b962c..a75e5eb3740e 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -2686,6 +2686,7 @@ again: } } else { int session_readonly = false; + int mds_wanted; if (ci->i_auth_cap && (need & (CEPH_CAP_FILE_WR | CEPH_CAP_FILE_EXCL))) { struct ceph_mds_session *s = ci->i_auth_cap->session; @@ -2694,32 +2695,27 @@ again: spin_unlock(&s->s_cap_lock); } if (session_readonly) { - dout("get_cap_refs %p needed %s but mds%d readonly\n", + dout("get_cap_refs %p need %s but mds%d readonly\n", inode, ceph_cap_string(need), ci->i_auth_cap->mds); ret = -EROFS; goto out_unlock; } - if (ci->i_ceph_flags & CEPH_I_CAP_DROPPED) { - int mds_wanted; - if (READ_ONCE(mdsc->fsc->mount_state) == - CEPH_MOUNT_SHUTDOWN) { - dout("get_cap_refs %p forced umount\n", inode); - ret = -EIO; - goto out_unlock; - } - mds_wanted = __ceph_caps_mds_wanted(ci, false); - if (need & ~(mds_wanted & need)) { - dout("get_cap_refs %p caps were dropped" - " (session killed?)\n", inode); - ret = -ESTALE; - goto out_unlock; - } - if (!(file_wanted & ~mds_wanted)) - ci->i_ceph_flags &= ~CEPH_I_CAP_DROPPED; + if (READ_ONCE(mdsc->fsc->mount_state) == CEPH_MOUNT_SHUTDOWN) { + dout("get_cap_refs %p forced umount\n", inode); + ret = -EIO; + goto out_unlock; + } + mds_wanted = __ceph_caps_mds_wanted(ci, false); + if (need & ~mds_wanted) { + dout("get_cap_refs %p need %s > mds_wanted %s\n", + inode, ceph_cap_string(need), + ceph_cap_string(mds_wanted)); + ret = -ESTALE; + goto out_unlock; } - dout("get_cap_refs %p have %s needed %s\n", inode, + dout("get_cap_refs %p have %s need %s\n", inode, ceph_cap_string(have), ceph_cap_string(need)); } out_unlock: @@ -3678,8 +3674,6 @@ retry: goto out_unlock; if (target < 0) { - if (cap->mds_wanted | cap->issued) - ci->i_ceph_flags |= CEPH_I_CAP_DROPPED; __ceph_remove_cap(cap, false); goto out_unlock; } diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index 177acb9f456b..68b8afded466 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -1514,8 +1514,6 @@ static int remove_session_caps_cb(struct inode *inode, struct ceph_cap *cap, dout("removing cap %p, ci is %p, inode is %p\n", cap, ci, &ci->vfs_inode); spin_lock(&ci->i_ceph_lock); - if (cap->mds_wanted | cap->issued) - ci->i_ceph_flags |= CEPH_I_CAP_DROPPED; __ceph_remove_cap(cap, false); if (!ci->i_auth_cap) { struct ceph_cap_flush *cf; @@ -1681,9 +1679,6 @@ static int wake_up_session_cb(struct inode *inode, struct ceph_cap *cap, /* mds did not re-issue stale cap */ spin_lock(&ci->i_ceph_lock); cap->issued = cap->implemented = CEPH_CAP_PIN; - /* make sure mds knows what we want */ - if (__ceph_caps_file_wanted(ci) & ~cap->mds_wanted) - ci->i_ceph_flags |= CEPH_I_CAP_DROPPED; spin_unlock(&ci->i_ceph_lock); } } else if (ev == FORCE_RO) { diff --git a/fs/ceph/super.h b/fs/ceph/super.h index 22b89eaeaad4..356ad7b46b85 100644 --- a/fs/ceph/super.h +++ b/fs/ceph/super.h @@ -522,13 +522,12 @@ static inline struct inode *ceph_find_inode(struct super_block *sb, #define CEPH_I_POOL_RD (1 << 4) /* can read from pool */ #define CEPH_I_POOL_WR (1 << 5) /* can write to pool */ #define CEPH_I_SEC_INITED (1 << 6) /* security initialized */ -#define CEPH_I_CAP_DROPPED (1 << 7) /* caps were forcibly dropped */ -#define CEPH_I_KICK_FLUSH (1 << 8) /* kick flushing caps */ -#define CEPH_I_FLUSH_SNAPS (1 << 9) /* need flush snapss */ -#define CEPH_I_ERROR_WRITE (1 << 10) /* have seen write errors */ -#define CEPH_I_ERROR_FILELOCK (1 << 11) /* have seen file lock errors */ -#define CEPH_I_ODIRECT (1 << 12) /* inode in direct I/O mode */ -#define CEPH_ASYNC_CREATE_BIT (13) /* async create in flight for this */ +#define CEPH_I_KICK_FLUSH (1 << 7) /* kick flushing caps */ +#define CEPH_I_FLUSH_SNAPS (1 << 8) /* need flush snapss */ +#define CEPH_I_ERROR_WRITE (1 << 9) /* have seen write errors */ +#define CEPH_I_ERROR_FILELOCK (1 << 10) /* have seen file lock errors */ +#define CEPH_I_ODIRECT (1 << 11) /* inode in direct I/O mode */ +#define CEPH_ASYNC_CREATE_BIT (12) /* async create in flight for this */ #define CEPH_I_ASYNC_CREATE (1 << CEPH_ASYNC_CREATE_BIT) /* From 719a2514e9bf313c3627078926d56bc2a8b290d1 Mon Sep 17 00:00:00 2001 From: "Yan, Zheng" Date: Thu, 5 Mar 2020 20:21:00 +0800 Subject: [PATCH 360/635] ceph: consider inode's last read/write when calculating wanted caps Add i_last_rd and i_last_wr to ceph_inode_info. These fields are used to track the last time the client acquired read/write caps for the inode. If there is no read/write on an inode for 'caps_wanted_delay_max' seconds, __ceph_caps_file_wanted() does not request caps for read/write even there are open files. Call __ceph_touch_fmode() for dir operations. __ceph_caps_file_wanted() calculates dir's wanted caps according to last dir read/modification. If there is recent dir read, dir inode wants CEPH_CAP_ANY_SHARED caps. If there is recent dir modification, also wants CEPH_CAP_FILE_EXCL. Readdir is a special case. Dir inode wants CEPH_CAP_FILE_EXCL after readdir, as with that, modifications do not need to release CEPH_CAP_FILE_SHARED or invalidate all dentry leases issued by readdir. Signed-off-by: "Yan, Zheng" Reviewed-by: Jeff Layton Signed-off-by: Ilya Dryomov --- fs/ceph/caps.c | 183 ++++++++++++++++++++++++----------- fs/ceph/dir.c | 21 ++-- fs/ceph/file.c | 21 ++-- fs/ceph/inode.c | 10 +- fs/ceph/ioctl.c | 2 + fs/ceph/mds_client.c | 11 ++- fs/ceph/super.h | 13 ++- include/linux/ceph/ceph_fs.h | 1 + 8 files changed, 188 insertions(+), 74 deletions(-) diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index a75e5eb3740e..aa5bb5a4de46 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -978,19 +978,67 @@ int __ceph_caps_used(struct ceph_inode_info *ci) return used; } +#define FMODE_WAIT_BIAS 1000 + /* * wanted, by virtue of open file modes */ int __ceph_caps_file_wanted(struct ceph_inode_info *ci) { - int i, bits = 0; - for (i = 0; i < CEPH_FILE_MODE_BITS; i++) { - if (ci->i_nr_by_mode[i]) - bits |= 1 << i; + const int PIN_SHIFT = ffs(CEPH_FILE_MODE_PIN); + const int RD_SHIFT = ffs(CEPH_FILE_MODE_RD); + const int WR_SHIFT = ffs(CEPH_FILE_MODE_WR); + const int LAZY_SHIFT = ffs(CEPH_FILE_MODE_LAZY); + struct ceph_mount_options *opt = + ceph_inode_to_client(&ci->vfs_inode)->mount_options; + unsigned long used_cutoff = jiffies - opt->caps_wanted_delay_max * HZ; + unsigned long idle_cutoff = jiffies - opt->caps_wanted_delay_min * HZ; + + if (S_ISDIR(ci->vfs_inode.i_mode)) { + int want = 0; + + /* use used_cutoff here, to keep dir's wanted caps longer */ + if (ci->i_nr_by_mode[RD_SHIFT] > 0 || + time_after(ci->i_last_rd, used_cutoff)) + want |= CEPH_CAP_ANY_SHARED; + + if (ci->i_nr_by_mode[WR_SHIFT] > 0 || + time_after(ci->i_last_wr, used_cutoff)) { + want |= CEPH_CAP_ANY_SHARED | CEPH_CAP_FILE_EXCL; + if (opt->flags & CEPH_MOUNT_OPT_ASYNC_DIROPS) + want |= CEPH_CAP_ANY_DIR_OPS; + } + + if (want || ci->i_nr_by_mode[PIN_SHIFT] > 0) + want |= CEPH_CAP_PIN; + + return want; + } else { + int bits = 0; + + if (ci->i_nr_by_mode[RD_SHIFT] > 0) { + if (ci->i_nr_by_mode[RD_SHIFT] >= FMODE_WAIT_BIAS || + time_after(ci->i_last_rd, used_cutoff)) + bits |= 1 << RD_SHIFT; + } else if (time_after(ci->i_last_rd, idle_cutoff)) { + bits |= 1 << RD_SHIFT; + } + + if (ci->i_nr_by_mode[WR_SHIFT] > 0) { + if (ci->i_nr_by_mode[WR_SHIFT] >= FMODE_WAIT_BIAS || + time_after(ci->i_last_wr, used_cutoff)) + bits |= 1 << WR_SHIFT; + } else if (time_after(ci->i_last_wr, idle_cutoff)) { + bits |= 1 << WR_SHIFT; + } + + /* check lazyio only when read/write is wanted */ + if ((bits & (CEPH_FILE_MODE_RDWR << 1)) && + ci->i_nr_by_mode[LAZY_SHIFT] > 0) + bits |= 1 << LAZY_SHIFT; + + return bits ? ceph_caps_for_mode(bits >> 1) : 0; } - if (bits == 0) - return 0; - return ceph_caps_for_mode(bits >> 1); } /* @@ -1032,14 +1080,6 @@ int __ceph_caps_mds_wanted(struct ceph_inode_info *ci, bool check) return mds_wanted; } -/* - * called under i_ceph_lock - */ -static int __ceph_is_single_caps(struct ceph_inode_info *ci) -{ - return rb_first(&ci->i_caps) == rb_last(&ci->i_caps); -} - int ceph_is_any_caps(struct inode *inode) { struct ceph_inode_info *ci = ceph_inode(inode); @@ -1877,10 +1917,6 @@ void ceph_check_caps(struct ceph_inode_info *ci, int flags, if (ci->i_ceph_flags & CEPH_I_FLUSH) flags |= CHECK_CAPS_FLUSH; - if (!(flags & CHECK_CAPS_AUTHONLY) || - (ci->i_auth_cap && __ceph_is_single_caps(ci))) - __cap_delay_cancel(mdsc, ci); - goto retry_locked; retry: spin_lock(&ci->i_ceph_lock); @@ -1907,9 +1943,7 @@ retry_locked: if (IS_RDONLY(inode)) { want = CEPH_CAP_ANY_SHARED; } else { - want = CEPH_CAP_ANY_SHARED | - CEPH_CAP_FILE_EXCL | - CEPH_CAP_ANY_DIR_OPS; + want |= CEPH_CAP_ANY_SHARED | CEPH_CAP_FILE_EXCL; } retain |= want; } else { @@ -2105,9 +2139,17 @@ ack: goto retry; /* retake i_ceph_lock and restart our cap scan. */ } - /* Reschedule delayed caps release if we delayed anything */ - if (delayed) - __cap_delay_requeue(mdsc, ci, false); + if (list_empty(&ci->i_cap_delay_list)) { + if (delayed) { + /* Reschedule delayed caps release if we delayed anything */ + __cap_delay_requeue(mdsc, ci, false); + } else if (__ceph_is_any_real_caps(ci) && + (file_wanted & ~CEPH_CAP_PIN) && + !(used & (CEPH_CAP_FILE_RD | CEPH_CAP_ANY_FILE_WR))) { + /* periodically re-calculate caps wanted by open files */ + __cap_delay_requeue(mdsc, ci, true); + } + } spin_unlock(&ci->i_ceph_lock); @@ -2573,8 +2615,9 @@ void ceph_take_cap_refs(struct ceph_inode_info *ci, int got, * FIXME: how does a 0 return differ from -EAGAIN? */ enum { - NON_BLOCKING = 1, - CHECK_FILELOCK = 2, + /* first 8 bits are reserved for CEPH_FILE_MODE_FOO */ + NON_BLOCKING = (1 << 8), + CHECK_FILELOCK = (1 << 9), }; static int try_get_cap_refs(struct inode *inode, int need, int want, @@ -2584,7 +2627,6 @@ static int try_get_cap_refs(struct inode *inode, int need, int want, struct ceph_mds_client *mdsc = ceph_inode_to_client(inode)->mdsc; int ret = 0; int have, implemented; - int file_wanted; bool snap_rwsem_locked = false; dout("get_cap_refs %p need %s want %s\n", inode, @@ -2600,15 +2642,6 @@ again: goto out_unlock; } - /* make sure file is actually open */ - file_wanted = __ceph_caps_file_wanted(ci); - if ((file_wanted & need) != need) { - dout("try_get_cap_refs need %s file_wanted %s, EBADF\n", - ceph_cap_string(need), ceph_cap_string(file_wanted)); - ret = -EBADF; - goto out_unlock; - } - /* finish pending truncate */ while (ci->i_truncate_pending) { spin_unlock(&ci->i_ceph_lock); @@ -2719,6 +2752,9 @@ again: ceph_cap_string(have), ceph_cap_string(need)); } out_unlock: + + __ceph_touch_fmode(ci, mdsc, flags); + spin_unlock(&ci->i_ceph_lock); if (snap_rwsem_locked) up_read(&mdsc->snap_rwsem); @@ -2756,10 +2792,20 @@ static void check_max_size(struct inode *inode, loff_t endoff) ceph_check_caps(ci, CHECK_CAPS_AUTHONLY, NULL); } +static inline int get_used_fmode(int caps) +{ + int fmode = 0; + if (caps & CEPH_CAP_FILE_RD) + fmode |= CEPH_FILE_MODE_RD; + if (caps & CEPH_CAP_FILE_WR) + fmode |= CEPH_FILE_MODE_WR; + return fmode; +} + int ceph_try_get_caps(struct inode *inode, int need, int want, bool nonblock, int *got) { - int ret; + int ret, flags; BUG_ON(need & ~CEPH_CAP_FILE_RD); BUG_ON(want & ~(CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_LAZYIO | @@ -2771,8 +2817,11 @@ int ceph_try_get_caps(struct inode *inode, int need, int want, return ret; } - ret = try_get_cap_refs(inode, need, want, 0, - (nonblock ? NON_BLOCKING : 0), got); + flags = get_used_fmode(need | want); + if (nonblock) + flags |= NON_BLOCKING; + + ret = try_get_cap_refs(inode, need, want, 0, flags, got); return ret == -EAGAIN ? 0 : ret; } @@ -2798,11 +2847,15 @@ int ceph_get_caps(struct file *filp, int need, int want, fi->filp_gen != READ_ONCE(fsc->filp_gen)) return -EBADF; + flags = get_used_fmode(need | want); + while (true) { if (endoff > 0) check_max_size(inode, endoff); - flags = atomic_read(&fi->num_locks) ? CHECK_FILELOCK : 0; + flags &= CEPH_FILE_MODE_MASK; + if (atomic_read(&fi->num_locks)) + flags |= CHECK_FILELOCK; _got = 0; ret = try_get_cap_refs(inode, need, want, endoff, flags, &_got); @@ -2822,6 +2875,8 @@ int ceph_get_caps(struct file *filp, int need, int want, list_add(&cw.list, &mdsc->cap_wait_list); spin_unlock(&mdsc->caps_list_lock); + /* make sure used fmode not timeout */ + ceph_get_fmode(ci, flags, FMODE_WAIT_BIAS); add_wait_queue(&ci->i_cap_wq, &wait); flags |= NON_BLOCKING; @@ -2835,6 +2890,7 @@ int ceph_get_caps(struct file *filp, int need, int want, } remove_wait_queue(&ci->i_cap_wq, &wait); + ceph_put_fmode(ci, flags, FMODE_WAIT_BIAS); spin_lock(&mdsc->caps_list_lock); list_del(&cw.list); @@ -2854,7 +2910,7 @@ int ceph_get_caps(struct file *filp, int need, int want, if (ret < 0) { if (ret == -ESTALE) { /* session was killed, try renew caps */ - ret = ceph_renew_caps(inode); + ret = ceph_renew_caps(inode, flags); if (ret == 0) continue; } @@ -4153,6 +4209,33 @@ void ceph_flush_dirty_caps(struct ceph_mds_client *mdsc) dout("flush_dirty_caps done\n"); } +void __ceph_touch_fmode(struct ceph_inode_info *ci, + struct ceph_mds_client *mdsc, int fmode) +{ + unsigned long now = jiffies; + if (fmode & CEPH_FILE_MODE_RD) + ci->i_last_rd = now; + if (fmode & CEPH_FILE_MODE_WR) + ci->i_last_wr = now; + /* queue periodic check */ + if (fmode && + __ceph_is_any_real_caps(ci) && + list_empty(&ci->i_cap_delay_list)) + __cap_delay_requeue(mdsc, ci, true); +} + +void ceph_get_fmode(struct ceph_inode_info *ci, int fmode, int count) +{ + int i; + int bits = (fmode << 1) | 1; + spin_lock(&ci->i_ceph_lock); + for (i = 0; i < CEPH_FILE_MODE_BITS; i++) { + if (bits & (1 << i)) + ci->i_nr_by_mode[i] += count; + } + spin_unlock(&ci->i_ceph_lock); +} + void __ceph_get_fmode(struct ceph_inode_info *ci, int fmode) { int i; @@ -4168,26 +4251,18 @@ void __ceph_get_fmode(struct ceph_inode_info *ci, int fmode) * we may need to release capabilities to the MDS (or schedule * their delayed release). */ -void ceph_put_fmode(struct ceph_inode_info *ci, int fmode) +void ceph_put_fmode(struct ceph_inode_info *ci, int fmode, int count) { - int i, last = 0; + int i; int bits = (fmode << 1) | 1; spin_lock(&ci->i_ceph_lock); for (i = 0; i < CEPH_FILE_MODE_BITS; i++) { if (bits & (1 << i)) { - BUG_ON(ci->i_nr_by_mode[i] == 0); - if (--ci->i_nr_by_mode[i] == 0) - last++; + BUG_ON(ci->i_nr_by_mode[i] < count); + ci->i_nr_by_mode[i] -= count; } } - dout("put_fmode %p fmode %d {%d,%d,%d,%d}\n", - &ci->vfs_inode, fmode, - ci->i_nr_by_mode[0], ci->i_nr_by_mode[1], - ci->i_nr_by_mode[2], ci->i_nr_by_mode[3]); spin_unlock(&ci->i_ceph_lock); - - if (last && ci->i_vino.snap == CEPH_NOSNAP) - ceph_check_caps(ci, 0, NULL); } /* diff --git a/fs/ceph/dir.c b/fs/ceph/dir.c index ee6b319e5481..d594c2627430 100644 --- a/fs/ceph/dir.c +++ b/fs/ceph/dir.c @@ -335,8 +335,11 @@ static int ceph_readdir(struct file *file, struct dir_context *ctx) ctx->pos = 2; } - /* can we use the dcache? */ spin_lock(&ci->i_ceph_lock); + /* request Fx cap. if have Fx, we don't need to release Fs cap + * for later create/unlink. */ + __ceph_touch_fmode(ci, mdsc, CEPH_FILE_MODE_WR); + /* can we use the dcache? */ if (ceph_test_mount_opt(fsc, DCACHE) && !ceph_test_mount_opt(fsc, NOASYNCREADDIR) && ceph_snap(inode) != CEPH_SNAPDIR && @@ -760,6 +763,7 @@ static struct dentry *ceph_lookup(struct inode *dir, struct dentry *dentry, ceph_test_mount_opt(fsc, DCACHE) && __ceph_dir_is_complete(ci) && (__ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 1))) { + __ceph_touch_fmode(ci, mdsc, CEPH_FILE_MODE_RD); spin_unlock(&ci->i_ceph_lock); dout(" dir %p complete, -ENOENT\n", dir); d_add(dentry, NULL); @@ -1621,7 +1625,8 @@ static int __dir_lease_try_check(const struct dentry *dentry) /* * Check if directory-wide content lease/cap is valid. */ -static int dir_lease_is_valid(struct inode *dir, struct dentry *dentry) +static int dir_lease_is_valid(struct inode *dir, struct dentry *dentry, + struct ceph_mds_client *mdsc) { struct ceph_inode_info *ci = ceph_inode(dir); int valid; @@ -1629,7 +1634,10 @@ static int dir_lease_is_valid(struct inode *dir, struct dentry *dentry) spin_lock(&ci->i_ceph_lock); valid = __ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 1); - shared_gen = atomic_read(&ci->i_shared_gen); + if (valid) { + __ceph_touch_fmode(ci, mdsc, CEPH_FILE_MODE_RD); + shared_gen = atomic_read(&ci->i_shared_gen); + } spin_unlock(&ci->i_ceph_lock); if (valid) { struct ceph_dentry_info *di; @@ -1655,6 +1663,7 @@ static int ceph_d_revalidate(struct dentry *dentry, unsigned int flags) int valid = 0; struct dentry *parent; struct inode *dir, *inode; + struct ceph_mds_client *mdsc; if (flags & LOOKUP_RCU) { parent = READ_ONCE(dentry->d_parent); @@ -1671,6 +1680,8 @@ static int ceph_d_revalidate(struct dentry *dentry, unsigned int flags) dout("d_revalidate %p '%pd' inode %p offset 0x%llx\n", dentry, dentry, inode, ceph_dentry(dentry)->offset); + mdsc = ceph_sb_to_client(dir->i_sb)->mdsc; + /* always trust cached snapped dentries, snapdir dentry */ if (ceph_snap(dir) != CEPH_NOSNAP) { dout("d_revalidate %p '%pd' inode %p is SNAPPED\n", dentry, @@ -1682,7 +1693,7 @@ static int ceph_d_revalidate(struct dentry *dentry, unsigned int flags) valid = dentry_lease_is_valid(dentry, flags); if (valid == -ECHILD) return valid; - if (valid || dir_lease_is_valid(dir, dentry)) { + if (valid || dir_lease_is_valid(dir, dentry, mdsc)) { if (inode) valid = ceph_is_any_caps(inode); else @@ -1691,8 +1702,6 @@ static int ceph_d_revalidate(struct dentry *dentry, unsigned int flags) } if (!valid) { - struct ceph_mds_client *mdsc = - ceph_sb_to_client(dir->i_sb)->mdsc; struct ceph_mds_request *req; int op, err; u32 mask; diff --git a/fs/ceph/file.c b/fs/ceph/file.c index a6c484123d98..6239cff491e2 100644 --- a/fs/ceph/file.c +++ b/fs/ceph/file.c @@ -213,7 +213,7 @@ static int ceph_init_file_info(struct inode *inode, struct file *file, struct ceph_dir_file_info *dfi = kmem_cache_zalloc(ceph_dir_file_cachep, GFP_KERNEL); if (!dfi) { - ceph_put_fmode(ci, fmode); /* clean up */ + ceph_put_fmode(ci, fmode, 1); /* clean up */ return -ENOMEM; } @@ -224,7 +224,7 @@ static int ceph_init_file_info(struct inode *inode, struct file *file, } else { fi = kmem_cache_zalloc(ceph_file_cachep, GFP_KERNEL); if (!fi) { - ceph_put_fmode(ci, fmode); /* clean up */ + ceph_put_fmode(ci, fmode, 1); /* clean up */ return -ENOMEM; } @@ -263,7 +263,7 @@ static int ceph_init_file(struct inode *inode, struct file *file, int fmode) case S_IFLNK: dout("init_file %p %p 0%o (symlink)\n", inode, file, inode->i_mode); - ceph_put_fmode(ceph_inode(inode), fmode); /* clean up */ + ceph_put_fmode(ceph_inode(inode), fmode, 1); /* clean up */ break; default: @@ -273,7 +273,7 @@ static int ceph_init_file(struct inode *inode, struct file *file, int fmode) * we need to drop the open ref now, since we don't * have .release set to ceph_release. */ - ceph_put_fmode(ceph_inode(inode), fmode); /* clean up */ + ceph_put_fmode(ceph_inode(inode), fmode, 1); /* clean up */ BUG_ON(inode->i_fop->release == ceph_release); /* call the proper open fop */ @@ -285,14 +285,15 @@ static int ceph_init_file(struct inode *inode, struct file *file, int fmode) /* * try renew caps after session gets killed. */ -int ceph_renew_caps(struct inode *inode) +int ceph_renew_caps(struct inode *inode, int fmode) { - struct ceph_mds_client *mdsc = ceph_sb_to_client(inode->i_sb)->mdsc; + struct ceph_mds_client *mdsc = ceph_inode_to_client(inode)->mdsc; struct ceph_inode_info *ci = ceph_inode(inode); struct ceph_mds_request *req; int err, flags, wanted; spin_lock(&ci->i_ceph_lock); + __ceph_touch_fmode(ci, mdsc, fmode); wanted = __ceph_caps_file_wanted(ci); if (__ceph_is_any_real_caps(ci) && (!(wanted & CEPH_CAP_ANY_WR) || ci->i_auth_cap)) { @@ -405,6 +406,7 @@ int ceph_open(struct inode *inode, struct file *file) } else if (ceph_snap(inode) != CEPH_NOSNAP && (ci->i_snap_caps & wanted) == wanted) { __ceph_get_fmode(ci, fmode); + __ceph_touch_fmode(ci, mdsc, fmode); spin_unlock(&ci->i_ceph_lock); return ceph_init_file(inode, file, fmode); } @@ -781,7 +783,7 @@ retry: } out_req: if (!req->r_err && req->r_target_inode) - ceph_put_fmode(ceph_inode(req->r_target_inode), req->r_fmode); + ceph_put_fmode(ceph_inode(req->r_target_inode), req->r_fmode, 1); ceph_mdsc_put_request(req); out_ctx: ceph_release_acl_sec_ctx(&as_ctx); @@ -798,7 +800,7 @@ int ceph_release(struct inode *inode, struct file *file) dout("release inode %p dir file %p\n", inode, file); WARN_ON(!list_empty(&dfi->file_info.rw_contexts)); - ceph_put_fmode(ci, dfi->file_info.fmode); + ceph_put_fmode(ci, dfi->file_info.fmode, 1); if (dfi->last_readdir) ceph_mdsc_put_request(dfi->last_readdir); @@ -810,7 +812,8 @@ int ceph_release(struct inode *inode, struct file *file) dout("release inode %p regular file %p\n", inode, file); WARN_ON(!list_empty(&fi->rw_contexts)); - ceph_put_fmode(ci, fi->fmode); + ceph_put_fmode(ci, fi->fmode, 1); + kmem_cache_free(ceph_file_cachep, fi); } diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c index 73f986efb1fd..a618e8ae9d0f 100644 --- a/fs/ceph/inode.c +++ b/fs/ceph/inode.c @@ -479,6 +479,7 @@ struct inode *ceph_alloc_inode(struct super_block *sb) ci->i_head_snapc = NULL; ci->i_snap_caps = 0; + ci->i_last_rd = ci->i_last_wr = jiffies - 3600 * HZ; for (i = 0; i < CEPH_FILE_MODE_BITS; i++) ci->i_nr_by_mode[i] = 0; @@ -639,7 +640,7 @@ int ceph_fill_file_size(struct inode *inode, int issued, if ((issued & (CEPH_CAP_FILE_CACHE| CEPH_CAP_FILE_BUFFER)) || mapping_mapped(inode->i_mapping) || - __ceph_caps_file_wanted(ci)) { + __ceph_is_file_opened(ci)) { ci->i_truncate_pending++; queue_trunc = 1; } @@ -1013,6 +1014,13 @@ int ceph_fill_inode(struct inode *inode, struct page *locked_page, fill_inline = true; } + if (cap_fmode >= 0) { + if (!info_caps) + pr_warn("mds issued no caps on %llx.%llx\n", + ceph_vinop(inode)); + __ceph_touch_fmode(ci, mdsc, cap_fmode); + } + spin_unlock(&ci->i_ceph_lock); if (fill_inline) diff --git a/fs/ceph/ioctl.c b/fs/ceph/ioctl.c index c90f03beb15d..6e061bf62ad4 100644 --- a/fs/ceph/ioctl.c +++ b/fs/ceph/ioctl.c @@ -243,11 +243,13 @@ static long ceph_ioctl_lazyio(struct file *file) struct ceph_file_info *fi = file->private_data; struct inode *inode = file_inode(file); struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_mds_client *mdsc = ceph_inode_to_client(inode)->mdsc; if ((fi->fmode & CEPH_FILE_MODE_LAZY) == 0) { spin_lock(&ci->i_ceph_lock); fi->fmode |= CEPH_FILE_MODE_LAZY; ci->i_nr_by_mode[ffs(CEPH_FILE_MODE_LAZY)]++; + __ceph_touch_fmode(ci, mdsc, fi->fmode); spin_unlock(&ci->i_ceph_lock); dout("ioctl_layzio: file %p marked lazy\n", file); diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index 68b8afded466..486f91f9685b 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -2486,7 +2486,8 @@ static struct ceph_msg *create_request_message(struct ceph_mds_client *mdsc, if (req->r_inode_drop) releases += ceph_encode_inode_release(&p, req->r_inode ? req->r_inode : d_inode(req->r_dentry), - mds, req->r_inode_drop, req->r_inode_unless, 0); + mds, req->r_inode_drop, req->r_inode_unless, + req->r_op == CEPH_MDS_OP_READDIR); if (req->r_dentry_drop) releases += ceph_encode_dentry_release(&p, req->r_dentry, req->r_parent, mds, req->r_dentry_drop, @@ -2833,7 +2834,13 @@ int ceph_mdsc_submit_request(struct ceph_mds_client *mdsc, struct inode *dir, if (req->r_inode) ceph_get_cap_refs(ceph_inode(req->r_inode), CEPH_CAP_PIN); if (req->r_parent) { - ceph_get_cap_refs(ceph_inode(req->r_parent), CEPH_CAP_PIN); + struct ceph_inode_info *ci = ceph_inode(req->r_parent); + int fmode = (req->r_op & CEPH_MDS_OP_WRITE) ? + CEPH_FILE_MODE_WR : CEPH_FILE_MODE_RD; + spin_lock(&ci->i_ceph_lock); + ceph_take_cap_refs(ci, CEPH_CAP_PIN, false); + __ceph_touch_fmode(ci, mdsc, fmode); + spin_unlock(&ci->i_ceph_lock); ihold(req->r_parent); } if (req->r_old_dentry_dir) diff --git a/fs/ceph/super.h b/fs/ceph/super.h index 356ad7b46b85..63dd8e61ba88 100644 --- a/fs/ceph/super.h +++ b/fs/ceph/super.h @@ -366,6 +366,8 @@ struct ceph_inode_info { dirty|flushing caps */ unsigned i_snap_caps; /* cap bits for snapped files */ + unsigned long i_last_rd; + unsigned long i_last_wr; int i_nr_by_mode[CEPH_FILE_MODE_BITS]; /* open file counts */ struct mutex i_truncate_mutex; @@ -680,6 +682,10 @@ extern int __ceph_caps_revoking_other(struct ceph_inode_info *ci, extern int ceph_caps_revoking(struct ceph_inode_info *ci, int mask); extern int __ceph_caps_used(struct ceph_inode_info *ci); +static inline bool __ceph_is_file_opened(struct ceph_inode_info *ci) +{ + return ci->i_nr_by_mode[0]; +} extern int __ceph_caps_file_wanted(struct ceph_inode_info *ci); extern int __ceph_caps_wanted(struct ceph_inode_info *ci); @@ -1093,7 +1099,10 @@ extern int ceph_try_get_caps(struct inode *inode, /* for counting open files by mode */ extern void __ceph_get_fmode(struct ceph_inode_info *ci, int mode); -extern void ceph_put_fmode(struct ceph_inode_info *ci, int mode); +extern void ceph_get_fmode(struct ceph_inode_info *ci, int mode, int count); +extern void ceph_put_fmode(struct ceph_inode_info *ci, int mode, int count); +extern void __ceph_touch_fmode(struct ceph_inode_info *ci, + struct ceph_mds_client *mdsc, int fmode); /* addr.c */ extern const struct address_space_operations ceph_aops; @@ -1105,7 +1114,7 @@ extern void ceph_pool_perm_destroy(struct ceph_mds_client* mdsc); /* file.c */ extern const struct file_operations ceph_file_fops; -extern int ceph_renew_caps(struct inode *inode); +extern int ceph_renew_caps(struct inode *inode, int fmode); extern int ceph_open(struct inode *inode, struct file *file); extern int ceph_atomic_open(struct inode *dir, struct dentry *dentry, struct file *file, unsigned flags, umode_t mode); diff --git a/include/linux/ceph/ceph_fs.h b/include/linux/ceph/ceph_fs.h index e035c5194005..ebf5ba62b772 100644 --- a/include/linux/ceph/ceph_fs.h +++ b/include/linux/ceph/ceph_fs.h @@ -568,6 +568,7 @@ struct ceph_filelock { #define CEPH_FILE_MODE_RDWR 3 /* RD | WR */ #define CEPH_FILE_MODE_LAZY 4 /* lazy io */ #define CEPH_FILE_MODE_BITS 4 +#define CEPH_FILE_MODE_MASK ((1 << CEPH_FILE_MODE_BITS) - 1) int ceph_flags_to_mode(int flags); From a0d93e327fc424b94e78a1b25287bb4e632ee054 Mon Sep 17 00:00:00 2001 From: "Yan, Zheng" Date: Thu, 5 Mar 2020 20:21:01 +0800 Subject: [PATCH 361/635] ceph: remove delay check logic from ceph_check_caps() __ceph_caps_file_wanted() already checks 'caps_wanted_delay_min' and 'caps_wanted_delay_max'. There is no need to duplicate the logic in ceph_check_caps() and __send_cap() Signed-off-by: "Yan, Zheng" Reviewed-by: Jeff Layton Signed-off-by: Ilya Dryomov --- fs/ceph/caps.c | 148 ++++++++++++------------------------------------ fs/ceph/file.c | 13 ++--- fs/ceph/inode.c | 1 - fs/ceph/super.h | 8 +-- 4 files changed, 44 insertions(+), 126 deletions(-) diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index aa5bb5a4de46..fe0f724d27e9 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -490,13 +490,10 @@ static void __cap_set_timeouts(struct ceph_mds_client *mdsc, struct ceph_inode_info *ci) { struct ceph_mount_options *opt = mdsc->fsc->mount_options; - - ci->i_hold_caps_min = round_jiffies(jiffies + - opt->caps_wanted_delay_min * HZ); ci->i_hold_caps_max = round_jiffies(jiffies + opt->caps_wanted_delay_max * HZ); - dout("__cap_set_timeouts %p min %lu max %lu\n", &ci->vfs_inode, - ci->i_hold_caps_min - jiffies, ci->i_hold_caps_max - jiffies); + dout("__cap_set_timeouts %p %lu\n", &ci->vfs_inode, + ci->i_hold_caps_max - jiffies); } /* @@ -508,8 +505,7 @@ static void __cap_set_timeouts(struct ceph_mds_client *mdsc, * -> we take mdsc->cap_delay_lock */ static void __cap_delay_requeue(struct ceph_mds_client *mdsc, - struct ceph_inode_info *ci, - bool set_timeout) + struct ceph_inode_info *ci) { dout("__cap_delay_requeue %p flags 0x%lx at %lu\n", &ci->vfs_inode, ci->i_ceph_flags, ci->i_hold_caps_max); @@ -520,8 +516,7 @@ static void __cap_delay_requeue(struct ceph_mds_client *mdsc, goto no_change; list_del_init(&ci->i_cap_delay_list); } - if (set_timeout) - __cap_set_timeouts(mdsc, ci); + __cap_set_timeouts(mdsc, ci); list_add_tail(&ci->i_cap_delay_list, &mdsc->cap_delay_list); no_change: spin_unlock(&mdsc->cap_delay_lock); @@ -733,7 +728,7 @@ void ceph_add_cap(struct inode *inode, dout(" issued %s, mds wanted %s, actual %s, queueing\n", ceph_cap_string(issued), ceph_cap_string(wanted), ceph_cap_string(actual_wanted)); - __cap_delay_requeue(mdsc, ci, true); + __cap_delay_requeue(mdsc, ci); } if (flags & CEPH_CAP_FLAG_AUTH) { @@ -1342,7 +1337,6 @@ static int __send_cap(struct ceph_mds_client *mdsc, struct ceph_cap *cap, struct cap_msg_args arg; int held, revoking; int wake = 0; - int delayed = 0; int ret; /* Don't send anything if it's still being created. Return delayed */ @@ -1362,28 +1356,7 @@ static int __send_cap(struct ceph_mds_client *mdsc, struct ceph_cap *cap, ceph_cap_string(revoking)); BUG_ON((retain & CEPH_CAP_PIN) == 0); - arg.session = cap->session; - - /* don't release wanted unless we've waited a bit. */ - if ((ci->i_ceph_flags & CEPH_I_NODELAY) == 0 && - time_before(jiffies, ci->i_hold_caps_min)) { - dout(" delaying issued %s -> %s, wanted %s -> %s on send\n", - ceph_cap_string(cap->issued), - ceph_cap_string(cap->issued & retain), - ceph_cap_string(cap->mds_wanted), - ceph_cap_string(want)); - want |= cap->mds_wanted; - retain |= cap->issued; - delayed = 1; - } - ci->i_ceph_flags &= ~(CEPH_I_NODELAY | CEPH_I_FLUSH); - if (want & ~cap->mds_wanted) { - /* user space may open/close single file frequently. - * This avoids droping mds_wanted immediately after - * requesting new mds_wanted. - */ - __cap_set_timeouts(mdsc, ci); - } + ci->i_ceph_flags &= ~CEPH_I_FLUSH; cap->issued &= retain; /* drop bits we don't want */ if (cap->implemented & ~cap->issued) { @@ -1398,6 +1371,7 @@ static int __send_cap(struct ceph_mds_client *mdsc, struct ceph_cap *cap, cap->implemented &= cap->issued | used; cap->mds_wanted = want; + arg.session = cap->session; arg.ino = ceph_vino(inode).ino; arg.cid = cap->cap_id; arg.follows = flushing ? ci->i_head_snapc->seq : 0; @@ -1458,14 +1432,19 @@ static int __send_cap(struct ceph_mds_client *mdsc, struct ceph_cap *cap, ret = send_cap_msg(&arg); if (ret < 0) { - dout("error sending cap msg, must requeue %p\n", inode); - delayed = 1; + pr_err("error sending cap msg, ino (%llx.%llx) " + "flushing %s tid %llu, requeue\n", + ceph_vinop(inode), ceph_cap_string(flushing), + flush_tid); + spin_lock(&ci->i_ceph_lock); + __cap_delay_requeue(mdsc, ci); + spin_unlock(&ci->i_ceph_lock); } if (wake) wake_up_all(&ci->i_cap_wq); - return delayed; + return ret; } static inline int __send_flush_snap(struct inode *inode, @@ -1731,7 +1710,7 @@ int __ceph_mark_dirty_caps(struct ceph_inode_info *ci, int mask, if (((was | ci->i_flushing_caps) & CEPH_CAP_FILE_BUFFER) && (mask & CEPH_CAP_FILE_BUFFER)) dirty |= I_DIRTY_DATASYNC; - __cap_delay_requeue(mdsc, ci, true); + __cap_delay_requeue(mdsc, ci); return dirty; } @@ -1883,8 +1862,6 @@ bool __ceph_should_report_size(struct ceph_inode_info *ci) * versus held caps. Release, flush, ack revoked caps to mds as * appropriate. * - * CHECK_CAPS_NODELAY - caller is delayed work and we should not delay - * cap release further. * CHECK_CAPS_AUTHONLY - we should only check the auth cap * CHECK_CAPS_FLUSH - we should flush any dirty caps immediately, without * further delay. @@ -1903,17 +1880,10 @@ void ceph_check_caps(struct ceph_inode_info *ci, int flags, int mds = -1; /* keep track of how far we've gone through i_caps list to avoid an infinite loop on retry */ struct rb_node *p; - int delayed = 0, sent = 0; - bool no_delay = flags & CHECK_CAPS_NODELAY; bool queue_invalidate = false; bool tried_invalidate = false; - /* if we are unmounting, flush any unused caps immediately. */ - if (mdsc->stopping) - no_delay = true; - spin_lock(&ci->i_ceph_lock); - if (ci->i_ceph_flags & CEPH_I_FLUSH) flags |= CHECK_CAPS_FLUSH; @@ -1960,14 +1930,13 @@ retry_locked: } dout("check_caps %p file_want %s used %s dirty %s flushing %s" - " issued %s revoking %s retain %s %s%s%s\n", inode, + " issued %s revoking %s retain %s %s%s\n", inode, ceph_cap_string(file_wanted), ceph_cap_string(used), ceph_cap_string(ci->i_dirty_caps), ceph_cap_string(ci->i_flushing_caps), ceph_cap_string(issued), ceph_cap_string(revoking), ceph_cap_string(retain), (flags & CHECK_CAPS_AUTHONLY) ? " AUTHONLY" : "", - (flags & CHECK_CAPS_NODELAY) ? " NODELAY" : "", (flags & CHECK_CAPS_FLUSH) ? " FLUSH" : ""); /* @@ -1975,7 +1944,7 @@ retry_locked: * have cached pages, but don't want them, then try to invalidate. * If we fail, it's because pages are locked.... try again later. */ - if ((!no_delay || mdsc->stopping) && + if ((!(flags & CHECK_CAPS_NOINVAL) || mdsc->stopping) && S_ISREG(inode->i_mode) && !(ci->i_wb_ref || ci->i_wrbuffer_ref) && /* no dirty pages... */ inode->i_data.nrpages && /* have cached pages */ @@ -2055,21 +2024,6 @@ retry_locked: if ((cap->issued & ~retain) == 0) continue; /* nope, all good */ - if (no_delay) - goto ack; - - /* delay? */ - if ((ci->i_ceph_flags & CEPH_I_NODELAY) == 0 && - time_before(jiffies, ci->i_hold_caps_max)) { - dout(" delaying issued %s -> %s, wanted %s -> %s\n", - ceph_cap_string(cap->issued), - ceph_cap_string(cap->issued & retain), - ceph_cap_string(cap->mds_wanted), - ceph_cap_string(want)); - delayed++; - continue; - } - ack: if (session && session != cap->session) { dout("oops, wrong session %p mutex\n", session); @@ -2130,25 +2084,19 @@ ack: } mds = cap->mds; /* remember mds, so we don't repeat */ - sent++; /* __send_cap drops i_ceph_lock */ - delayed += __send_cap(mdsc, cap, CEPH_CAP_OP_UPDATE, 0, - cap_used, want, retain, flushing, - flush_tid, oldest_flush_tid); + __send_cap(mdsc, cap, CEPH_CAP_OP_UPDATE, 0, cap_used, want, + retain, flushing, flush_tid, oldest_flush_tid); goto retry; /* retake i_ceph_lock and restart our cap scan. */ } - if (list_empty(&ci->i_cap_delay_list)) { - if (delayed) { - /* Reschedule delayed caps release if we delayed anything */ - __cap_delay_requeue(mdsc, ci, false); - } else if (__ceph_is_any_real_caps(ci) && - (file_wanted & ~CEPH_CAP_PIN) && - !(used & (CEPH_CAP_FILE_RD | CEPH_CAP_ANY_FILE_WR))) { - /* periodically re-calculate caps wanted by open files */ - __cap_delay_requeue(mdsc, ci, true); - } + /* periodically re-calculate caps wanted by open files */ + if (__ceph_is_any_real_caps(ci) && + list_empty(&ci->i_cap_delay_list) && + (file_wanted & ~CEPH_CAP_PIN) && + !(used & (CEPH_CAP_FILE_RD | CEPH_CAP_ANY_FILE_WR))) { + __cap_delay_requeue(mdsc, ci); } spin_unlock(&ci->i_ceph_lock); @@ -2178,7 +2126,6 @@ retry: retry_locked: if (ci->i_dirty_caps && ci->i_auth_cap) { struct ceph_cap *cap = ci->i_auth_cap; - int delayed; if (session != cap->session) { spin_unlock(&ci->i_ceph_lock); @@ -2207,18 +2154,10 @@ retry_locked: &oldest_flush_tid); /* __send_cap drops i_ceph_lock */ - delayed = __send_cap(mdsc, cap, CEPH_CAP_OP_FLUSH, - CEPH_CLIENT_CAPS_SYNC, - __ceph_caps_used(ci), - __ceph_caps_wanted(ci), - (cap->issued | cap->implemented), - flushing, flush_tid, oldest_flush_tid); - - if (delayed) { - spin_lock(&ci->i_ceph_lock); - __cap_delay_requeue(mdsc, ci, true); - spin_unlock(&ci->i_ceph_lock); - } + __send_cap(mdsc, cap, CEPH_CAP_OP_FLUSH, CEPH_CLIENT_CAPS_SYNC, + __ceph_caps_used(ci), __ceph_caps_wanted(ci), + (cap->issued | cap->implemented), + flushing, flush_tid, oldest_flush_tid); } else { if (!list_empty(&ci->i_cap_flush_list)) { struct ceph_cap_flush *cf = @@ -2422,22 +2361,13 @@ static void __kick_flushing_caps(struct ceph_mds_client *mdsc, if (cf->caps) { dout("kick_flushing_caps %p cap %p tid %llu %s\n", inode, cap, cf->tid, ceph_cap_string(cf->caps)); - ci->i_ceph_flags |= CEPH_I_NODELAY; - - ret = __send_cap(mdsc, cap, CEPH_CAP_OP_FLUSH, + __send_cap(mdsc, cap, CEPH_CAP_OP_FLUSH, (cf->tid < last_snap_flush ? CEPH_CLIENT_CAPS_PENDING_CAPSNAP : 0), __ceph_caps_used(ci), __ceph_caps_wanted(ci), (cap->issued | cap->implemented), cf->caps, cf->tid, oldest_flush_tid); - if (ret) { - pr_err("kick_flushing_caps: error sending " - "cap flush, ino (%llx.%llx) " - "tid %llu flushing %s\n", - ceph_vinop(inode), cf->tid, - ceph_cap_string(cf->caps)); - } } else { struct ceph_cap_snap *capsnap = container_of(cf, struct ceph_cap_snap, @@ -3059,7 +2989,7 @@ void ceph_put_cap_refs(struct ceph_inode_info *ci, int had) dout("put_cap_refs %p had %s%s%s\n", inode, ceph_cap_string(had), last ? " last" : "", put ? " put" : ""); - if (last && !flushsnaps) + if (last) ceph_check_caps(ci, 0, NULL); else if (flushsnaps) ceph_flush_snaps(ci, NULL); @@ -3478,10 +3408,10 @@ static void handle_cap_grant(struct inode *inode, wake_up_all(&ci->i_cap_wq); if (check_caps == 1) - ceph_check_caps(ci, CHECK_CAPS_NODELAY|CHECK_CAPS_AUTHONLY, + ceph_check_caps(ci, CHECK_CAPS_AUTHONLY | CHECK_CAPS_NOINVAL, session); else if (check_caps == 2) - ceph_check_caps(ci, CHECK_CAPS_NODELAY, session); + ceph_check_caps(ci, CHECK_CAPS_NOINVAL, session); else mutex_unlock(&session->s_mutex); } @@ -4156,7 +4086,6 @@ void ceph_check_delayed_caps(struct ceph_mds_client *mdsc) { struct inode *inode; struct ceph_inode_info *ci; - int flags = CHECK_CAPS_NODELAY; dout("check_delayed_caps\n"); while (1) { @@ -4176,7 +4105,7 @@ void ceph_check_delayed_caps(struct ceph_mds_client *mdsc) if (inode) { dout("check_delayed_caps on %p\n", inode); - ceph_check_caps(ci, flags, NULL); + ceph_check_caps(ci, 0, NULL); /* avoid calling iput_final() in tick thread */ ceph_async_iput(inode); } @@ -4201,7 +4130,7 @@ void ceph_flush_dirty_caps(struct ceph_mds_client *mdsc) ihold(inode); dout("flush_dirty_caps %p\n", inode); spin_unlock(&mdsc->cap_dirty_lock); - ceph_check_caps(ci, CHECK_CAPS_NODELAY|CHECK_CAPS_FLUSH, NULL); + ceph_check_caps(ci, CHECK_CAPS_FLUSH, NULL); iput(inode); spin_lock(&mdsc->cap_dirty_lock); } @@ -4221,7 +4150,7 @@ void __ceph_touch_fmode(struct ceph_inode_info *ci, if (fmode && __ceph_is_any_real_caps(ci) && list_empty(&ci->i_cap_delay_list)) - __cap_delay_requeue(mdsc, ci, true); + __cap_delay_requeue(mdsc, ci); } void ceph_get_fmode(struct ceph_inode_info *ci, int fmode, int count) @@ -4280,7 +4209,6 @@ int ceph_drop_caps_for_unlink(struct inode *inode) if (inode->i_nlink == 1) { drop |= ~(__ceph_caps_wanted(ci) | CEPH_CAP_PIN); - ci->i_ceph_flags |= CEPH_I_NODELAY; if (__ceph_caps_dirty(ci)) { struct ceph_mds_client *mdsc = ceph_inode_to_client(inode)->mdsc; @@ -4336,8 +4264,6 @@ int ceph_encode_inode_release(void **p, struct inode *inode, if (force || (cap->issued & drop)) { if (cap->issued & drop) { int wanted = __ceph_caps_wanted(ci); - if ((ci->i_ceph_flags & CEPH_I_NODELAY) == 0) - wanted |= cap->mds_wanted; dout("encode_inode_release %p cap %p " "%s -> %s, wanted %s -> %s\n", inode, cap, ceph_cap_string(cap->issued), diff --git a/fs/ceph/file.c b/fs/ceph/file.c index 6239cff491e2..39b364a51c17 100644 --- a/fs/ceph/file.c +++ b/fs/ceph/file.c @@ -1826,7 +1826,7 @@ retry_snap: if (dirty) __mark_inode_dirty(inode, dirty); if (ceph_quota_is_max_bytes_approaching(inode, iocb->ki_pos)) - ceph_check_caps(ci, CHECK_CAPS_NODELAY, NULL); + ceph_check_caps(ci, 0, NULL); } dout("aio_write %p %llx.%llx %llu~%u dropping cap refs on %s\n", @@ -2427,15 +2427,10 @@ static ssize_t __ceph_copy_file_range(struct file *src_file, loff_t src_off, inode_inc_iversion_raw(dst_inode); if (dst_off > size) { - int caps_flags = 0; - /* Let the MDS know about dst file size change */ - if (ceph_quota_is_max_bytes_approaching(dst_inode, dst_off)) - caps_flags |= CHECK_CAPS_NODELAY; - if (ceph_inode_set_size(dst_inode, dst_off)) - caps_flags |= CHECK_CAPS_AUTHONLY; - if (caps_flags) - ceph_check_caps(dst_ci, caps_flags, NULL); + if (ceph_inode_set_size(dst_inode, dst_off) || + ceph_quota_is_max_bytes_approaching(dst_inode, dst_off)) + ceph_check_caps(dst_ci, CHECK_CAPS_AUTHONLY, NULL); } /* Mark Fw dirty */ spin_lock(&dst_ci->i_ceph_lock); diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c index a618e8ae9d0f..c4dd534aaec3 100644 --- a/fs/ceph/inode.c +++ b/fs/ceph/inode.c @@ -472,7 +472,6 @@ struct inode *ceph_alloc_inode(struct super_block *sb) ci->i_prealloc_cap_flush = NULL; INIT_LIST_HEAD(&ci->i_cap_flush_list); init_waitqueue_head(&ci->i_cap_wq); - ci->i_hold_caps_min = 0; ci->i_hold_caps_max = 0; INIT_LIST_HEAD(&ci->i_cap_delay_list); INIT_LIST_HEAD(&ci->i_cap_snaps); diff --git a/fs/ceph/super.h b/fs/ceph/super.h index 63dd8e61ba88..edceedfee00a 100644 --- a/fs/ceph/super.h +++ b/fs/ceph/super.h @@ -173,9 +173,9 @@ struct ceph_cap { struct list_head caps_item; }; -#define CHECK_CAPS_NODELAY 1 /* do not delay any further */ -#define CHECK_CAPS_AUTHONLY 2 /* only check auth cap */ -#define CHECK_CAPS_FLUSH 4 /* flush any dirty caps */ +#define CHECK_CAPS_AUTHONLY 1 /* only check auth cap */ +#define CHECK_CAPS_FLUSH 2 /* flush any dirty caps */ +#define CHECK_CAPS_NOINVAL 4 /* don't invalidate pagecache */ struct ceph_cap_flush { u64 tid; @@ -357,7 +357,6 @@ struct ceph_inode_info { struct ceph_cap_flush *i_prealloc_cap_flush; struct list_head i_cap_flush_list; wait_queue_head_t i_cap_wq; /* threads waiting on a capability */ - unsigned long i_hold_caps_min; /* jiffies */ unsigned long i_hold_caps_max; /* jiffies */ struct list_head i_cap_delay_list; /* for delayed cap release to mds */ struct ceph_cap_reservation i_cap_migration_resv; @@ -518,7 +517,6 @@ static inline struct inode *ceph_find_inode(struct super_block *sb, * Ceph inode. */ #define CEPH_I_DIR_ORDERED (1 << 0) /* dentries in dir are ordered */ -#define CEPH_I_NODELAY (1 << 1) /* do not delay cap release */ #define CEPH_I_FLUSH (1 << 2) /* do not delay flush of dirty metadata */ #define CEPH_I_POOL_PERM (1 << 3) /* pool rd/wr bits are valid */ #define CEPH_I_POOL_RD (1 << 4) /* can read from pool */ From 135e671e54fcb46cb4f8b7be9673d3ea7b8ac315 Mon Sep 17 00:00:00 2001 From: "Yan, Zheng" Date: Thu, 5 Mar 2020 20:21:02 +0800 Subject: [PATCH 362/635] ceph: simplify calling of ceph_get_fmode() Originally, calling ceph_get_fmode() for open files is by thread that handles request reply. There is a small window between updating caps and and waking the request initiator. We need to prevent ceph_check_caps() from releasing wanted caps in the window. Previous patches made fill_inode() call __ceph_touch_fmode() for open file requests. This prevented ceph_check_caps() from releasing wanted caps for 'caps_wanted_delay_min' seconds, enough for request initiator to get woken up and call ceph_get_fmode(). This allows us to now call ceph_get_fmode() in ceph_open() instead. Signed-off-by: "Yan, Zheng" Reviewed-by: Jeff Layton Signed-off-by: Ilya Dryomov --- fs/ceph/caps.c | 26 +++----------------------- fs/ceph/file.c | 21 +++++---------------- fs/ceph/inode.c | 8 +------- fs/ceph/super.h | 3 +-- 4 files changed, 10 insertions(+), 48 deletions(-) diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index fe0f724d27e9..02dd2a9bd252 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -608,7 +608,7 @@ static void __check_cap_issue(struct ceph_inode_info *ci, struct ceph_cap *cap, */ void ceph_add_cap(struct inode *inode, struct ceph_mds_session *session, u64 cap_id, - int fmode, unsigned issued, unsigned wanted, + unsigned issued, unsigned wanted, unsigned seq, unsigned mseq, u64 realmino, int flags, struct ceph_cap **new_cap) { @@ -624,13 +624,6 @@ void ceph_add_cap(struct inode *inode, dout("add_cap %p mds%d cap %llx %s seq %d\n", inode, session->s_mds, cap_id, ceph_cap_string(issued), seq); - /* - * If we are opening the file, include file mode wanted bits - * in wanted. - */ - if (fmode >= 0) - wanted |= ceph_caps_for_mode(fmode); - spin_lock(&session->s_gen_ttl_lock); gen = session->s_cap_gen; spin_unlock(&session->s_gen_ttl_lock); @@ -755,9 +748,6 @@ void ceph_add_cap(struct inode *inode, cap->issue_seq = seq; cap->mseq = mseq; cap->cap_gen = gen; - - if (fmode >= 0) - __ceph_get_fmode(ci, fmode); } /* @@ -3707,7 +3697,7 @@ retry: /* add placeholder for the export tagert */ int flag = (cap == ci->i_auth_cap) ? CEPH_CAP_FLAG_AUTH : 0; tcap = new_cap; - ceph_add_cap(inode, tsession, t_cap_id, -1, issued, 0, + ceph_add_cap(inode, tsession, t_cap_id, issued, 0, t_seq - 1, t_mseq, (u64)-1, flag, &new_cap); if (!list_empty(&ci->i_cap_flush_list) && @@ -3812,7 +3802,7 @@ retry: __ceph_caps_issued(ci, &issued); issued |= __ceph_caps_dirty(ci); - ceph_add_cap(inode, session, cap_id, -1, caps, wanted, seq, mseq, + ceph_add_cap(inode, session, cap_id, caps, wanted, seq, mseq, realmino, CEPH_CAP_FLAG_AUTH, &new_cap); ocap = peer >= 0 ? __get_cap_for_mds(ci, peer) : NULL; @@ -4165,16 +4155,6 @@ void ceph_get_fmode(struct ceph_inode_info *ci, int fmode, int count) spin_unlock(&ci->i_ceph_lock); } -void __ceph_get_fmode(struct ceph_inode_info *ci, int fmode) -{ - int i; - int bits = (fmode << 1) | 1; - for (i = 0; i < CEPH_FILE_MODE_BITS; i++) { - if (bits & (1 << i)) - ci->i_nr_by_mode[i]++; - } -} - /* * Drop open file reference. If we were the last open file, * we may need to release capabilities to the MDS (or schedule diff --git a/fs/ceph/file.c b/fs/ceph/file.c index 39b364a51c17..4a5ccbb7e808 100644 --- a/fs/ceph/file.c +++ b/fs/ceph/file.c @@ -212,10 +212,8 @@ static int ceph_init_file_info(struct inode *inode, struct file *file, if (isdir) { struct ceph_dir_file_info *dfi = kmem_cache_zalloc(ceph_dir_file_cachep, GFP_KERNEL); - if (!dfi) { - ceph_put_fmode(ci, fmode, 1); /* clean up */ + if (!dfi) return -ENOMEM; - } file->private_data = dfi; fi = &dfi->file_info; @@ -223,15 +221,15 @@ static int ceph_init_file_info(struct inode *inode, struct file *file, dfi->readdir_cache_idx = -1; } else { fi = kmem_cache_zalloc(ceph_file_cachep, GFP_KERNEL); - if (!fi) { - ceph_put_fmode(ci, fmode, 1); /* clean up */ + if (!fi) return -ENOMEM; - } file->private_data = fi; } + ceph_get_fmode(ci, fmode, 1); fi->fmode = fmode; + spin_lock_init(&fi->rw_contexts_lock); INIT_LIST_HEAD(&fi->rw_contexts); fi->meta_err = errseq_sample(&ci->i_meta_err); @@ -263,7 +261,6 @@ static int ceph_init_file(struct inode *inode, struct file *file, int fmode) case S_IFLNK: dout("init_file %p %p 0%o (symlink)\n", inode, file, inode->i_mode); - ceph_put_fmode(ceph_inode(inode), fmode, 1); /* clean up */ break; default: @@ -273,7 +270,6 @@ static int ceph_init_file(struct inode *inode, struct file *file, int fmode) * we need to drop the open ref now, since we don't * have .release set to ceph_release. */ - ceph_put_fmode(ceph_inode(inode), fmode, 1); /* clean up */ BUG_ON(inode->i_fop->release == ceph_release); /* call the proper open fop */ @@ -327,7 +323,6 @@ int ceph_renew_caps(struct inode *inode, int fmode) req->r_inode = inode; ihold(inode); req->r_num_caps = 1; - req->r_fmode = -1; err = ceph_mdsc_do_request(mdsc, NULL, req); ceph_mdsc_put_request(req); @@ -373,9 +368,6 @@ int ceph_open(struct inode *inode, struct file *file) /* trivially open snapdir */ if (ceph_snap(inode) == CEPH_SNAPDIR) { - spin_lock(&ci->i_ceph_lock); - __ceph_get_fmode(ci, fmode); - spin_unlock(&ci->i_ceph_lock); return ceph_init_file(inode, file, fmode); } @@ -393,7 +385,7 @@ int ceph_open(struct inode *inode, struct file *file) dout("open %p fmode %d want %s issued %s using existing\n", inode, fmode, ceph_cap_string(wanted), ceph_cap_string(issued)); - __ceph_get_fmode(ci, fmode); + __ceph_touch_fmode(ci, mdsc, fmode); spin_unlock(&ci->i_ceph_lock); /* adjust wanted? */ @@ -405,7 +397,6 @@ int ceph_open(struct inode *inode, struct file *file) return ceph_init_file(inode, file, fmode); } else if (ceph_snap(inode) != CEPH_NOSNAP && (ci->i_snap_caps & wanted) == wanted) { - __ceph_get_fmode(ci, fmode); __ceph_touch_fmode(ci, mdsc, fmode); spin_unlock(&ci->i_ceph_lock); return ceph_init_file(inode, file, fmode); @@ -782,8 +773,6 @@ retry: err = finish_open(file, dentry, ceph_open); } out_req: - if (!req->r_err && req->r_target_inode) - ceph_put_fmode(ceph_inode(req->r_target_inode), req->r_fmode, 1); ceph_mdsc_put_request(req); out_ctx: ceph_release_acl_sec_ctx(&as_ctx); diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c index c4dd534aaec3..c7ff9f7067f6 100644 --- a/fs/ceph/inode.c +++ b/fs/ceph/inode.c @@ -970,7 +970,7 @@ int ceph_fill_inode(struct inode *inode, struct page *locked_page, if (ceph_snap(inode) == CEPH_NOSNAP) { ceph_add_cap(inode, session, le64_to_cpu(info->cap.cap_id), - cap_fmode, info_caps, + info_caps, le32_to_cpu(info->cap.wanted), le32_to_cpu(info->cap.seq), le32_to_cpu(info->cap.mseq), @@ -995,13 +995,7 @@ int ceph_fill_inode(struct inode *inode, struct page *locked_page, dout(" %p got snap_caps %s\n", inode, ceph_cap_string(info_caps)); ci->i_snap_caps |= info_caps; - if (cap_fmode >= 0) - __ceph_get_fmode(ci, cap_fmode); } - } else if (cap_fmode >= 0) { - pr_warn("mds issued no caps on %llx.%llx\n", - ceph_vinop(inode)); - __ceph_get_fmode(ci, cap_fmode); } if (iinfo->inline_version > 0 && diff --git a/fs/ceph/super.h b/fs/ceph/super.h index edceedfee00a..60aac3aee055 100644 --- a/fs/ceph/super.h +++ b/fs/ceph/super.h @@ -1050,7 +1050,7 @@ extern struct ceph_cap *ceph_get_cap(struct ceph_mds_client *mdsc, struct ceph_cap_reservation *ctx); extern void ceph_add_cap(struct inode *inode, struct ceph_mds_session *session, u64 cap_id, - int fmode, unsigned issued, unsigned wanted, + unsigned issued, unsigned wanted, unsigned cap, unsigned seq, u64 realmino, int flags, struct ceph_cap **new_cap); extern void __ceph_remove_cap(struct ceph_cap *cap, bool queue_release); @@ -1096,7 +1096,6 @@ extern int ceph_try_get_caps(struct inode *inode, int need, int want, bool nonblock, int *got); /* for counting open files by mode */ -extern void __ceph_get_fmode(struct ceph_inode_info *ci, int mode); extern void ceph_get_fmode(struct ceph_inode_info *ci, int mode, int count); extern void ceph_put_fmode(struct ceph_inode_info *ci, int mode, int count); extern void __ceph_touch_fmode(struct ceph_inode_info *ci, From 11ba6b9ceeb4e20e0610ad249f9187dba412f787 Mon Sep 17 00:00:00 2001 From: "Yan, Zheng" Date: Thu, 5 Mar 2020 20:21:03 +0800 Subject: [PATCH 363/635] ceph: update i_requested_max_size only when sending cap msg to auth mds Non-auth mds can't do anything to 'update max' cap message. Signed-off-by: "Yan, Zheng" Reviewed-by: Jeff Layton Signed-off-by: Ilya Dryomov --- fs/ceph/caps.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index 02dd2a9bd252..0b3a36310033 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -1371,7 +1371,8 @@ static int __send_cap(struct ceph_mds_client *mdsc, struct ceph_cap *cap, arg.size = inode->i_size; ci->i_reported_size = arg.size; arg.max_size = ci->i_wanted_max_size; - ci->i_requested_max_size = arg.max_size; + if (cap == ci->i_auth_cap) + ci->i_requested_max_size = arg.max_size; if (flushing & CEPH_CAP_XATTR_EXCL) { old_blob = __ceph_build_xattrs_blob(ci); From bf73c62e7ff8dc220f3f57df4e42f9947320c447 Mon Sep 17 00:00:00 2001 From: "Yan, Zheng" Date: Thu, 5 Mar 2020 20:21:04 +0800 Subject: [PATCH 364/635] ceph: check all mds' caps after page writeback If an inode has caps from multiple mds's, the following can happen: - non-auth mds revokes Fsc. Fcb is used, so page writeback is queued. - when writeback finishes, ceph_check_caps() is called with auth only flag. ceph_check_caps() invalidates pagecache, but skips checking any non-auth caps. Signed-off-by: "Yan, Zheng" Reviewed-by: Jeff Layton Signed-off-by: Ilya Dryomov --- fs/ceph/caps.c | 2 +- fs/ceph/inode.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index 0b3a36310033..342a32c74c64 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -3062,7 +3062,7 @@ void ceph_put_wrbuffer_cap_refs(struct ceph_inode_info *ci, int nr, spin_unlock(&ci->i_ceph_lock); if (last) { - ceph_check_caps(ci, CHECK_CAPS_AUTHONLY, NULL); + ceph_check_caps(ci, 0, NULL); } else if (flush_snaps) { ceph_flush_snaps(ci, NULL); } diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c index c7ff9f7067f6..ee40ba7e0e77 100644 --- a/fs/ceph/inode.c +++ b/fs/ceph/inode.c @@ -1984,7 +1984,7 @@ retry: mutex_unlock(&ci->i_truncate_mutex); if (wrbuffer_refs == 0) - ceph_check_caps(ci, CHECK_CAPS_AUTHONLY, NULL); + ceph_check_caps(ci, 0, NULL); wake_up_all(&ci->i_cap_wq); } From c6d50296032f0b97473eb2e274dc7cc5d0173847 Mon Sep 17 00:00:00 2001 From: Qiujun Huang Date: Fri, 6 Mar 2020 09:34:20 +0800 Subject: [PATCH 365/635] ceph: return ceph_mdsc_do_request() errors from __get_parent() Return the error returned by ceph_mdsc_do_request(). Otherwise, r_target_inode ends up being NULL this ends up returning ENOENT regardless of the error. Signed-off-by: Qiujun Huang Reviewed-by: Jeff Layton Signed-off-by: Ilya Dryomov --- fs/ceph/export.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fs/ceph/export.c b/fs/ceph/export.c index b6bfa94332c3..79dc06881e78 100644 --- a/fs/ceph/export.c +++ b/fs/ceph/export.c @@ -315,6 +315,11 @@ static struct dentry *__get_parent(struct super_block *sb, req->r_num_caps = 1; err = ceph_mdsc_do_request(mdsc, NULL, req); + if (err) { + ceph_mdsc_put_request(req); + return ERR_PTR(err); + } + inode = req->r_target_inode; if (inode) ihold(inode); From 546d402085d6ea3c13bdea62a98b48a53c18eed4 Mon Sep 17 00:00:00 2001 From: "Yan, Zheng" Date: Tue, 10 Mar 2020 19:34:18 +0800 Subject: [PATCH 366/635] ceph: cleanup return error of try_get_cap_refs() Returns 0 if caps were not able to be acquired (yet), 1 if cap acquisition succeeded, or a negative error code. There are 3 special error codes: -EAGAIN: need to sleep but non-blocking is specified -EFBIG: ask caller to call check_max_size() and try again. -ESTALE: ask caller to call ceph_renew_caps() and try again. [ jlayton: add WARN_ON_ONCE check for -EAGAIN ] Signed-off-by: "Yan, Zheng" Reviewed-by: Jeff Layton Signed-off-by: Ilya Dryomov --- fs/ceph/caps.c | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index 342a32c74c64..78a305068cac 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -2530,10 +2530,11 @@ void ceph_take_cap_refs(struct ceph_inode_info *ci, int got, * Note that caller is responsible for ensuring max_size increases are * requested from the MDS. * - * Returns 0 if caps were not able to be acquired (yet), a 1 if they were, - * or a negative error code. - * - * FIXME: how does a 0 return differ from -EAGAIN? + * Returns 0 if caps were not able to be acquired (yet), 1 if succeed, + * or a negative error code. There are 3 speical error codes: + * -EAGAIN: need to sleep but non-blocking is specified + * -EFBIG: ask caller to call check_max_size() and try again. + * -ESTALE: ask caller to call ceph_renew_caps() and try again. */ enum { /* first 8 bits are reserved for CEPH_FILE_MODE_FOO */ @@ -2581,7 +2582,7 @@ again: dout("get_cap_refs %p endoff %llu > maxsize %llu\n", inode, endoff, ci->i_max_size); if (endoff > ci->i_requested_max_size) - ret = -EAGAIN; + ret = -EFBIG; goto out_unlock; } /* @@ -2743,7 +2744,10 @@ int ceph_try_get_caps(struct inode *inode, int need, int want, flags |= NON_BLOCKING; ret = try_get_cap_refs(inode, need, want, 0, flags, got); - return ret == -EAGAIN ? 0 : ret; + /* three special error codes */ + if (ret == -EAGAIN || ret == -EFBIG || ret == -EAGAIN) + ret = 0; + return ret; } /* @@ -2771,17 +2775,13 @@ int ceph_get_caps(struct file *filp, int need, int want, flags = get_used_fmode(need | want); while (true) { - if (endoff > 0) - check_max_size(inode, endoff); - flags &= CEPH_FILE_MODE_MASK; if (atomic_read(&fi->num_locks)) flags |= CHECK_FILELOCK; _got = 0; ret = try_get_cap_refs(inode, need, want, endoff, flags, &_got); - if (ret == -EAGAIN) - continue; + WARN_ON_ONCE(ret == -EAGAIN); if (!ret) { struct ceph_mds_client *mdsc = fsc->mdsc; struct cap_wait cw; @@ -2829,6 +2829,10 @@ int ceph_get_caps(struct file *filp, int need, int want, } if (ret < 0) { + if (ret == -EFBIG) { + check_max_size(inode, endoff); + continue; + } if (ret == -ESTALE) { /* session was killed, try renew caps */ ret = ceph_renew_caps(inode, flags); From 42d70f8e31dfd3b795259b0d5c7ff4d4b45c24fd Mon Sep 17 00:00:00 2001 From: "Yan, Zheng" Date: Tue, 10 Mar 2020 19:34:19 +0800 Subject: [PATCH 367/635] ceph: request new max size only when there is auth cap When there is no auth cap, check_max_size() can't do anything and may cause an infinite loop inside ceph_get_caps(). Signed-off-by: "Yan, Zheng" Reviewed-by: Jeff Layton Signed-off-by: Ilya Dryomov --- fs/ceph/caps.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index 78a305068cac..086bb3adc08a 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -2582,7 +2582,7 @@ again: dout("get_cap_refs %p endoff %llu > maxsize %llu\n", inode, endoff, ci->i_max_size); if (endoff > ci->i_requested_max_size) - ret = -EFBIG; + ret = ci->i_auth_cap ? -EFBIG : -ESTALE; goto out_unlock; } /* From 0aa971b6fd3f92afef6afe24ef78d9bb14471519 Mon Sep 17 00:00:00 2001 From: "Yan, Zheng" Date: Tue, 10 Mar 2020 19:34:20 +0800 Subject: [PATCH 368/635] ceph: don't skip updating wanted caps when cap is stale 1. try_get_cap_refs() fails to get caps and finds that mds_wanted does not include what it wants. It returns -ESTALE. 2. ceph_get_caps() calls ceph_renew_caps(). ceph_renew_caps() finds that inode has cap, so it calls ceph_check_caps(). 3. ceph_check_caps() finds that issued caps (without checking if it's stale) already includes caps wanted by open file, so it skips updating wanted caps. Above events can cause an infinite loop inside ceph_get_caps(). Signed-off-by: "Yan, Zheng" Reviewed-by: Jeff Layton Signed-off-by: Ilya Dryomov --- fs/ceph/caps.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index 086bb3adc08a..be1ae4b92cfd 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -2008,8 +2008,12 @@ retry_locked: } /* want more caps from mds? */ - if (want & ~(cap->mds_wanted | cap->issued)) - goto ack; + if (want & ~cap->mds_wanted) { + if (want & ~(cap->mds_wanted | cap->issued)) + goto ack; + if (!__cap_is_valid(cap)) + goto ack; + } /* things we might delay */ if ((cap->issued & ~retain) == 0) From 9bccb76574ad9ba6f5868b09053179fa2e72f8e0 Mon Sep 17 00:00:00 2001 From: "Yan, Zheng" Date: Tue, 10 Mar 2020 19:34:21 +0800 Subject: [PATCH 369/635] ceph: wait for async creating inode before requesting new max size ceph_check_caps() can't request new max size for async creating inode. This may make ceph_get_caps() loop busily until getting reply of the async create. Also, wait for async creating reply before calling ceph_renew_caps(). Signed-off-by: "Yan, Zheng" Reviewed-by: Jeff Layton Signed-off-by: Ilya Dryomov --- fs/ceph/caps.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index be1ae4b92cfd..185db76300b3 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -2833,6 +2833,11 @@ int ceph_get_caps(struct file *filp, int need, int want, } if (ret < 0) { + if (ret == -EFBIG || ret == -ESTALE) { + int ret2 = ceph_wait_on_async_create(inode); + if (ret2 < 0) + return ret2; + } if (ret == -EFBIG) { check_max_size(inode, endoff); continue; From ef9157259fb7bb3bc2c61df227e36f1b861a4753 Mon Sep 17 00:00:00 2001 From: Luis Henriques Date: Wed, 11 Mar 2020 11:05:38 +0000 Subject: [PATCH 370/635] ceph: fix snapshot directory timestamps The .snap directory timestamps are kept at 0 (1970-01-01 00:00), which isn't consistent with what the fuse client does. This patch makes the behaviour consistent, by setting these timestamps (atime, btime, ctime, mtime) to those of the parent directory. Cc: Marc Roos Signed-off-by: Luis Henriques Reviewed-by: Jeff Layton Signed-off-by: Ilya Dryomov --- fs/ceph/inode.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c index ee40ba7e0e77..7fef94fd1e55 100644 --- a/fs/ceph/inode.c +++ b/fs/ceph/inode.c @@ -82,10 +82,14 @@ struct inode *ceph_get_snapdir(struct inode *parent) inode->i_mode = parent->i_mode; inode->i_uid = parent->i_uid; inode->i_gid = parent->i_gid; + inode->i_mtime = parent->i_mtime; + inode->i_ctime = parent->i_ctime; + inode->i_atime = parent->i_atime; inode->i_op = &ceph_snapdir_iops; inode->i_fop = &ceph_snapdir_fops; ci->i_snap_caps = CEPH_CAP_PIN; /* so we can open */ ci->i_rbytes = 0; + ci->i_btime = ceph_inode(parent)->i_btime; if (inode->i_state & I_NEW) unlock_new_inode(inode); From 98878d9dfc7acbad56c3433f4166472fee56884f Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Fri, 13 Mar 2020 13:07:27 -0500 Subject: [PATCH 371/635] dt-bindings: display: ti: Fix dtc unit-address warnings in examples Extra dtc warnings (roughly what W=1 enables) are now enabled by default when building the binding examples. These were fixed treewide in 5.6-rc5, but some new display bindings have been added with new warnings: Documentation/devicetree/bindings/display/ti/ti,am65x-dss.example.dts:21.27-49.11: Warning (unit_address_format): /example-0/dss@04a00000: unit name should not have leading 0s Documentation/devicetree/bindings/display/ti/ti,j721e-dss.example.dts:21.27-72.11: Warning (unit_address_format): /example-0/dss@04a00000: unit name should not have leading 0s Documentation/devicetree/bindings/display/ti/ti,k2g-dss.example.dts:20.27-42.11: Warning (unit_address_format): /example-0/dss@02540000: unit name should not have leading 0s Cc: Jyri Sarha Cc: Tomi Valkeinen Signed-off-by: Rob Herring Reviewed-by: Sam Ravnborg Acked-by: Jyri Sarha Link: https://patchwork.freedesktop.org/patch/msgid/20200313180727.23044-1-robh@kernel.org --- Documentation/devicetree/bindings/display/ti/ti,am65x-dss.yaml | 2 +- Documentation/devicetree/bindings/display/ti/ti,j721e-dss.yaml | 2 +- Documentation/devicetree/bindings/display/ti/ti,k2g-dss.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/display/ti/ti,am65x-dss.yaml b/Documentation/devicetree/bindings/display/ti/ti,am65x-dss.yaml index cac61a998203..aa5543a64526 100644 --- a/Documentation/devicetree/bindings/display/ti/ti,am65x-dss.yaml +++ b/Documentation/devicetree/bindings/display/ti/ti,am65x-dss.yaml @@ -121,7 +121,7 @@ examples: #include #include - dss: dss@04a00000 { + dss: dss@4a00000 { compatible = "ti,am65x-dss"; reg = <0x0 0x04a00000 0x0 0x1000>, /* common */ <0x0 0x04a02000 0x0 0x1000>, /* vidl1 */ diff --git a/Documentation/devicetree/bindings/display/ti/ti,j721e-dss.yaml b/Documentation/devicetree/bindings/display/ti/ti,j721e-dss.yaml index ade9b2f513f5..6d47cd7206c2 100644 --- a/Documentation/devicetree/bindings/display/ti/ti,j721e-dss.yaml +++ b/Documentation/devicetree/bindings/display/ti/ti,j721e-dss.yaml @@ -154,7 +154,7 @@ examples: #include #include - dss: dss@04a00000 { + dss: dss@4a00000 { compatible = "ti,j721e-dss"; reg = <0x00 0x04a00000 0x00 0x10000>, /* common_m */ <0x00 0x04a10000 0x00 0x10000>, /* common_s0*/ diff --git a/Documentation/devicetree/bindings/display/ti/ti,k2g-dss.yaml b/Documentation/devicetree/bindings/display/ti/ti,k2g-dss.yaml index 385bd060ccf9..7cb37053e95b 100644 --- a/Documentation/devicetree/bindings/display/ti/ti,k2g-dss.yaml +++ b/Documentation/devicetree/bindings/display/ti/ti,k2g-dss.yaml @@ -81,7 +81,7 @@ examples: #include #include - dss: dss@02540000 { + dss: dss@2540000 { compatible = "ti,k2g-dss"; reg = <0x02540000 0x400>, <0x02550000 0x1000>, From 1e361632da12ac00cb86c25a857ba251fdf2de95 Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Thu, 19 Mar 2020 18:07:37 -0500 Subject: [PATCH 372/635] libnvdimm/label: Replace zero-length array with flexible-array member The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertently introduced[3] to the codebase from now on. Also, notice that, dynamic memory allocations won't be affected by this change: "Flexible array members have incomplete type, and so the sizeof operator may not be applied. As a quirk of the original implementation of zero-length arrays, sizeof evaluates to zero."[1] This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Link: https://lore.kernel.org/r/20200319230737.GA16452@embeddedor.com Signed-off-by: Dan Williams --- drivers/nvdimm/label.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/nvdimm/label.h b/drivers/nvdimm/label.h index 4c7b775c2811..956b6d1bd8cc 100644 --- a/drivers/nvdimm/label.h +++ b/drivers/nvdimm/label.h @@ -62,7 +62,7 @@ struct nd_namespace_index { __le16 major; __le16 minor; __le64 checksum; - u8 free[0]; + u8 free[]; }; /** From 9106137c6f0d0d959a855ad6885c6b3cb010ff98 Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Thu, 19 Mar 2020 18:09:37 -0500 Subject: [PATCH 373/635] libnvdimm/region: Replace zero-length array with flexible-array member The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertently introduced[3] to the codebase from now on. Also, notice that, dynamic memory allocations won't be affected by this change: "Flexible array members have incomplete type, and so the sizeof operator may not be applied. As a quirk of the original implementation of zero-length arrays, sizeof evaluates to zero."[1] This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Link: https://lore.kernel.org/r/20200319230937.GA16648@embeddedor.com Signed-off-by: Dan Williams --- drivers/nvdimm/nd.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/nvdimm/nd.h b/drivers/nvdimm/nd.h index c9f6a5b5253a..2cf77bc859c1 100644 --- a/drivers/nvdimm/nd.h +++ b/drivers/nvdimm/nd.h @@ -39,7 +39,7 @@ struct nd_region_data { int ns_count; int ns_active; unsigned int hints_shift; - void __iomem *flush_wpq[0]; + void __iomem *flush_wpq[]; }; static inline void __iomem *ndrd_get_flush_wpq(struct nd_region_data *ndrd, @@ -156,7 +156,7 @@ struct nd_region { struct nd_interleave_set *nd_set; struct nd_percpu_lane __percpu *lane; int (*flush)(struct nd_region *nd_region, struct bio *bio); - struct nd_mapping mapping[0]; + struct nd_mapping mapping[]; }; struct nd_blk_region { From 9488ef75ef44ee4afdea2cfd2629101c4dc133c8 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 17 Mar 2020 14:10:49 +0100 Subject: [PATCH 374/635] docs: dt: display/ti: fix typos at the devicetree/ directory name The name of the devicetree directory is wrong on those three TI bindings: Signed-off-by: Mauro Carvalho Chehab Acked-by: Jyri Sarha Signed-off-by: Rob Herring Link: https://patchwork.freedesktop.org/patch/msgid/875b824ac97bd76dfe77b6227ff9b6b2671a6abf.1584450500.git.mchehab+huawei@kernel.org --- Documentation/devicetree/bindings/display/ti/ti,am65x-dss.yaml | 2 +- Documentation/devicetree/bindings/display/ti/ti,j721e-dss.yaml | 2 +- Documentation/devicetree/bindings/display/ti/ti,k2g-dss.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/display/ti/ti,am65x-dss.yaml b/Documentation/devicetree/bindings/display/ti/ti,am65x-dss.yaml index aa5543a64526..eb04c2330698 100644 --- a/Documentation/devicetree/bindings/display/ti/ti,am65x-dss.yaml +++ b/Documentation/devicetree/bindings/display/ti/ti,am65x-dss.yaml @@ -65,7 +65,7 @@ properties: ports: type: object description: - Ports as described in Documentation/devictree/bindings/graph.txt + Ports as described in Documentation/devicetree/bindings/graph.txt properties: "#address-cells": const: 1 diff --git a/Documentation/devicetree/bindings/display/ti/ti,j721e-dss.yaml b/Documentation/devicetree/bindings/display/ti/ti,j721e-dss.yaml index 6d47cd7206c2..eb4b1a266210 100644 --- a/Documentation/devicetree/bindings/display/ti/ti,j721e-dss.yaml +++ b/Documentation/devicetree/bindings/display/ti/ti,j721e-dss.yaml @@ -98,7 +98,7 @@ properties: ports: type: object description: - Ports as described in Documentation/devictree/bindings/graph.txt + Ports as described in Documentation/devicetree/bindings/graph.txt properties: "#address-cells": const: 1 diff --git a/Documentation/devicetree/bindings/display/ti/ti,k2g-dss.yaml b/Documentation/devicetree/bindings/display/ti/ti,k2g-dss.yaml index 7cb37053e95b..8f87b82c6695 100644 --- a/Documentation/devicetree/bindings/display/ti/ti,k2g-dss.yaml +++ b/Documentation/devicetree/bindings/display/ti/ti,k2g-dss.yaml @@ -56,7 +56,7 @@ properties: port: type: object description: - Port as described in Documentation/devictree/bindings/graph.txt. + Port as described in Documentation/devicetree/bindings/graph.txt. The DSS DPI output port node max-memory-bandwidth: From b139f8b00db4a8ea75a4174346eafa48041aa489 Mon Sep 17 00:00:00 2001 From: Qiujun Huang Date: Sun, 29 Mar 2020 16:56:47 +0800 Subject: [PATCH 375/635] fbcon: fix null-ptr-deref in fbcon_switch Set logo_shown to FBCON_LOGO_CANSHOW when the vc was deallocated. syzkaller report: https://lkml.org/lkml/2020/3/27/403 general protection fault, probably for non-canonical address 0xdffffc000000006c: 0000 [#1] SMP KASAN KASAN: null-ptr-deref in range [0x0000000000000360-0x0000000000000367] RIP: 0010:fbcon_switch+0x28f/0x1740 drivers/video/fbdev/core/fbcon.c:2260 Call Trace: redraw_screen+0x2a8/0x770 drivers/tty/vt/vt.c:1008 vc_do_resize+0xfe7/0x1360 drivers/tty/vt/vt.c:1295 fbcon_init+0x1221/0x1ab0 drivers/video/fbdev/core/fbcon.c:1219 visual_init+0x305/0x5c0 drivers/tty/vt/vt.c:1062 do_bind_con_driver+0x536/0x890 drivers/tty/vt/vt.c:3542 do_take_over_console+0x453/0x5b0 drivers/tty/vt/vt.c:4122 do_fbcon_takeover+0x10b/0x210 drivers/video/fbdev/core/fbcon.c:588 fbcon_fb_registered+0x26b/0x340 drivers/video/fbdev/core/fbcon.c:3259 do_register_framebuffer drivers/video/fbdev/core/fbmem.c:1664 [inline] register_framebuffer+0x56e/0x980 drivers/video/fbdev/core/fbmem.c:1832 dlfb_usb_probe.cold+0x1743/0x1ba3 drivers/video/fbdev/udlfb.c:1735 usb_probe_interface+0x310/0x800 drivers/usb/core/driver.c:374 accessing vc_cons[logo_shown].d->vc_top causes the bug. Reported-by: syzbot+732528bae351682f1f27@syzkaller.appspotmail.com Signed-off-by: Qiujun Huang Acked-by: Sam Ravnborg Cc: stable@vger.kernel.org Signed-off-by: Daniel Vetter Link: https://patchwork.freedesktop.org/patch/msgid/20200329085647.25133-1-hqjagain@gmail.com --- drivers/video/fbdev/core/fbcon.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c index 28335788e76e..9d28a8e3328f 100644 --- a/drivers/video/fbdev/core/fbcon.c +++ b/drivers/video/fbdev/core/fbcon.c @@ -1282,6 +1282,9 @@ finished: if (!con_is_bound(&fb_con)) fbcon_exit(); + if (vc->vc_num == logo_shown) + logo_shown = FBCON_LOGO_CANSHOW; + return; } From 4c1cb04e0e7ac4ba1ef5457929ef9b5671d9eed3 Mon Sep 17 00:00:00 2001 From: Ding Xiang Date: Mon, 30 Mar 2020 17:59:07 +0800 Subject: [PATCH 376/635] drm/xen: fix passing zero to 'PTR_ERR' warning Fix a static code checker warning: drivers/gpu/drm/xen/xen_drm_front.c:404 xen_drm_drv_dumb_create() warn: passing zero to 'PTR_ERR' Signed-off-by: Ding Xiang Reviewed-by: Oleksandr Andrushchenko Signed-off-by: Daniel Vetter Link: https://patchwork.freedesktop.org/patch/msgid/1585562347-30214-1-git-send-email-dingxiang@cmss.chinamobile.com --- drivers/gpu/drm/xen/xen_drm_front.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xen/xen_drm_front.c b/drivers/gpu/drm/xen/xen_drm_front.c index 4be49c1aef51..374142018171 100644 --- a/drivers/gpu/drm/xen/xen_drm_front.c +++ b/drivers/gpu/drm/xen/xen_drm_front.c @@ -401,7 +401,7 @@ static int xen_drm_drv_dumb_create(struct drm_file *filp, obj = xen_drm_front_gem_create(dev, args->size); if (IS_ERR_OR_NULL(obj)) { - ret = PTR_ERR(obj); + ret = PTR_ERR_OR_ZERO(obj); goto fail; } From d8a26d8fc37c5b8b9e95f2fa194f287cf8cab3f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 30 Mar 2020 14:30:41 +0200 Subject: [PATCH 377/635] drm/mm: revert "Break long searches in fragmented address spaces" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 7be1b9b8e9d1e9ef0342d2e001f44eec4030aa4d. The drm_mm is supposed to work in atomic context, so calling schedule() or in this case cond_resched() is illegal. Signed-off-by: Christian König Acked-by: Daniel Vetter Link: https://patchwork.freedesktop.org/patch/359278/ --- drivers/gpu/drm/drm_mm.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/drivers/gpu/drm/drm_mm.c b/drivers/gpu/drm/drm_mm.c index bc6e208949e8..8981abe8b7c9 100644 --- a/drivers/gpu/drm/drm_mm.c +++ b/drivers/gpu/drm/drm_mm.c @@ -45,7 +45,6 @@ #include #include #include -#include #include #include @@ -367,11 +366,6 @@ next_hole(struct drm_mm *mm, struct drm_mm_node *node, enum drm_mm_insert_mode mode) { - /* Searching is slow; check if we ran out of time/patience */ - cond_resched(); - if (fatal_signal_pending(current)) - return NULL; - switch (mode) { default: case DRM_MM_INSERT_BEST: @@ -563,7 +557,7 @@ int drm_mm_insert_node_in_range(struct drm_mm * const mm, return 0; } - return signal_pending(current) ? -ERESTARTSYS : -ENOSPC; + return -ENOSPC; } EXPORT_SYMBOL(drm_mm_insert_node_in_range); From 987ed8e938d11fcf4b90ab97cad71c7ffd272776 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 25 Mar 2020 17:06:14 +0800 Subject: [PATCH 378/635] drm/amdgpu: fix hpd bo size calculation error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit the HPD bo size calculation error. the "mem.size" can't present actual BO size all time. Signed-off-by: Kevin Wang Reviewed-by: Alex Deucher Acked-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c index c8f2aa1db13b..f6e3f59efa2f 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c @@ -1113,7 +1113,7 @@ static int gfx_v10_0_mec_init(struct amdgpu_device *adev) return r; } - memset(hpd, 0, adev->gfx.mec.hpd_eop_obj->tbo.mem.size); + memset(hpd, 0, mec_hpd_size); amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c index 37c8231f1407..608ffe3b684e 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c @@ -1946,7 +1946,7 @@ static int gfx_v9_0_mec_init(struct amdgpu_device *adev) return r; } - memset(hpd, 0, adev->gfx.mec.hpd_eop_obj->tbo.mem.size); + memset(hpd, 0, mec_hpd_size); amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); From 33eef72f8a781b227ba413c453c51e7703a1c481 Mon Sep 17 00:00:00 2001 From: Eric Bernstein Date: Fri, 6 Mar 2020 17:07:12 -0500 Subject: [PATCH 379/635] drm/amd/display: Update function to get optimal number of taps [Why] Diagnostics scaling test failing to set required number of vertical taps in 4:2:0 surface case [How] In dpp3_get_optimal_number_of_taps() need to use LB_MEMORY_CONFIG_3 for 4:2:0 surface case. In resource_build_scaling_params() make sure to also set plane res alpha enable based on updated surface state Signed-off-by: Eric Bernstein Reviewed-by: Dmytro Laktyushkin Acked-by: Rodrigo Siqueira Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/core/dc_resource.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c index 75c7ce4c7581..f4bcc71b2920 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c @@ -1077,6 +1077,7 @@ bool resource_build_scaling_params(struct pipe_ctx *pipe_ctx) * on certain displays, such as the Sharp 4k */ pipe_ctx->plane_res.scl_data.lb_params.depth = LB_PIXEL_DEPTH_30BPP; + pipe_ctx->plane_res.scl_data.lb_params.alpha_en = plane_state->per_pixel_alpha; pipe_ctx->plane_res.scl_data.recout.x += timing->h_border_left; pipe_ctx->plane_res.scl_data.recout.y += timing->v_border_top; From cbec6477ced5638c6121ee1110f7f457575be954 Mon Sep 17 00:00:00 2001 From: Stylon Wang Date: Fri, 6 Mar 2020 09:55:29 -0500 Subject: [PATCH 380/635] drm/amd/display: Support P010 pixel format [Why] P010 pixel format is not declared as supported in DRM and DM. [How] Add P010 format to the support list presented to DRM and checked in DM Signed-off-by: Stylon Wang Reviewed-by: Nicholas Kazlauskas Acked-by: Rodrigo Siqueira Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c | 5 +++++ drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c | 3 ++- drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c | 3 ++- drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c | 3 ++- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index d3674d805a0a..ad14c1a30843 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -3639,6 +3639,9 @@ fill_dc_plane_info_and_addr(struct amdgpu_device *adev, case DRM_FORMAT_NV12: plane_info->format = SURFACE_PIXEL_FORMAT_VIDEO_420_YCrCb; break; + case DRM_FORMAT_P010: + plane_info->format = SURFACE_PIXEL_FORMAT_VIDEO_420_10bpc_YCrCb; + break; default: DRM_ERROR( "Unsupported screen format %s\n", @@ -5535,6 +5538,8 @@ static int get_plane_formats(const struct drm_plane *plane, if (plane_cap && plane_cap->pixel_format_support.nv12) formats[num_formats++] = DRM_FORMAT_NV12; + if (plane_cap && plane_cap->pixel_format_support.p010) + formats[num_formats++] = DRM_FORMAT_P010; break; case DRM_PLANE_TYPE_OVERLAY: diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c index 261bdc3a8218..8b7122249ddc 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c @@ -552,7 +552,8 @@ static const struct dc_plane_cap plane_cap = { .pixel_format_support = { .argb8888 = true, .nv12 = true, - .fp16 = true + .fp16 = true, + .p010 = true }, .max_upscale_factor = { diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c index a67395208991..54e096055c6f 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c @@ -1012,7 +1012,8 @@ static const struct dc_plane_cap plane_cap = { .pixel_format_support = { .argb8888 = true, .nv12 = true, - .fp16 = true + .fp16 = true, + .p010 = true }, .max_upscale_factor = { diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c index 51b5910cd05f..deda1a6b603b 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c @@ -838,7 +838,8 @@ static const struct dc_plane_cap plane_cap = { .pixel_format_support = { .argb8888 = true, .nv12 = true, - .fp16 = true + .fp16 = true, + .p010 = true }, .max_upscale_factor = { From 8f43965f79ea7eabd2ea1a341670630ab4d0d70e Mon Sep 17 00:00:00 2001 From: Nicholas Kazlauskas Date: Thu, 5 Mar 2020 14:43:00 -0500 Subject: [PATCH 381/635] drm/amd/display: Use double buffered DRR timing update by default [Why] For some monitors extreme flickering can occur while using LFC for if we're not doing the DRR timing update for V_TOTAL_MIN / V_TOTAL_MAX at the DP start of frame. Hardware can default to any time in the frame which isn't the behavior we want. [How] Add a new function for setting the double buffering mode for DRR timing. Default to DP start of frame double buffering on timing generator init. Signed-off-by: Nicholas Kazlauskas Reviewed-by: Aric Cyr Acked-by: Rodrigo Siqueira Signed-off-by: Alex Deucher --- .../gpu/drm/amd/display/dc/dcn10/dcn10_optc.c | 18 ++++++++++++++++++ .../gpu/drm/amd/display/dc/dcn10/dcn10_optc.h | 3 +++ 2 files changed, 21 insertions(+) diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c index 63acb8ff7462..17d96ec6acd8 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c @@ -342,6 +342,23 @@ void optc1_set_blank_data_double_buffer(struct timing_generator *optc, bool enab OTG_BLANK_DATA_DOUBLE_BUFFER_EN, blank_data_double_buffer_enable); } +/** + * optc1_set_timing_double_buffer() - DRR double buffering control + * + * Sets double buffer point for V_TOTAL, H_TOTAL, VTOTAL_MIN, + * VTOTAL_MAX, VTOTAL_MIN_SEL and VTOTAL_MAX_SEL registers. + * + * Options: any time, start of frame, dp start of frame (range timing) + */ +void optc1_set_timing_double_buffer(struct timing_generator *optc, bool enable) +{ + struct optc *optc1 = DCN10TG_FROM_TG(optc); + uint32_t mode = enable ? 2 : 0; + + REG_UPDATE(OTG_DOUBLE_BUFFER_CONTROL, + OTG_RANGE_TIMING_DBUF_UPDATE_MODE, mode); +} + /** * unblank_crtc * Call ASIC Control Object to UnBlank CRTC. @@ -1353,6 +1370,7 @@ void optc1_clear_optc_underflow(struct timing_generator *optc) void optc1_tg_init(struct timing_generator *optc) { optc1_set_blank_data_double_buffer(optc, true); + optc1_set_timing_double_buffer(optc, true); optc1_clear_optc_underflow(optc); } diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h index f277656d5464..9a459a8fe8a0 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h @@ -185,6 +185,7 @@ struct dcn_optc_registers { SF(OTG0_OTG_GLOBAL_CONTROL0, OTG_MASTER_UPDATE_LOCK_SEL, mask_sh),\ SF(OTG0_OTG_DOUBLE_BUFFER_CONTROL, OTG_UPDATE_PENDING, mask_sh),\ SF(OTG0_OTG_DOUBLE_BUFFER_CONTROL, OTG_BLANK_DATA_DOUBLE_BUFFER_EN, mask_sh),\ + SF(OTG0_OTG_DOUBLE_BUFFER_CONTROL, OTG_RANGE_TIMING_DBUF_UPDATE_MODE, mask_sh),\ SF(OTG0_OTG_H_TOTAL, OTG_H_TOTAL, mask_sh),\ SF(OTG0_OTG_H_BLANK_START_END, OTG_H_BLANK_START, mask_sh),\ SF(OTG0_OTG_H_BLANK_START_END, OTG_H_BLANK_END, mask_sh),\ @@ -643,6 +644,8 @@ bool optc1_is_optc_underflow_occurred(struct timing_generator *optc); void optc1_set_blank_data_double_buffer(struct timing_generator *optc, bool enable); +void optc1_set_timing_double_buffer(struct timing_generator *optc, bool enable); + bool optc1_get_otg_active_size(struct timing_generator *optc, uint32_t *otg_active_width, uint32_t *otg_active_height); From 9941b8129030c9202aaf39114477a0e58c0d6ffc Mon Sep 17 00:00:00 2001 From: Yongqiang Sun Date: Mon, 9 Mar 2020 17:13:02 -0400 Subject: [PATCH 382/635] drm/amd/display: Not doing optimize bandwidth if flip pending. [Why] In some scenario like 1366x768 VSR enabled connected with a 4K monitor and playing 4K video in clone mode, underflow will be observed due to decrease dppclk when previouse surface scan isn't finished [How] In this use case, surface flip is switching between 4K and 1366x768, 1366x768 needs smaller dppclk, and when decrease the clk and previous surface scan is for 4K and scan isn't done, underflow will happen. Not doing optimize bandwidth in case of flip pending. Signed-off-by: Yongqiang Sun Reviewed-by: Tony Cheng Acked-by: Rodrigo Siqueira Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/core/dc.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index 2ffb22177df9..1a2c2e3dd6d2 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -1360,6 +1360,26 @@ bool dc_commit_state(struct dc *dc, struct dc_state *context) return (result == DC_OK); } +static bool is_flip_pending_in_pipes(struct dc *dc, struct dc_state *context) +{ + int i; + struct pipe_ctx *pipe; + + for (i = 0; i < MAX_PIPES; i++) { + pipe = &context->res_ctx.pipe_ctx[i]; + + if (!pipe->plane_state) + continue; + + /* Must set to false to start with, due to OR in update function */ + pipe->plane_state->status.is_flip_pending = false; + dc->hwss.update_pending_status(pipe); + if (pipe->plane_state->status.is_flip_pending) + return true; + } + return false; +} + bool dc_post_update_surfaces_to_stream(struct dc *dc) { int i; @@ -1370,6 +1390,9 @@ bool dc_post_update_surfaces_to_stream(struct dc *dc) post_surface_trace(dc); + if (is_flip_pending_in_pipes(dc, context)) + return true; + for (i = 0; i < dc->res_pool->pipe_count; i++) if (context->res_ctx.pipe_ctx[i].stream == NULL || context->res_ctx.pipe_ctx[i].plane_state == NULL) { From 58edb079102efa2b4029e5fe3dbdad78079bf425 Mon Sep 17 00:00:00 2001 From: Isabel Zhang Date: Wed, 11 Mar 2020 15:59:41 -0400 Subject: [PATCH 383/635] drm/amd/display: Revert change to HDCP display states [Why] Change is causing a regression where the OPC app no longer functions properly. [How] Revert the changelist causing the issue. Signed-off-by: Isabel Zhang Reviewed-by: Yongqiang Sun Acked-by: Rodrigo Siqueira Signed-off-by: Alex Deucher --- .../gpu/drm/amd/display/modules/hdcp/hdcp.c | 5 +-- .../gpu/drm/amd/display/modules/hdcp/hdcp.h | 28 ++++++++++--- .../display/modules/hdcp/hdcp1_execution.c | 2 +- .../display/modules/hdcp/hdcp2_execution.c | 2 +- .../drm/amd/display/modules/hdcp/hdcp_psp.c | 39 +++++++++++-------- .../drm/amd/display/modules/inc/mod_hdcp.h | 1 + 6 files changed, 49 insertions(+), 28 deletions(-) diff --git a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp.c b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp.c index e9fbd94f8635..cc1d3f470b99 100644 --- a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp.c +++ b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp.c @@ -328,8 +328,7 @@ enum mod_hdcp_status mod_hdcp_add_display(struct mod_hdcp *hdcp, /* add display to connection */ hdcp->connection.link = *link; *display_container = *display; - status = mod_hdcp_add_display_to_topology(hdcp, display_container); - + status = mod_hdcp_add_display_to_topology(hdcp, display->index); if (status != MOD_HDCP_STATUS_SUCCESS) goto out; @@ -375,7 +374,7 @@ enum mod_hdcp_status mod_hdcp_remove_display(struct mod_hdcp *hdcp, status = mod_hdcp_remove_display_from_topology(hdcp, index); if (status != MOD_HDCP_STATUS_SUCCESS) goto out; - memset(display, 0, sizeof(struct mod_hdcp_display)); + display->state = MOD_HDCP_DISPLAY_INACTIVE; /* request authentication when connection is not reset */ if (current_state(hdcp) != HDCP_UNINITIALIZED) diff --git a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp.h b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp.h index 60ff1a0028ac..5cb4546be0ef 100644 --- a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp.h +++ b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp.h @@ -328,7 +328,7 @@ void mod_hdcp_dump_binary_message(uint8_t *msg, uint32_t msg_size, /* psp functions */ enum mod_hdcp_status mod_hdcp_add_display_to_topology( - struct mod_hdcp *hdcp, struct mod_hdcp_display *display); + struct mod_hdcp *hdcp, uint8_t index); enum mod_hdcp_status mod_hdcp_remove_display_from_topology( struct mod_hdcp *hdcp, uint8_t index); enum mod_hdcp_status mod_hdcp_hdcp1_create_session(struct mod_hdcp *hdcp); @@ -503,6 +503,11 @@ static inline uint8_t is_display_active(struct mod_hdcp_display *display) return display->state >= MOD_HDCP_DISPLAY_ACTIVE; } +static inline uint8_t is_display_added(struct mod_hdcp_display *display) +{ + return display->state >= MOD_HDCP_DISPLAY_ACTIVE_AND_ADDED; +} + static inline uint8_t is_display_encryption_enabled(struct mod_hdcp_display *display) { return display->state >= MOD_HDCP_DISPLAY_ENCRYPTION_ENABLED; @@ -510,23 +515,34 @@ static inline uint8_t is_display_encryption_enabled(struct mod_hdcp_display *dis static inline uint8_t get_active_display_count(struct mod_hdcp *hdcp) { - uint8_t active_count = 0; + uint8_t added_count = 0; uint8_t i; for (i = 0; i < MAX_NUM_OF_DISPLAYS; i++) if (is_display_active(&hdcp->displays[i])) - active_count++; - return active_count; + added_count++; + return added_count; } -static inline struct mod_hdcp_display *get_first_active_display( +static inline uint8_t get_added_display_count(struct mod_hdcp *hdcp) +{ + uint8_t added_count = 0; + uint8_t i; + + for (i = 0; i < MAX_NUM_OF_DISPLAYS; i++) + if (is_display_added(&hdcp->displays[i])) + added_count++; + return added_count; +} + +static inline struct mod_hdcp_display *get_first_added_display( struct mod_hdcp *hdcp) { uint8_t i; struct mod_hdcp_display *display = NULL; for (i = 0; i < MAX_NUM_OF_DISPLAYS; i++) - if (is_display_active(&hdcp->displays[i])) { + if (is_display_added(&hdcp->displays[i])) { display = &hdcp->displays[i]; break; } diff --git a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp1_execution.c b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp1_execution.c index f244b72e74e0..37c8c05497d6 100644 --- a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp1_execution.c +++ b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp1_execution.c @@ -129,7 +129,7 @@ static inline uint8_t get_device_count(struct mod_hdcp *hdcp) static inline enum mod_hdcp_status check_device_count(struct mod_hdcp *hdcp) { /* device count must be greater than or equal to tracked hdcp displays */ - return (get_device_count(hdcp) < get_active_display_count(hdcp)) ? + return (get_device_count(hdcp) < get_added_display_count(hdcp)) ? MOD_HDCP_STATUS_HDCP1_DEVICE_COUNT_MISMATCH_FAILURE : MOD_HDCP_STATUS_SUCCESS; } diff --git a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp2_execution.c b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp2_execution.c index 549c113abcf7..491c00f48026 100644 --- a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp2_execution.c +++ b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp2_execution.c @@ -208,7 +208,7 @@ static inline uint8_t get_device_count(struct mod_hdcp *hdcp) static enum mod_hdcp_status check_device_count(struct mod_hdcp *hdcp) { /* device count must be greater than or equal to tracked hdcp displays */ - return (get_device_count(hdcp) < get_active_display_count(hdcp)) ? + return (get_device_count(hdcp) < get_added_display_count(hdcp)) ? MOD_HDCP_STATUS_HDCP2_DEVICE_COUNT_MISMATCH_FAILURE : MOD_HDCP_STATUS_SUCCESS; } diff --git a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_psp.c b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_psp.c index 836e47954938..c2929815c3ee 100644 --- a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_psp.c +++ b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_psp.c @@ -54,7 +54,7 @@ enum mod_hdcp_status mod_hdcp_remove_display_from_topology( dtm_cmd = (struct ta_dtm_shared_memory *)psp->dtm_context.dtm_shared_buf; - if (!display || !is_display_active(display)) + if (!display || !is_display_added(display)) return MOD_HDCP_STATUS_DISPLAY_NOT_FOUND; memset(dtm_cmd, 0, sizeof(struct ta_dtm_shared_memory)); @@ -73,21 +73,25 @@ enum mod_hdcp_status mod_hdcp_remove_display_from_topology( HDCP_TOP_REMOVE_DISPLAY_TRACE(hdcp, display->index); return MOD_HDCP_STATUS_SUCCESS; - } - -enum mod_hdcp_status mod_hdcp_add_display_to_topology( - struct mod_hdcp *hdcp, struct mod_hdcp_display *display) + +} +enum mod_hdcp_status mod_hdcp_add_display_to_topology(struct mod_hdcp *hdcp, + uint8_t index) { struct psp_context *psp = hdcp->config.psp.handle; struct ta_dtm_shared_memory *dtm_cmd; + struct mod_hdcp_display *display = + get_active_display_at_index(hdcp, index); struct mod_hdcp_link *link = &hdcp->connection.link; if (!psp->dtm_context.dtm_initialized) { DRM_ERROR("Failed to add display topology, DTM TA is not initialized."); - display->state = MOD_HDCP_DISPLAY_INACTIVE; return MOD_HDCP_STATUS_FAILURE; } + if (!display || is_display_added(display)) + return MOD_HDCP_STATUS_UPDATE_TOPOLOGY_FAILURE; + dtm_cmd = (struct ta_dtm_shared_memory *)psp->dtm_context.dtm_shared_buf; memset(dtm_cmd, 0, sizeof(struct ta_dtm_shared_memory)); @@ -109,11 +113,10 @@ enum mod_hdcp_status mod_hdcp_add_display_to_topology( psp_dtm_invoke(psp, dtm_cmd->cmd_id); - if (dtm_cmd->dtm_status != TA_DTM_STATUS__SUCCESS) { - display->state = MOD_HDCP_DISPLAY_INACTIVE; + if (dtm_cmd->dtm_status != TA_DTM_STATUS__SUCCESS) return MOD_HDCP_STATUS_UPDATE_TOPOLOGY_FAILURE; - } + display->state = MOD_HDCP_DISPLAY_ACTIVE_AND_ADDED; HDCP_TOP_ADD_DISPLAY_TRACE(hdcp, display->index); return MOD_HDCP_STATUS_SUCCESS; @@ -123,7 +126,7 @@ enum mod_hdcp_status mod_hdcp_hdcp1_create_session(struct mod_hdcp *hdcp) { struct psp_context *psp = hdcp->config.psp.handle; - struct mod_hdcp_display *display = get_first_active_display(hdcp); + struct mod_hdcp_display *display = get_first_added_display(hdcp); struct ta_hdcp_shared_memory *hdcp_cmd; if (!psp->hdcp_context.hdcp_initialized) { @@ -176,7 +179,7 @@ enum mod_hdcp_status mod_hdcp_hdcp1_destroy_session(struct mod_hdcp *hdcp) if (is_display_encryption_enabled( &hdcp->displays[i])) { hdcp->displays[i].state = - MOD_HDCP_DISPLAY_ACTIVE; + MOD_HDCP_DISPLAY_ACTIVE_AND_ADDED; HDCP_HDCP1_DISABLED_TRACE(hdcp, hdcp->displays[i].index); } @@ -228,7 +231,7 @@ enum mod_hdcp_status mod_hdcp_hdcp1_enable_encryption(struct mod_hdcp *hdcp) { struct psp_context *psp = hdcp->config.psp.handle; struct ta_hdcp_shared_memory *hdcp_cmd; - struct mod_hdcp_display *display = get_first_active_display(hdcp); + struct mod_hdcp_display *display = get_first_added_display(hdcp); hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.hdcp_shared_buf; memset(hdcp_cmd, 0, sizeof(struct ta_hdcp_shared_memory)); @@ -298,7 +301,8 @@ enum mod_hdcp_status mod_hdcp_hdcp1_enable_dp_stream_encryption(struct mod_hdcp for (i = 0; i < MAX_NUM_OF_DISPLAYS; i++) { - if (hdcp->displays[i].adjust.disable) + if (hdcp->displays[i].state != MOD_HDCP_DISPLAY_ACTIVE_AND_ADDED || + hdcp->displays[i].adjust.disable) continue; memset(hdcp_cmd, 0, sizeof(struct ta_hdcp_shared_memory)); @@ -360,7 +364,7 @@ enum mod_hdcp_status mod_hdcp_hdcp2_create_session(struct mod_hdcp *hdcp) { struct psp_context *psp = hdcp->config.psp.handle; struct ta_hdcp_shared_memory *hdcp_cmd; - struct mod_hdcp_display *display = get_first_active_display(hdcp); + struct mod_hdcp_display *display = get_first_added_display(hdcp); if (!psp->hdcp_context.hdcp_initialized) { DRM_ERROR("Failed to create hdcp session, HDCP TA is not initialized"); @@ -419,7 +423,7 @@ enum mod_hdcp_status mod_hdcp_hdcp2_destroy_session(struct mod_hdcp *hdcp) if (is_display_encryption_enabled( &hdcp->displays[i])) { hdcp->displays[i].state = - MOD_HDCP_DISPLAY_ACTIVE; + MOD_HDCP_DISPLAY_ACTIVE_AND_ADDED; HDCP_HDCP2_DISABLED_TRACE(hdcp, hdcp->displays[i].index); } @@ -658,7 +662,7 @@ enum mod_hdcp_status mod_hdcp_hdcp2_enable_encryption(struct mod_hdcp *hdcp) { struct psp_context *psp = hdcp->config.psp.handle; struct ta_hdcp_shared_memory *hdcp_cmd; - struct mod_hdcp_display *display = get_first_active_display(hdcp); + struct mod_hdcp_display *display = get_first_added_display(hdcp); hdcp_cmd = (struct ta_hdcp_shared_memory *)psp->hdcp_context.hdcp_shared_buf; memset(hdcp_cmd, 0, sizeof(struct ta_hdcp_shared_memory)); @@ -743,7 +747,8 @@ enum mod_hdcp_status mod_hdcp_hdcp2_enable_dp_stream_encryption(struct mod_hdcp for (i = 0; i < MAX_NUM_OF_DISPLAYS; i++) { - if (hdcp->displays[i].adjust.disable) + if (hdcp->displays[i].state != MOD_HDCP_DISPLAY_ACTIVE_AND_ADDED || + hdcp->displays[i].adjust.disable) continue; hdcp_cmd->in_msg.hdcp2_enable_dp_stream_encryption.display_handle = hdcp->displays[i].index; hdcp_cmd->in_msg.hdcp2_enable_dp_stream_encryption.session_handle = hdcp->auth.id; diff --git a/drivers/gpu/drm/amd/display/modules/inc/mod_hdcp.h b/drivers/gpu/drm/amd/display/modules/inc/mod_hdcp.h index eae9309cfb24..c088602bc1a0 100644 --- a/drivers/gpu/drm/amd/display/modules/inc/mod_hdcp.h +++ b/drivers/gpu/drm/amd/display/modules/inc/mod_hdcp.h @@ -117,6 +117,7 @@ enum mod_hdcp_operation_mode { enum mod_hdcp_display_state { MOD_HDCP_DISPLAY_INACTIVE = 0, MOD_HDCP_DISPLAY_ACTIVE, + MOD_HDCP_DISPLAY_ACTIVE_AND_ADDED, MOD_HDCP_DISPLAY_ENCRYPTION_ENABLED }; From 5c41c023f8b213d425af973a4dd83f5b9bc0bfb2 Mon Sep 17 00:00:00 2001 From: Stylon Wang Date: Tue, 10 Mar 2020 15:09:29 -0400 Subject: [PATCH 384/635] drm/amd/display: Support plane level CTM [Why] CTM was only supported at CRTC level and we need color space conversion in linear space at plane level. [How] - Add plane-level CTM to dc interface - Program plane-level CTM in DCN Signed-off-by: Stylon Wang Reviewed-by: Nicholas Kazlauskas Acked-by: Rodrigo Siqueira Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/core/dc.c | 10 +++++++++- drivers/gpu/drm/amd/display/dc/dc.h | 3 +++ .../gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c | 6 ++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index 1a2c2e3dd6d2..f21bbb295ad3 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -1726,6 +1726,9 @@ static enum surface_update_type det_surface_update(const struct dc *dc, if (u->coeff_reduction_factor) update_flags->bits.coeff_reduction_change = 1; + if (u->gamut_remap_matrix) + update_flags->bits.gamut_remap_change = 1; + if (u->gamma) { enum surface_pixel_format format = SURFACE_PIXEL_FORMAT_GRPH_BEGIN; @@ -1751,7 +1754,8 @@ static enum surface_update_type det_surface_update(const struct dc *dc, if (update_flags->bits.input_csc_change || update_flags->bits.coeff_reduction_change - || update_flags->bits.gamma_change) { + || update_flags->bits.gamma_change + || update_flags->bits.gamut_remap_change) { type = UPDATE_TYPE_FULL; elevate_update_type(&overall_type, type); } @@ -1996,6 +2000,10 @@ static void copy_surface_update_to_plane( if (srf_update->coeff_reduction_factor) surface->coeff_reduction_factor = *srf_update->coeff_reduction_factor; + + if (srf_update->gamut_remap_matrix) + surface->gamut_remap_matrix = + *srf_update->gamut_remap_matrix; } static void copy_stream_update_to_stream(struct dc *dc, diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index d3ceb39e428e..1935cf6601eb 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -726,6 +726,7 @@ union surface_update_flags { uint32_t output_tf_change:1; uint32_t pixel_format_change:1; uint32_t plane_size_change:1; + uint32_t gamut_remap_change:1; /* Full updates */ uint32_t new_plane:1; @@ -760,6 +761,7 @@ struct dc_plane_state { struct dc_csc_transform input_csc_color_matrix; struct fixed31_32 coeff_reduction_factor; struct fixed31_32 hdr_mult; + struct colorspace_transform gamut_remap_matrix; // TODO: No longer used, remove struct dc_hdr_static_metadata hdr_static_ctx; @@ -839,6 +841,7 @@ struct dc_surface_update { const struct dc_transfer_func *func_shaper; const struct dc_3dlut *lut3d_func; const struct dc_transfer_func *blend_tf; + const struct colorspace_transform *gamut_remap_matrix; }; /* diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c index 9cc3314966bd..0be010085575 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c @@ -2004,6 +2004,12 @@ void dcn10_program_gamut_remap(struct pipe_ctx *pipe_ctx) for (i = 0; i < CSC_TEMPERATURE_MATRIX_SIZE; i++) adjust.temperature_matrix[i] = pipe_ctx->stream->gamut_remap_matrix.matrix[i]; + } else if (pipe_ctx->plane_state && + pipe_ctx->plane_state->gamut_remap_matrix.enable_remap == true) { + adjust.gamut_adjust_type = GRAPHICS_GAMUT_ADJUST_TYPE_SW; + for (i = 0; i < CSC_TEMPERATURE_MATRIX_SIZE; i++) + adjust.temperature_matrix[i] = + pipe_ctx->plane_state->gamut_remap_matrix.matrix[i]; } pipe_ctx->plane_res.dpp->funcs->dpp_set_gamut_remap(pipe_ctx->plane_res.dpp, &adjust); From 5a6b5458699d8e6fdb5ae09d950935a48b79f8c7 Mon Sep 17 00:00:00 2001 From: Aric Cyr Date: Wed, 11 Mar 2020 18:10:20 -0400 Subject: [PATCH 385/635] drm/amd/display: LFC not working on 2.0x range monitors (v2) [Why] Nominal pixel clock and EDID information differ in precision so although monitor reports maximum refresh is 2x minimum, LFC was not being enabled. [How] Use minimum refresh rate as nominal/2 when EDID dictates that min refresh = max refresh/2. v2: squash in 64 bit divide fix Signed-off-by: Aric Cyr Reviewed-by: Nicholas Kazlauskas Acked-by: Rodrigo Siqueira Signed-off-by: Alex Deucher --- .../amd/display/modules/freesync/freesync.c | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/amd/display/modules/freesync/freesync.c b/drivers/gpu/drm/amd/display/modules/freesync/freesync.c index 4e542826cd26..c33454a9e0b4 100644 --- a/drivers/gpu/drm/amd/display/modules/freesync/freesync.c +++ b/drivers/gpu/drm/amd/display/modules/freesync/freesync.c @@ -734,6 +734,7 @@ void mod_freesync_build_vrr_params(struct mod_freesync *mod_freesync, { struct core_freesync *core_freesync = NULL; unsigned long long nominal_field_rate_in_uhz = 0; + unsigned long long rounded_nominal_in_uhz = 0; unsigned int refresh_range = 0; unsigned long long min_refresh_in_uhz = 0; unsigned long long max_refresh_in_uhz = 0; @@ -750,17 +751,20 @@ void mod_freesync_build_vrr_params(struct mod_freesync *mod_freesync, min_refresh_in_uhz = in_config->min_refresh_in_uhz; max_refresh_in_uhz = in_config->max_refresh_in_uhz; - // Don't allow min > max - if (min_refresh_in_uhz > max_refresh_in_uhz) - min_refresh_in_uhz = max_refresh_in_uhz; - // Full range may be larger than current video timing, so cap at nominal if (max_refresh_in_uhz > nominal_field_rate_in_uhz) max_refresh_in_uhz = nominal_field_rate_in_uhz; // Full range may be larger than current video timing, so cap at nominal - if (min_refresh_in_uhz > nominal_field_rate_in_uhz) - min_refresh_in_uhz = nominal_field_rate_in_uhz; + if (min_refresh_in_uhz > max_refresh_in_uhz) + min_refresh_in_uhz = max_refresh_in_uhz; + + // If a monitor reports exactly max refresh of 2x of min, enforce it on nominal + rounded_nominal_in_uhz = + div_u64(nominal_field_rate_in_uhz + 50000, 100000) * 100000; + if (in_config->max_refresh_in_uhz == (2 * in_config->min_refresh_in_uhz) && + in_config->max_refresh_in_uhz == rounded_nominal_in_uhz) + min_refresh_in_uhz = div_u64(nominal_field_rate_in_uhz, 2); if (!vrr_settings_require_update(core_freesync, in_config, (unsigned int)min_refresh_in_uhz, (unsigned int)max_refresh_in_uhz, @@ -792,11 +796,6 @@ void mod_freesync_build_vrr_params(struct mod_freesync *mod_freesync, refresh_range = in_out_vrr->max_refresh_in_uhz - in_out_vrr->min_refresh_in_uhz; - in_out_vrr->btr.margin_in_us = in_out_vrr->max_duration_in_us - - 2 * in_out_vrr->min_duration_in_us; - if (in_out_vrr->btr.margin_in_us > BTR_MAX_MARGIN) - in_out_vrr->btr.margin_in_us = BTR_MAX_MARGIN; - in_out_vrr->supported = true; } @@ -804,9 +803,14 @@ void mod_freesync_build_vrr_params(struct mod_freesync *mod_freesync, in_out_vrr->btr.btr_enabled = in_config->btr; - if (in_out_vrr->max_refresh_in_uhz < - 2 * in_out_vrr->min_refresh_in_uhz) + if (in_out_vrr->max_refresh_in_uhz < (2 * in_out_vrr->min_refresh_in_uhz)) in_out_vrr->btr.btr_enabled = false; + else { + in_out_vrr->btr.margin_in_us = in_out_vrr->max_duration_in_us - + 2 * in_out_vrr->min_duration_in_us; + if (in_out_vrr->btr.margin_in_us > BTR_MAX_MARGIN) + in_out_vrr->btr.margin_in_us = BTR_MAX_MARGIN; + } in_out_vrr->btr.btr_active = false; in_out_vrr->btr.inserted_duration_in_us = 0; @@ -1008,8 +1012,8 @@ unsigned long long mod_freesync_calc_nominal_field_rate( unsigned int total = stream->timing.h_total * stream->timing.v_total; /* Calculate nominal field rate for stream, rounded up to nearest integer */ - nominal_field_rate_in_uhz = stream->timing.pix_clk_100hz / 10; - nominal_field_rate_in_uhz *= 1000ULL * 1000ULL * 1000ULL; + nominal_field_rate_in_uhz = stream->timing.pix_clk_100hz; + nominal_field_rate_in_uhz *= 100000000ULL; nominal_field_rate_in_uhz = div_u64(nominal_field_rate_in_uhz, total); From 00755bb79cb239905780965c9bc128de7073a9f0 Mon Sep 17 00:00:00 2001 From: Stylon Wang Date: Fri, 13 Mar 2020 10:21:38 -0400 Subject: [PATCH 386/635] drm/amd/display: Enable BT2020 in COLOR_ENCODING property [Why] BT2020 is not supported in COLOR_ENCODING property of planes. Only BT601 and BT709 was available. [How] Allow BT2020 as legit value in setting COLOR_ENCODING property. Signed-off-by: Stylon Wang Reviewed-by: Nicholas Kazlauskas Acked-by: Rodrigo Siqueira Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index ad14c1a30843..bab587ab6e8d 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -5592,12 +5592,15 @@ static int amdgpu_dm_plane_init(struct amdgpu_display_manager *dm, } if (plane->type == DRM_PLANE_TYPE_PRIMARY && - plane_cap && plane_cap->pixel_format_support.nv12) { + plane_cap && + (plane_cap->pixel_format_support.nv12 || + plane_cap->pixel_format_support.p010)) { /* This only affects YUV formats. */ drm_plane_create_color_properties( plane, BIT(DRM_COLOR_YCBCR_BT601) | - BIT(DRM_COLOR_YCBCR_BT709), + BIT(DRM_COLOR_YCBCR_BT709) | + BIT(DRM_COLOR_YCBCR_BT2020), BIT(DRM_COLOR_YCBCR_LIMITED_RANGE) | BIT(DRM_COLOR_YCBCR_FULL_RANGE), DRM_COLOR_YCBCR_BT709, DRM_COLOR_YCBCR_LIMITED_RANGE); From c42656f8fc52de4f0321c5054896b1ccbb3c6cd2 Mon Sep 17 00:00:00 2001 From: Dmytro Laktyushkin Date: Mon, 9 Mar 2020 17:11:16 -0400 Subject: [PATCH 387/635] drm/amd/display: Fix dcn21 num_states [Why] DML expects num_states to exclude the duplicate state. [How] Set num_states to correct value to prevent array off-by-one error. Also refactor max clock level code for diags. Signed-off-by: Dmytro Laktyushkin Signed-off-by: George Shen Reviewed-by: Dmytro Laktyushkin Acked-by: Rodrigo Siqueira Signed-off-by: Alex Deucher --- .../drm/amd/display/dc/dcn20/dcn20_resource.c | 2 +- .../drm/amd/display/dc/dcn21/dcn21_resource.c | 37 ++++++------------- .../gpu/drm/amd/display/dc/dml/dc_features.h | 2 +- .../amd/display/dc/dml/display_mode_structs.h | 7 ++-- 4 files changed, 18 insertions(+), 30 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c index 54e096055c6f..5cdbba0cd873 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c @@ -3343,7 +3343,7 @@ void dcn20_cap_soc_clocks( void dcn20_update_bounding_box(struct dc *dc, struct _vcs_dpi_soc_bounding_box_st *bb, struct pp_smu_nv_clock_table *max_clocks, unsigned int *uclk_states, unsigned int num_states) { - struct _vcs_dpi_voltage_scaling_st calculated_states[MAX_CLOCK_LIMIT_STATES]; + struct _vcs_dpi_voltage_scaling_st calculated_states[DC__VOLTAGE_STATES]; int i; int num_calculated_states = 0; int min_dcfclk = 0; diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c index deda1a6b603b..b25484aa8222 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c @@ -300,7 +300,7 @@ struct _vcs_dpi_soc_bounding_box_st dcn2_1_soc = { .xfc_bus_transport_time_us = 4, .xfc_xbuf_latency_tolerance_us = 4, .use_urgent_burst_bw = 1, - .num_states = 9 + .num_states = 8 }; #ifndef MAX @@ -1377,21 +1377,8 @@ static void update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_param unsigned int i, j, k; int closest_clk_lvl; - // diags does not retrieve proper values from SMU - // cap states to 5 and make state 5 the max state - if (IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment) || IS_DIAG_DC(dc->ctx->dce_environment)) { - dcn2_1_soc.num_states = 5; - - dcn2_1_soc.clock_limits[5].state = 5; - dcn2_1_soc.clock_limits[5].dcfclk_mhz = 810.0; - dcn2_1_soc.clock_limits[5].fabricclk_mhz = 1600.0; - dcn2_1_soc.clock_limits[5].dispclk_mhz = 1395.0; - dcn2_1_soc.clock_limits[5].dppclk_mhz = 1285.0; - dcn2_1_soc.clock_limits[5].phyclk_mhz = 1325.0; - dcn2_1_soc.clock_limits[5].socclk_mhz = 953.0; - dcn2_1_soc.clock_limits[5].dscclk_mhz = 489.0; - dcn2_1_soc.clock_limits[5].dram_speed_mts = 4266.0; - } else { + // Default clock levels are used for diags, which may lead to overclocking. + if (!IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment) && !IS_DIAG_DC(dc->ctx->dce_environment)) { dcn2_1_ip.max_num_otg = pool->base.res_cap->num_timing_generator; dcn2_1_ip.max_num_dpp = pool->base.pipe_count; dcn2_1_soc.num_chans = bw_params->num_channels; @@ -1404,16 +1391,16 @@ static void update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_param dcn2_1_soc.clock_limits[0].dram_speed_mts = clk_table->entries[0].memclk_mhz * 2; /* - * Other levels: find cloest DCN clocks that fit the given clock limit using dcfclk - * as indicater + * Other levels: find closest DCN clocks that fit the given clock limit using dcfclk + * as indicator */ closest_clk_lvl = -1; /* index currently being filled */ k = 1; for (i = 1; i < clk_table->num_entries; i++) { - /* loop backwards, skip duplicate state, +1 because SMU has precision issue */ - for (j = dcn2_1_soc.num_states - 2; j >= k; j--) { + /* loop backwards, skip duplicate state*/ + for (j = dcn2_1_soc.num_states - 1; j >= k; j--) { if ((unsigned int) dcn2_1_soc.clock_limits[j].dcfclk_mhz <= clk_table->entries[i].dcfclk_mhz) { closest_clk_lvl = j; break; @@ -1438,13 +1425,13 @@ static void update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_param k++; } } - - /* duplicate last level */ - dcn2_1_soc.clock_limits[k] = dcn2_1_soc.clock_limits[k - 1]; - dcn2_1_soc.clock_limits[k].state = k; - dcn2_1_soc.num_states = k + 1; + dcn2_1_soc.num_states = k; } + /* duplicate last level */ + dcn2_1_soc.clock_limits[dcn2_1_soc.num_states] = dcn2_1_soc.clock_limits[dcn2_1_soc.num_states - 1]; + dcn2_1_soc.clock_limits[dcn2_1_soc.num_states].state = dcn2_1_soc.num_states; + dml_init_instance(&dc->dml, &dcn2_1_soc, &dcn2_1_ip, DML_PROJECT_DCN21); } diff --git a/drivers/gpu/drm/amd/display/dc/dml/dc_features.h b/drivers/gpu/drm/amd/display/dc/dml/dc_features.h index ea4cde952f4f..2a1983324629 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dc_features.h +++ b/drivers/gpu/drm/amd/display/dc/dml/dc_features.h @@ -29,7 +29,7 @@ #define DC__PRESENT 1 #define DC__PRESENT__1 1 #define DC__NUM_DPP 4 -#define DC__VOLTAGE_STATES 7 +#define DC__VOLTAGE_STATES 9 #define DC__NUM_DPP__4 1 #define DC__NUM_DPP__0_PRESENT 1 #define DC__NUM_DPP__1_PRESENT 1 diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h index dfd3be452766..687010c17324 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h +++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h @@ -22,11 +22,12 @@ * Authors: AMD * */ + +#include "dc_features.h" + #ifndef __DISPLAY_MODE_STRUCTS_H__ #define __DISPLAY_MODE_STRUCTS_H__ -#define MAX_CLOCK_LIMIT_STATES 9 - typedef struct _vcs_dpi_voltage_scaling_st voltage_scaling_st; typedef struct _vcs_dpi_soc_bounding_box_st soc_bounding_box_st; typedef struct _vcs_dpi_ip_params_st ip_params_st; @@ -68,7 +69,7 @@ struct _vcs_dpi_voltage_scaling_st { }; struct _vcs_dpi_soc_bounding_box_st { - struct _vcs_dpi_voltage_scaling_st clock_limits[MAX_CLOCK_LIMIT_STATES]; + struct _vcs_dpi_voltage_scaling_st clock_limits[DC__VOLTAGE_STATES]; unsigned int num_states; double sr_exit_time_us; double sr_enter_plus_exit_time_us; From e4295adc84a342cf6f627128cbbd939f320cf687 Mon Sep 17 00:00:00 2001 From: Evan Quan Date: Fri, 27 Mar 2020 15:05:09 +0800 Subject: [PATCH 388/635] drm/amd/powerplay: drop redundant BIF doorbell interrupt operations This is already done in soc15.c. And this is really ASIC specific and should not be placed here. Signed-off-by: Evan Quan Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/powerplay/smu_v11_0.c | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/drivers/gpu/drm/amd/powerplay/smu_v11_0.c b/drivers/gpu/drm/amd/powerplay/smu_v11_0.c index d19e1d0d56c0..b60d57ac1cd4 100644 --- a/drivers/gpu/drm/amd/powerplay/smu_v11_0.c +++ b/drivers/gpu/drm/amd/powerplay/smu_v11_0.c @@ -1700,11 +1700,9 @@ enum smu_baco_state smu_v11_0_baco_get_state(struct smu_context *smu) int smu_v11_0_baco_set_state(struct smu_context *smu, enum smu_baco_state state) { - struct smu_baco_context *smu_baco = &smu->smu_baco; struct amdgpu_device *adev = smu->adev; struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); - uint32_t bif_doorbell_intr_cntl; uint32_t data; int ret = 0; @@ -1713,14 +1711,7 @@ int smu_v11_0_baco_set_state(struct smu_context *smu, enum smu_baco_state state) mutex_lock(&smu_baco->mutex); - bif_doorbell_intr_cntl = RREG32_SOC15(NBIO, 0, mmBIF_DOORBELL_INT_CNTL); - if (state == SMU_BACO_STATE_ENTER) { - bif_doorbell_intr_cntl = REG_SET_FIELD(bif_doorbell_intr_cntl, - BIF_DOORBELL_INT_CNTL, - DOORBELL_INTERRUPT_DISABLE, 1); - WREG32_SOC15(NBIO, 0, mmBIF_DOORBELL_INT_CNTL, bif_doorbell_intr_cntl); - if (!ras || !ras->supported) { data = RREG32_SOC15(THM, 0, mmTHM_BACO_CNTL); data |= 0x80000000; @@ -1735,11 +1726,6 @@ int smu_v11_0_baco_set_state(struct smu_context *smu, enum smu_baco_state state) if (ret) goto out; - bif_doorbell_intr_cntl = REG_SET_FIELD(bif_doorbell_intr_cntl, - BIF_DOORBELL_INT_CNTL, - DOORBELL_INTERRUPT_DISABLE, 0); - WREG32_SOC15(NBIO, 0, mmBIF_DOORBELL_INT_CNTL, bif_doorbell_intr_cntl); - /* clear vbios scratch 6 and 7 for coming asic reinit */ WREG32(adev->bios_scratch_reg_offset + 6, 0); WREG32(adev->bios_scratch_reg_offset + 7, 0); From 49e78c820a025a2c6e849b34fe113727194a7034 Mon Sep 17 00:00:00 2001 From: Evan Quan Date: Fri, 27 Mar 2020 15:33:00 +0800 Subject: [PATCH 389/635] drm/amd/powerplay: move the ASIC specific nbio operation out of smu_v11_0.c This is ASIC specific and should be placed in _ppt.c of each ASIC. Signed-off-by: Evan Quan Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/powerplay/arcturus_ppt.c | 15 ++++++++++++++- drivers/gpu/drm/amd/powerplay/navi10_ppt.c | 18 ++++++++++++++++-- drivers/gpu/drm/amd/powerplay/smu_v11_0.c | 10 +--------- drivers/gpu/drm/amd/powerplay/vega20_ppt.c | 14 +++++++++++++- 4 files changed, 44 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/amd/powerplay/arcturus_ppt.c b/drivers/gpu/drm/amd/powerplay/arcturus_ppt.c index c6d3bef15320..5db8c56066ee 100644 --- a/drivers/gpu/drm/amd/powerplay/arcturus_ppt.c +++ b/drivers/gpu/drm/amd/powerplay/arcturus_ppt.c @@ -35,6 +35,7 @@ #include "arcturus_ppt.h" #include "smu_v11_0_pptable.h" #include "arcturus_ppsmc.h" +#include "nbio/nbio_7_4_offset.h" #include "nbio/nbio_7_4_sh_mask.h" #include "amdgpu_xgmi.h" #include @@ -2210,6 +2211,18 @@ static void arcturus_i2c_eeprom_control_fini(struct i2c_adapter *control) i2c_del_adapter(control); } +static bool arcturus_is_baco_supported(struct smu_context *smu) +{ + struct amdgpu_device *adev = smu->adev; + uint32_t val; + + if (!smu_v11_0_baco_is_support(smu)) + return false; + + val = RREG32_SOC15(NBIO, 0, mmRCC_BIF_STRAP0); + return (val & RCC_BIF_STRAP0__STRAP_PX_CAPABLE_MASK) ? true : false; +} + static uint32_t arcturus_get_pptable_power_limit(struct smu_context *smu) { PPTable_t *pptable = smu->smu_table.driver_pptable; @@ -2321,7 +2334,7 @@ static const struct pptable_funcs arcturus_ppt_funcs = { .register_irq_handler = smu_v11_0_register_irq_handler, .set_azalia_d3_pme = smu_v11_0_set_azalia_d3_pme, .get_max_sustainable_clocks_by_dc = smu_v11_0_get_max_sustainable_clocks_by_dc, - .baco_is_support= smu_v11_0_baco_is_support, + .baco_is_support= arcturus_is_baco_supported, .baco_get_state = smu_v11_0_baco_get_state, .baco_set_state = smu_v11_0_baco_set_state, .baco_enter = smu_v11_0_baco_enter, diff --git a/drivers/gpu/drm/amd/powerplay/navi10_ppt.c b/drivers/gpu/drm/amd/powerplay/navi10_ppt.c index 9c60b38ab53a..15030284b444 100644 --- a/drivers/gpu/drm/amd/powerplay/navi10_ppt.c +++ b/drivers/gpu/drm/amd/powerplay/navi10_ppt.c @@ -28,13 +28,15 @@ #include "smu_internal.h" #include "atomfirmware.h" #include "amdgpu_atomfirmware.h" +#include "soc15_common.h" #include "smu_v11_0.h" #include "smu11_driver_if_navi10.h" #include "atom.h" #include "navi10_ppt.h" #include "smu_v11_0_pptable.h" #include "smu_v11_0_ppsmc.h" -#include "nbio/nbio_7_4_sh_mask.h" +#include "nbio/nbio_2_3_offset.h" +#include "nbio/nbio_2_3_sh_mask.h" #include "asic_reg/mp/mp_11_0_sh_mask.h" @@ -1985,6 +1987,18 @@ static int navi10_setup_od_limits(struct smu_context *smu) { return 0; } +static bool navi10_is_baco_supported(struct smu_context *smu) +{ + struct amdgpu_device *adev = smu->adev; + uint32_t val; + + if (!smu_v11_0_baco_is_support(smu)) + return false; + + val = RREG32_SOC15(NBIO, 0, mmRCC_BIF_STRAP0); + return (val & RCC_BIF_STRAP0__STRAP_PX_CAPABLE_MASK) ? true : false; +} + static int navi10_set_default_od_settings(struct smu_context *smu, bool initialize) { OverDriveTable_t *od_table, *boot_od_table; int ret = 0; @@ -2361,7 +2375,7 @@ static const struct pptable_funcs navi10_ppt_funcs = { .register_irq_handler = smu_v11_0_register_irq_handler, .set_azalia_d3_pme = smu_v11_0_set_azalia_d3_pme, .get_max_sustainable_clocks_by_dc = smu_v11_0_get_max_sustainable_clocks_by_dc, - .baco_is_support= smu_v11_0_baco_is_support, + .baco_is_support= navi10_is_baco_supported, .baco_get_state = smu_v11_0_baco_get_state, .baco_set_state = smu_v11_0_baco_set_state, .baco_enter = smu_v11_0_baco_enter, diff --git a/drivers/gpu/drm/amd/powerplay/smu_v11_0.c b/drivers/gpu/drm/amd/powerplay/smu_v11_0.c index b60d57ac1cd4..541c932a6005 100644 --- a/drivers/gpu/drm/amd/powerplay/smu_v11_0.c +++ b/drivers/gpu/drm/amd/powerplay/smu_v11_0.c @@ -42,8 +42,6 @@ #include "asic_reg/thm/thm_11_0_2_sh_mask.h" #include "asic_reg/mp/mp_11_0_offset.h" #include "asic_reg/mp/mp_11_0_sh_mask.h" -#include "asic_reg/nbio/nbio_7_4_offset.h" -#include "asic_reg/nbio/nbio_7_4_sh_mask.h" #include "asic_reg/smuio/smuio_11_0_0_offset.h" #include "asic_reg/smuio/smuio_11_0_0_sh_mask.h" @@ -1662,9 +1660,7 @@ static int smu_v11_0_baco_set_armd3_sequence(struct smu_context *smu, enum smu_v bool smu_v11_0_baco_is_support(struct smu_context *smu) { - struct amdgpu_device *adev = smu->adev; struct smu_baco_context *smu_baco = &smu->smu_baco; - uint32_t val; bool baco_support; mutex_lock(&smu_baco->mutex); @@ -1679,11 +1675,7 @@ bool smu_v11_0_baco_is_support(struct smu_context *smu) !smu_feature_is_enabled(smu, SMU_FEATURE_BACO_BIT)) return false; - val = RREG32_SOC15(NBIO, 0, mmRCC_BIF_STRAP0); - if (val & RCC_BIF_STRAP0__STRAP_PX_CAPABLE_MASK) - return true; - - return false; + return true; } enum smu_baco_state smu_v11_0_baco_get_state(struct smu_context *smu) diff --git a/drivers/gpu/drm/amd/powerplay/vega20_ppt.c b/drivers/gpu/drm/amd/powerplay/vega20_ppt.c index 49ff3756bd9f..3f1044326dcb 100644 --- a/drivers/gpu/drm/amd/powerplay/vega20_ppt.c +++ b/drivers/gpu/drm/amd/powerplay/vega20_ppt.c @@ -35,6 +35,7 @@ #include "vega20_ppt.h" #include "vega20_pptable.h" #include "vega20_ppsmc.h" +#include "nbio/nbio_7_4_offset.h" #include "nbio/nbio_7_4_sh_mask.h" #include "asic_reg/thm/thm_11_0_2_offset.h" #include "asic_reg/thm/thm_11_0_2_sh_mask.h" @@ -3174,6 +3175,17 @@ static int vega20_update_pcie_parameters(struct smu_context *smu, return ret; } +static bool vega20_is_baco_supported(struct smu_context *smu) +{ + struct amdgpu_device *adev = smu->adev; + uint32_t val; + + if (!smu_v11_0_baco_is_support(smu)) + return false; + + val = RREG32_SOC15(NBIO, 0, mmRCC_BIF_STRAP0); + return (val & RCC_BIF_STRAP0__STRAP_PX_CAPABLE_MASK) ? true : false; +} static const struct pptable_funcs vega20_ppt_funcs = { .tables_init = vega20_tables_init, @@ -3262,7 +3274,7 @@ static const struct pptable_funcs vega20_ppt_funcs = { .register_irq_handler = smu_v11_0_register_irq_handler, .set_azalia_d3_pme = smu_v11_0_set_azalia_d3_pme, .get_max_sustainable_clocks_by_dc = smu_v11_0_get_max_sustainable_clocks_by_dc, - .baco_is_support= smu_v11_0_baco_is_support, + .baco_is_support= vega20_is_baco_supported, .baco_get_state = smu_v11_0_baco_get_state, .baco_set_state = smu_v11_0_baco_set_state, .baco_enter = smu_v11_0_baco_enter, From 04ff4863e1261187ef6152d2cf1f9a20e9b4cc04 Mon Sep 17 00:00:00 2001 From: YueHaibing Date: Tue, 31 Mar 2020 19:50:24 +0800 Subject: [PATCH 390/635] libnvdimm/region: Fix build error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On CONFIG_PPC32=y build fails: drivers/nvdimm/region_devs.c:1034:14: note: in expansion of macro ‘do_div’ remainder = do_div(per_mapping, mappings); ^~~~~~ In file included from ./arch/powerpc/include/generated/asm/div64.h:1:0, from ./include/linux/kernel.h:18, from ./include/asm-generic/bug.h:19, from ./arch/powerpc/include/asm/bug.h:109, from ./include/linux/bug.h:5, from ./include/linux/scatterlist.h:7, from drivers/nvdimm/region_devs.c:5: ./include/asm-generic/div64.h:243:22: error: passing argument 1 of ‘__div64_32’ from incompatible pointer type [-Werror=incompatible-pointer-types] __rem = __div64_32(&(n), __base); \ Use div_u64 instead of do_div to fix this. Fixes: 2522afb86a8c ("libnvdimm/region: Introduce an 'align' attribute") Signed-off-by: YueHaibing Link: https://lore.kernel.org/r/20200331115024.31628-1-yuehaibing@huawei.com Signed-off-by: Dan Williams --- drivers/nvdimm/region_devs.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/drivers/nvdimm/region_devs.c b/drivers/nvdimm/region_devs.c index bf239e783940..ccbb5b43b8b2 100644 --- a/drivers/nvdimm/region_devs.c +++ b/drivers/nvdimm/region_devs.c @@ -563,8 +563,7 @@ static ssize_t align_store(struct device *dev, * contribute to the tail capacity in system-physical-address * space for the namespace. */ - dpa = val; - remainder = do_div(dpa, nd_region->ndr_mappings); + dpa = div_u64_rem(val, nd_region->ndr_mappings, &remainder); if (!is_power_of_2(dpa) || dpa < PAGE_SIZE || val > region_size(nd_region) || remainder) return -EINVAL; @@ -1010,7 +1009,7 @@ EXPORT_SYMBOL(nd_region_release_lane); static unsigned long default_align(struct nd_region *nd_region) { - unsigned long align, per_mapping; + unsigned long align; int i, mappings; u32 remainder; @@ -1030,8 +1029,7 @@ static unsigned long default_align(struct nd_region *nd_region) } mappings = max_t(u16, 1, nd_region->ndr_mappings); - per_mapping = align; - remainder = do_div(per_mapping, mappings); + div_u64_rem(align, mappings, &remainder); if (remainder) align *= mappings; From c2850c125d919efbb3a9ab46410d23912934f585 Mon Sep 17 00:00:00 2001 From: Bhawanpreet Lakha Date: Mon, 30 Mar 2020 13:29:46 -0400 Subject: [PATCH 391/635] drm/amd/display: Don't try hdcp1.4 when content_type is set to type1 [Why] When content type property is set to 1. We should enable hdcp2.2 and if we cant then stop. Currently the way it works in DC is that if we fail hdcp2, we will try hdcp1 after. [How] Use link config to force disable hdcp1.4 when type1 is set. Signed-off-by: Bhawanpreet Lakha Reviewed-by: Nicholas Kazlauskas Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c index 5b70ed3cdb88..41fd1f5acff4 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c @@ -192,10 +192,13 @@ void hdcp_update_display(struct hdcp_workqueue *hdcp_work, &hdcp_work->srm_version); display->adjust.disable = 0; - if (content_type == DRM_MODE_HDCP_CONTENT_TYPE0) + if (content_type == DRM_MODE_HDCP_CONTENT_TYPE0) { + hdcp_w->link.adjust.hdcp1.disable = 0; hdcp_w->link.adjust.hdcp2.force_type = MOD_HDCP_FORCE_TYPE_0; - else if (content_type == DRM_MODE_HDCP_CONTENT_TYPE1) + } else if (content_type == DRM_MODE_HDCP_CONTENT_TYPE1) { + hdcp_w->link.adjust.hdcp1.disable = 1; hdcp_w->link.adjust.hdcp2.force_type = MOD_HDCP_FORCE_TYPE_1; + } schedule_delayed_work(&hdcp_w->property_validate_dwork, msecs_to_jiffies(DRM_HDCP_CHECK_PERIOD_MS)); @@ -415,6 +418,7 @@ static void update_config(void *handle, struct cp_psp_stream_config *config) link->dp.mst_supported = config->mst_supported; display->adjust.disable = 1; link->adjust.auth_delay = 2; + link->adjust.hdcp1.disable = 0; hdcp_update_display(hdcp_work, link_index, aconnector, DRM_MODE_HDCP_CONTENT_TYPE0, false); } From 2ebbe7c9d11d7fea296c4997b80d2590a0928424 Mon Sep 17 00:00:00 2001 From: Bhawanpreet Lakha Date: Mon, 30 Mar 2020 13:37:07 -0400 Subject: [PATCH 392/635] drm/amd/display: Correctly cancel future watchdog and callback events [Why] -We need to cancel future callbacks/watchdogs events when a callback/watchdog event happens [How] -fix typo in event_callback() -cancel callback, not watchdog -cancel watchdog events in event_watchdog_timer(). Signed-off-by: Bhawanpreet Lakha Reviewed-by: Nicholas Kazlauskas Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c index 41fd1f5acff4..39804d2df9df 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c @@ -266,7 +266,7 @@ static void event_callback(struct work_struct *work) mutex_lock(&hdcp_work->mutex); - cancel_delayed_work(&hdcp_work->watchdog_timer_dwork); + cancel_delayed_work(&hdcp_work->callback_dwork); mod_hdcp_process_event(&hdcp_work->hdcp, MOD_HDCP_EVENT_CALLBACK, &hdcp_work->output); @@ -347,6 +347,8 @@ static void event_watchdog_timer(struct work_struct *work) mutex_lock(&hdcp_work->mutex); + cancel_delayed_work(&hdcp_work->watchdog_timer_dwork); + mod_hdcp_process_event(&hdcp_work->hdcp, MOD_HDCP_EVENT_WATCHDOG_TIMEOUT, &hdcp_work->output); From d6e7042f4e80f2f139c23237da63ffcb060b0710 Mon Sep 17 00:00:00 2001 From: Bhawanpreet Lakha Date: Mon, 30 Mar 2020 13:43:23 -0400 Subject: [PATCH 393/635] drm/amd/display: increase HDCP authentication delay [Why] Some displays have an issue where the hdcp chips are initialized after the display has already lit up. This means we can sometimes authentication too early and cause authentication failures. This happens when HDCP is enabled and the display is power cycled. Normally we will authenticate 2 seconds after the display is lit, but some displays need a bit more time. [How] Increase delay to 3 second before we start authentication. Signed-off-by: Bhawanpreet Lakha Reviewed-by: Nicholas Kazlauskas Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c index 39804d2df9df..78e1c11d4ae5 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c @@ -419,7 +419,7 @@ static void update_config(void *handle, struct cp_psp_stream_config *config) link->dp.rev = aconnector->dc_link->dpcd_caps.dpcd_rev.raw; link->dp.mst_supported = config->mst_supported; display->adjust.disable = 1; - link->adjust.auth_delay = 2; + link->adjust.auth_delay = 3; link->adjust.hdcp1.disable = 0; hdcp_update_display(hdcp_work, link_index, aconnector, DRM_MODE_HDCP_CONTENT_TYPE0, false); From 1f776799628139d0da47e710ad86eb58d987ff66 Mon Sep 17 00:00:00 2001 From: Santosh Sivaraj Date: Tue, 14 Jan 2020 11:10:51 +0530 Subject: [PATCH 394/635] tools/test/nvdimm: Fix out of tree build Out of tree build using make M=tools/test/nvdimm O=/tmp/build -C /tmp/build fails with the following error make: Entering directory '/tmp/build' CC [M] tools/testing/nvdimm/test/nfit.o linux/tools/testing/nvdimm/test/nfit.c:19:10: fatal error: nd-core.h: No such file or directory 19 | #include | ^~~~~~~~~~~ compilation terminated. That is because the kbuild file uses $(src) which points to tools/testing/nvdimm, $(srctree) correctly points to root of the linux source tree. Reported-by: Aneesh Kumar K.V Signed-off-by: Santosh Sivaraj Link: https://lore.kernel.org/r/20200114054051.4115790-1-santosh@fossix.org Signed-off-by: Dan Williams --- tools/testing/nvdimm/Kbuild | 4 ++-- tools/testing/nvdimm/test/Kbuild | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/testing/nvdimm/Kbuild b/tools/testing/nvdimm/Kbuild index dbebf05f5931..47f9cc9dcd94 100644 --- a/tools/testing/nvdimm/Kbuild +++ b/tools/testing/nvdimm/Kbuild @@ -21,8 +21,8 @@ DRIVERS := ../../../drivers NVDIMM_SRC := $(DRIVERS)/nvdimm ACPI_SRC := $(DRIVERS)/acpi/nfit DAX_SRC := $(DRIVERS)/dax -ccflags-y := -I$(src)/$(NVDIMM_SRC)/ -ccflags-y += -I$(src)/$(ACPI_SRC)/ +ccflags-y := -I$(srctree)/drivers/nvdimm/ +ccflags-y += -I$(srctree)/drivers/acpi/nfit/ obj-$(CONFIG_LIBNVDIMM) += libnvdimm.o obj-$(CONFIG_BLK_DEV_PMEM) += nd_pmem.o diff --git a/tools/testing/nvdimm/test/Kbuild b/tools/testing/nvdimm/test/Kbuild index fb3c3d7cdb9b..75baebf8f4ba 100644 --- a/tools/testing/nvdimm/test/Kbuild +++ b/tools/testing/nvdimm/test/Kbuild @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 -ccflags-y := -I$(src)/../../../../drivers/nvdimm/ -ccflags-y += -I$(src)/../../../../drivers/acpi/nfit/ +ccflags-y := -I$(srctree)/drivers/nvdimm/ +ccflags-y += -I$(srctree)/drivers/acpi/nfit/ obj-m += nfit_test.o obj-m += nfit_test_iomap.o From 338f6dac8585beaf4d913de8847e430808fb7596 Mon Sep 17 00:00:00 2001 From: "Aneesh Kumar K.V" Date: Tue, 24 Mar 2020 09:18:21 +0530 Subject: [PATCH 395/635] libnvdimm: Update persistence domain value for of_pmem and papr_scm device Currently, kernel shows the below values "persistence_domain":"cpu_cache" "persistence_domain":"memory_controller" "persistence_domain":"unknown" "cpu_cache" indicates no extra instructions is needed to ensure the persistence of data in the pmem media on power failure. "memory_controller" indicates cpu cache flush instructions are required to flush the data. Platform provides mechanisms to automatically flush outstanding write data from memory controler to pmem on system power loss. Based on the above use memory_controller for non volatile regions on ppc64. Signed-off-by: Aneesh Kumar K.V Link: https://lore.kernel.org/r/20200324034821.60869-1-aneesh.kumar@linux.ibm.com Signed-off-by: Dan Williams --- arch/powerpc/platforms/pseries/papr_scm.c | 4 +++- drivers/nvdimm/of_pmem.c | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/arch/powerpc/platforms/pseries/papr_scm.c b/arch/powerpc/platforms/pseries/papr_scm.c index 0b4467e378e5..922a4fc3b61b 100644 --- a/arch/powerpc/platforms/pseries/papr_scm.c +++ b/arch/powerpc/platforms/pseries/papr_scm.c @@ -361,8 +361,10 @@ static int papr_scm_nvdimm_init(struct papr_scm_priv *p) if (p->is_volatile) p->region = nvdimm_volatile_region_create(p->bus, &ndr_desc); - else + else { + set_bit(ND_REGION_PERSIST_MEMCTRL, &ndr_desc.flags); p->region = nvdimm_pmem_region_create(p->bus, &ndr_desc); + } if (!p->region) { dev_err(dev, "Error registering region %pR from %pOF\n", ndr_desc.res, p->dn); diff --git a/drivers/nvdimm/of_pmem.c b/drivers/nvdimm/of_pmem.c index 8224d1431ea9..6826a274a1f1 100644 --- a/drivers/nvdimm/of_pmem.c +++ b/drivers/nvdimm/of_pmem.c @@ -62,8 +62,10 @@ static int of_pmem_region_probe(struct platform_device *pdev) if (is_volatile) region = nvdimm_volatile_region_create(bus, &ndr_desc); - else + else { + set_bit(ND_REGION_PERSIST_MEMCTRL, &ndr_desc.flags); region = nvdimm_pmem_region_create(bus, &ndr_desc); + } if (!region) dev_warn(&pdev->dev, "Unable to register region %pR from %pOF\n", From bb914088bd8a91c382f54d469367b2e5508b5493 Mon Sep 17 00:00:00 2001 From: Chris Packham Date: Fri, 13 Mar 2020 16:13:12 +1300 Subject: [PATCH 396/635] watchdog: orion: use 0 for unset heartbeat If the heartbeat module param is not specified we would get an error message watchdog: f1020300.watchdog: driver supplied timeout (4294967295) out of range watchdog: f1020300.watchdog: falling back to default timeout (171) This is because we were initialising heartbeat to -1. By removing the initialisation (thus letting the C run time initialise it to 0) we silence the warning message and the default timeout is still used. Signed-off-by: Chris Packham Reviewed-by: Guenter Roeck Link: https://lore.kernel.org/r/20200313031312.1485-1-chris.packham@alliedtelesis.co.nz Signed-off-by: Guenter Roeck Signed-off-by: Wim Van Sebroeck --- drivers/watchdog/orion_wdt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/watchdog/orion_wdt.c b/drivers/watchdog/orion_wdt.c index 8e6dfe76f9c9..4ddb4ea2e4a3 100644 --- a/drivers/watchdog/orion_wdt.c +++ b/drivers/watchdog/orion_wdt.c @@ -52,7 +52,7 @@ #define WDT_A370_RATIO (1 << WDT_A370_RATIO_SHIFT) static bool nowayout = WATCHDOG_NOWAYOUT; -static int heartbeat = -1; /* module parameter (seconds) */ +static int heartbeat; /* module parameter (seconds) */ struct orion_watchdog; From e2ad9bca45496681b0e954845da49e6e1a731b79 Mon Sep 17 00:00:00 2001 From: Lucas Stach Date: Fri, 13 Mar 2020 11:11:38 +0100 Subject: [PATCH 397/635] watchdog: ziirave_wdt: change name to be more specific The RAVE watchdog is not a full system watchdog, but is used to reset ethernet switch when required. Change the name to better reflect this usage. Signed-off-by: Lucas Stach Reviewed-by: Guenter Roeck Link: https://lore.kernel.org/r/20200313101138.25915-1-l.stach@pengutronix.de Signed-off-by: Guenter Roeck Signed-off-by: Wim Van Sebroeck --- drivers/watchdog/ziirave_wdt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/watchdog/ziirave_wdt.c b/drivers/watchdog/ziirave_wdt.c index 4a363a8b2d20..cab86a08456b 100644 --- a/drivers/watchdog/ziirave_wdt.c +++ b/drivers/watchdog/ziirave_wdt.c @@ -422,7 +422,7 @@ static int ziirave_firm_upload(struct watchdog_device *wdd, static const struct watchdog_info ziirave_wdt_info = { .options = WDIOF_SETTIMEOUT | WDIOF_MAGICCLOSE | WDIOF_KEEPALIVEPING, - .identity = "Zodiac RAVE Watchdog", + .identity = "RAVE Switch Watchdog", }; static const struct watchdog_ops ziirave_wdt_ops = { From 936253d7e904d64767819fc7c8c97bbe7821ff12 Mon Sep 17 00:00:00 2001 From: Tero Kristo Date: Thu, 12 Mar 2020 11:58:05 +0200 Subject: [PATCH 398/635] dt-bindings: watchdog: Add support for TI K3 RTI watchdog TI K3 SoCs contain an RTI (Real Time Interrupt) module which can be used to implement a windowed watchdog functionality. Windowed watchdog will generate an error if it is petted outside the time window, either too early or too late. Cc: Rob Herring Cc: devicetree@vger.kernel.org Signed-off-by: Tero Kristo Reviewed-by: Rob Herring Link: https://lore.kernel.org/r/20200312095808.19907-2-t-kristo@ti.com Signed-off-by: Guenter Roeck Signed-off-by: Wim Van Sebroeck --- .../bindings/watchdog/ti,rti-wdt.yaml | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 Documentation/devicetree/bindings/watchdog/ti,rti-wdt.yaml diff --git a/Documentation/devicetree/bindings/watchdog/ti,rti-wdt.yaml b/Documentation/devicetree/bindings/watchdog/ti,rti-wdt.yaml new file mode 100644 index 000000000000..e83026fef2e9 --- /dev/null +++ b/Documentation/devicetree/bindings/watchdog/ti,rti-wdt.yaml @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/watchdog/ti,rti-wdt.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Texas Instruments K3 SoC Watchdog Timer + +maintainers: + - Tero Kristo + +description: + The TI K3 SoC watchdog timer is implemented via the RTI (Real Time + Interrupt) IP module. This timer adds a support for windowed watchdog + mode, which will signal an error if it is pinged outside the watchdog + time window, meaning either too early or too late. The error signal + generated can be routed to either interrupt a safety controller or + to directly reset the SoC. + +allOf: + - $ref: "watchdog.yaml#" + +properties: + compatible: + enum: + - ti,j7-rti-wdt + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + + power-domains: + maxItems: 1 + + assigned-clocks: + maxItems: 1 + + assigned-clocks-parents: + maxItems: 1 + +required: + - compatible + - reg + - clocks + - power-domains + +examples: + - | + /* + * RTI WDT in main domain on J721e SoC. Assigned clocks are used to + * select the source clock for the watchdog, forcing it to tick with + * a 32kHz clock in this case. + */ + #include + + watchdog0: rti@2200000 { + compatible = "ti,rti-wdt"; + reg = <0x0 0x2200000 0x0 0x100>; + clocks = <&k3_clks 252 1>; + power-domains = <&k3_pds 252 TI_SCI_PD_EXCLUSIVE>; + assigned-clocks = <&k3_clks 252 1>; + assigned-clock-parents = <&k3_clks 252 5>; + }; From 2d63908bdbfbce0d98195b22236ad5105dc6eba2 Mon Sep 17 00:00:00 2001 From: Tero Kristo Date: Thu, 12 Mar 2020 11:58:07 +0200 Subject: [PATCH 399/635] watchdog: Add K3 RTI watchdog support Texas Instruments K3 SoCs contain an RTI (Real Time Interrupt) module which can be used as a watchdog. This IP provides a support for windowed watchdog mode, in which the watchdog must be petted within a certain time window. If it is petted either too soon, or too late, a watchdog error will be triggered. Signed-off-by: Tero Kristo Reviewed-by: Guenter Roeck Link: https://lore.kernel.org/r/20200312095808.19907-4-t-kristo@ti.com Signed-off-by: Guenter Roeck Signed-off-by: Wim Van Sebroeck --- drivers/watchdog/Kconfig | 8 ++ drivers/watchdog/Makefile | 1 + drivers/watchdog/rti_wdt.c | 255 +++++++++++++++++++++++++++++++++++++ 3 files changed, 264 insertions(+) create mode 100644 drivers/watchdog/rti_wdt.c diff --git a/drivers/watchdog/Kconfig b/drivers/watchdog/Kconfig index 9ea2b43d4b01..0663c604bd64 100644 --- a/drivers/watchdog/Kconfig +++ b/drivers/watchdog/Kconfig @@ -584,6 +584,14 @@ config DAVINCI_WATCHDOG NOTE: once enabled, this timer cannot be disabled. Say N if you are unsure. +config K3_RTI_WATCHDOG + tristate "Texas Instruments K3 RTI watchdog" + depends on ARCH_K3 || COMPILE_TEST + select WATCHDOG_CORE + help + Say Y here if you want to include support for the K3 watchdog + timer (RTI module) available in the K3 generation of processors. + config ORION_WATCHDOG tristate "Orion watchdog" depends on ARCH_ORION5X || ARCH_DOVE || MACH_DOVE || ARCH_MVEBU || (COMPILE_TEST && !ARCH_EBSA110) diff --git a/drivers/watchdog/Makefile b/drivers/watchdog/Makefile index 2ee352bf3372..6de2e4ceef19 100644 --- a/drivers/watchdog/Makefile +++ b/drivers/watchdog/Makefile @@ -57,6 +57,7 @@ obj-$(CONFIG_EP93XX_WATCHDOG) += ep93xx_wdt.o obj-$(CONFIG_PNX4008_WATCHDOG) += pnx4008_wdt.o obj-$(CONFIG_IOP_WATCHDOG) += iop_wdt.o obj-$(CONFIG_DAVINCI_WATCHDOG) += davinci_wdt.o +obj-$(CONFIG_K3_RTI_WATCHDOG) += rti_wdt.o obj-$(CONFIG_ORION_WATCHDOG) += orion_wdt.o obj-$(CONFIG_SUNXI_WATCHDOG) += sunxi_wdt.o obj-$(CONFIG_RN5T618_WATCHDOG) += rn5t618_wdt.o diff --git a/drivers/watchdog/rti_wdt.c b/drivers/watchdog/rti_wdt.c new file mode 100644 index 000000000000..d456dd72d99a --- /dev/null +++ b/drivers/watchdog/rti_wdt.c @@ -0,0 +1,255 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Watchdog driver for the K3 RTI module + * + * (c) Copyright 2019-2020 Texas Instruments Inc. + * All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DEFAULT_HEARTBEAT 60 + +/* Max heartbeat is calculated at 32kHz source clock */ +#define MAX_HEARTBEAT 1000 + +/* Timer register set definition */ +#define RTIDWDCTRL 0x90 +#define RTIDWDPRLD 0x94 +#define RTIWDSTATUS 0x98 +#define RTIWDKEY 0x9c +#define RTIDWDCNTR 0xa0 +#define RTIWWDRXCTRL 0xa4 +#define RTIWWDSIZECTRL 0xa8 + +#define RTIWWDRX_NMI 0xa + +#define RTIWWDSIZE_50P 0x50 + +#define WDENABLE_KEY 0xa98559da + +#define WDKEY_SEQ0 0xe51a +#define WDKEY_SEQ1 0xa35c + +#define WDT_PRELOAD_SHIFT 13 + +#define WDT_PRELOAD_MAX 0xfff + +#define DWDST BIT(1) + +static int heartbeat; + +/* + * struct to hold data for each WDT device + * @base - base io address of WD device + * @freq - source clock frequency of WDT + * @wdd - hold watchdog device as is in WDT core + */ +struct rti_wdt_device { + void __iomem *base; + unsigned long freq; + struct watchdog_device wdd; +}; + +static int rti_wdt_start(struct watchdog_device *wdd) +{ + u32 timer_margin; + struct rti_wdt_device *wdt = watchdog_get_drvdata(wdd); + + /* set timeout period */ + timer_margin = (u64)wdd->timeout * wdt->freq; + timer_margin >>= WDT_PRELOAD_SHIFT; + if (timer_margin > WDT_PRELOAD_MAX) + timer_margin = WDT_PRELOAD_MAX; + writel_relaxed(timer_margin, wdt->base + RTIDWDPRLD); + + /* + * RTI only supports a windowed mode, where the watchdog can only + * be petted during the open window; not too early or not too late. + * The HW configuration options only allow for the open window size + * to be 50% or less than that; we obviouly want to configure the open + * window as large as possible so we select the 50% option. To avoid + * any glitches, we accommodate 5% safety margin also, so we setup + * the min_hw_hearbeat at 55% of the timeout period. + */ + wdd->min_hw_heartbeat_ms = 11 * wdd->timeout * 1000 / 20; + + /* Generate NMI when wdt expires */ + writel_relaxed(RTIWWDRX_NMI, wdt->base + RTIWWDRXCTRL); + + /* Open window size 50%; this is the largest window size available */ + writel_relaxed(RTIWWDSIZE_50P, wdt->base + RTIWWDSIZECTRL); + + readl_relaxed(wdt->base + RTIWWDSIZECTRL); + + /* enable watchdog */ + writel_relaxed(WDENABLE_KEY, wdt->base + RTIDWDCTRL); + return 0; +} + +static int rti_wdt_ping(struct watchdog_device *wdd) +{ + struct rti_wdt_device *wdt = watchdog_get_drvdata(wdd); + + /* put watchdog in service state */ + writel_relaxed(WDKEY_SEQ0, wdt->base + RTIWDKEY); + /* put watchdog in active state */ + writel_relaxed(WDKEY_SEQ1, wdt->base + RTIWDKEY); + + return 0; +} + +static unsigned int rti_wdt_get_timeleft(struct watchdog_device *wdd) +{ + u64 timer_counter; + u32 val; + struct rti_wdt_device *wdt = watchdog_get_drvdata(wdd); + + /* if timeout has occurred then return 0 */ + val = readl_relaxed(wdt->base + RTIWDSTATUS); + if (val & DWDST) + return 0; + + timer_counter = readl_relaxed(wdt->base + RTIDWDCNTR); + + do_div(timer_counter, wdt->freq); + + return timer_counter; +} + +static const struct watchdog_info rti_wdt_info = { + .options = WDIOF_KEEPALIVEPING, + .identity = "K3 RTI Watchdog", +}; + +static const struct watchdog_ops rti_wdt_ops = { + .owner = THIS_MODULE, + .start = rti_wdt_start, + .ping = rti_wdt_ping, + .get_timeleft = rti_wdt_get_timeleft, +}; + +static int rti_wdt_probe(struct platform_device *pdev) +{ + int ret = 0; + struct device *dev = &pdev->dev; + struct resource *wdt_mem; + struct watchdog_device *wdd; + struct rti_wdt_device *wdt; + struct clk *clk; + + wdt = devm_kzalloc(dev, sizeof(*wdt), GFP_KERNEL); + if (!wdt) + return -ENOMEM; + + clk = clk_get(dev, NULL); + if (IS_ERR(clk)) { + if (PTR_ERR(clk) != -EPROBE_DEFER) + dev_err(dev, "failed to get clock\n"); + return PTR_ERR(clk); + } + + wdt->freq = clk_get_rate(clk); + + clk_put(clk); + + if (!wdt->freq) { + dev_err(dev, "Failed to get fck rate.\n"); + return -EINVAL; + } + + pm_runtime_enable(dev); + ret = pm_runtime_get_sync(dev); + if (ret) { + if (ret != -EPROBE_DEFER) + dev_err(&pdev->dev, "runtime pm failed\n"); + return ret; + } + + platform_set_drvdata(pdev, wdt); + + wdd = &wdt->wdd; + wdd->info = &rti_wdt_info; + wdd->ops = &rti_wdt_ops; + wdd->min_timeout = 1; + wdd->max_hw_heartbeat_ms = (WDT_PRELOAD_MAX << WDT_PRELOAD_SHIFT) / + wdt->freq * 1000; + wdd->timeout = DEFAULT_HEARTBEAT; + wdd->parent = dev; + + watchdog_init_timeout(wdd, heartbeat, dev); + + watchdog_set_drvdata(wdd, wdt); + watchdog_set_nowayout(wdd, 1); + watchdog_set_restart_priority(wdd, 128); + + wdt_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); + wdt->base = devm_ioremap_resource(dev, wdt_mem); + if (IS_ERR(wdt->base)) { + ret = PTR_ERR(wdt->base); + goto err_iomap; + } + + ret = watchdog_register_device(wdd); + if (ret) { + dev_err(dev, "cannot register watchdog device\n"); + goto err_iomap; + } + + return 0; + +err_iomap: + pm_runtime_put_sync(&pdev->dev); + + return ret; +} + +static int rti_wdt_remove(struct platform_device *pdev) +{ + struct rti_wdt_device *wdt = platform_get_drvdata(pdev); + + watchdog_unregister_device(&wdt->wdd); + pm_runtime_put(&pdev->dev); + + return 0; +} + +static const struct of_device_id rti_wdt_of_match[] = { + { .compatible = "ti,j7-rti-wdt", }, + {}, +}; +MODULE_DEVICE_TABLE(of, rti_wdt_of_match); + +static struct platform_driver rti_wdt_driver = { + .driver = { + .name = "rti-wdt", + .of_match_table = rti_wdt_of_match, + }, + .probe = rti_wdt_probe, + .remove = rti_wdt_remove, +}; + +module_platform_driver(rti_wdt_driver); + +MODULE_AUTHOR("Tero Kristo "); +MODULE_DESCRIPTION("K3 RTI Watchdog Driver"); + +module_param(heartbeat, int, 0); +MODULE_PARM_DESC(heartbeat, + "Watchdog heartbeat period in seconds from 1 to " + __MODULE_STRING(MAX_HEARTBEAT) ", default " + __MODULE_STRING(DEFAULT_HEARTBEAT)); + +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:rti-wdt"); From add42de31721fa29ed77a7ce388674d69f9d31a4 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Wed, 1 Apr 2020 10:07:16 -0400 Subject: [PATCH 400/635] NFS: Fix a page leak in nfs_destroy_unlinked_subrequests() When we detach a subrequest from the list, we must also release the reference it holds to the parent. Fixes: 5b2b5187fa85 ("NFS: Fix nfs_page_group_destroy() and nfs_lock_and_join_requests() race cases") Cc: stable@vger.kernel.org # v4.14+ Signed-off-by: Trond Myklebust --- fs/nfs/write.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/nfs/write.c b/fs/nfs/write.c index 03b7f64f7c4f..626e99cbb50e 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -444,6 +444,7 @@ nfs_destroy_unlinked_subrequests(struct nfs_page *destroy_list, } subreq->wb_head = subreq; + nfs_release_request(old_head); if (test_and_clear_bit(PG_INODE_REF, &subreq->wb_flags)) { nfs_release_request(subreq); From 20c384f1ea1a0bc7320bc445c72dd02d2970d594 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Thu, 26 Mar 2020 22:01:17 +0800 Subject: [PATCH 401/635] vhost: refine vhost and vringh kconfig Currently, CONFIG_VHOST depends on CONFIG_VIRTUALIZATION. But vhost is not necessarily for VM since it's a generic userspace and kernel communication protocol. Such dependency may prevent archs without virtualization support from using vhost. To solve this, a dedicated vhost menu is created under drivers so CONIFG_VHOST can be decoupled out of CONFIG_VIRTUALIZATION. While at it, also squash Kconfig.vringh into vhost Kconfig file. This avoids the trick of conditional inclusion from VOP or CAIF. Then it will be easier to introduce new vringh users and common dependency for both vringh and vhost. Signed-off-by: Jason Wang Link: https://lore.kernel.org/r/20200326140125.19794-2-jasowang@redhat.com Signed-off-by: Michael S. Tsirkin --- arch/arm/kvm/Kconfig | 2 -- arch/arm64/kvm/Kconfig | 2 -- arch/mips/kvm/Kconfig | 2 -- arch/powerpc/kvm/Kconfig | 2 -- arch/s390/kvm/Kconfig | 4 ---- arch/x86/kvm/Kconfig | 4 ---- drivers/Kconfig | 2 ++ drivers/misc/mic/Kconfig | 4 ---- drivers/net/caif/Kconfig | 4 ---- drivers/vhost/Kconfig | 28 +++++++++++++++++++++------- drivers/vhost/Kconfig.vringh | 6 ------ 11 files changed, 23 insertions(+), 37 deletions(-) delete mode 100644 drivers/vhost/Kconfig.vringh diff --git a/arch/arm/kvm/Kconfig b/arch/arm/kvm/Kconfig index f591026347a5..be97393761bf 100644 --- a/arch/arm/kvm/Kconfig +++ b/arch/arm/kvm/Kconfig @@ -54,6 +54,4 @@ config KVM_ARM_HOST ---help--- Provides host support for ARM processors. -source "drivers/vhost/Kconfig" - endif # VIRTUALIZATION diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig index a475c68cbfec..449386d76441 100644 --- a/arch/arm64/kvm/Kconfig +++ b/arch/arm64/kvm/Kconfig @@ -64,6 +64,4 @@ config KVM_ARM_PMU config KVM_INDIRECT_VECTORS def_bool KVM && (HARDEN_BRANCH_PREDICTOR || HARDEN_EL2_VECTORS) -source "drivers/vhost/Kconfig" - endif # VIRTUALIZATION diff --git a/arch/mips/kvm/Kconfig b/arch/mips/kvm/Kconfig index eac25aef21e0..b91d145aa2d5 100644 --- a/arch/mips/kvm/Kconfig +++ b/arch/mips/kvm/Kconfig @@ -72,6 +72,4 @@ config KVM_MIPS_DEBUG_COP0_COUNTERS If unsure, say N. -source "drivers/vhost/Kconfig" - endif # VIRTUALIZATION diff --git a/arch/powerpc/kvm/Kconfig b/arch/powerpc/kvm/Kconfig index 711fca9bc6f0..12885eda324e 100644 --- a/arch/powerpc/kvm/Kconfig +++ b/arch/powerpc/kvm/Kconfig @@ -204,6 +204,4 @@ config KVM_XIVE default y depends on KVM_XICS && PPC_XIVE_NATIVE && KVM_BOOK3S_HV_POSSIBLE -source "drivers/vhost/Kconfig" - endif # VIRTUALIZATION diff --git a/arch/s390/kvm/Kconfig b/arch/s390/kvm/Kconfig index d3db3d7ed077..def3b60f1fe8 100644 --- a/arch/s390/kvm/Kconfig +++ b/arch/s390/kvm/Kconfig @@ -55,8 +55,4 @@ config KVM_S390_UCONTROL If unsure, say N. -# OK, it's a little counter-intuitive to do this, but it puts it neatly under -# the virtualization menu. -source "drivers/vhost/Kconfig" - endif # VIRTUALIZATION diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index 9fea0757db92..d8154e0684b6 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -107,8 +107,4 @@ config KVM_MMU_AUDIT This option adds a R/W kVM module parameter 'mmu_audit', which allows auditing of KVM MMU events at runtime. -# OK, it's a little counter-intuitive to do this, but it puts it neatly under -# the virtualization menu. -source "drivers/vhost/Kconfig" - endif # VIRTUALIZATION diff --git a/drivers/Kconfig b/drivers/Kconfig index 8befa53f43be..7a6d8b2b68b4 100644 --- a/drivers/Kconfig +++ b/drivers/Kconfig @@ -138,6 +138,8 @@ source "drivers/virt/Kconfig" source "drivers/virtio/Kconfig" +source "drivers/vhost/Kconfig" + source "drivers/hv/Kconfig" source "drivers/xen/Kconfig" diff --git a/drivers/misc/mic/Kconfig b/drivers/misc/mic/Kconfig index b6841ba6d922..8f201d019f5a 100644 --- a/drivers/misc/mic/Kconfig +++ b/drivers/misc/mic/Kconfig @@ -133,8 +133,4 @@ config VOP OS and tools for MIC to use with this driver are available from . -if VOP -source "drivers/vhost/Kconfig.vringh" -endif - endmenu diff --git a/drivers/net/caif/Kconfig b/drivers/net/caif/Kconfig index e74e2bb61236..9db0570c5beb 100644 --- a/drivers/net/caif/Kconfig +++ b/drivers/net/caif/Kconfig @@ -58,8 +58,4 @@ config CAIF_VIRTIO ---help--- The CAIF driver for CAIF over Virtio. -if CAIF_VIRTIO -source "drivers/vhost/Kconfig.vringh" -endif - endif # CAIF_DRIVERS diff --git a/drivers/vhost/Kconfig b/drivers/vhost/Kconfig index 3d03ccbd1adc..e775beddc36a 100644 --- a/drivers/vhost/Kconfig +++ b/drivers/vhost/Kconfig @@ -1,4 +1,23 @@ # SPDX-License-Identifier: GPL-2.0-only +config VHOST_RING + tristate + help + This option is selected by any driver which needs to access + the host side of a virtio ring. + +config VHOST + tristate + select VHOST_IOTLB + help + This option is selected by any driver which needs to access + the core of vhost. + +menuconfig VHOST_MENU + bool "VHOST drivers" + default y + +if VHOST_MENU + config VHOST_NET tristate "Host kernel accelerator for virtio net" depends on NET && EVENTFD && (TUN || !TUN) && (TAP || !TAP) @@ -23,8 +42,8 @@ config VHOST_SCSI config VHOST_VSOCK tristate "vhost virtio-vsock driver" depends on VSOCKETS && EVENTFD - select VIRTIO_VSOCKETS_COMMON select VHOST + select VIRTIO_VSOCKETS_COMMON default n ---help--- This kernel module can be loaded in the host kernel to provide AF_VSOCK @@ -34,12 +53,6 @@ config VHOST_VSOCK To compile this driver as a module, choose M here: the module will be called vhost_vsock. -config VHOST - tristate - ---help--- - This option is selected by any driver which needs to access - the core of vhost. - config VHOST_CROSS_ENDIAN_LEGACY bool "Cross-endian support for vhost" default n @@ -54,3 +67,4 @@ config VHOST_CROSS_ENDIAN_LEGACY adds some overhead, it is disabled by default. If unsure, say "N". +endif diff --git a/drivers/vhost/Kconfig.vringh b/drivers/vhost/Kconfig.vringh deleted file mode 100644 index c1fe36a9b8d4..000000000000 --- a/drivers/vhost/Kconfig.vringh +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -config VHOST_RING - tristate - ---help--- - This option is selected by any driver which needs to access - the host side of a virtio ring. From 792a4f2ed24fcdf0a1956e84fe2a71ada318ba7c Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Thu, 26 Mar 2020 22:01:18 +0800 Subject: [PATCH 402/635] vhost: allow per device message handler This patch allow device to register its own message handler during vhost_dev_init(). vDPA device will use it to implement its own DMA mapping logic. Signed-off-by: Jason Wang Link: https://lore.kernel.org/r/20200326140125.19794-3-jasowang@redhat.com Signed-off-by: Michael S. Tsirkin --- drivers/vhost/net.c | 3 ++- drivers/vhost/scsi.c | 2 +- drivers/vhost/vhost.c | 12 ++++++++++-- drivers/vhost/vhost.h | 6 +++++- drivers/vhost/vsock.c | 2 +- 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c index 18e205eeb9af..7b1d2dfec7f2 100644 --- a/drivers/vhost/net.c +++ b/drivers/vhost/net.c @@ -1324,7 +1324,8 @@ static int vhost_net_open(struct inode *inode, struct file *f) } vhost_dev_init(dev, vqs, VHOST_NET_VQ_MAX, UIO_MAXIOV + VHOST_NET_BATCH, - VHOST_NET_PKT_WEIGHT, VHOST_NET_WEIGHT); + VHOST_NET_PKT_WEIGHT, VHOST_NET_WEIGHT, + NULL); vhost_poll_init(n->poll + VHOST_NET_VQ_TX, handle_tx_net, EPOLLOUT, dev); vhost_poll_init(n->poll + VHOST_NET_VQ_RX, handle_rx_net, EPOLLIN, dev); diff --git a/drivers/vhost/scsi.c b/drivers/vhost/scsi.c index 0b949a14bce3..7653667a8cdc 100644 --- a/drivers/vhost/scsi.c +++ b/drivers/vhost/scsi.c @@ -1628,7 +1628,7 @@ static int vhost_scsi_open(struct inode *inode, struct file *f) vs->vqs[i].vq.handle_kick = vhost_scsi_handle_kick; } vhost_dev_init(&vs->dev, vqs, VHOST_SCSI_MAX_VQ, UIO_MAXIOV, - VHOST_SCSI_WEIGHT, 0); + VHOST_SCSI_WEIGHT, 0, NULL); vhost_scsi_init_inflight(vs, NULL); diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index f44340b41494..8e9e2341e40a 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -457,7 +457,9 @@ static size_t vhost_get_desc_size(struct vhost_virtqueue *vq, void vhost_dev_init(struct vhost_dev *dev, struct vhost_virtqueue **vqs, int nvqs, - int iov_limit, int weight, int byte_weight) + int iov_limit, int weight, int byte_weight, + int (*msg_handler)(struct vhost_dev *dev, + struct vhost_iotlb_msg *msg)) { struct vhost_virtqueue *vq; int i; @@ -473,6 +475,7 @@ void vhost_dev_init(struct vhost_dev *dev, dev->iov_limit = iov_limit; dev->weight = weight; dev->byte_weight = byte_weight; + dev->msg_handler = msg_handler; init_llist_head(&dev->work_list); init_waitqueue_head(&dev->wait); INIT_LIST_HEAD(&dev->read_list); @@ -1178,7 +1181,12 @@ ssize_t vhost_chr_write_iter(struct vhost_dev *dev, ret = -EINVAL; goto done; } - if (vhost_process_iotlb_msg(dev, &msg)) { + + if (dev->msg_handler) + ret = dev->msg_handler(dev, &msg); + else + ret = vhost_process_iotlb_msg(dev, &msg); + if (ret) { ret = -EFAULT; goto done; } diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h index a123fd70847e..f9d1a03dd153 100644 --- a/drivers/vhost/vhost.h +++ b/drivers/vhost/vhost.h @@ -174,11 +174,15 @@ struct vhost_dev { int weight; int byte_weight; u64 kcov_handle; + int (*msg_handler)(struct vhost_dev *dev, + struct vhost_iotlb_msg *msg); }; bool vhost_exceeds_weight(struct vhost_virtqueue *vq, int pkts, int total_len); void vhost_dev_init(struct vhost_dev *, struct vhost_virtqueue **vqs, - int nvqs, int iov_limit, int weight, int byte_weight); + int nvqs, int iov_limit, int weight, int byte_weight, + int (*msg_handler)(struct vhost_dev *dev, + struct vhost_iotlb_msg *msg)); long vhost_dev_set_owner(struct vhost_dev *dev); bool vhost_dev_has_owner(struct vhost_dev *dev); long vhost_dev_check_owner(struct vhost_dev *); diff --git a/drivers/vhost/vsock.c b/drivers/vhost/vsock.c index c2d7d57e98cf..97669484a3f6 100644 --- a/drivers/vhost/vsock.c +++ b/drivers/vhost/vsock.c @@ -621,7 +621,7 @@ static int vhost_vsock_dev_open(struct inode *inode, struct file *file) vhost_dev_init(&vsock->dev, vqs, ARRAY_SIZE(vsock->vqs), UIO_MAXIOV, VHOST_VSOCK_PKT_WEIGHT, - VHOST_VSOCK_WEIGHT); + VHOST_VSOCK_WEIGHT, NULL); file->private_data = vsock; spin_lock_init(&vsock->send_pkt_list_lock); From 0bbe30668d89ec8a309f28ced6d092c90fb23e8c Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Thu, 26 Mar 2020 22:01:19 +0800 Subject: [PATCH 403/635] vhost: factor out IOTLB This patch factors out IOTLB into a dedicated module in order to be reused by other modules like vringh. User may choose to enable the automatic retiring by specifying VHOST_IOTLB_FLAG_RETIRE flag to fit for the case of vhost device IOTLB implementation. Signed-off-by: Jason Wang Link: https://lore.kernel.org/r/20200326140125.19794-4-jasowang@redhat.com Signed-off-by: Michael S. Tsirkin --- MAINTAINERS | 1 + drivers/vhost/Kconfig | 6 + drivers/vhost/Makefile | 3 + drivers/vhost/iotlb.c | 177 +++++++++++++++++++++++++++++ drivers/vhost/net.c | 2 +- drivers/vhost/vhost.c | 221 +++++++++++------------------------- drivers/vhost/vhost.h | 39 ++----- include/linux/vhost_iotlb.h | 47 ++++++++ 8 files changed, 315 insertions(+), 181 deletions(-) create mode 100644 drivers/vhost/iotlb.c create mode 100644 include/linux/vhost_iotlb.h diff --git a/MAINTAINERS b/MAINTAINERS index cc1d18cb5d18..19363ed5e723 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -17766,6 +17766,7 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost.git S: Maintained F: drivers/vhost/ F: include/uapi/linux/vhost.h +F: include/linux/vhost_iotlb.h VIRTIO INPUT DRIVER M: Gerd Hoffmann diff --git a/drivers/vhost/Kconfig b/drivers/vhost/Kconfig index e775beddc36a..37400a1655b4 100644 --- a/drivers/vhost/Kconfig +++ b/drivers/vhost/Kconfig @@ -1,4 +1,9 @@ # SPDX-License-Identifier: GPL-2.0-only +config VHOST_IOTLB + tristate + help + Generic IOTLB implementation for vhost and vringh. + config VHOST_RING tristate help @@ -67,4 +72,5 @@ config VHOST_CROSS_ENDIAN_LEGACY adds some overhead, it is disabled by default. If unsure, say "N". + endif diff --git a/drivers/vhost/Makefile b/drivers/vhost/Makefile index 6c6df24f770c..fb831002bcf0 100644 --- a/drivers/vhost/Makefile +++ b/drivers/vhost/Makefile @@ -11,3 +11,6 @@ vhost_vsock-y := vsock.o obj-$(CONFIG_VHOST_RING) += vringh.o obj-$(CONFIG_VHOST) += vhost.o + +obj-$(CONFIG_VHOST_IOTLB) += vhost_iotlb.o +vhost_iotlb-y := iotlb.o diff --git a/drivers/vhost/iotlb.c b/drivers/vhost/iotlb.c new file mode 100644 index 000000000000..1f0ca6e44410 --- /dev/null +++ b/drivers/vhost/iotlb.c @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Red Hat, Inc. + * Author: Jason Wang + * + * IOTLB implementation for vhost. + */ +#include +#include +#include + +#define MOD_VERSION "0.1" +#define MOD_DESC "VHOST IOTLB" +#define MOD_AUTHOR "Jason Wang " +#define MOD_LICENSE "GPL v2" + +#define START(map) ((map)->start) +#define LAST(map) ((map)->last) + +INTERVAL_TREE_DEFINE(struct vhost_iotlb_map, + rb, __u64, __subtree_last, + START, LAST, static inline, vhost_iotlb_itree); + +/** + * vhost_iotlb_map_free - remove a map node and free it + * @iotlb: the IOTLB + * @map: the map that want to be remove and freed + */ +void vhost_iotlb_map_free(struct vhost_iotlb *iotlb, + struct vhost_iotlb_map *map) +{ + vhost_iotlb_itree_remove(map, &iotlb->root); + list_del(&map->link); + kfree(map); + iotlb->nmaps--; +} +EXPORT_SYMBOL_GPL(vhost_iotlb_map_free); + +/** + * vhost_iotlb_add_range - add a new range to vhost IOTLB + * @iotlb: the IOTLB + * @start: start of the IOVA range + * @last: last of IOVA range + * @addr: the address that is mapped to @start + * @perm: access permission of this range + * + * Returns an error last is smaller than start or memory allocation + * fails + */ +int vhost_iotlb_add_range(struct vhost_iotlb *iotlb, + u64 start, u64 last, + u64 addr, unsigned int perm) +{ + struct vhost_iotlb_map *map; + + if (last < start) + return -EFAULT; + + if (iotlb->limit && + iotlb->nmaps == iotlb->limit && + iotlb->flags & VHOST_IOTLB_FLAG_RETIRE) { + map = list_first_entry(&iotlb->list, typeof(*map), link); + vhost_iotlb_map_free(iotlb, map); + } + + map = kmalloc(sizeof(*map), GFP_ATOMIC); + if (!map) + return -ENOMEM; + + map->start = start; + map->size = last - start + 1; + map->last = last; + map->addr = addr; + map->perm = perm; + + iotlb->nmaps++; + vhost_iotlb_itree_insert(map, &iotlb->root); + + INIT_LIST_HEAD(&map->link); + list_add_tail(&map->link, &iotlb->list); + + return 0; +} +EXPORT_SYMBOL_GPL(vhost_iotlb_add_range); + +/** + * vring_iotlb_del_range - delete overlapped ranges from vhost IOTLB + * @iotlb: the IOTLB + * @start: start of the IOVA range + * @last: last of IOVA range + */ +void vhost_iotlb_del_range(struct vhost_iotlb *iotlb, u64 start, u64 last) +{ + struct vhost_iotlb_map *map; + + while ((map = vhost_iotlb_itree_iter_first(&iotlb->root, + start, last))) + vhost_iotlb_map_free(iotlb, map); +} +EXPORT_SYMBOL_GPL(vhost_iotlb_del_range); + +/** + * vhost_iotlb_alloc - add a new vhost IOTLB + * @limit: maximum number of IOTLB entries + * @flags: VHOST_IOTLB_FLAG_XXX + * + * Returns an error is memory allocation fails + */ +struct vhost_iotlb *vhost_iotlb_alloc(unsigned int limit, unsigned int flags) +{ + struct vhost_iotlb *iotlb = kzalloc(sizeof(*iotlb), GFP_KERNEL); + + if (!iotlb) + return NULL; + + iotlb->root = RB_ROOT_CACHED; + iotlb->limit = limit; + iotlb->nmaps = 0; + iotlb->flags = flags; + INIT_LIST_HEAD(&iotlb->list); + + return iotlb; +} +EXPORT_SYMBOL_GPL(vhost_iotlb_alloc); + +/** + * vhost_iotlb_reset - reset vhost IOTLB (free all IOTLB entries) + * @iotlb: the IOTLB to be reset + */ +void vhost_iotlb_reset(struct vhost_iotlb *iotlb) +{ + vhost_iotlb_del_range(iotlb, 0ULL, 0ULL - 1); +} +EXPORT_SYMBOL_GPL(vhost_iotlb_reset); + +/** + * vhost_iotlb_free - reset and free vhost IOTLB + * @iotlb: the IOTLB to be freed + */ +void vhost_iotlb_free(struct vhost_iotlb *iotlb) +{ + if (iotlb) { + vhost_iotlb_reset(iotlb); + kfree(iotlb); + } +} +EXPORT_SYMBOL_GPL(vhost_iotlb_free); + +/** + * vhost_iotlb_itree_first - return the first overlapped range + * @iotlb: the IOTLB + * @start: start of IOVA range + * @end: end of IOVA range + */ +struct vhost_iotlb_map * +vhost_iotlb_itree_first(struct vhost_iotlb *iotlb, u64 start, u64 last) +{ + return vhost_iotlb_itree_iter_first(&iotlb->root, start, last); +} +EXPORT_SYMBOL_GPL(vhost_iotlb_itree_first); + +/** + * vhost_iotlb_itree_first - return the next overlapped range + * @iotlb: the IOTLB + * @start: start of IOVA range + * @end: end of IOVA range + */ +struct vhost_iotlb_map * +vhost_iotlb_itree_next(struct vhost_iotlb_map *map, u64 start, u64 last) +{ + return vhost_iotlb_itree_iter_next(map, start, last); +} +EXPORT_SYMBOL_GPL(vhost_iotlb_itree_next); + +MODULE_VERSION(MOD_VERSION); +MODULE_DESCRIPTION(MOD_DESC); +MODULE_AUTHOR(MOD_AUTHOR); +MODULE_LICENSE(MOD_LICENSE); diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c index 7b1d2dfec7f2..87469d67ede8 100644 --- a/drivers/vhost/net.c +++ b/drivers/vhost/net.c @@ -1587,7 +1587,7 @@ static long vhost_net_reset_owner(struct vhost_net *n) struct socket *tx_sock = NULL; struct socket *rx_sock = NULL; long err; - struct vhost_umem *umem; + struct vhost_iotlb *umem; mutex_lock(&n->dev.mutex); err = vhost_dev_check_owner(&n->dev); diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index 8e9e2341e40a..d450e16c5c25 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -50,10 +50,6 @@ enum { #define vhost_used_event(vq) ((__virtio16 __user *)&vq->avail->ring[vq->num]) #define vhost_avail_event(vq) ((__virtio16 __user *)&vq->used->ring[vq->num]) -INTERVAL_TREE_DEFINE(struct vhost_umem_node, - rb, __u64, __subtree_last, - START, LAST, static inline, vhost_umem_interval_tree); - #ifdef CONFIG_VHOST_CROSS_ENDIAN_LEGACY static void vhost_disable_cross_endian(struct vhost_virtqueue *vq) { @@ -584,21 +580,25 @@ err_mm: } EXPORT_SYMBOL_GPL(vhost_dev_set_owner); -struct vhost_umem *vhost_dev_reset_owner_prepare(void) +static struct vhost_iotlb *iotlb_alloc(void) { - return kvzalloc(sizeof(struct vhost_umem), GFP_KERNEL); + return vhost_iotlb_alloc(max_iotlb_entries, + VHOST_IOTLB_FLAG_RETIRE); +} + +struct vhost_iotlb *vhost_dev_reset_owner_prepare(void) +{ + return iotlb_alloc(); } EXPORT_SYMBOL_GPL(vhost_dev_reset_owner_prepare); /* Caller should have device mutex */ -void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_umem *umem) +void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_iotlb *umem) { int i; vhost_dev_cleanup(dev); - /* Restore memory to default empty mapping. */ - INIT_LIST_HEAD(&umem->umem_list); dev->umem = umem; /* We don't need VQ locks below since vhost_dev_cleanup makes sure * VQs aren't running. @@ -621,28 +621,6 @@ void vhost_dev_stop(struct vhost_dev *dev) } EXPORT_SYMBOL_GPL(vhost_dev_stop); -static void vhost_umem_free(struct vhost_umem *umem, - struct vhost_umem_node *node) -{ - vhost_umem_interval_tree_remove(node, &umem->umem_tree); - list_del(&node->link); - kfree(node); - umem->numem--; -} - -static void vhost_umem_clean(struct vhost_umem *umem) -{ - struct vhost_umem_node *node, *tmp; - - if (!umem) - return; - - list_for_each_entry_safe(node, tmp, &umem->umem_list, link) - vhost_umem_free(umem, node); - - kvfree(umem); -} - static void vhost_clear_msg(struct vhost_dev *dev) { struct vhost_msg_node *node, *n; @@ -680,9 +658,9 @@ void vhost_dev_cleanup(struct vhost_dev *dev) eventfd_ctx_put(dev->log_ctx); dev->log_ctx = NULL; /* No one will access memory at this point */ - vhost_umem_clean(dev->umem); + vhost_iotlb_free(dev->umem); dev->umem = NULL; - vhost_umem_clean(dev->iotlb); + vhost_iotlb_free(dev->iotlb); dev->iotlb = NULL; vhost_clear_msg(dev); wake_up_interruptible_poll(&dev->wait, EPOLLIN | EPOLLRDNORM); @@ -718,27 +696,26 @@ static bool vhost_overflow(u64 uaddr, u64 size) } /* Caller should have vq mutex and device mutex. */ -static bool vq_memory_access_ok(void __user *log_base, struct vhost_umem *umem, +static bool vq_memory_access_ok(void __user *log_base, struct vhost_iotlb *umem, int log_all) { - struct vhost_umem_node *node; + struct vhost_iotlb_map *map; if (!umem) return false; - list_for_each_entry(node, &umem->umem_list, link) { - unsigned long a = node->userspace_addr; + list_for_each_entry(map, &umem->list, link) { + unsigned long a = map->addr; - if (vhost_overflow(node->userspace_addr, node->size)) + if (vhost_overflow(map->addr, map->size)) return false; - if (!access_ok((void __user *)a, - node->size)) + if (!access_ok((void __user *)a, map->size)) return false; else if (log_all && !log_access_ok(log_base, - node->start, - node->size)) + map->start, + map->size)) return false; } return true; @@ -748,17 +725,17 @@ static inline void __user *vhost_vq_meta_fetch(struct vhost_virtqueue *vq, u64 addr, unsigned int size, int type) { - const struct vhost_umem_node *node = vq->meta_iotlb[type]; + const struct vhost_iotlb_map *map = vq->meta_iotlb[type]; - if (!node) + if (!map) return NULL; - return (void *)(uintptr_t)(node->userspace_addr + addr - node->start); + return (void *)(uintptr_t)(map->addr + addr - map->start); } /* Can we switch to this memory table? */ /* Caller should have device mutex but not vq mutex */ -static bool memory_access_ok(struct vhost_dev *d, struct vhost_umem *umem, +static bool memory_access_ok(struct vhost_dev *d, struct vhost_iotlb *umem, int log_all) { int i; @@ -1023,47 +1000,6 @@ static inline int vhost_get_desc(struct vhost_virtqueue *vq, return vhost_copy_from_user(vq, desc, vq->desc + idx, sizeof(*desc)); } -static int vhost_new_umem_range(struct vhost_umem *umem, - u64 start, u64 size, u64 end, - u64 userspace_addr, int perm) -{ - struct vhost_umem_node *tmp, *node; - - if (!size) - return -EFAULT; - - node = kmalloc(sizeof(*node), GFP_ATOMIC); - if (!node) - return -ENOMEM; - - if (umem->numem == max_iotlb_entries) { - tmp = list_first_entry(&umem->umem_list, typeof(*tmp), link); - vhost_umem_free(umem, tmp); - } - - node->start = start; - node->size = size; - node->last = end; - node->userspace_addr = userspace_addr; - node->perm = perm; - INIT_LIST_HEAD(&node->link); - list_add_tail(&node->link, &umem->umem_list); - vhost_umem_interval_tree_insert(node, &umem->umem_tree); - umem->numem++; - - return 0; -} - -static void vhost_del_umem_range(struct vhost_umem *umem, - u64 start, u64 end) -{ - struct vhost_umem_node *node; - - while ((node = vhost_umem_interval_tree_iter_first(&umem->umem_tree, - start, end))) - vhost_umem_free(umem, node); -} - static void vhost_iotlb_notify_vq(struct vhost_dev *d, struct vhost_iotlb_msg *msg) { @@ -1120,9 +1056,9 @@ static int vhost_process_iotlb_msg(struct vhost_dev *dev, break; } vhost_vq_meta_reset(dev); - if (vhost_new_umem_range(dev->iotlb, msg->iova, msg->size, - msg->iova + msg->size - 1, - msg->uaddr, msg->perm)) { + if (vhost_iotlb_add_range(dev->iotlb, msg->iova, + msg->iova + msg->size - 1, + msg->uaddr, msg->perm)) { ret = -ENOMEM; break; } @@ -1134,8 +1070,8 @@ static int vhost_process_iotlb_msg(struct vhost_dev *dev, break; } vhost_vq_meta_reset(dev); - vhost_del_umem_range(dev->iotlb, msg->iova, - msg->iova + msg->size - 1); + vhost_iotlb_del_range(dev->iotlb, msg->iova, + msg->iova + msg->size - 1); break; default: ret = -EINVAL; @@ -1319,44 +1255,42 @@ static bool vq_access_ok(struct vhost_virtqueue *vq, unsigned int num, } static void vhost_vq_meta_update(struct vhost_virtqueue *vq, - const struct vhost_umem_node *node, + const struct vhost_iotlb_map *map, int type) { int access = (type == VHOST_ADDR_USED) ? VHOST_ACCESS_WO : VHOST_ACCESS_RO; - if (likely(node->perm & access)) - vq->meta_iotlb[type] = node; + if (likely(map->perm & access)) + vq->meta_iotlb[type] = map; } static bool iotlb_access_ok(struct vhost_virtqueue *vq, int access, u64 addr, u64 len, int type) { - const struct vhost_umem_node *node; - struct vhost_umem *umem = vq->iotlb; + const struct vhost_iotlb_map *map; + struct vhost_iotlb *umem = vq->iotlb; u64 s = 0, size, orig_addr = addr, last = addr + len - 1; if (vhost_vq_meta_fetch(vq, addr, len, type)) return true; while (len > s) { - node = vhost_umem_interval_tree_iter_first(&umem->umem_tree, - addr, - last); - if (node == NULL || node->start > addr) { + map = vhost_iotlb_itree_first(umem, addr, last); + if (map == NULL || map->start > addr) { vhost_iotlb_miss(vq, addr, access); return false; - } else if (!(node->perm & access)) { + } else if (!(map->perm & access)) { /* Report the possible access violation by * request another translation from userspace. */ return false; } - size = node->size - addr + node->start; + size = map->size - addr + map->start; if (orig_addr == addr && size >= len) - vhost_vq_meta_update(vq, node, type); + vhost_vq_meta_update(vq, map, type); s += size; addr += size; @@ -1372,12 +1306,12 @@ int vq_meta_prefetch(struct vhost_virtqueue *vq) if (!vq->iotlb) return 1; - return iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->desc, + return iotlb_access_ok(vq, VHOST_MAP_RO, (u64)(uintptr_t)vq->desc, vhost_get_desc_size(vq, num), VHOST_ADDR_DESC) && - iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->avail, + iotlb_access_ok(vq, VHOST_MAP_RO, (u64)(uintptr_t)vq->avail, vhost_get_avail_size(vq, num), VHOST_ADDR_AVAIL) && - iotlb_access_ok(vq, VHOST_ACCESS_WO, (u64)(uintptr_t)vq->used, + iotlb_access_ok(vq, VHOST_MAP_WO, (u64)(uintptr_t)vq->used, vhost_get_used_size(vq, num), VHOST_ADDR_USED); } EXPORT_SYMBOL_GPL(vq_meta_prefetch); @@ -1416,25 +1350,11 @@ bool vhost_vq_access_ok(struct vhost_virtqueue *vq) } EXPORT_SYMBOL_GPL(vhost_vq_access_ok); -static struct vhost_umem *vhost_umem_alloc(void) -{ - struct vhost_umem *umem = kvzalloc(sizeof(*umem), GFP_KERNEL); - - if (!umem) - return NULL; - - umem->umem_tree = RB_ROOT_CACHED; - umem->numem = 0; - INIT_LIST_HEAD(&umem->umem_list); - - return umem; -} - static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m) { struct vhost_memory mem, *newmem; struct vhost_memory_region *region; - struct vhost_umem *newumem, *oldumem; + struct vhost_iotlb *newumem, *oldumem; unsigned long size = offsetof(struct vhost_memory, regions); int i; @@ -1456,7 +1376,7 @@ static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m) return -EFAULT; } - newumem = vhost_umem_alloc(); + newumem = iotlb_alloc(); if (!newumem) { kvfree(newmem); return -ENOMEM; @@ -1465,13 +1385,12 @@ static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m) for (region = newmem->regions; region < newmem->regions + mem.nregions; region++) { - if (vhost_new_umem_range(newumem, - region->guest_phys_addr, - region->memory_size, - region->guest_phys_addr + - region->memory_size - 1, - region->userspace_addr, - VHOST_ACCESS_RW)) + if (vhost_iotlb_add_range(newumem, + region->guest_phys_addr, + region->guest_phys_addr + + region->memory_size - 1, + region->userspace_addr, + VHOST_MAP_RW)) goto err; } @@ -1489,11 +1408,11 @@ static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m) } kvfree(newmem); - vhost_umem_clean(oldumem); + vhost_iotlb_free(oldumem); return 0; err: - vhost_umem_clean(newumem); + vhost_iotlb_free(newumem); kvfree(newmem); return -EFAULT; } @@ -1734,10 +1653,10 @@ EXPORT_SYMBOL_GPL(vhost_vring_ioctl); int vhost_init_device_iotlb(struct vhost_dev *d, bool enabled) { - struct vhost_umem *niotlb, *oiotlb; + struct vhost_iotlb *niotlb, *oiotlb; int i; - niotlb = vhost_umem_alloc(); + niotlb = iotlb_alloc(); if (!niotlb) return -ENOMEM; @@ -1753,7 +1672,7 @@ int vhost_init_device_iotlb(struct vhost_dev *d, bool enabled) mutex_unlock(&vq->mutex); } - vhost_umem_clean(oiotlb); + vhost_iotlb_free(oiotlb); return 0; } @@ -1883,8 +1802,8 @@ static int log_write(void __user *log_base, static int log_write_hva(struct vhost_virtqueue *vq, u64 hva, u64 len) { - struct vhost_umem *umem = vq->umem; - struct vhost_umem_node *u; + struct vhost_iotlb *umem = vq->umem; + struct vhost_iotlb_map *u; u64 start, end, l, min; int r; bool hit = false; @@ -1894,16 +1813,15 @@ static int log_write_hva(struct vhost_virtqueue *vq, u64 hva, u64 len) /* More than one GPAs can be mapped into a single HVA. So * iterate all possible umems here to be safe. */ - list_for_each_entry(u, &umem->umem_list, link) { - if (u->userspace_addr > hva - 1 + len || - u->userspace_addr - 1 + u->size < hva) + list_for_each_entry(u, &umem->list, link) { + if (u->addr > hva - 1 + len || + u->addr - 1 + u->size < hva) continue; - start = max(u->userspace_addr, hva); - end = min(u->userspace_addr - 1 + u->size, - hva - 1 + len); + start = max(u->addr, hva); + end = min(u->addr - 1 + u->size, hva - 1 + len); l = end - start + 1; r = log_write(vq->log_base, - u->start + start - u->userspace_addr, + u->start + start - u->addr, l); if (r < 0) return r; @@ -2054,9 +1972,9 @@ EXPORT_SYMBOL_GPL(vhost_vq_init_access); static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len, struct iovec iov[], int iov_size, int access) { - const struct vhost_umem_node *node; + const struct vhost_iotlb_map *map; struct vhost_dev *dev = vq->dev; - struct vhost_umem *umem = dev->iotlb ? dev->iotlb : dev->umem; + struct vhost_iotlb *umem = dev->iotlb ? dev->iotlb : dev->umem; struct iovec *_iov; u64 s = 0; int ret = 0; @@ -2068,25 +1986,24 @@ static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len, break; } - node = vhost_umem_interval_tree_iter_first(&umem->umem_tree, - addr, addr + len - 1); - if (node == NULL || node->start > addr) { + map = vhost_iotlb_itree_first(umem, addr, addr + len - 1); + if (map == NULL || map->start > addr) { if (umem != dev->iotlb) { ret = -EFAULT; break; } ret = -EAGAIN; break; - } else if (!(node->perm & access)) { + } else if (!(map->perm & access)) { ret = -EPERM; break; } _iov = iov + ret; - size = node->size - addr + node->start; + size = map->size - addr + map->start; _iov->iov_len = min((u64)len - s, size); _iov->iov_base = (void __user *)(unsigned long) - (node->userspace_addr + addr - node->start); + (map->addr + addr - map->start); s += size; addr += size; ++ret; diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h index f9d1a03dd153..181382185bbc 100644 --- a/drivers/vhost/vhost.h +++ b/drivers/vhost/vhost.h @@ -12,6 +12,7 @@ #include #include #include +#include struct vhost_work; typedef void (*vhost_work_fn_t)(struct vhost_work *work); @@ -52,27 +53,6 @@ struct vhost_log { u64 len; }; -#define START(node) ((node)->start) -#define LAST(node) ((node)->last) - -struct vhost_umem_node { - struct rb_node rb; - struct list_head link; - __u64 start; - __u64 last; - __u64 size; - __u64 userspace_addr; - __u32 perm; - __u32 flags_padding; - __u64 __subtree_last; -}; - -struct vhost_umem { - struct rb_root_cached umem_tree; - struct list_head umem_list; - int numem; -}; - enum vhost_uaddr_type { VHOST_ADDR_DESC = 0, VHOST_ADDR_AVAIL = 1, @@ -90,7 +70,7 @@ struct vhost_virtqueue { struct vring_desc __user *desc; struct vring_avail __user *avail; struct vring_used __user *used; - const struct vhost_umem_node *meta_iotlb[VHOST_NUM_ADDRS]; + const struct vhost_iotlb_map *meta_iotlb[VHOST_NUM_ADDRS]; struct file *kick; struct eventfd_ctx *call_ctx; struct eventfd_ctx *error_ctx; @@ -128,8 +108,8 @@ struct vhost_virtqueue { struct iovec *indirect; struct vring_used_elem *heads; /* Protected by virtqueue mutex. */ - struct vhost_umem *umem; - struct vhost_umem *iotlb; + struct vhost_iotlb *umem; + struct vhost_iotlb *iotlb; void *private_data; u64 acked_features; u64 acked_backend_features; @@ -164,8 +144,8 @@ struct vhost_dev { struct eventfd_ctx *log_ctx; struct llist_head work_list; struct task_struct *worker; - struct vhost_umem *umem; - struct vhost_umem *iotlb; + struct vhost_iotlb *umem; + struct vhost_iotlb *iotlb; spinlock_t iotlb_lock; struct list_head read_list; struct list_head pending_list; @@ -186,8 +166,8 @@ void vhost_dev_init(struct vhost_dev *, struct vhost_virtqueue **vqs, long vhost_dev_set_owner(struct vhost_dev *dev); bool vhost_dev_has_owner(struct vhost_dev *dev); long vhost_dev_check_owner(struct vhost_dev *); -struct vhost_umem *vhost_dev_reset_owner_prepare(void); -void vhost_dev_reset_owner(struct vhost_dev *, struct vhost_umem *); +struct vhost_iotlb *vhost_dev_reset_owner_prepare(void); +void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_iotlb *iotlb); void vhost_dev_cleanup(struct vhost_dev *); void vhost_dev_stop(struct vhost_dev *); long vhost_dev_ioctl(struct vhost_dev *, unsigned int ioctl, void __user *argp); @@ -233,6 +213,9 @@ ssize_t vhost_chr_write_iter(struct vhost_dev *dev, struct iov_iter *from); int vhost_init_device_iotlb(struct vhost_dev *d, bool enabled); +void vhost_iotlb_map_free(struct vhost_iotlb *iotlb, + struct vhost_iotlb_map *map); + #define vq_err(vq, fmt, ...) do { \ pr_debug(pr_fmt(fmt), ##__VA_ARGS__); \ if ((vq)->error_ctx) \ diff --git a/include/linux/vhost_iotlb.h b/include/linux/vhost_iotlb.h new file mode 100644 index 000000000000..6b09b786a762 --- /dev/null +++ b/include/linux/vhost_iotlb.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_VHOST_IOTLB_H +#define _LINUX_VHOST_IOTLB_H + +#include + +struct vhost_iotlb_map { + struct rb_node rb; + struct list_head link; + u64 start; + u64 last; + u64 size; + u64 addr; +#define VHOST_MAP_RO 0x1 +#define VHOST_MAP_WO 0x2 +#define VHOST_MAP_RW 0x3 + u32 perm; + u32 flags_padding; + u64 __subtree_last; +}; + +#define VHOST_IOTLB_FLAG_RETIRE 0x1 + +struct vhost_iotlb { + struct rb_root_cached root; + struct list_head list; + unsigned int limit; + unsigned int nmaps; + unsigned int flags; +}; + +int vhost_iotlb_add_range(struct vhost_iotlb *iotlb, u64 start, u64 last, + u64 addr, unsigned int perm); +void vhost_iotlb_del_range(struct vhost_iotlb *iotlb, u64 start, u64 last); + +struct vhost_iotlb *vhost_iotlb_alloc(unsigned int limit, unsigned int flags); +void vhost_iotlb_free(struct vhost_iotlb *iotlb); +void vhost_iotlb_reset(struct vhost_iotlb *iotlb); + +struct vhost_iotlb_map * +vhost_iotlb_itree_first(struct vhost_iotlb *iotlb, u64 start, u64 last); +struct vhost_iotlb_map * +vhost_iotlb_itree_next(struct vhost_iotlb_map *map, u64 start, u64 last); + +void vhost_iotlb_map_free(struct vhost_iotlb *iotlb, + struct vhost_iotlb_map *map); +#endif From 9ad9c49cfe970b053bb0ef323b682dd1b4d4f8a0 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Thu, 26 Mar 2020 22:01:20 +0800 Subject: [PATCH 404/635] vringh: IOTLB support This patch implements the third memory accessor for vringh besides current kernel and userspace accessors. This idea is to allow vringh to do the address translation through an IOTLB which is implemented via vhost_map interval tree. Users should setup and IOVA to PA mapping in this IOTLB. This allows us to: - Use vringh to access virtqueues with vIOMMU - Use vringh to implement software virtqueues for vDPA devices Signed-off-by: Jason Wang Link: https://lore.kernel.org/r/20200326140125.19794-5-jasowang@redhat.com Signed-off-by: Michael S. Tsirkin --- drivers/vhost/Kconfig | 1 + drivers/vhost/vringh.c | 421 ++++++++++++++++++++++++++++++++++++++--- include/linux/vringh.h | 36 ++++ 3 files changed, 435 insertions(+), 23 deletions(-) diff --git a/drivers/vhost/Kconfig b/drivers/vhost/Kconfig index 37400a1655b4..128238488078 100644 --- a/drivers/vhost/Kconfig +++ b/drivers/vhost/Kconfig @@ -6,6 +6,7 @@ config VHOST_IOTLB config VHOST_RING tristate + select VHOST_IOTLB help This option is selected by any driver which needs to access the host side of a virtio ring. diff --git a/drivers/vhost/vringh.c b/drivers/vhost/vringh.c index a0a2d74967ef..ee0491f579ac 100644 --- a/drivers/vhost/vringh.c +++ b/drivers/vhost/vringh.c @@ -13,6 +13,9 @@ #include #include #include +#include +#include +#include #include static __printf(1,2) __cold void vringh_bad(const char *fmt, ...) @@ -71,9 +74,11 @@ static inline int __vringh_get_head(const struct vringh *vrh, } /* Copy some bytes to/from the iovec. Returns num copied. */ -static inline ssize_t vringh_iov_xfer(struct vringh_kiov *iov, +static inline ssize_t vringh_iov_xfer(struct vringh *vrh, + struct vringh_kiov *iov, void *ptr, size_t len, - int (*xfer)(void *addr, void *ptr, + int (*xfer)(const struct vringh *vrh, + void *addr, void *ptr, size_t len)) { int err, done = 0; @@ -82,7 +87,7 @@ static inline ssize_t vringh_iov_xfer(struct vringh_kiov *iov, size_t partlen; partlen = min(iov->iov[iov->i].iov_len, len); - err = xfer(iov->iov[iov->i].iov_base, ptr, partlen); + err = xfer(vrh, iov->iov[iov->i].iov_base, ptr, partlen); if (err) return err; done += partlen; @@ -96,6 +101,7 @@ static inline ssize_t vringh_iov_xfer(struct vringh_kiov *iov, /* Fix up old iov element then increment. */ iov->iov[iov->i].iov_len = iov->consumed; iov->iov[iov->i].iov_base -= iov->consumed; + iov->consumed = 0; iov->i++; @@ -227,7 +233,8 @@ static int slow_copy(struct vringh *vrh, void *dst, const void *src, u64 addr, struct vringh_range *r), struct vringh_range *range, - int (*copy)(void *dst, const void *src, size_t len)) + int (*copy)(const struct vringh *vrh, + void *dst, const void *src, size_t len)) { size_t part, len = sizeof(struct vring_desc); @@ -241,7 +248,7 @@ static int slow_copy(struct vringh *vrh, void *dst, const void *src, if (!rcheck(vrh, addr, &part, range, getrange)) return -EINVAL; - err = copy(dst, src, part); + err = copy(vrh, dst, src, part); if (err) return err; @@ -262,7 +269,8 @@ __vringh_iov(struct vringh *vrh, u16 i, struct vringh_range *)), bool (*getrange)(struct vringh *, u64, struct vringh_range *), gfp_t gfp, - int (*copy)(void *dst, const void *src, size_t len)) + int (*copy)(const struct vringh *vrh, + void *dst, const void *src, size_t len)) { int err, count = 0, up_next, desc_max; struct vring_desc desc, *descs; @@ -291,7 +299,7 @@ __vringh_iov(struct vringh *vrh, u16 i, err = slow_copy(vrh, &desc, &descs[i], rcheck, getrange, &slowrange, copy); else - err = copy(&desc, &descs[i], sizeof(desc)); + err = copy(vrh, &desc, &descs[i], sizeof(desc)); if (unlikely(err)) goto fail; @@ -404,7 +412,8 @@ static inline int __vringh_complete(struct vringh *vrh, unsigned int num_used, int (*putu16)(const struct vringh *vrh, __virtio16 *p, u16 val), - int (*putused)(struct vring_used_elem *dst, + int (*putused)(const struct vringh *vrh, + struct vring_used_elem *dst, const struct vring_used_elem *src, unsigned num)) { @@ -420,12 +429,12 @@ static inline int __vringh_complete(struct vringh *vrh, /* Compiler knows num_used == 1 sometimes, hence extra check */ if (num_used > 1 && unlikely(off + num_used >= vrh->vring.num)) { u16 part = vrh->vring.num - off; - err = putused(&used_ring->ring[off], used, part); + err = putused(vrh, &used_ring->ring[off], used, part); if (!err) - err = putused(&used_ring->ring[0], used + part, + err = putused(vrh, &used_ring->ring[0], used + part, num_used - part); } else - err = putused(&used_ring->ring[off], used, num_used); + err = putused(vrh, &used_ring->ring[off], used, num_used); if (err) { vringh_bad("Failed to write %u used entries %u at %p", @@ -564,13 +573,15 @@ static inline int putu16_user(const struct vringh *vrh, __virtio16 *p, u16 val) return put_user(v, (__force __virtio16 __user *)p); } -static inline int copydesc_user(void *dst, const void *src, size_t len) +static inline int copydesc_user(const struct vringh *vrh, + void *dst, const void *src, size_t len) { return copy_from_user(dst, (__force void __user *)src, len) ? -EFAULT : 0; } -static inline int putused_user(struct vring_used_elem *dst, +static inline int putused_user(const struct vringh *vrh, + struct vring_used_elem *dst, const struct vring_used_elem *src, unsigned int num) { @@ -578,13 +589,15 @@ static inline int putused_user(struct vring_used_elem *dst, sizeof(*dst) * num) ? -EFAULT : 0; } -static inline int xfer_from_user(void *src, void *dst, size_t len) +static inline int xfer_from_user(const struct vringh *vrh, void *src, + void *dst, size_t len) { return copy_from_user(dst, (__force void __user *)src, len) ? -EFAULT : 0; } -static inline int xfer_to_user(void *dst, void *src, size_t len) +static inline int xfer_to_user(const struct vringh *vrh, + void *dst, void *src, size_t len) { return copy_to_user((__force void __user *)dst, src, len) ? -EFAULT : 0; @@ -706,7 +719,7 @@ EXPORT_SYMBOL(vringh_getdesc_user); */ ssize_t vringh_iov_pull_user(struct vringh_iov *riov, void *dst, size_t len) { - return vringh_iov_xfer((struct vringh_kiov *)riov, + return vringh_iov_xfer(NULL, (struct vringh_kiov *)riov, dst, len, xfer_from_user); } EXPORT_SYMBOL(vringh_iov_pull_user); @@ -722,7 +735,7 @@ EXPORT_SYMBOL(vringh_iov_pull_user); ssize_t vringh_iov_push_user(struct vringh_iov *wiov, const void *src, size_t len) { - return vringh_iov_xfer((struct vringh_kiov *)wiov, + return vringh_iov_xfer(NULL, (struct vringh_kiov *)wiov, (void *)src, len, xfer_to_user); } EXPORT_SYMBOL(vringh_iov_push_user); @@ -832,13 +845,15 @@ static inline int putu16_kern(const struct vringh *vrh, __virtio16 *p, u16 val) return 0; } -static inline int copydesc_kern(void *dst, const void *src, size_t len) +static inline int copydesc_kern(const struct vringh *vrh, + void *dst, const void *src, size_t len) { memcpy(dst, src, len); return 0; } -static inline int putused_kern(struct vring_used_elem *dst, +static inline int putused_kern(const struct vringh *vrh, + struct vring_used_elem *dst, const struct vring_used_elem *src, unsigned int num) { @@ -846,13 +861,15 @@ static inline int putused_kern(struct vring_used_elem *dst, return 0; } -static inline int xfer_kern(void *src, void *dst, size_t len) +static inline int xfer_kern(const struct vringh *vrh, void *src, + void *dst, size_t len) { memcpy(dst, src, len); return 0; } -static inline int kern_xfer(void *dst, void *src, size_t len) +static inline int kern_xfer(const struct vringh *vrh, void *dst, + void *src, size_t len) { memcpy(dst, src, len); return 0; @@ -949,7 +966,7 @@ EXPORT_SYMBOL(vringh_getdesc_kern); */ ssize_t vringh_iov_pull_kern(struct vringh_kiov *riov, void *dst, size_t len) { - return vringh_iov_xfer(riov, dst, len, xfer_kern); + return vringh_iov_xfer(NULL, riov, dst, len, xfer_kern); } EXPORT_SYMBOL(vringh_iov_pull_kern); @@ -964,7 +981,7 @@ EXPORT_SYMBOL(vringh_iov_pull_kern); ssize_t vringh_iov_push_kern(struct vringh_kiov *wiov, const void *src, size_t len) { - return vringh_iov_xfer(wiov, (void *)src, len, kern_xfer); + return vringh_iov_xfer(NULL, wiov, (void *)src, len, kern_xfer); } EXPORT_SYMBOL(vringh_iov_push_kern); @@ -1042,4 +1059,362 @@ int vringh_need_notify_kern(struct vringh *vrh) } EXPORT_SYMBOL(vringh_need_notify_kern); +static int iotlb_translate(const struct vringh *vrh, + u64 addr, u64 len, struct bio_vec iov[], + int iov_size, u32 perm) +{ + struct vhost_iotlb_map *map; + struct vhost_iotlb *iotlb = vrh->iotlb; + int ret = 0; + u64 s = 0; + + while (len > s) { + u64 size, pa, pfn; + + if (unlikely(ret >= iov_size)) { + ret = -ENOBUFS; + break; + } + + map = vhost_iotlb_itree_first(iotlb, addr, + addr + len - 1); + if (!map || map->start > addr) { + ret = -EINVAL; + break; + } else if (!(map->perm & perm)) { + ret = -EPERM; + break; + } + + size = map->size - addr + map->start; + pa = map->addr + addr - map->start; + pfn = pa >> PAGE_SHIFT; + iov[ret].bv_page = pfn_to_page(pfn); + iov[ret].bv_len = min(len - s, size); + iov[ret].bv_offset = pa & (PAGE_SIZE - 1); + s += size; + addr += size; + ++ret; + } + + return ret; +} + +static inline int copy_from_iotlb(const struct vringh *vrh, void *dst, + void *src, size_t len) +{ + struct iov_iter iter; + struct bio_vec iov[16]; + int ret; + + ret = iotlb_translate(vrh, (u64)(uintptr_t)src, + len, iov, 16, VHOST_MAP_RO); + if (ret < 0) + return ret; + + iov_iter_bvec(&iter, READ, iov, ret, len); + + ret = copy_from_iter(dst, len, &iter); + + return ret; +} + +static inline int copy_to_iotlb(const struct vringh *vrh, void *dst, + void *src, size_t len) +{ + struct iov_iter iter; + struct bio_vec iov[16]; + int ret; + + ret = iotlb_translate(vrh, (u64)(uintptr_t)dst, + len, iov, 16, VHOST_MAP_WO); + if (ret < 0) + return ret; + + iov_iter_bvec(&iter, WRITE, iov, ret, len); + + return copy_to_iter(src, len, &iter); +} + +static inline int getu16_iotlb(const struct vringh *vrh, + u16 *val, const __virtio16 *p) +{ + struct bio_vec iov; + void *kaddr, *from; + int ret; + + /* Atomic read is needed for getu16 */ + ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p), + &iov, 1, VHOST_MAP_RO); + if (ret < 0) + return ret; + + kaddr = kmap_atomic(iov.bv_page); + from = kaddr + iov.bv_offset; + *val = vringh16_to_cpu(vrh, READ_ONCE(*(__virtio16 *)from)); + kunmap_atomic(kaddr); + + return 0; +} + +static inline int putu16_iotlb(const struct vringh *vrh, + __virtio16 *p, u16 val) +{ + struct bio_vec iov; + void *kaddr, *to; + int ret; + + /* Atomic write is needed for putu16 */ + ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p), + &iov, 1, VHOST_MAP_WO); + if (ret < 0) + return ret; + + kaddr = kmap_atomic(iov.bv_page); + to = kaddr + iov.bv_offset; + WRITE_ONCE(*(__virtio16 *)to, cpu_to_vringh16(vrh, val)); + kunmap_atomic(kaddr); + + return 0; +} + +static inline int copydesc_iotlb(const struct vringh *vrh, + void *dst, const void *src, size_t len) +{ + int ret; + + ret = copy_from_iotlb(vrh, dst, (void *)src, len); + if (ret != len) + return -EFAULT; + + return 0; +} + +static inline int xfer_from_iotlb(const struct vringh *vrh, void *src, + void *dst, size_t len) +{ + int ret; + + ret = copy_from_iotlb(vrh, dst, src, len); + if (ret != len) + return -EFAULT; + + return 0; +} + +static inline int xfer_to_iotlb(const struct vringh *vrh, + void *dst, void *src, size_t len) +{ + int ret; + + ret = copy_to_iotlb(vrh, dst, src, len); + if (ret != len) + return -EFAULT; + + return 0; +} + +static inline int putused_iotlb(const struct vringh *vrh, + struct vring_used_elem *dst, + const struct vring_used_elem *src, + unsigned int num) +{ + int size = num * sizeof(*dst); + int ret; + + ret = copy_to_iotlb(vrh, dst, (void *)src, num * sizeof(*dst)); + if (ret != size) + return -EFAULT; + + return 0; +} + +/** + * vringh_init_iotlb - initialize a vringh for a ring with IOTLB. + * @vrh: the vringh to initialize. + * @features: the feature bits for this ring. + * @num: the number of elements. + * @weak_barriers: true if we only need memory barriers, not I/O. + * @desc: the userpace descriptor pointer. + * @avail: the userpace avail pointer. + * @used: the userpace used pointer. + * + * Returns an error if num is invalid. + */ +int vringh_init_iotlb(struct vringh *vrh, u64 features, + unsigned int num, bool weak_barriers, + struct vring_desc *desc, + struct vring_avail *avail, + struct vring_used *used) +{ + return vringh_init_kern(vrh, features, num, weak_barriers, + desc, avail, used); +} +EXPORT_SYMBOL(vringh_init_iotlb); + +/** + * vringh_set_iotlb - initialize a vringh for a ring with IOTLB. + * @vrh: the vring + * @iotlb: iotlb associated with this vring + */ +void vringh_set_iotlb(struct vringh *vrh, struct vhost_iotlb *iotlb) +{ + vrh->iotlb = iotlb; +} +EXPORT_SYMBOL(vringh_set_iotlb); + +/** + * vringh_getdesc_iotlb - get next available descriptor from ring with + * IOTLB. + * @vrh: the kernelspace vring. + * @riov: where to put the readable descriptors (or NULL) + * @wiov: where to put the writable descriptors (or NULL) + * @head: head index we received, for passing to vringh_complete_iotlb(). + * @gfp: flags for allocating larger riov/wiov. + * + * Returns 0 if there was no descriptor, 1 if there was, or -errno. + * + * Note that on error return, you can tell the difference between an + * invalid ring and a single invalid descriptor: in the former case, + * *head will be vrh->vring.num. You may be able to ignore an invalid + * descriptor, but there's not much you can do with an invalid ring. + * + * Note that you may need to clean up riov and wiov, even on error! + */ +int vringh_getdesc_iotlb(struct vringh *vrh, + struct vringh_kiov *riov, + struct vringh_kiov *wiov, + u16 *head, + gfp_t gfp) +{ + int err; + + err = __vringh_get_head(vrh, getu16_iotlb, &vrh->last_avail_idx); + if (err < 0) + return err; + + /* Empty... */ + if (err == vrh->vring.num) + return 0; + + *head = err; + err = __vringh_iov(vrh, *head, riov, wiov, no_range_check, NULL, + gfp, copydesc_iotlb); + if (err) + return err; + + return 1; +} +EXPORT_SYMBOL(vringh_getdesc_iotlb); + +/** + * vringh_iov_pull_iotlb - copy bytes from vring_iov. + * @vrh: the vring. + * @riov: the riov as passed to vringh_getdesc_iotlb() (updated as we consume) + * @dst: the place to copy. + * @len: the maximum length to copy. + * + * Returns the bytes copied <= len or a negative errno. + */ +ssize_t vringh_iov_pull_iotlb(struct vringh *vrh, + struct vringh_kiov *riov, + void *dst, size_t len) +{ + return vringh_iov_xfer(vrh, riov, dst, len, xfer_from_iotlb); +} +EXPORT_SYMBOL(vringh_iov_pull_iotlb); + +/** + * vringh_iov_push_iotlb - copy bytes into vring_iov. + * @vrh: the vring. + * @wiov: the wiov as passed to vringh_getdesc_iotlb() (updated as we consume) + * @dst: the place to copy. + * @len: the maximum length to copy. + * + * Returns the bytes copied <= len or a negative errno. + */ +ssize_t vringh_iov_push_iotlb(struct vringh *vrh, + struct vringh_kiov *wiov, + const void *src, size_t len) +{ + return vringh_iov_xfer(vrh, wiov, (void *)src, len, xfer_to_iotlb); +} +EXPORT_SYMBOL(vringh_iov_push_iotlb); + +/** + * vringh_abandon_iotlb - we've decided not to handle the descriptor(s). + * @vrh: the vring. + * @num: the number of descriptors to put back (ie. num + * vringh_get_iotlb() to undo). + * + * The next vringh_get_iotlb() will return the old descriptor(s) again. + */ +void vringh_abandon_iotlb(struct vringh *vrh, unsigned int num) +{ + /* We only update vring_avail_event(vr) when we want to be notified, + * so we haven't changed that yet. + */ + vrh->last_avail_idx -= num; +} +EXPORT_SYMBOL(vringh_abandon_iotlb); + +/** + * vringh_complete_iotlb - we've finished with descriptor, publish it. + * @vrh: the vring. + * @head: the head as filled in by vringh_getdesc_iotlb. + * @len: the length of data we have written. + * + * You should check vringh_need_notify_iotlb() after one or more calls + * to this function. + */ +int vringh_complete_iotlb(struct vringh *vrh, u16 head, u32 len) +{ + struct vring_used_elem used; + + used.id = cpu_to_vringh32(vrh, head); + used.len = cpu_to_vringh32(vrh, len); + + return __vringh_complete(vrh, &used, 1, putu16_iotlb, putused_iotlb); +} +EXPORT_SYMBOL(vringh_complete_iotlb); + +/** + * vringh_notify_enable_iotlb - we want to know if something changes. + * @vrh: the vring. + * + * This always enables notifications, but returns false if there are + * now more buffers available in the vring. + */ +bool vringh_notify_enable_iotlb(struct vringh *vrh) +{ + return __vringh_notify_enable(vrh, getu16_iotlb, putu16_iotlb); +} +EXPORT_SYMBOL(vringh_notify_enable_iotlb); + +/** + * vringh_notify_disable_iotlb - don't tell us if something changes. + * @vrh: the vring. + * + * This is our normal running state: we disable and then only enable when + * we're going to sleep. + */ +void vringh_notify_disable_iotlb(struct vringh *vrh) +{ + __vringh_notify_disable(vrh, putu16_iotlb); +} +EXPORT_SYMBOL(vringh_notify_disable_iotlb); + +/** + * vringh_need_notify_iotlb - must we tell the other side about used buffers? + * @vrh: the vring we've called vringh_complete_iotlb() on. + * + * Returns -errno or 0 if we don't need to tell the other side, 1 if we do. + */ +int vringh_need_notify_iotlb(struct vringh *vrh) +{ + return __vringh_need_notify(vrh, getu16_iotlb); +} +EXPORT_SYMBOL(vringh_need_notify_iotlb); + + MODULE_LICENSE("GPL"); diff --git a/include/linux/vringh.h b/include/linux/vringh.h index d237087eb257..bd0503ca6f8f 100644 --- a/include/linux/vringh.h +++ b/include/linux/vringh.h @@ -14,6 +14,8 @@ #include #include #include +#include +#include #include /* virtio_ring with information needed for host access. */ @@ -39,6 +41,9 @@ struct vringh { /* The vring (note: it may contain user pointers!) */ struct vring vring; + /* IOTLB for this vring */ + struct vhost_iotlb *iotlb; + /* The function to call to notify the guest about added buffers */ void (*notify)(struct vringh *); }; @@ -248,4 +253,35 @@ static inline __virtio64 cpu_to_vringh64(const struct vringh *vrh, u64 val) { return __cpu_to_virtio64(vringh_is_little_endian(vrh), val); } + +void vringh_set_iotlb(struct vringh *vrh, struct vhost_iotlb *iotlb); + +int vringh_init_iotlb(struct vringh *vrh, u64 features, + unsigned int num, bool weak_barriers, + struct vring_desc *desc, + struct vring_avail *avail, + struct vring_used *used); + +int vringh_getdesc_iotlb(struct vringh *vrh, + struct vringh_kiov *riov, + struct vringh_kiov *wiov, + u16 *head, + gfp_t gfp); + +ssize_t vringh_iov_pull_iotlb(struct vringh *vrh, + struct vringh_kiov *riov, + void *dst, size_t len); +ssize_t vringh_iov_push_iotlb(struct vringh *vrh, + struct vringh_kiov *wiov, + const void *src, size_t len); + +void vringh_abandon_iotlb(struct vringh *vrh, unsigned int num); + +int vringh_complete_iotlb(struct vringh *vrh, u16 head, u32 len); + +bool vringh_notify_enable_iotlb(struct vringh *vrh); +void vringh_notify_disable_iotlb(struct vringh *vrh); + +int vringh_need_notify_iotlb(struct vringh *vrh); + #endif /* _LINUX_VRINGH_H */ From 961e9c84077f6c8579d7a628cbe94a675cb67ae4 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Thu, 26 Mar 2020 22:01:21 +0800 Subject: [PATCH 405/635] vDPA: introduce vDPA bus vDPA device is a device that uses a datapath which complies with the virtio specifications with vendor specific control path. vDPA devices can be both physically located on the hardware or emulated by software. vDPA hardware devices are usually implemented through PCIE with the following types: - PF (Physical Function) - A single Physical Function - VF (Virtual Function) - Device that supports single root I/O virtualization (SR-IOV). Its Virtual Function (VF) represents a virtualized instance of the device that can be assigned to different partitions - ADI (Assignable Device Interface) and its equivalents - With technologies such as Intel Scalable IOV, a virtual device (VDEV) composed by host OS utilizing one or more ADIs. Or its equivalent like SF (Sub function) from Mellanox. >From a driver's perspective, depends on how and where the DMA translation is done, vDPA devices are split into two types: - Platform specific DMA translation - From the driver's perspective, the device can be used on a platform where device access to data in memory is limited and/or translated. An example is a PCIE vDPA whose DMA request was tagged via a bus (e.g PCIE) specific way. DMA translation and protection are done at PCIE bus IOMMU level. - Device specific DMA translation - The device implements DMA isolation and protection through its own logic. An example is a vDPA device which uses on-chip IOMMU. To hide the differences and complexity of the above types for a vDPA device/IOMMU options and in order to present a generic virtio device to the upper layer, a device agnostic framework is required. This patch introduces a software vDPA bus which abstracts the common attributes of vDPA device, vDPA bus driver and the communication method (vdpa_config_ops) between the vDPA device abstraction and the vDPA bus driver. This allows multiple types of drivers to be used for vDPA device like the virtio_vdpa and vhost_vdpa driver to operate on the bus and allow vDPA device could be used by either kernel virtio driver or userspace vhost drivers as: virtio drivers vhost drivers | | [virtio bus] [vhost uAPI] | | virtio device vhost device virtio_vdpa drv vhost_vdpa drv \ / [vDPA bus] | vDPA device hardware drv | [hardware bus] | vDPA hardware With the abstraction of vDPA bus and vDPA bus operations, the difference and complexity of the under layer hardware is hidden from upper layer. The vDPA bus drivers on top can use a unified vdpa_config_ops to control different types of vDPA device. Signed-off-by: Jason Wang Link: https://lore.kernel.org/r/20200326140125.19794-6-jasowang@redhat.com Signed-off-by: Michael S. Tsirkin --- MAINTAINERS | 1 + drivers/virtio/Kconfig | 2 + drivers/virtio/Makefile | 1 + drivers/virtio/vdpa/Kconfig | 7 + drivers/virtio/vdpa/Makefile | 2 + drivers/virtio/vdpa/vdpa.c | 180 +++++++++++++++++++++++++ include/linux/vdpa.h | 253 +++++++++++++++++++++++++++++++++++ 7 files changed, 446 insertions(+) create mode 100644 drivers/virtio/vdpa/Kconfig create mode 100644 drivers/virtio/vdpa/Makefile create mode 100644 drivers/virtio/vdpa/vdpa.c create mode 100644 include/linux/vdpa.h diff --git a/MAINTAINERS b/MAINTAINERS index 19363ed5e723..70c47bc55343 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -17699,6 +17699,7 @@ F: tools/virtio/ F: drivers/net/virtio_net.c F: drivers/block/virtio_blk.c F: include/linux/virtio*.h +F: include/linux/vdpa.h F: include/uapi/linux/virtio_*.h F: drivers/crypto/virtio/ F: mm/balloon_compaction.c diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig index 078615cf2afc..9c4fdb64d9ac 100644 --- a/drivers/virtio/Kconfig +++ b/drivers/virtio/Kconfig @@ -96,3 +96,5 @@ config VIRTIO_MMIO_CMDLINE_DEVICES If unsure, say 'N'. endif # VIRTIO_MENU + +source "drivers/virtio/vdpa/Kconfig" diff --git a/drivers/virtio/Makefile b/drivers/virtio/Makefile index 3a2b5c5dcf46..fdf5eacd0d0a 100644 --- a/drivers/virtio/Makefile +++ b/drivers/virtio/Makefile @@ -6,3 +6,4 @@ virtio_pci-y := virtio_pci_modern.o virtio_pci_common.o virtio_pci-$(CONFIG_VIRTIO_PCI_LEGACY) += virtio_pci_legacy.o obj-$(CONFIG_VIRTIO_BALLOON) += virtio_balloon.o obj-$(CONFIG_VIRTIO_INPUT) += virtio_input.o +obj-$(CONFIG_VDPA) += vdpa/ diff --git a/drivers/virtio/vdpa/Kconfig b/drivers/virtio/vdpa/Kconfig new file mode 100644 index 000000000000..351617723d12 --- /dev/null +++ b/drivers/virtio/vdpa/Kconfig @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0-only +config VDPA + tristate + help + Enable this module to support vDPA device that uses a + datapath which complies with virtio specifications with + vendor specific control path. diff --git a/drivers/virtio/vdpa/Makefile b/drivers/virtio/vdpa/Makefile new file mode 100644 index 000000000000..ee6a35e8a4fb --- /dev/null +++ b/drivers/virtio/vdpa/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_VDPA) += vdpa.o diff --git a/drivers/virtio/vdpa/vdpa.c b/drivers/virtio/vdpa/vdpa.c new file mode 100644 index 000000000000..e9ed6a2b635b --- /dev/null +++ b/drivers/virtio/vdpa/vdpa.c @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * vDPA bus. + * + * Copyright (c) 2020, Red Hat. All rights reserved. + * Author: Jason Wang + * + */ + +#include +#include +#include +#include + +static DEFINE_IDA(vdpa_index_ida); + +static int vdpa_dev_probe(struct device *d) +{ + struct vdpa_device *vdev = dev_to_vdpa(d); + struct vdpa_driver *drv = drv_to_vdpa(vdev->dev.driver); + int ret = 0; + + if (drv && drv->probe) + ret = drv->probe(vdev); + + return ret; +} + +static int vdpa_dev_remove(struct device *d) +{ + struct vdpa_device *vdev = dev_to_vdpa(d); + struct vdpa_driver *drv = drv_to_vdpa(vdev->dev.driver); + + if (drv && drv->remove) + drv->remove(vdev); + + return 0; +} + +static struct bus_type vdpa_bus = { + .name = "vdpa", + .probe = vdpa_dev_probe, + .remove = vdpa_dev_remove, +}; + +static void vdpa_release_dev(struct device *d) +{ + struct vdpa_device *vdev = dev_to_vdpa(d); + const struct vdpa_config_ops *ops = vdev->config; + + if (ops->free) + ops->free(vdev); + + ida_simple_remove(&vdpa_index_ida, vdev->index); + kfree(vdev); +} + +/** + * __vdpa_alloc_device - allocate and initilaize a vDPA device + * This allows driver to some prepartion after device is + * initialized but before registered. + * @parent: the parent device + * @config: the bus operations that is supported by this device + * @size: size of the parent structure that contains private data + * + * Drvier should use vdap_alloc_device() wrapper macro instead of + * using this directly. + * + * Returns an error when parent/config/dma_dev is not set or fail to get + * ida. + */ +struct vdpa_device *__vdpa_alloc_device(struct device *parent, + const struct vdpa_config_ops *config, + size_t size) +{ + struct vdpa_device *vdev; + int err = -EINVAL; + + if (!config) + goto err; + + if (!!config->dma_map != !!config->dma_unmap) + goto err; + + err = -ENOMEM; + vdev = kzalloc(size, GFP_KERNEL); + if (!vdev) + goto err; + + err = ida_simple_get(&vdpa_index_ida, 0, 0, GFP_KERNEL); + if (err < 0) + goto err_ida; + + vdev->dev.bus = &vdpa_bus; + vdev->dev.parent = parent; + vdev->dev.release = vdpa_release_dev; + vdev->index = err; + vdev->config = config; + + err = dev_set_name(&vdev->dev, "vdpa%u", vdev->index); + if (err) + goto err_name; + + device_initialize(&vdev->dev); + + return vdev; + +err_name: + ida_simple_remove(&vdpa_index_ida, vdev->index); +err_ida: + kfree(vdev); +err: + return ERR_PTR(err); +} +EXPORT_SYMBOL_GPL(__vdpa_alloc_device); + +/** + * vdpa_register_device - register a vDPA device + * Callers must have a succeed call of vdpa_init_device() before. + * @vdev: the vdpa device to be registered to vDPA bus + * + * Returns an error when fail to add to vDPA bus + */ +int vdpa_register_device(struct vdpa_device *vdev) +{ + return device_add(&vdev->dev); +} +EXPORT_SYMBOL_GPL(vdpa_register_device); + +/** + * vdpa_unregister_device - unregister a vDPA device + * @vdev: the vdpa device to be unregisted from vDPA bus + */ +void vdpa_unregister_device(struct vdpa_device *vdev) +{ + device_unregister(&vdev->dev); +} +EXPORT_SYMBOL_GPL(vdpa_unregister_device); + +/** + * __vdpa_register_driver - register a vDPA device driver + * @drv: the vdpa device driver to be registered + * @owner: module owner of the driver + * + * Returns an err when fail to do the registration + */ +int __vdpa_register_driver(struct vdpa_driver *drv, struct module *owner) +{ + drv->driver.bus = &vdpa_bus; + drv->driver.owner = owner; + + return driver_register(&drv->driver); +} +EXPORT_SYMBOL_GPL(__vdpa_register_driver); + +/** + * vdpa_unregister_driver - unregister a vDPA device driver + * @drv: the vdpa device driver to be unregistered + */ +void vdpa_unregister_driver(struct vdpa_driver *drv) +{ + driver_unregister(&drv->driver); +} +EXPORT_SYMBOL_GPL(vdpa_unregister_driver); + +static int vdpa_init(void) +{ + return bus_register(&vdpa_bus); +} + +static void __exit vdpa_exit(void) +{ + bus_unregister(&vdpa_bus); + ida_destroy(&vdpa_index_ida); +} +core_initcall(vdpa_init); +module_exit(vdpa_exit); + +MODULE_AUTHOR("Jason Wang "); +MODULE_LICENSE("GPL v2"); diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h new file mode 100644 index 000000000000..733acfb7ef84 --- /dev/null +++ b/include/linux/vdpa.h @@ -0,0 +1,253 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_VDPA_H +#define _LINUX_VDPA_H + +#include +#include +#include +#include + +/** + * vDPA callback definition. + * @callback: interrupt callback function + * @private: the data passed to the callback function + */ +struct vdpa_callback { + irqreturn_t (*callback)(void *data); + void *private; +}; + +/** + * vDPA device - representation of a vDPA device + * @dev: underlying device + * @dma_dev: the actual device that is performing DMA + * @config: the configuration ops for this device. + * @index: device index + */ +struct vdpa_device { + struct device dev; + struct device *dma_dev; + const struct vdpa_config_ops *config; + unsigned int index; +}; + +/** + * vDPA_config_ops - operations for configuring a vDPA device. + * Note: vDPA device drivers are required to implement all of the + * operations unless it is mentioned to be optional in the following + * list. + * + * @set_vq_address: Set the address of virtqueue + * @vdev: vdpa device + * @idx: virtqueue index + * @desc_area: address of desc area + * @driver_area: address of driver area + * @device_area: address of device area + * Returns integer: success (0) or error (< 0) + * @set_vq_num: Set the size of virtqueue + * @vdev: vdpa device + * @idx: virtqueue index + * @num: the size of virtqueue + * @kick_vq: Kick the virtqueue + * @vdev: vdpa device + * @idx: virtqueue index + * @set_vq_cb: Set the interrupt callback function for + * a virtqueue + * @vdev: vdpa device + * @idx: virtqueue index + * @cb: virtio-vdev interrupt callback structure + * @set_vq_ready: Set ready status for a virtqueue + * @vdev: vdpa device + * @idx: virtqueue index + * @ready: ready (true) not ready(false) + * @get_vq_ready: Get ready status for a virtqueue + * @vdev: vdpa device + * @idx: virtqueue index + * Returns boolean: ready (true) or not (false) + * @set_vq_state: Set the state for a virtqueue + * @vdev: vdpa device + * @idx: virtqueue index + * @state: virtqueue state (last_avail_idx) + * Returns integer: success (0) or error (< 0) + * @get_vq_state: Get the state for a virtqueue + * @vdev: vdpa device + * @idx: virtqueue index + * Returns virtqueue state (last_avail_idx) + * @get_vq_align: Get the virtqueue align requirement + * for the device + * @vdev: vdpa device + * Returns virtqueue algin requirement + * @get_features: Get virtio features supported by the device + * @vdev: vdpa device + * Returns the virtio features support by the + * device + * @set_features: Set virtio features supported by the driver + * @vdev: vdpa device + * @features: feature support by the driver + * Returns integer: success (0) or error (< 0) + * @set_config_cb: Set the config interrupt callback + * @vdev: vdpa device + * @cb: virtio-vdev interrupt callback structure + * @get_vq_num_max: Get the max size of virtqueue + * @vdev: vdpa device + * Returns u16: max size of virtqueue + * @get_device_id: Get virtio device id + * @vdev: vdpa device + * Returns u32: virtio device id + * @get_vendor_id: Get id for the vendor that provides this device + * @vdev: vdpa device + * Returns u32: virtio vendor id + * @get_status: Get the device status + * @vdev: vdpa device + * Returns u8: virtio device status + * @set_status: Set the device status + * @vdev: vdpa device + * @status: virtio device status + * @get_config: Read from device specific configuration space + * @vdev: vdpa device + * @offset: offset from the beginning of + * configuration space + * @buf: buffer used to read to + * @len: the length to read from + * configuration space + * @set_config: Write to device specific configuration space + * @vdev: vdpa device + * @offset: offset from the beginning of + * configuration space + * @buf: buffer used to write from + * @len: the length to write to + * configuration space + * @get_generation: Get device config generation (optional) + * @vdev: vdpa device + * Returns u32: device generation + * @set_map: Set device memory mapping (optional) + * Needed for device that using device + * specific DMA translation (on-chip IOMMU) + * @vdev: vdpa device + * @iotlb: vhost memory mapping to be + * used by the vDPA + * Returns integer: success (0) or error (< 0) + * @dma_map: Map an area of PA to IOVA (optional) + * Needed for device that using device + * specific DMA translation (on-chip IOMMU) + * and preferring incremental map. + * @vdev: vdpa device + * @iova: iova to be mapped + * @size: size of the area + * @pa: physical address for the map + * @perm: device access permission (VHOST_MAP_XX) + * Returns integer: success (0) or error (< 0) + * @dma_unmap: Unmap an area of IOVA (optional but + * must be implemented with dma_map) + * Needed for device that using device + * specific DMA translation (on-chip IOMMU) + * and preferring incremental unmap. + * @vdev: vdpa device + * @iova: iova to be unmapped + * @size: size of the area + * Returns integer: success (0) or error (< 0) + * @free: Free resources that belongs to vDPA (optional) + * @vdev: vdpa device + */ +struct vdpa_config_ops { + /* Virtqueue ops */ + int (*set_vq_address)(struct vdpa_device *vdev, + u16 idx, u64 desc_area, u64 driver_area, + u64 device_area); + void (*set_vq_num)(struct vdpa_device *vdev, u16 idx, u32 num); + void (*kick_vq)(struct vdpa_device *vdev, u16 idx); + void (*set_vq_cb)(struct vdpa_device *vdev, u16 idx, + struct vdpa_callback *cb); + void (*set_vq_ready)(struct vdpa_device *vdev, u16 idx, bool ready); + bool (*get_vq_ready)(struct vdpa_device *vdev, u16 idx); + int (*set_vq_state)(struct vdpa_device *vdev, u16 idx, u64 state); + u64 (*get_vq_state)(struct vdpa_device *vdev, u16 idx); + + /* Device ops */ + u16 (*get_vq_align)(struct vdpa_device *vdev); + u64 (*get_features)(struct vdpa_device *vdev); + int (*set_features)(struct vdpa_device *vdev, u64 features); + void (*set_config_cb)(struct vdpa_device *vdev, + struct vdpa_callback *cb); + u16 (*get_vq_num_max)(struct vdpa_device *vdev); + u32 (*get_device_id)(struct vdpa_device *vdev); + u32 (*get_vendor_id)(struct vdpa_device *vdev); + u8 (*get_status)(struct vdpa_device *vdev); + void (*set_status)(struct vdpa_device *vdev, u8 status); + void (*get_config)(struct vdpa_device *vdev, unsigned int offset, + void *buf, unsigned int len); + void (*set_config)(struct vdpa_device *vdev, unsigned int offset, + const void *buf, unsigned int len); + u32 (*get_generation)(struct vdpa_device *vdev); + + /* DMA ops */ + int (*set_map)(struct vdpa_device *vdev, struct vhost_iotlb *iotlb); + int (*dma_map)(struct vdpa_device *vdev, u64 iova, u64 size, + u64 pa, u32 perm); + int (*dma_unmap)(struct vdpa_device *vdev, u64 iova, u64 size); + + /* Free device resources */ + void (*free)(struct vdpa_device *vdev); +}; + +struct vdpa_device *__vdpa_alloc_device(struct device *parent, + const struct vdpa_config_ops *config, + size_t size); + +#define vdpa_alloc_device(dev_struct, member, parent, config) \ + container_of(__vdpa_alloc_device( \ + parent, config, \ + sizeof(dev_struct) + \ + BUILD_BUG_ON_ZERO(offsetof( \ + dev_struct, member))), \ + dev_struct, member) + +int vdpa_register_device(struct vdpa_device *vdev); +void vdpa_unregister_device(struct vdpa_device *vdev); + +/** + * vdpa_driver - operations for a vDPA driver + * @driver: underlying device driver + * @probe: the function to call when a device is found. Returns 0 or -errno. + * @remove: the function to call when a device is removed. + */ +struct vdpa_driver { + struct device_driver driver; + int (*probe)(struct vdpa_device *vdev); + void (*remove)(struct vdpa_device *vdev); +}; + +#define vdpa_register_driver(drv) \ + __vdpa_register_driver(drv, THIS_MODULE) +int __vdpa_register_driver(struct vdpa_driver *drv, struct module *owner); +void vdpa_unregister_driver(struct vdpa_driver *drv); + +#define module_vdpa_driver(__vdpa_driver) \ + module_driver(__vdpa_driver, vdpa_register_driver, \ + vdpa_unregister_driver) + +static inline struct vdpa_driver *drv_to_vdpa(struct device_driver *driver) +{ + return container_of(driver, struct vdpa_driver, driver); +} + +static inline struct vdpa_device *dev_to_vdpa(struct device *_dev) +{ + return container_of(_dev, struct vdpa_device, dev); +} + +static inline void *vdpa_get_drvdata(const struct vdpa_device *vdev) +{ + return dev_get_drvdata(&vdev->dev); +} + +static inline void vdpa_set_drvdata(struct vdpa_device *vdev, void *data) +{ + dev_set_drvdata(&vdev->dev, data); +} + +static inline struct device *vdpa_get_dma_dev(struct vdpa_device *vdev) +{ + return vdev->dma_dev; +} +#endif /* _LINUX_VDPA_H */ From c043b4a8cf3b16fbdcaec1126841431c33b16e98 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Thu, 26 Mar 2020 22:01:22 +0800 Subject: [PATCH 406/635] virtio: introduce a vDPA based transport This patch introduces a vDPA transport for virtio. This is used to use kernel virtio driver to drive the vDPA device that is capable of populating virtqueue directly. A new virtio-vdpa driver will be registered to the vDPA bus, when a new virtio-vdpa device is probed, it will register the device with vdpa based config ops. This means it is a software transport between vDPA driver and vDPA device. The transport was implemented through bus_ops of vDPA parent. Signed-off-by: Jason Wang Link: https://lore.kernel.org/r/20200326140125.19794-7-jasowang@redhat.com Signed-off-by: Michael S. Tsirkin --- drivers/virtio/Kconfig | 13 ++ drivers/virtio/Makefile | 1 + drivers/virtio/virtio_vdpa.c | 396 +++++++++++++++++++++++++++++++++++ 3 files changed, 410 insertions(+) create mode 100644 drivers/virtio/virtio_vdpa.c diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig index 9c4fdb64d9ac..99e424570644 100644 --- a/drivers/virtio/Kconfig +++ b/drivers/virtio/Kconfig @@ -43,6 +43,19 @@ config VIRTIO_PCI_LEGACY If unsure, say Y. +config VIRTIO_VDPA + tristate "vDPA driver for virtio devices" + select VDPA + select VIRTIO + help + This driver provides support for virtio based paravirtual + device driver over vDPA bus. For this to be useful, you need + an appropriate vDPA device implementation that operates on a + physical device to allow the datapath of virtio to be + offloaded to hardware. + + If unsure, say M. + config VIRTIO_PMEM tristate "Support for virtio pmem driver" depends on VIRTIO diff --git a/drivers/virtio/Makefile b/drivers/virtio/Makefile index fdf5eacd0d0a..3407ac03fe60 100644 --- a/drivers/virtio/Makefile +++ b/drivers/virtio/Makefile @@ -6,4 +6,5 @@ virtio_pci-y := virtio_pci_modern.o virtio_pci_common.o virtio_pci-$(CONFIG_VIRTIO_PCI_LEGACY) += virtio_pci_legacy.o obj-$(CONFIG_VIRTIO_BALLOON) += virtio_balloon.o obj-$(CONFIG_VIRTIO_INPUT) += virtio_input.o +obj-$(CONFIG_VIRTIO_VDPA) += virtio_vdpa.o obj-$(CONFIG_VDPA) += vdpa/ diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c new file mode 100644 index 000000000000..c30eb55030be --- /dev/null +++ b/drivers/virtio/virtio_vdpa.c @@ -0,0 +1,396 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * VIRTIO based driver for vDPA device + * + * Copyright (c) 2020, Red Hat. All rights reserved. + * Author: Jason Wang + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MOD_VERSION "0.1" +#define MOD_AUTHOR "Jason Wang " +#define MOD_DESC "vDPA bus driver for virtio devices" +#define MOD_LICENSE "GPL v2" + +struct virtio_vdpa_device { + struct virtio_device vdev; + struct vdpa_device *vdpa; + u64 features; + + /* The lock to protect virtqueue list */ + spinlock_t lock; + /* List of virtio_vdpa_vq_info */ + struct list_head virtqueues; +}; + +struct virtio_vdpa_vq_info { + /* the actual virtqueue */ + struct virtqueue *vq; + + /* the list node for the virtqueues list */ + struct list_head node; +}; + +static inline struct virtio_vdpa_device * +to_virtio_vdpa_device(struct virtio_device *dev) +{ + return container_of(dev, struct virtio_vdpa_device, vdev); +} + +static struct vdpa_device *vd_get_vdpa(struct virtio_device *vdev) +{ + return to_virtio_vdpa_device(vdev)->vdpa; +} + +static void virtio_vdpa_get(struct virtio_device *vdev, unsigned offset, + void *buf, unsigned len) +{ + struct vdpa_device *vdpa = vd_get_vdpa(vdev); + const struct vdpa_config_ops *ops = vdpa->config; + + ops->get_config(vdpa, offset, buf, len); +} + +static void virtio_vdpa_set(struct virtio_device *vdev, unsigned offset, + const void *buf, unsigned len) +{ + struct vdpa_device *vdpa = vd_get_vdpa(vdev); + const struct vdpa_config_ops *ops = vdpa->config; + + ops->set_config(vdpa, offset, buf, len); +} + +static u32 virtio_vdpa_generation(struct virtio_device *vdev) +{ + struct vdpa_device *vdpa = vd_get_vdpa(vdev); + const struct vdpa_config_ops *ops = vdpa->config; + + if (ops->get_generation) + return ops->get_generation(vdpa); + + return 0; +} + +static u8 virtio_vdpa_get_status(struct virtio_device *vdev) +{ + struct vdpa_device *vdpa = vd_get_vdpa(vdev); + const struct vdpa_config_ops *ops = vdpa->config; + + return ops->get_status(vdpa); +} + +static void virtio_vdpa_set_status(struct virtio_device *vdev, u8 status) +{ + struct vdpa_device *vdpa = vd_get_vdpa(vdev); + const struct vdpa_config_ops *ops = vdpa->config; + + return ops->set_status(vdpa, status); +} + +static void virtio_vdpa_reset(struct virtio_device *vdev) +{ + struct vdpa_device *vdpa = vd_get_vdpa(vdev); + const struct vdpa_config_ops *ops = vdpa->config; + + return ops->set_status(vdpa, 0); +} + +static bool virtio_vdpa_notify(struct virtqueue *vq) +{ + struct vdpa_device *vdpa = vd_get_vdpa(vq->vdev); + const struct vdpa_config_ops *ops = vdpa->config; + + ops->kick_vq(vdpa, vq->index); + + return true; +} + +static irqreturn_t virtio_vdpa_config_cb(void *private) +{ + struct virtio_vdpa_device *vd_dev = private; + + virtio_config_changed(&vd_dev->vdev); + + return IRQ_HANDLED; +} + +static irqreturn_t virtio_vdpa_virtqueue_cb(void *private) +{ + struct virtio_vdpa_vq_info *info = private; + + return vring_interrupt(0, info->vq); +} + +static struct virtqueue * +virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, + void (*callback)(struct virtqueue *vq), + const char *name, bool ctx) +{ + struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev); + struct vdpa_device *vdpa = vd_get_vdpa(vdev); + const struct vdpa_config_ops *ops = vdpa->config; + struct virtio_vdpa_vq_info *info; + struct vdpa_callback cb; + struct virtqueue *vq; + u64 desc_addr, driver_addr, device_addr; + unsigned long flags; + u32 align, num; + int err; + + if (!name) + return NULL; + + /* Queue shouldn't already be set up. */ + if (ops->get_vq_ready(vdpa, index)) + return ERR_PTR(-ENOENT); + + /* Allocate and fill out our active queue description */ + info = kmalloc(sizeof(*info), GFP_KERNEL); + if (!info) + return ERR_PTR(-ENOMEM); + + num = ops->get_vq_num_max(vdpa); + if (num == 0) { + err = -ENOENT; + goto error_new_virtqueue; + } + + /* Create the vring */ + align = ops->get_vq_align(vdpa); + vq = vring_create_virtqueue(index, num, align, vdev, + true, true, ctx, + virtio_vdpa_notify, callback, name); + if (!vq) { + err = -ENOMEM; + goto error_new_virtqueue; + } + + /* Setup virtqueue callback */ + cb.callback = virtio_vdpa_virtqueue_cb; + cb.private = info; + ops->set_vq_cb(vdpa, index, &cb); + ops->set_vq_num(vdpa, index, virtqueue_get_vring_size(vq)); + + desc_addr = virtqueue_get_desc_addr(vq); + driver_addr = virtqueue_get_avail_addr(vq); + device_addr = virtqueue_get_used_addr(vq); + + if (ops->set_vq_address(vdpa, index, + desc_addr, driver_addr, + device_addr)) { + err = -EINVAL; + goto err_vq; + } + + ops->set_vq_ready(vdpa, index, 1); + + vq->priv = info; + info->vq = vq; + + spin_lock_irqsave(&vd_dev->lock, flags); + list_add(&info->node, &vd_dev->virtqueues); + spin_unlock_irqrestore(&vd_dev->lock, flags); + + return vq; + +err_vq: + vring_del_virtqueue(vq); +error_new_virtqueue: + ops->set_vq_ready(vdpa, index, 0); + /* VDPA driver should make sure vq is stopeed here */ + WARN_ON(ops->get_vq_ready(vdpa, index)); + kfree(info); + return ERR_PTR(err); +} + +static void virtio_vdpa_del_vq(struct virtqueue *vq) +{ + struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev); + struct vdpa_device *vdpa = vd_dev->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + struct virtio_vdpa_vq_info *info = vq->priv; + unsigned int index = vq->index; + unsigned long flags; + + spin_lock_irqsave(&vd_dev->lock, flags); + list_del(&info->node); + spin_unlock_irqrestore(&vd_dev->lock, flags); + + /* Select and deactivate the queue */ + ops->set_vq_ready(vdpa, index, 0); + WARN_ON(ops->get_vq_ready(vdpa, index)); + + vring_del_virtqueue(vq); + + kfree(info); +} + +static void virtio_vdpa_del_vqs(struct virtio_device *vdev) +{ + struct virtqueue *vq, *n; + + list_for_each_entry_safe(vq, n, &vdev->vqs, list) + virtio_vdpa_del_vq(vq); +} + +static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned nvqs, + struct virtqueue *vqs[], + vq_callback_t *callbacks[], + const char * const names[], + const bool *ctx, + struct irq_affinity *desc) +{ + struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev); + struct vdpa_device *vdpa = vd_get_vdpa(vdev); + const struct vdpa_config_ops *ops = vdpa->config; + struct vdpa_callback cb; + int i, err, queue_idx = 0; + + for (i = 0; i < nvqs; ++i) { + if (!names[i]) { + vqs[i] = NULL; + continue; + } + + vqs[i] = virtio_vdpa_setup_vq(vdev, queue_idx++, + callbacks[i], names[i], ctx ? + ctx[i] : false); + if (IS_ERR(vqs[i])) { + err = PTR_ERR(vqs[i]); + goto err_setup_vq; + } + } + + cb.callback = virtio_vdpa_config_cb; + cb.private = vd_dev; + ops->set_config_cb(vdpa, &cb); + + return 0; + +err_setup_vq: + virtio_vdpa_del_vqs(vdev); + return err; +} + +static u64 virtio_vdpa_get_features(struct virtio_device *vdev) +{ + struct vdpa_device *vdpa = vd_get_vdpa(vdev); + const struct vdpa_config_ops *ops = vdpa->config; + + return ops->get_features(vdpa); +} + +static int virtio_vdpa_finalize_features(struct virtio_device *vdev) +{ + struct vdpa_device *vdpa = vd_get_vdpa(vdev); + const struct vdpa_config_ops *ops = vdpa->config; + + /* Give virtio_ring a chance to accept features. */ + vring_transport_features(vdev); + + return ops->set_features(vdpa, vdev->features); +} + +static const char *virtio_vdpa_bus_name(struct virtio_device *vdev) +{ + struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev); + struct vdpa_device *vdpa = vd_dev->vdpa; + + return dev_name(&vdpa->dev); +} + +static const struct virtio_config_ops virtio_vdpa_config_ops = { + .get = virtio_vdpa_get, + .set = virtio_vdpa_set, + .generation = virtio_vdpa_generation, + .get_status = virtio_vdpa_get_status, + .set_status = virtio_vdpa_set_status, + .reset = virtio_vdpa_reset, + .find_vqs = virtio_vdpa_find_vqs, + .del_vqs = virtio_vdpa_del_vqs, + .get_features = virtio_vdpa_get_features, + .finalize_features = virtio_vdpa_finalize_features, + .bus_name = virtio_vdpa_bus_name, +}; + +static void virtio_vdpa_release_dev(struct device *_d) +{ + struct virtio_device *vdev = + container_of(_d, struct virtio_device, dev); + struct virtio_vdpa_device *vd_dev = + container_of(vdev, struct virtio_vdpa_device, vdev); + + kfree(vd_dev); +} + +static int virtio_vdpa_probe(struct vdpa_device *vdpa) +{ + const struct vdpa_config_ops *ops = vdpa->config; + struct virtio_vdpa_device *vd_dev, *reg_dev = NULL; + int ret = -EINVAL; + + vd_dev = kzalloc(sizeof(*vd_dev), GFP_KERNEL); + if (!vd_dev) + return -ENOMEM; + + vd_dev->vdev.dev.parent = vdpa_get_dma_dev(vdpa); + vd_dev->vdev.dev.release = virtio_vdpa_release_dev; + vd_dev->vdev.config = &virtio_vdpa_config_ops; + vd_dev->vdpa = vdpa; + INIT_LIST_HEAD(&vd_dev->virtqueues); + spin_lock_init(&vd_dev->lock); + + vd_dev->vdev.id.device = ops->get_device_id(vdpa); + if (vd_dev->vdev.id.device == 0) + goto err; + + vd_dev->vdev.id.vendor = ops->get_vendor_id(vdpa); + ret = register_virtio_device(&vd_dev->vdev); + reg_dev = vd_dev; + if (ret) + goto err; + + vdpa_set_drvdata(vdpa, vd_dev); + + return 0; + +err: + if (reg_dev) + put_device(&vd_dev->vdev.dev); + else + kfree(vd_dev); + return ret; +} + +static void virtio_vdpa_remove(struct vdpa_device *vdpa) +{ + struct virtio_vdpa_device *vd_dev = vdpa_get_drvdata(vdpa); + + unregister_virtio_device(&vd_dev->vdev); +} + +static struct vdpa_driver virtio_vdpa_driver = { + .driver = { + .name = "virtio_vdpa", + }, + .probe = virtio_vdpa_probe, + .remove = virtio_vdpa_remove, +}; + +module_vdpa_driver(virtio_vdpa_driver); + +MODULE_VERSION(MOD_VERSION); +MODULE_LICENSE(MOD_LICENSE); +MODULE_AUTHOR(MOD_AUTHOR); +MODULE_DESCRIPTION(MOD_DESC); From 08ca8b21f760c0ed5034a5c122092eec22ccf8f4 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Wed, 1 Apr 2020 13:04:49 -0400 Subject: [PATCH 407/635] NFS: Fix races nfs_page_group_destroy() vs nfs_destroy_unlinked_subrequests() When a subrequest is being detached from the subgroup, we want to ensure that it is not holding the group lock, or in the process of waiting for the group lock. Fixes: 5b2b5187fa85 ("NFS: Fix nfs_page_group_destroy() and nfs_lock_and_join_requests() race cases") Signed-off-by: Trond Myklebust --- fs/nfs/pagelist.c | 67 +++++++++++++++++++++++++++------------- fs/nfs/write.c | 10 ++++-- include/linux/nfs_page.h | 2 ++ 3 files changed, 55 insertions(+), 24 deletions(-) diff --git a/fs/nfs/pagelist.c b/fs/nfs/pagelist.c index be5e209399ea..0e3f0f241d83 100644 --- a/fs/nfs/pagelist.c +++ b/fs/nfs/pagelist.c @@ -132,9 +132,44 @@ nfs_async_iocounter_wait(struct rpc_task *task, struct nfs_lock_context *l_ctx) } EXPORT_SYMBOL_GPL(nfs_async_iocounter_wait); +/* + * nfs_page_set_headlock - set the request PG_HEADLOCK + * @req: request that is to be locked + * + * this lock must be held when modifying req->wb_head + * + * return 0 on success, < 0 on error + */ +int +nfs_page_set_headlock(struct nfs_page *req) +{ + if (!test_and_set_bit(PG_HEADLOCK, &req->wb_flags)) + return 0; + + set_bit(PG_CONTENDED1, &req->wb_flags); + smp_mb__after_atomic(); + return wait_on_bit_lock(&req->wb_flags, PG_HEADLOCK, + TASK_UNINTERRUPTIBLE); +} + +/* + * nfs_page_clear_headlock - clear the request PG_HEADLOCK + * @req: request that is to be locked + */ +void +nfs_page_clear_headlock(struct nfs_page *req) +{ + smp_mb__before_atomic(); + clear_bit(PG_HEADLOCK, &req->wb_flags); + smp_mb__after_atomic(); + if (!test_bit(PG_CONTENDED1, &req->wb_flags)) + return; + wake_up_bit(&req->wb_flags, PG_HEADLOCK); +} + /* * nfs_page_group_lock - lock the head of the page group - * @req - request in group that is to be locked + * @req: request in group that is to be locked * * this lock must be held when traversing or modifying the page * group list @@ -144,36 +179,24 @@ EXPORT_SYMBOL_GPL(nfs_async_iocounter_wait); int nfs_page_group_lock(struct nfs_page *req) { - struct nfs_page *head = req->wb_head; + int ret; - WARN_ON_ONCE(head != head->wb_head); - - if (!test_and_set_bit(PG_HEADLOCK, &head->wb_flags)) - return 0; - - set_bit(PG_CONTENDED1, &head->wb_flags); - smp_mb__after_atomic(); - return wait_on_bit_lock(&head->wb_flags, PG_HEADLOCK, - TASK_UNINTERRUPTIBLE); + ret = nfs_page_set_headlock(req); + if (ret || req->wb_head == req) + return ret; + return nfs_page_set_headlock(req->wb_head); } /* * nfs_page_group_unlock - unlock the head of the page group - * @req - request in group that is to be unlocked + * @req: request in group that is to be unlocked */ void nfs_page_group_unlock(struct nfs_page *req) { - struct nfs_page *head = req->wb_head; - - WARN_ON_ONCE(head != head->wb_head); - - smp_mb__before_atomic(); - clear_bit(PG_HEADLOCK, &head->wb_flags); - smp_mb__after_atomic(); - if (!test_bit(PG_CONTENDED1, &head->wb_flags)) - return; - wake_up_bit(&head->wb_flags, PG_HEADLOCK); + if (req != req->wb_head) + nfs_page_clear_headlock(req->wb_head); + nfs_page_clear_headlock(req); } /* diff --git a/fs/nfs/write.c b/fs/nfs/write.c index 626e99cbb50e..a6d7926b0653 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -428,22 +428,28 @@ nfs_destroy_unlinked_subrequests(struct nfs_page *destroy_list, destroy_list = (subreq->wb_this_page == old_head) ? NULL : subreq->wb_this_page; + /* Note: lock subreq in order to change subreq->wb_head */ + nfs_page_set_headlock(subreq); WARN_ON_ONCE(old_head != subreq->wb_head); /* make sure old group is not used */ subreq->wb_this_page = subreq; + subreq->wb_head = subreq; clear_bit(PG_REMOVE, &subreq->wb_flags); /* Note: races with nfs_page_group_destroy() */ if (!kref_read(&subreq->wb_kref)) { /* Check if we raced with nfs_page_group_destroy() */ - if (test_and_clear_bit(PG_TEARDOWN, &subreq->wb_flags)) + if (test_and_clear_bit(PG_TEARDOWN, &subreq->wb_flags)) { + nfs_page_clear_headlock(subreq); nfs_free_request(subreq); + } else + nfs_page_clear_headlock(subreq); continue; } + nfs_page_clear_headlock(subreq); - subreq->wb_head = subreq; nfs_release_request(old_head); if (test_and_clear_bit(PG_INODE_REF, &subreq->wb_flags)) { diff --git a/include/linux/nfs_page.h b/include/linux/nfs_page.h index 0bbd587fac6a..7e9419d74b86 100644 --- a/include/linux/nfs_page.h +++ b/include/linux/nfs_page.h @@ -142,6 +142,8 @@ extern void nfs_unlock_and_release_request(struct nfs_page *); extern int nfs_page_group_lock(struct nfs_page *); extern void nfs_page_group_unlock(struct nfs_page *); extern bool nfs_page_group_sync_on_bit(struct nfs_page *, unsigned int); +extern int nfs_page_set_headlock(struct nfs_page *req); +extern void nfs_page_clear_headlock(struct nfs_page *req); extern bool nfs_async_iocounter_wait(struct rpc_task *, struct nfs_lock_context *); /* From dc9dc2febb17f72e9878eb540ad3996f7984239a Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Sun, 29 Mar 2020 19:55:05 -0400 Subject: [PATCH 408/635] NFS: Fix use-after-free issues in nfs_pageio_add_request() We need to ensure that we create the mirror requests before calling nfs_pageio_add_request_mirror() on the request we are adding. Otherwise, we can end up with a use-after-free if the call to nfs_pageio_add_request_mirror() triggers I/O. Fixes: c917cfaf9bbe ("NFS: Fix up NFS I/O subrequest creation") Cc: stable@vger.kernel.org Signed-off-by: Trond Myklebust --- fs/nfs/pagelist.c | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/fs/nfs/pagelist.c b/fs/nfs/pagelist.c index 0e3f0f241d83..99eb839c5778 100644 --- a/fs/nfs/pagelist.c +++ b/fs/nfs/pagelist.c @@ -1191,38 +1191,38 @@ int nfs_pageio_add_request(struct nfs_pageio_descriptor *desc, if (desc->pg_error < 0) goto out_failed; - for (midx = 0; midx < desc->pg_mirror_count; midx++) { - if (midx) { - nfs_page_group_lock(req); + /* Create the mirror instances first, and fire them off */ + for (midx = 1; midx < desc->pg_mirror_count; midx++) { + nfs_page_group_lock(req); - /* find the last request */ - for (lastreq = req->wb_head; - lastreq->wb_this_page != req->wb_head; - lastreq = lastreq->wb_this_page) - ; + /* find the last request */ + for (lastreq = req->wb_head; + lastreq->wb_this_page != req->wb_head; + lastreq = lastreq->wb_this_page) + ; - dupreq = nfs_create_subreq(req, lastreq, - pgbase, offset, bytes); + dupreq = nfs_create_subreq(req, lastreq, + pgbase, offset, bytes); - nfs_page_group_unlock(req); - if (IS_ERR(dupreq)) { - desc->pg_error = PTR_ERR(dupreq); - goto out_failed; - } - } else - dupreq = req; + nfs_page_group_unlock(req); + if (IS_ERR(dupreq)) { + desc->pg_error = PTR_ERR(dupreq); + goto out_failed; + } - if (nfs_pgio_has_mirroring(desc)) - desc->pg_mirror_idx = midx; + desc->pg_mirror_idx = midx; if (!nfs_pageio_add_request_mirror(desc, dupreq)) goto out_cleanup_subreq; } + desc->pg_mirror_idx = 0; + if (!nfs_pageio_add_request_mirror(desc, req)) + goto out_failed; + return 1; out_cleanup_subreq: - if (req != dupreq) - nfs_pageio_cleanup_request(desc, dupreq); + nfs_pageio_cleanup_request(desc, dupreq); out_failed: nfs_pageio_error_cleanup(desc); return 0; From f02cec9d33e0069c11e58f97529c1d697255889d Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 30 Mar 2020 20:13:48 -0400 Subject: [PATCH 409/635] NFS: Fix a request reference leak in nfs_direct_write_clear_reqs() nfs_direct_write_scan_commit_list() will lock the request and bump the reference count, but we also need to account for the reference that was taken when we initially added the request to the commit list. Fixes: fb5f7f20cdb9 ("NFS: commit errors should be fatal") Signed-off-by: Trond Myklebust --- fs/nfs/direct.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c index 51ab4627c4d6..8074304fd5b4 100644 --- a/fs/nfs/direct.c +++ b/fs/nfs/direct.c @@ -646,6 +646,7 @@ static void nfs_direct_write_clear_reqs(struct nfs_direct_req *dreq) while (!list_empty(&reqs)) { req = nfs_list_entry(reqs.next); nfs_list_remove_request(req); + nfs_release_request(req); nfs_unlock_and_release_request(req); } } From 862f35c94730c9270833f3ad05bd758a29f204ed Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Sun, 29 Mar 2020 20:06:45 -0400 Subject: [PATCH 410/635] NFS: Fix memory leaks in nfs_pageio_stop_mirroring() If we just set the mirror count to 1 without first clearing out the mirrors, we can leak queued up requests. Signed-off-by: Trond Myklebust --- fs/nfs/pagelist.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/fs/nfs/pagelist.c b/fs/nfs/pagelist.c index 99eb839c5778..1fd4862217e0 100644 --- a/fs/nfs/pagelist.c +++ b/fs/nfs/pagelist.c @@ -900,15 +900,6 @@ static void nfs_pageio_setup_mirroring(struct nfs_pageio_descriptor *pgio, pgio->pg_mirror_count = mirror_count; } -/* - * nfs_pageio_stop_mirroring - stop using mirroring (set mirror count to 1) - */ -void nfs_pageio_stop_mirroring(struct nfs_pageio_descriptor *pgio) -{ - pgio->pg_mirror_count = 1; - pgio->pg_mirror_idx = 0; -} - static void nfs_pageio_cleanup_mirroring(struct nfs_pageio_descriptor *pgio) { pgio->pg_mirror_count = 1; @@ -1334,6 +1325,14 @@ void nfs_pageio_cond_complete(struct nfs_pageio_descriptor *desc, pgoff_t index) } } +/* + * nfs_pageio_stop_mirroring - stop using mirroring (set mirror count to 1) + */ +void nfs_pageio_stop_mirroring(struct nfs_pageio_descriptor *pgio) +{ + nfs_pageio_complete(pgio); +} + int __init nfs_init_nfspagecache(void) { nfs_page_cachep = kmem_cache_create("nfs_page", From 377840ee48cde0700678ef14141106bbd13e00b5 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Sun, 29 Mar 2020 20:03:33 -0400 Subject: [PATCH 411/635] NFS: Remove the redundant function nfs_pgio_has_mirroring() We need to trust that desc->pg_mirror_idx is set correctly, whether or not mirroring is enabled. Signed-off-by: Trond Myklebust --- fs/nfs/internal.h | 6 ------ fs/nfs/pagelist.c | 7 ++----- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h index 78f317fac940..1f32a9fbfdaf 100644 --- a/fs/nfs/internal.h +++ b/fs/nfs/internal.h @@ -274,12 +274,6 @@ void nfs_free_request(struct nfs_page *req); struct nfs_pgio_mirror * nfs_pgio_current_mirror(struct nfs_pageio_descriptor *desc); -static inline bool nfs_pgio_has_mirroring(struct nfs_pageio_descriptor *desc) -{ - WARN_ON_ONCE(desc->pg_mirror_count < 1); - return desc->pg_mirror_count > 1; -} - static inline bool nfs_match_open_context(const struct nfs_open_context *ctx1, const struct nfs_open_context *ctx2) { diff --git a/fs/nfs/pagelist.c b/fs/nfs/pagelist.c index 1fd4862217e0..f535a92403bf 100644 --- a/fs/nfs/pagelist.c +++ b/fs/nfs/pagelist.c @@ -33,9 +33,7 @@ static const struct rpc_call_ops nfs_pgio_common_ops; struct nfs_pgio_mirror * nfs_pgio_current_mirror(struct nfs_pageio_descriptor *desc) { - return nfs_pgio_has_mirroring(desc) ? - &desc->pg_mirrors[desc->pg_mirror_idx] : - &desc->pg_mirrors[0]; + return &desc->pg_mirrors[desc->pg_mirror_idx]; } EXPORT_SYMBOL_GPL(nfs_pgio_current_mirror); @@ -1231,8 +1229,7 @@ static void nfs_pageio_complete_mirror(struct nfs_pageio_descriptor *desc, struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[mirror_idx]; u32 restore_idx = desc->pg_mirror_idx; - if (nfs_pgio_has_mirroring(desc)) - desc->pg_mirror_idx = mirror_idx; + desc->pg_mirror_idx = mirror_idx; for (;;) { nfs_pageio_doio(desc); if (desc->pg_error < 0 || !mirror->pg_recoalesce) From a62f8e3bd836bf1abde1648a45e14afd050dbd23 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 30 Mar 2020 11:12:16 -0400 Subject: [PATCH 412/635] NFS: Clean up nfs_lock_and_join_requests() Clean up nfs_lock_and_join_requests() to simplify the calculation of the range covered by the page group, taking into account the presence of mirrors. Signed-off-by: Trond Myklebust --- fs/nfs/pagelist.c | 74 ++++++++++++++++++++++++++++++++ fs/nfs/write.c | 91 +++++++++------------------------------- include/linux/nfs_page.h | 1 + 3 files changed, 95 insertions(+), 71 deletions(-) diff --git a/fs/nfs/pagelist.c b/fs/nfs/pagelist.c index f535a92403bf..261236157e33 100644 --- a/fs/nfs/pagelist.c +++ b/fs/nfs/pagelist.c @@ -130,6 +130,80 @@ nfs_async_iocounter_wait(struct rpc_task *task, struct nfs_lock_context *l_ctx) } EXPORT_SYMBOL_GPL(nfs_async_iocounter_wait); +/* + * nfs_unroll_locks - unlock all newly locked reqs and wait on @req + * @head: head request of page group, must be holding head lock + * @req: request that couldn't lock and needs to wait on the req bit lock + * + * This is a helper function for nfs_lock_and_join_requests + * returns 0 on success, < 0 on error. + */ +static void +nfs_unroll_locks(struct nfs_page *head, struct nfs_page *req) +{ + struct nfs_page *tmp; + + /* relinquish all the locks successfully grabbed this run */ + for (tmp = head->wb_this_page ; tmp != req; tmp = tmp->wb_this_page) { + if (!kref_read(&tmp->wb_kref)) + continue; + nfs_unlock_and_release_request(tmp); + } +} + +/* + * nfs_page_group_lock_subreq - try to lock a subrequest + * @head: head request of page group + * @subreq: request to lock + * + * This is a helper function for nfs_lock_and_join_requests which + * must be called with the head request and page group both locked. + * On error, it returns with the page group unlocked. + */ +static int +nfs_page_group_lock_subreq(struct nfs_page *head, struct nfs_page *subreq) +{ + int ret; + + if (!kref_get_unless_zero(&subreq->wb_kref)) + return 0; + while (!nfs_lock_request(subreq)) { + nfs_page_group_unlock(head); + ret = nfs_wait_on_request(subreq); + if (!ret) + ret = nfs_page_group_lock(head); + if (ret < 0) { + nfs_unroll_locks(head, subreq); + nfs_release_request(subreq); + return ret; + } + } + return 0; +} + +/* + * nfs_page_group_lock_subrequests - try to lock the subrequests + * @head: head request of page group + * + * This is a helper function for nfs_lock_and_join_requests which + * must be called with the head request and page group both locked. + * On error, it returns with the page group unlocked. + */ +int nfs_page_group_lock_subrequests(struct nfs_page *head) +{ + struct nfs_page *subreq; + int ret; + + /* lock each request in the page group */ + for (subreq = head->wb_this_page; subreq != head; + subreq = subreq->wb_this_page) { + ret = nfs_page_group_lock_subreq(head, subreq); + if (ret < 0) + return ret; + } + return 0; +} + /* * nfs_page_set_headlock - set the request PG_HEADLOCK * @req: request that is to be locked diff --git a/fs/nfs/write.c b/fs/nfs/write.c index a6d7926b0653..832cf57ea442 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -379,34 +379,6 @@ static void nfs_end_page_writeback(struct nfs_page *req) clear_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC); } -/* - * nfs_unroll_locks_and_wait - unlock all newly locked reqs and wait on @req - * - * this is a helper function for nfs_lock_and_join_requests - * - * @inode - inode associated with request page group, must be holding inode lock - * @head - head request of page group, must be holding head lock - * @req - request that couldn't lock and needs to wait on the req bit lock - * - * NOTE: this must be called holding page_group bit lock - * which will be released before returning. - * - * returns 0 on success, < 0 on error. - */ -static void -nfs_unroll_locks(struct inode *inode, struct nfs_page *head, - struct nfs_page *req) -{ - struct nfs_page *tmp; - - /* relinquish all the locks successfully grabbed this run */ - for (tmp = head->wb_this_page ; tmp != req; tmp = tmp->wb_this_page) { - if (!kref_read(&tmp->wb_kref)) - continue; - nfs_unlock_and_release_request(tmp); - } -} - /* * nfs_destroy_unlinked_subrequests - destroy recently unlinked subrequests * @@ -487,7 +459,7 @@ nfs_lock_and_join_requests(struct page *page) struct inode *inode = page_file_mapping(page)->host; struct nfs_page *head, *subreq; struct nfs_page *destroy_list = NULL; - unsigned int total_bytes; + unsigned int pgbase, off, bytes; int ret; try_again: @@ -520,49 +492,30 @@ try_again: goto release_request; /* lock each request in the page group */ - total_bytes = head->wb_bytes; + ret = nfs_page_group_lock_subrequests(head); + if (ret < 0) + goto release_request; + + pgbase = head->wb_pgbase; + bytes = head->wb_bytes; + off = head->wb_offset; for (subreq = head->wb_this_page; subreq != head; subreq = subreq->wb_this_page) { - - if (!kref_get_unless_zero(&subreq->wb_kref)) { - if (subreq->wb_offset == head->wb_offset + total_bytes) - total_bytes += subreq->wb_bytes; - continue; - } - - while (!nfs_lock_request(subreq)) { - /* - * Unlock page to allow nfs_page_group_sync_on_bit() - * to succeed - */ - nfs_page_group_unlock(head); - ret = nfs_wait_on_request(subreq); - if (!ret) - ret = nfs_page_group_lock(head); - if (ret < 0) { - nfs_unroll_locks(inode, head, subreq); - nfs_release_request(subreq); - goto release_request; - } - } - /* - * Subrequests are always contiguous, non overlapping - * and in order - but may be repeated (mirrored writes). - */ - if (subreq->wb_offset == (head->wb_offset + total_bytes)) { - /* keep track of how many bytes this group covers */ - total_bytes += subreq->wb_bytes; - } else if (WARN_ON_ONCE(subreq->wb_offset < head->wb_offset || - ((subreq->wb_offset + subreq->wb_bytes) > - (head->wb_offset + total_bytes)))) { - nfs_page_group_unlock(head); - nfs_unroll_locks(inode, head, subreq); - nfs_unlock_and_release_request(subreq); - ret = -EIO; - goto release_request; + /* Subrequests should always form a contiguous range */ + if (pgbase > subreq->wb_pgbase) { + off -= pgbase - subreq->wb_pgbase; + bytes += pgbase - subreq->wb_pgbase; + pgbase = subreq->wb_pgbase; } + bytes = max(subreq->wb_pgbase + subreq->wb_bytes + - pgbase, bytes); } + /* Set the head request's range to cover the former page group */ + head->wb_pgbase = pgbase; + head->wb_bytes = bytes; + head->wb_offset = off; + /* Now that all requests are locked, make sure they aren't on any list. * Commit list removal accounting is done after locks are dropped */ subreq = head; @@ -576,10 +529,6 @@ try_again: /* destroy list will be terminated by head */ destroy_list = head->wb_this_page; head->wb_this_page = head; - - /* change head request to cover whole range that - * the former page group covered */ - head->wb_bytes = total_bytes; } /* Postpone destruction of this request */ diff --git a/include/linux/nfs_page.h b/include/linux/nfs_page.h index 7e9419d74b86..dd205bc6bc58 100644 --- a/include/linux/nfs_page.h +++ b/include/linux/nfs_page.h @@ -139,6 +139,7 @@ extern size_t nfs_generic_pg_test(struct nfs_pageio_descriptor *desc, extern int nfs_wait_on_request(struct nfs_page *); extern void nfs_unlock_request(struct nfs_page *req); extern void nfs_unlock_and_release_request(struct nfs_page *); +extern int nfs_page_group_lock_subrequests(struct nfs_page *head); extern int nfs_page_group_lock(struct nfs_page *); extern void nfs_page_group_unlock(struct nfs_page *); extern bool nfs_page_group_sync_on_bit(struct nfs_page *, unsigned int); From 44a65a0c278336719892287a185836fddeabb933 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Tue, 31 Mar 2020 18:27:26 -0400 Subject: [PATCH 413/635] NFS: Reverse the submission order of requests in __nfs_pageio_add_request() If we have to split the request up into subrequests, we have to submit the request pointed to by the function call parameter last, in case there is an error or other issue that causes us to exit before the last request is submitted. The reason is that the caller is expected to perform cleanup in those cases. Signed-off-by: Trond Myklebust --- fs/nfs/pagelist.c | 133 ++++++++++++++++++++++------------------------ 1 file changed, 64 insertions(+), 69 deletions(-) diff --git a/fs/nfs/pagelist.c b/fs/nfs/pagelist.c index 261236157e33..b9805d1dac75 100644 --- a/fs/nfs/pagelist.c +++ b/fs/nfs/pagelist.c @@ -454,15 +454,23 @@ nfs_create_request(struct nfs_open_context *ctx, struct page *page, } static struct nfs_page * -nfs_create_subreq(struct nfs_page *req, struct nfs_page *last, - unsigned int pgbase, unsigned int offset, +nfs_create_subreq(struct nfs_page *req, + unsigned int pgbase, + unsigned int offset, unsigned int count) { + struct nfs_page *last; struct nfs_page *ret; ret = __nfs_create_request(req->wb_lock_context, req->wb_page, pgbase, offset, count); if (!IS_ERR(ret)) { + /* find the last request */ + for (last = req->wb_head; + last->wb_this_page != req->wb_head; + last = last->wb_this_page) + ; + nfs_lock_request(ret); ret->wb_index = req->wb_index; nfs_page_group_init(ret, last); @@ -988,7 +996,7 @@ static bool nfs_match_lock_context(const struct nfs_lock_context *l1, } /** - * nfs_can_coalesce_requests - test two requests for compatibility + * nfs_coalesce_size - test two requests for compatibility * @prev: pointer to nfs_page * @req: pointer to nfs_page * @pgio: pointer to nfs_pagio_descriptor @@ -997,41 +1005,36 @@ static bool nfs_match_lock_context(const struct nfs_lock_context *l1, * page data area they describe is contiguous, and that their RPC * credentials, NFSv4 open state, and lockowners are the same. * - * Return 'true' if this is the case, else return 'false'. + * Returns size of the request that can be coalesced */ -static bool nfs_can_coalesce_requests(struct nfs_page *prev, +static unsigned int nfs_coalesce_size(struct nfs_page *prev, struct nfs_page *req, struct nfs_pageio_descriptor *pgio) { - size_t size; struct file_lock_context *flctx; if (prev) { if (!nfs_match_open_context(nfs_req_openctx(req), nfs_req_openctx(prev))) - return false; + return 0; flctx = d_inode(nfs_req_openctx(req)->dentry)->i_flctx; if (flctx != NULL && !(list_empty_careful(&flctx->flc_posix) && list_empty_careful(&flctx->flc_flock)) && !nfs_match_lock_context(req->wb_lock_context, prev->wb_lock_context)) - return false; + return 0; if (req_offset(req) != req_offset(prev) + prev->wb_bytes) - return false; + return 0; if (req->wb_page == prev->wb_page) { if (req->wb_pgbase != prev->wb_pgbase + prev->wb_bytes) - return false; + return 0; } else { if (req->wb_pgbase != 0 || prev->wb_pgbase + prev->wb_bytes != PAGE_SIZE) - return false; + return 0; } } - size = pgio->pg_ops->pg_test(pgio, prev, req); - WARN_ON_ONCE(size > req->wb_bytes); - if (size && size < req->wb_bytes) - req->wb_bytes = size; - return size > 0; + return pgio->pg_ops->pg_test(pgio, prev, req); } /** @@ -1039,15 +1042,16 @@ static bool nfs_can_coalesce_requests(struct nfs_page *prev, * @desc: destination io descriptor * @req: request * - * Returns true if the request 'req' was successfully coalesced into the - * existing list of pages 'desc'. + * If the request 'req' was successfully coalesced into the existing list + * of pages 'desc', it returns the size of req. */ -static int nfs_pageio_do_add_request(struct nfs_pageio_descriptor *desc, - struct nfs_page *req) +static unsigned int +nfs_pageio_do_add_request(struct nfs_pageio_descriptor *desc, + struct nfs_page *req) { struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); - struct nfs_page *prev = NULL; + unsigned int size; if (mirror->pg_count != 0) { prev = nfs_list_entry(mirror->pg_list.prev); @@ -1067,11 +1071,12 @@ static int nfs_pageio_do_add_request(struct nfs_pageio_descriptor *desc, return 0; } - if (!nfs_can_coalesce_requests(prev, req, desc)) - return 0; + size = nfs_coalesce_size(prev, req, desc); + if (size < req->wb_bytes) + return size; nfs_list_move_request(req, &mirror->pg_list); mirror->pg_count += req->wb_bytes; - return 1; + return req->wb_bytes; } /* @@ -1111,7 +1116,8 @@ nfs_pageio_cleanup_request(struct nfs_pageio_descriptor *desc, * @req: request * * This may split a request into subrequests which are all part of the - * same page group. + * same page group. If so, it will submit @req as the last one, to ensure + * the pointer to @req is still valid in case of failure. * * Returns true if the request 'req' was successfully coalesced into the * existing list of pages 'desc'. @@ -1120,51 +1126,50 @@ static int __nfs_pageio_add_request(struct nfs_pageio_descriptor *desc, struct nfs_page *req) { struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); - struct nfs_page *subreq; - unsigned int bytes_left = 0; - unsigned int offset, pgbase; + unsigned int size, subreq_size; nfs_page_group_lock(req); subreq = req; - bytes_left = subreq->wb_bytes; - offset = subreq->wb_offset; - pgbase = subreq->wb_pgbase; - - do { - if (!nfs_pageio_do_add_request(desc, subreq)) { - /* make sure pg_test call(s) did nothing */ - WARN_ON_ONCE(subreq->wb_bytes != bytes_left); - WARN_ON_ONCE(subreq->wb_offset != offset); - WARN_ON_ONCE(subreq->wb_pgbase != pgbase); - + subreq_size = subreq->wb_bytes; + for(;;) { + size = nfs_pageio_do_add_request(desc, subreq); + if (size == subreq_size) { + /* We successfully submitted a request */ + if (subreq == req) + break; + req->wb_pgbase += size; + req->wb_bytes -= size; + req->wb_offset += size; + subreq_size = req->wb_bytes; + subreq = req; + continue; + } + if (WARN_ON_ONCE(subreq != req)) { + nfs_page_group_unlock(req); + nfs_pageio_cleanup_request(desc, subreq); + subreq = req; + subreq_size = req->wb_bytes; + nfs_page_group_lock(req); + } + if (!size) { + /* Can't coalesce any more, so do I/O */ nfs_page_group_unlock(req); desc->pg_moreio = 1; nfs_pageio_doio(desc); if (desc->pg_error < 0 || mirror->pg_recoalesce) - goto out_cleanup_subreq; + return 0; /* retry add_request for this subreq */ nfs_page_group_lock(req); continue; } - - /* check for buggy pg_test call(s) */ - WARN_ON_ONCE(subreq->wb_bytes + subreq->wb_pgbase > PAGE_SIZE); - WARN_ON_ONCE(subreq->wb_bytes > bytes_left); - WARN_ON_ONCE(subreq->wb_bytes == 0); - - bytes_left -= subreq->wb_bytes; - offset += subreq->wb_bytes; - pgbase += subreq->wb_bytes; - - if (bytes_left) { - subreq = nfs_create_subreq(req, subreq, pgbase, - offset, bytes_left); - if (IS_ERR(subreq)) - goto err_ptr; - } - } while (bytes_left > 0); + subreq = nfs_create_subreq(req, req->wb_pgbase, + req->wb_offset, size); + if (IS_ERR(subreq)) + goto err_ptr; + subreq_size = size; + } nfs_page_group_unlock(req); return 1; @@ -1172,10 +1177,6 @@ err_ptr: desc->pg_error = PTR_ERR(subreq); nfs_page_group_unlock(req); return 0; -out_cleanup_subreq: - if (req != subreq) - nfs_pageio_cleanup_request(desc, subreq); - return 0; } static int nfs_do_recoalesce(struct nfs_pageio_descriptor *desc) @@ -1244,7 +1245,7 @@ int nfs_pageio_add_request(struct nfs_pageio_descriptor *desc, { u32 midx; unsigned int pgbase, offset, bytes; - struct nfs_page *dupreq, *lastreq; + struct nfs_page *dupreq; pgbase = req->wb_pgbase; offset = req->wb_offset; @@ -1258,13 +1259,7 @@ int nfs_pageio_add_request(struct nfs_pageio_descriptor *desc, for (midx = 1; midx < desc->pg_mirror_count; midx++) { nfs_page_group_lock(req); - /* find the last request */ - for (lastreq = req->wb_head; - lastreq->wb_this_page != req->wb_head; - lastreq = lastreq->wb_this_page) - ; - - dupreq = nfs_create_subreq(req, lastreq, + dupreq = nfs_create_subreq(req, pgbase, offset, bytes); nfs_page_group_unlock(req); From e00ed89d7bd59c4ae49d6aeeee567187b1357a4b Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 30 Mar 2020 12:40:47 -0400 Subject: [PATCH 414/635] NFS: Refactor nfs_lock_and_join_requests() Refactor nfs_lock_and_join_requests() in order to separate out the subrequest merging into its own function nfs_lock_and_join_group() that can be used by O_DIRECT. Signed-off-by: Trond Myklebust --- fs/nfs/pagelist.c | 26 +++++- fs/nfs/write.c | 166 +++++++++++++++++++++++---------------- include/linux/nfs_page.h | 1 + 3 files changed, 124 insertions(+), 69 deletions(-) diff --git a/fs/nfs/pagelist.c b/fs/nfs/pagelist.c index b9805d1dac75..f61f96603df7 100644 --- a/fs/nfs/pagelist.c +++ b/fs/nfs/pagelist.c @@ -130,6 +130,25 @@ nfs_async_iocounter_wait(struct rpc_task *task, struct nfs_lock_context *l_ctx) } EXPORT_SYMBOL_GPL(nfs_async_iocounter_wait); +/* + * nfs_page_lock_head_request - page lock the head of the page group + * @req: any member of the page group + */ +struct nfs_page * +nfs_page_group_lock_head(struct nfs_page *req) +{ + struct nfs_page *head = req->wb_head; + + while (!nfs_lock_request(head)) { + int ret = nfs_wait_on_request(head); + if (ret < 0) + return ERR_PTR(ret); + } + if (head != req) + kref_get(&head->wb_kref); + return head; +} + /* * nfs_unroll_locks - unlock all newly locked reqs and wait on @req * @head: head request of page group, must be holding head lock @@ -186,14 +205,16 @@ nfs_page_group_lock_subreq(struct nfs_page *head, struct nfs_page *subreq) * @head: head request of page group * * This is a helper function for nfs_lock_and_join_requests which - * must be called with the head request and page group both locked. - * On error, it returns with the page group unlocked. + * must be called with the head request locked. */ int nfs_page_group_lock_subrequests(struct nfs_page *head) { struct nfs_page *subreq; int ret; + ret = nfs_page_group_lock(head); + if (ret < 0) + return ret; /* lock each request in the page group */ for (subreq = head->wb_this_page; subreq != head; subreq = subreq->wb_this_page) { @@ -201,6 +222,7 @@ int nfs_page_group_lock_subrequests(struct nfs_page *head) if (ret < 0) return ret; } + nfs_page_group_unlock(head); return 0; } diff --git a/fs/nfs/write.c b/fs/nfs/write.c index 832cf57ea442..63b64333c3ea 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -149,6 +149,31 @@ static void nfs_io_completion_put(struct nfs_io_completion *ioc) kref_put(&ioc->refcount, nfs_io_completion_release); } +static void +nfs_page_set_inode_ref(struct nfs_page *req, struct inode *inode) +{ + if (!test_and_set_bit(PG_INODE_REF, &req->wb_flags)) { + kref_get(&req->wb_kref); + atomic_long_inc(&NFS_I(inode)->nrequests); + } +} + +static int +nfs_cancel_remove_inode(struct nfs_page *req, struct inode *inode) +{ + int ret; + + if (!test_bit(PG_REMOVE, &req->wb_flags)) + return 0; + ret = nfs_page_group_lock(req); + if (ret) + return ret; + if (test_and_clear_bit(PG_REMOVE, &req->wb_flags)) + nfs_page_set_inode_ref(req, inode); + nfs_page_group_unlock(req); + return 0; +} + static struct nfs_page * nfs_page_private_request(struct page *page) { @@ -218,6 +243,36 @@ static struct nfs_page *nfs_page_find_head_request(struct page *page) return req; } +static struct nfs_page *nfs_find_and_lock_page_request(struct page *page) +{ + struct inode *inode = page_file_mapping(page)->host; + struct nfs_page *req, *head; + int ret; + + for (;;) { + req = nfs_page_find_head_request(page); + if (!req) + return req; + head = nfs_page_group_lock_head(req); + if (head != req) + nfs_release_request(req); + if (IS_ERR(head)) + return head; + ret = nfs_cancel_remove_inode(head, inode); + if (ret < 0) { + nfs_unlock_and_release_request(head); + return ERR_PTR(ret); + } + /* Ensure that nobody removed the request before we locked it */ + if (head == nfs_page_private_request(page)) + break; + if (PageSwapCache(page)) + break; + nfs_unlock_and_release_request(head); + } + return head; +} + /* Adjust the file length if we're writing beyond the end */ static void nfs_grow_file(struct page *page, unsigned int offset, unsigned int count) { @@ -436,65 +491,22 @@ nfs_destroy_unlinked_subrequests(struct nfs_page *destroy_list, } /* - * nfs_lock_and_join_requests - join all subreqs to the head req and return - * a locked reference, cancelling any pending - * operations for this page. - * - * @page - the page used to lookup the "page group" of nfs_page structures + * nfs_join_page_group - destroy subrequests of the head req + * @head: the page used to lookup the "page group" of nfs_page structures + * @inode: Inode to which the request belongs. * * This function joins all sub requests to the head request by first * locking all requests in the group, cancelling any pending operations * and finally updating the head request to cover the whole range covered by * the (former) group. All subrequests are removed from any write or commit * lists, unlinked from the group and destroyed. - * - * Returns a locked, referenced pointer to the head request - which after - * this call is guaranteed to be the only request associated with the page. - * Returns NULL if no requests are found for @page, or a ERR_PTR if an - * error was encountered. */ -static struct nfs_page * -nfs_lock_and_join_requests(struct page *page) +static void +nfs_join_page_group(struct nfs_page *head, struct inode *inode) { - struct inode *inode = page_file_mapping(page)->host; - struct nfs_page *head, *subreq; + struct nfs_page *subreq; struct nfs_page *destroy_list = NULL; unsigned int pgbase, off, bytes; - int ret; - -try_again: - /* - * A reference is taken only on the head request which acts as a - * reference to the whole page group - the group will not be destroyed - * until the head reference is released. - */ - head = nfs_page_find_head_request(page); - if (!head) - return NULL; - - /* lock the page head first in order to avoid an ABBA inefficiency */ - if (!nfs_lock_request(head)) { - ret = nfs_wait_on_request(head); - nfs_release_request(head); - if (ret < 0) - return ERR_PTR(ret); - goto try_again; - } - - /* Ensure that nobody removed the request before we locked it */ - if (head != nfs_page_private_request(page) && !PageSwapCache(page)) { - nfs_unlock_and_release_request(head); - goto try_again; - } - - ret = nfs_page_group_lock(head); - if (ret < 0) - goto release_request; - - /* lock each request in the page group */ - ret = nfs_page_group_lock_subrequests(head); - if (ret < 0) - goto release_request; pgbase = head->wb_pgbase; bytes = head->wb_bytes; @@ -531,30 +543,50 @@ try_again: head->wb_this_page = head; } - /* Postpone destruction of this request */ - if (test_and_clear_bit(PG_REMOVE, &head->wb_flags)) { - set_bit(PG_INODE_REF, &head->wb_flags); - kref_get(&head->wb_kref); - atomic_long_inc(&NFS_I(inode)->nrequests); - } - - nfs_page_group_unlock(head); - nfs_destroy_unlinked_subrequests(destroy_list, head, inode); +} - /* Did we lose a race with nfs_inode_remove_request()? */ - if (!(PagePrivate(page) || PageSwapCache(page))) { +/* + * nfs_lock_and_join_requests - join all subreqs to the head req + * @page: the page used to lookup the "page group" of nfs_page structures + * + * This function joins all sub requests to the head request by first + * locking all requests in the group, cancelling any pending operations + * and finally updating the head request to cover the whole range covered by + * the (former) group. All subrequests are removed from any write or commit + * lists, unlinked from the group and destroyed. + * + * Returns a locked, referenced pointer to the head request - which after + * this call is guaranteed to be the only request associated with the page. + * Returns NULL if no requests are found for @page, or a ERR_PTR if an + * error was encountered. + */ +static struct nfs_page * +nfs_lock_and_join_requests(struct page *page) +{ + struct inode *inode = page_file_mapping(page)->host; + struct nfs_page *head; + int ret; + + /* + * A reference is taken only on the head request which acts as a + * reference to the whole page group - the group will not be destroyed + * until the head reference is released. + */ + head = nfs_find_and_lock_page_request(page); + if (IS_ERR_OR_NULL(head)) + return head; + + /* lock each request in the page group */ + ret = nfs_page_group_lock_subrequests(head); + if (ret < 0) { nfs_unlock_and_release_request(head); - return NULL; + return ERR_PTR(ret); } - /* still holds ref on head from nfs_page_find_head_request - * and still has lock on head from lock loop */ - return head; + nfs_join_page_group(head, inode); -release_request: - nfs_unlock_and_release_request(head); - return ERR_PTR(ret); + return head; } static void nfs_write_error(struct nfs_page *req, int error) diff --git a/include/linux/nfs_page.h b/include/linux/nfs_page.h index dd205bc6bc58..99198c039bd6 100644 --- a/include/linux/nfs_page.h +++ b/include/linux/nfs_page.h @@ -139,6 +139,7 @@ extern size_t nfs_generic_pg_test(struct nfs_pageio_descriptor *desc, extern int nfs_wait_on_request(struct nfs_page *); extern void nfs_unlock_request(struct nfs_page *req); extern void nfs_unlock_and_release_request(struct nfs_page *); +extern struct nfs_page *nfs_page_group_lock_head(struct nfs_page *req); extern int nfs_page_group_lock_subrequests(struct nfs_page *head); extern int nfs_page_group_lock(struct nfs_page *); extern void nfs_page_group_unlock(struct nfs_page *); From ed5d588fe47feef290f271022820e255d8371561 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 30 Mar 2020 20:57:49 -0400 Subject: [PATCH 415/635] NFS: Try to join page groups before an O_DIRECT retransmission If we have to retransmit requests, try to join their page groups first. Signed-off-by: Trond Myklebust --- fs/nfs/direct.c | 20 ++++++++++++++++++++ fs/nfs/write.c | 2 +- include/linux/nfs_page.h | 1 + 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c index 8074304fd5b4..a57e7c72c7f4 100644 --- a/fs/nfs/direct.c +++ b/fs/nfs/direct.c @@ -505,6 +505,24 @@ out: return result; } +static void +nfs_direct_join_group(struct list_head *list, struct inode *inode) +{ + struct nfs_page *req, *next; + + list_for_each_entry(req, list, wb_list) { + if (req->wb_head != req || req->wb_this_page == req) + continue; + for (next = req->wb_this_page; + next != req->wb_head; + next = next->wb_this_page) { + nfs_list_remove_request(next); + nfs_release_request(next); + } + nfs_join_page_group(req, inode); + } +} + static void nfs_direct_write_scan_commit_list(struct inode *inode, struct list_head *list, @@ -527,6 +545,8 @@ static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq) nfs_init_cinfo_from_dreq(&cinfo, dreq); nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo); + nfs_direct_join_group(&reqs, dreq->inode); + dreq->count = 0; dreq->max_count = 0; list_for_each_entry(req, &reqs, wb_list) diff --git a/fs/nfs/write.c b/fs/nfs/write.c index 63b64333c3ea..df4b87c30ac9 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -501,7 +501,7 @@ nfs_destroy_unlinked_subrequests(struct nfs_page *destroy_list, * the (former) group. All subrequests are removed from any write or commit * lists, unlinked from the group and destroyed. */ -static void +void nfs_join_page_group(struct nfs_page *head, struct inode *inode) { struct nfs_page *subreq; diff --git a/include/linux/nfs_page.h b/include/linux/nfs_page.h index 99198c039bd6..c32c15216da3 100644 --- a/include/linux/nfs_page.h +++ b/include/linux/nfs_page.h @@ -141,6 +141,7 @@ extern void nfs_unlock_request(struct nfs_page *req); extern void nfs_unlock_and_release_request(struct nfs_page *); extern struct nfs_page *nfs_page_group_lock_head(struct nfs_page *req); extern int nfs_page_group_lock_subrequests(struct nfs_page *head); +extern void nfs_join_page_group(struct nfs_page *head, struct inode *inode); extern int nfs_page_group_lock(struct nfs_page *); extern void nfs_page_group_unlock(struct nfs_page *); extern bool nfs_page_group_sync_on_bit(struct nfs_page *, unsigned int); From 3148a6a0ef3cf93570f30a477292768f7eb5d3c3 Mon Sep 17 00:00:00 2001 From: Jack Zhang Date: Wed, 1 Apr 2020 20:06:58 +0800 Subject: [PATCH 416/635] drm/amdkfd: kfree the wrong pointer Originally, it kfrees the wrong pointer for mem_obj. It would cause memory leak under stress test. Signed-off-by: Jack Zhang Acked-by: Nirmoy Das Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_device.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c index d5386f15c4a5..05bc6d96ec52 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c @@ -1112,9 +1112,9 @@ kfd_gtt_out: return 0; kfd_gtt_no_free_chunk: - pr_debug("Allocation failed with mem_obj = %p\n", mem_obj); + pr_debug("Allocation failed with mem_obj = %p\n", *mem_obj); mutex_unlock(&kfd->gtt_sa_lock); - kfree(mem_obj); + kfree(*mem_obj); return -ENOMEM; } From c9c20ee3cfce924e7b2a8138df2d6958c500858a Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Wed, 1 Apr 2020 12:53:36 -0700 Subject: [PATCH 417/635] Input: goodix - fix compilation when ACPI support is disabled acpi_evaluate_object() and acpi_execute_simple_method() are not part of the group of ACPI related functions which get stubbed by include/linux/acpi.h when ACPI support is disabled, so the IRQ_PIN_ACCESS_ACPI_METHOD handling code must be stubbed out. For consistency use the same #if condition as which is used to replace goodix_add_acpi_gpio_mappings with a stub. Fixes: c5fca485320e ("Input: goodix - add support for controlling the IRQ pin through ACPI methods") Reported-by: kbuild test robot Signed-off-by: Hans de Goede Link: https://lore.kernel.org/r/20200401014529.GL75430@dtor-ws [dtor: stubbed out the ACPI method accessors] Reviewed-by: Bastien Nocera Signed-off-by: Dmitry Torokhov --- drivers/input/touchscreen/goodix.c | 55 +++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 13 deletions(-) diff --git a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c index 47f812b804c8..02c75ea385e0 100644 --- a/drivers/input/touchscreen/goodix.c +++ b/drivers/input/touchscreen/goodix.c @@ -63,6 +63,11 @@ #define MAX_CONTACTS_LOC 5 #define TRIGGER_LOC 6 +/* Our special handling for GPIO accesses through ACPI is x86 specific */ +#if defined CONFIG_X86 && defined CONFIG_ACPI +#define ACPI_GPIO_SUPPORT +#endif + struct goodix_ts_data; enum goodix_irq_pin_access_method { @@ -600,12 +605,42 @@ static int goodix_send_cfg(struct goodix_ts_data *ts, const u8 *cfg, int len) return 0; } -static int goodix_irq_direction_output(struct goodix_ts_data *ts, - int value) +#ifdef ACPI_GPIO_SUPPORT +static int goodix_pin_acpi_direction_input(struct goodix_ts_data *ts) { - struct device *dev = &ts->client->dev; + acpi_handle handle = ACPI_HANDLE(&ts->client->dev); acpi_status status; + status = acpi_evaluate_object(handle, "INTI", NULL, NULL); + return ACPI_SUCCESS(status) ? 0 : -EIO; +} + +static int goodix_pin_acpi_output_method(struct goodix_ts_data *ts, int value) +{ + acpi_handle handle = ACPI_HANDLE(&ts->client->dev); + acpi_status status; + + status = acpi_execute_simple_method(handle, "INTO", value); + return ACPI_SUCCESS(status) ? 0 : -EIO; +} +#else +static int goodix_pin_acpi_direction_input(struct goodix_ts_data *ts) +{ + dev_err(&ts->client->dev, + "%s called on device without ACPI support\n", __func__); + return -EINVAL; +} + +static int goodix_pin_acpi_output_method(struct goodix_ts_data *ts, int value) +{ + dev_err(&ts->client->dev, + "%s called on device without ACPI support\n", __func__); + return -EINVAL; +} +#endif + +static int goodix_irq_direction_output(struct goodix_ts_data *ts, int value) +{ switch (ts->irq_pin_access_method) { case IRQ_PIN_ACCESS_NONE: dev_err(&ts->client->dev, @@ -621,9 +656,7 @@ static int goodix_irq_direction_output(struct goodix_ts_data *ts, */ return gpiod_direction_output_raw(ts->gpiod_int, value); case IRQ_PIN_ACCESS_ACPI_METHOD: - status = acpi_execute_simple_method(ACPI_HANDLE(dev), - "INTO", value); - return ACPI_SUCCESS(status) ? 0 : -EIO; + return goodix_pin_acpi_output_method(ts, value); } return -EINVAL; /* Never reached */ @@ -631,9 +664,6 @@ static int goodix_irq_direction_output(struct goodix_ts_data *ts, static int goodix_irq_direction_input(struct goodix_ts_data *ts) { - struct device *dev = &ts->client->dev; - acpi_status status; - switch (ts->irq_pin_access_method) { case IRQ_PIN_ACCESS_NONE: dev_err(&ts->client->dev, @@ -641,12 +671,11 @@ static int goodix_irq_direction_input(struct goodix_ts_data *ts) __func__); return -EINVAL; case IRQ_PIN_ACCESS_GPIO: + return gpiod_direction_input(ts->gpiod_int); case IRQ_PIN_ACCESS_ACPI_GPIO: return gpiod_direction_input(ts->gpiod_int); case IRQ_PIN_ACCESS_ACPI_METHOD: - status = acpi_evaluate_object(ACPI_HANDLE(dev), "INTI", - NULL, NULL); - return ACPI_SUCCESS(status) ? 0 : -EIO; + return goodix_pin_acpi_direction_input(ts); } return -EINVAL; /* Never reached */ @@ -710,7 +739,7 @@ static int goodix_reset(struct goodix_ts_data *ts) return 0; } -#if defined CONFIG_X86 && defined CONFIG_ACPI +#ifdef ACPI_GPIO_SUPPORT #include #include From ebc68cedec4aead47d8d11623d013cca9bf8e825 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Wed, 1 Apr 2020 13:23:06 -0700 Subject: [PATCH 418/635] Input: i8042 - add Acer Aspire 5738z to nomux list The Acer Aspire 5738z has a button to disable (and re-enable) the touchpad next to the touchpad. When this button is pressed a LED underneath indicates that the touchpad is disabled (and an event is send to userspace and GNOME shows its touchpad enabled / disable OSD thingie). So far so good, but after re-enabling the touchpad it no longer works. The laptop does not have an external ps2 port, so mux mode is not needed and disabling mux mode fixes the touchpad no longer working after toggling it off and back on again, so lets add this laptop model to the nomux list. Signed-off-by: Hans de Goede Link: https://lore.kernel.org/r/20200331123947.318908-1-hdegoede@redhat.com Cc: stable@vger.kernel.org Signed-off-by: Dmitry Torokhov --- drivers/input/serio/i8042-x86ia64io.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/input/serio/i8042-x86ia64io.h b/drivers/input/serio/i8042-x86ia64io.h index dc974c288e88..08e919dbeb5d 100644 --- a/drivers/input/serio/i8042-x86ia64io.h +++ b/drivers/input/serio/i8042-x86ia64io.h @@ -530,6 +530,17 @@ static const struct dmi_system_id __initconst i8042_dmi_nomux_table[] = { DMI_MATCH(DMI_PRODUCT_VERSION, "Lenovo LaVie Z"), }, }, + { + /* + * Acer Aspire 5738z + * Touchpad stops working in mux mode when dis- + re-enabled + * with the touchpad enable/disable toggle hotkey + */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Acer"), + DMI_MATCH(DMI_PRODUCT_NAME, "Aspire 5738"), + }, + }, { } }; From 3a8579629665bd6c0b37afcb2291080e959b885d Mon Sep 17 00:00:00 2001 From: Rajat Jain Date: Wed, 1 Apr 2020 13:24:11 -0700 Subject: [PATCH 419/635] Input: update SPDX tag for input-event-codes.h Replace the /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ with /* SPDX-License-Identifier: GPL-2.0-only WITH Linux-syscall-note */ to help coreboot community consume this file without relaxing their licensing checks. Signed-off-by: Rajat Jain Link: https://lore.kernel.org/r/20200329172513.133548-1-rajatja@google.com Signed-off-by: Dmitry Torokhov --- include/uapi/linux/input-event-codes.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/uapi/linux/input-event-codes.h b/include/uapi/linux/input-event-codes.h index 0f1db1cccc3f..f6cc830b31f2 100644 --- a/include/uapi/linux/input-event-codes.h +++ b/include/uapi/linux/input-event-codes.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* SPDX-License-Identifier: GPL-2.0-only WITH Linux-syscall-note */ /* * Input event codes * From 4c8cf31885f69e86be0b5b9e6677a26797365e1d Mon Sep 17 00:00:00 2001 From: Tiwei Bie Date: Thu, 26 Mar 2020 22:01:23 +0800 Subject: [PATCH 420/635] vhost: introduce vDPA-based backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch introduces a vDPA-based vhost backend. This backend is built on top of the same interface defined in virtio-vDPA and provides a generic vhost interface for userspace to accelerate the virtio devices in guest. This backend is implemented as a vDPA device driver on top of the same ops used in virtio-vDPA. It will create char device entry named vhost-vdpa-$index for userspace to use. Userspace can use vhost ioctls on top of this char device to setup the backend. Vhost ioctls are extended to make it type agnostic and behave like a virtio device, this help to eliminate type specific API like what vhost_net/scsi/vsock did: - VHOST_VDPA_GET_DEVICE_ID: get the virtio device ID which is defined by virtio specification to differ from different type of devices - VHOST_VDPA_GET_VRING_NUM: get the maximum size of virtqueue supported by the vDPA device - VHSOT_VDPA_SET/GET_STATUS: set and get virtio status of vDPA device - VHOST_VDPA_SET/GET_CONFIG: access virtio config space - VHOST_VDPA_SET_VRING_ENABLE: enable a specific virtqueue For memory mapping, IOTLB API is mandated for vhost-vDPA which means userspace drivers are required to use VHOST_IOTLB_UPDATE/VHOST_IOTLB_INVALIDATE to add or remove mapping for a specific userspace memory region. The vhost-vDPA API is designed to be type agnostic, but it allows net device only in current stage. Due to the lacking of control virtqueue support, some features were filter out by vhost-vdpa. We will enable more features and devices in the near future. Signed-off-by: Tiwei Bie Signed-off-by: Eugenio Pérez Signed-off-by: Jason Wang Link: https://lore.kernel.org/r/20200326140125.19794-8-jasowang@redhat.com Signed-off-by: Michael S. Tsirkin --- drivers/vhost/Kconfig | 12 + drivers/vhost/Makefile | 3 + drivers/vhost/vdpa.c | 883 +++++++++++++++++++++++++++++++ include/uapi/linux/vhost.h | 24 + include/uapi/linux/vhost_types.h | 8 + 5 files changed, 930 insertions(+) create mode 100644 drivers/vhost/vdpa.c diff --git a/drivers/vhost/Kconfig b/drivers/vhost/Kconfig index 128238488078..362b832f5338 100644 --- a/drivers/vhost/Kconfig +++ b/drivers/vhost/Kconfig @@ -59,6 +59,18 @@ config VHOST_VSOCK To compile this driver as a module, choose M here: the module will be called vhost_vsock. +config VHOST_VDPA + tristate "Vhost driver for vDPA-based backend" + depends on EVENTFD + select VHOST + select VDPA + help + This kernel module can be loaded in host kernel to accelerate + guest virtio devices with the vDPA-based backends. + + To compile this driver as a module, choose M here: the module + will be called vhost_vdpa. + config VHOST_CROSS_ENDIAN_LEGACY bool "Cross-endian support for vhost" default n diff --git a/drivers/vhost/Makefile b/drivers/vhost/Makefile index fb831002bcf0..f3e1897cce85 100644 --- a/drivers/vhost/Makefile +++ b/drivers/vhost/Makefile @@ -10,6 +10,9 @@ vhost_vsock-y := vsock.o obj-$(CONFIG_VHOST_RING) += vringh.o +obj-$(CONFIG_VHOST_VDPA) += vhost_vdpa.o +vhost_vdpa-y := vdpa.o + obj-$(CONFIG_VHOST) += vhost.o obj-$(CONFIG_VHOST_IOTLB) += vhost_iotlb.o diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c new file mode 100644 index 000000000000..421f02a8530a --- /dev/null +++ b/drivers/vhost/vdpa.c @@ -0,0 +1,883 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2018-2020 Intel Corporation. + * Copyright (C) 2020 Red Hat, Inc. + * + * Author: Tiwei Bie + * Jason Wang + * + * Thanks Michael S. Tsirkin for the valuable comments and + * suggestions. And thanks to Cunming Liang and Zhihong Wang for all + * their supports. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "vhost.h" + +enum { + VHOST_VDPA_FEATURES = + (1ULL << VIRTIO_F_NOTIFY_ON_EMPTY) | + (1ULL << VIRTIO_F_ANY_LAYOUT) | + (1ULL << VIRTIO_F_VERSION_1) | + (1ULL << VIRTIO_F_IOMMU_PLATFORM) | + (1ULL << VIRTIO_F_RING_PACKED) | + (1ULL << VIRTIO_F_ORDER_PLATFORM) | + (1ULL << VIRTIO_RING_F_INDIRECT_DESC) | + (1ULL << VIRTIO_RING_F_EVENT_IDX), + + VHOST_VDPA_NET_FEATURES = VHOST_VDPA_FEATURES | + (1ULL << VIRTIO_NET_F_CSUM) | + (1ULL << VIRTIO_NET_F_GUEST_CSUM) | + (1ULL << VIRTIO_NET_F_MTU) | + (1ULL << VIRTIO_NET_F_MAC) | + (1ULL << VIRTIO_NET_F_GUEST_TSO4) | + (1ULL << VIRTIO_NET_F_GUEST_TSO6) | + (1ULL << VIRTIO_NET_F_GUEST_ECN) | + (1ULL << VIRTIO_NET_F_GUEST_UFO) | + (1ULL << VIRTIO_NET_F_HOST_TSO4) | + (1ULL << VIRTIO_NET_F_HOST_TSO6) | + (1ULL << VIRTIO_NET_F_HOST_ECN) | + (1ULL << VIRTIO_NET_F_HOST_UFO) | + (1ULL << VIRTIO_NET_F_MRG_RXBUF) | + (1ULL << VIRTIO_NET_F_STATUS) | + (1ULL << VIRTIO_NET_F_SPEED_DUPLEX), +}; + +/* Currently, only network backend w/o multiqueue is supported. */ +#define VHOST_VDPA_VQ_MAX 2 + +#define VHOST_VDPA_DEV_MAX (1U << MINORBITS) + +struct vhost_vdpa { + struct vhost_dev vdev; + struct iommu_domain *domain; + struct vhost_virtqueue *vqs; + struct completion completion; + struct vdpa_device *vdpa; + struct device dev; + struct cdev cdev; + atomic_t opened; + int nvqs; + int virtio_id; + int minor; +}; + +static DEFINE_IDA(vhost_vdpa_ida); + +static dev_t vhost_vdpa_major; + +static const u64 vhost_vdpa_features[] = { + [VIRTIO_ID_NET] = VHOST_VDPA_NET_FEATURES, +}; + +static void handle_vq_kick(struct vhost_work *work) +{ + struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, + poll.work); + struct vhost_vdpa *v = container_of(vq->dev, struct vhost_vdpa, vdev); + const struct vdpa_config_ops *ops = v->vdpa->config; + + ops->kick_vq(v->vdpa, vq - v->vqs); +} + +static irqreturn_t vhost_vdpa_virtqueue_cb(void *private) +{ + struct vhost_virtqueue *vq = private; + struct eventfd_ctx *call_ctx = vq->call_ctx; + + if (call_ctx) + eventfd_signal(call_ctx, 1); + + return IRQ_HANDLED; +} + +static void vhost_vdpa_reset(struct vhost_vdpa *v) +{ + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + + ops->set_status(vdpa, 0); +} + +static long vhost_vdpa_get_device_id(struct vhost_vdpa *v, u8 __user *argp) +{ + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + u32 device_id; + + device_id = ops->get_device_id(vdpa); + + if (copy_to_user(argp, &device_id, sizeof(device_id))) + return -EFAULT; + + return 0; +} + +static long vhost_vdpa_get_status(struct vhost_vdpa *v, u8 __user *statusp) +{ + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + u8 status; + + status = ops->get_status(vdpa); + + if (copy_to_user(statusp, &status, sizeof(status))) + return -EFAULT; + + return 0; +} + +static long vhost_vdpa_set_status(struct vhost_vdpa *v, u8 __user *statusp) +{ + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + u8 status; + + if (copy_from_user(&status, statusp, sizeof(status))) + return -EFAULT; + + /* + * Userspace shouldn't remove status bits unless reset the + * status to 0. + */ + if (status != 0 && (ops->get_status(vdpa) & ~status) != 0) + return -EINVAL; + + ops->set_status(vdpa, status); + + return 0; +} + +static int vhost_vdpa_config_validate(struct vhost_vdpa *v, + struct vhost_vdpa_config *c) +{ + long size = 0; + + switch (v->virtio_id) { + case VIRTIO_ID_NET: + size = sizeof(struct virtio_net_config); + break; + } + + if (c->len == 0) + return -EINVAL; + + if (c->len > size - c->off) + return -E2BIG; + + return 0; +} + +static long vhost_vdpa_get_config(struct vhost_vdpa *v, + struct vhost_vdpa_config __user *c) +{ + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + struct vhost_vdpa_config config; + unsigned long size = offsetof(struct vhost_vdpa_config, buf); + u8 *buf; + + if (copy_from_user(&config, c, size)) + return -EFAULT; + if (vhost_vdpa_config_validate(v, &config)) + return -EINVAL; + buf = kvzalloc(config.len, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + ops->get_config(vdpa, config.off, buf, config.len); + + if (copy_to_user(c->buf, buf, config.len)) { + kvfree(buf); + return -EFAULT; + } + + kvfree(buf); + return 0; +} + +static long vhost_vdpa_set_config(struct vhost_vdpa *v, + struct vhost_vdpa_config __user *c) +{ + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + struct vhost_vdpa_config config; + unsigned long size = offsetof(struct vhost_vdpa_config, buf); + u8 *buf; + + if (copy_from_user(&config, c, size)) + return -EFAULT; + if (vhost_vdpa_config_validate(v, &config)) + return -EINVAL; + buf = kvzalloc(config.len, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + if (copy_from_user(buf, c->buf, config.len)) { + kvfree(buf); + return -EFAULT; + } + + ops->set_config(vdpa, config.off, buf, config.len); + + kvfree(buf); + return 0; +} + +static long vhost_vdpa_get_features(struct vhost_vdpa *v, u64 __user *featurep) +{ + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + u64 features; + + features = ops->get_features(vdpa); + features &= vhost_vdpa_features[v->virtio_id]; + + if (copy_to_user(featurep, &features, sizeof(features))) + return -EFAULT; + + return 0; +} + +static long vhost_vdpa_set_features(struct vhost_vdpa *v, u64 __user *featurep) +{ + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + u64 features; + + /* + * It's not allowed to change the features after they have + * been negotiated. + */ + if (ops->get_status(vdpa) & VIRTIO_CONFIG_S_FEATURES_OK) + return -EBUSY; + + if (copy_from_user(&features, featurep, sizeof(features))) + return -EFAULT; + + if (features & ~vhost_vdpa_features[v->virtio_id]) + return -EINVAL; + + if (ops->set_features(vdpa, features)) + return -EINVAL; + + return 0; +} + +static long vhost_vdpa_get_vring_num(struct vhost_vdpa *v, u16 __user *argp) +{ + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + u16 num; + + num = ops->get_vq_num_max(vdpa); + + if (copy_to_user(argp, &num, sizeof(num))) + return -EFAULT; + + return 0; +} + +static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd, + void __user *argp) +{ + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + struct vdpa_callback cb; + struct vhost_virtqueue *vq; + struct vhost_vring_state s; + u8 status; + u32 idx; + long r; + + r = get_user(idx, (u32 __user *)argp); + if (r < 0) + return r; + + if (idx >= v->nvqs) + return -ENOBUFS; + + idx = array_index_nospec(idx, v->nvqs); + vq = &v->vqs[idx]; + + status = ops->get_status(vdpa); + + if (cmd == VHOST_VDPA_SET_VRING_ENABLE) { + if (copy_from_user(&s, argp, sizeof(s))) + return -EFAULT; + ops->set_vq_ready(vdpa, idx, s.num); + return 0; + } + + if (cmd == VHOST_GET_VRING_BASE) + vq->last_avail_idx = ops->get_vq_state(v->vdpa, idx); + + r = vhost_vring_ioctl(&v->vdev, cmd, argp); + if (r) + return r; + + switch (cmd) { + case VHOST_SET_VRING_ADDR: + if (ops->set_vq_address(vdpa, idx, + (u64)(uintptr_t)vq->desc, + (u64)(uintptr_t)vq->avail, + (u64)(uintptr_t)vq->used)) + r = -EINVAL; + break; + + case VHOST_SET_VRING_BASE: + if (ops->set_vq_state(vdpa, idx, vq->last_avail_idx)) + r = -EINVAL; + break; + + case VHOST_SET_VRING_CALL: + if (vq->call_ctx) { + cb.callback = vhost_vdpa_virtqueue_cb; + cb.private = vq; + } else { + cb.callback = NULL; + cb.private = NULL; + } + ops->set_vq_cb(vdpa, idx, &cb); + break; + + case VHOST_SET_VRING_NUM: + ops->set_vq_num(vdpa, idx, vq->num); + break; + } + + return r; +} + +static long vhost_vdpa_unlocked_ioctl(struct file *filep, + unsigned int cmd, unsigned long arg) +{ + struct vhost_vdpa *v = filep->private_data; + struct vhost_dev *d = &v->vdev; + void __user *argp = (void __user *)arg; + long r; + + mutex_lock(&d->mutex); + + switch (cmd) { + case VHOST_VDPA_GET_DEVICE_ID: + r = vhost_vdpa_get_device_id(v, argp); + break; + case VHOST_VDPA_GET_STATUS: + r = vhost_vdpa_get_status(v, argp); + break; + case VHOST_VDPA_SET_STATUS: + r = vhost_vdpa_set_status(v, argp); + break; + case VHOST_VDPA_GET_CONFIG: + r = vhost_vdpa_get_config(v, argp); + break; + case VHOST_VDPA_SET_CONFIG: + r = vhost_vdpa_set_config(v, argp); + break; + case VHOST_GET_FEATURES: + r = vhost_vdpa_get_features(v, argp); + break; + case VHOST_SET_FEATURES: + r = vhost_vdpa_set_features(v, argp); + break; + case VHOST_VDPA_GET_VRING_NUM: + r = vhost_vdpa_get_vring_num(v, argp); + break; + case VHOST_SET_LOG_BASE: + case VHOST_SET_LOG_FD: + r = -ENOIOCTLCMD; + break; + default: + r = vhost_dev_ioctl(&v->vdev, cmd, argp); + if (r == -ENOIOCTLCMD) + r = vhost_vdpa_vring_ioctl(v, cmd, argp); + break; + } + + mutex_unlock(&d->mutex); + return r; +} + +static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v, u64 start, u64 last) +{ + struct vhost_dev *dev = &v->vdev; + struct vhost_iotlb *iotlb = dev->iotlb; + struct vhost_iotlb_map *map; + struct page *page; + unsigned long pfn, pinned; + + while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) { + pinned = map->size >> PAGE_SHIFT; + for (pfn = map->addr >> PAGE_SHIFT; + pinned > 0; pfn++, pinned--) { + page = pfn_to_page(pfn); + if (map->perm & VHOST_ACCESS_WO) + set_page_dirty_lock(page); + unpin_user_page(page); + } + atomic64_sub(map->size >> PAGE_SHIFT, &dev->mm->pinned_vm); + vhost_iotlb_map_free(iotlb, map); + } +} + +static void vhost_vdpa_iotlb_free(struct vhost_vdpa *v) +{ + struct vhost_dev *dev = &v->vdev; + + vhost_vdpa_iotlb_unmap(v, 0ULL, 0ULL - 1); + kfree(dev->iotlb); + dev->iotlb = NULL; +} + +static int perm_to_iommu_flags(u32 perm) +{ + int flags = 0; + + switch (perm) { + case VHOST_ACCESS_WO: + flags |= IOMMU_WRITE; + break; + case VHOST_ACCESS_RO: + flags |= IOMMU_READ; + break; + case VHOST_ACCESS_RW: + flags |= (IOMMU_WRITE | IOMMU_READ); + break; + default: + WARN(1, "invalidate vhost IOTLB permission\n"); + break; + } + + return flags | IOMMU_CACHE; +} + +static int vhost_vdpa_map(struct vhost_vdpa *v, + u64 iova, u64 size, u64 pa, u32 perm) +{ + struct vhost_dev *dev = &v->vdev; + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + int r = 0; + + r = vhost_iotlb_add_range(dev->iotlb, iova, iova + size - 1, + pa, perm); + if (r) + return r; + + if (ops->dma_map) + r = ops->dma_map(vdpa, iova, size, pa, perm); + else if (ops->set_map) + r = ops->set_map(vdpa, dev->iotlb); + else + r = iommu_map(v->domain, iova, pa, size, + perm_to_iommu_flags(perm)); + + return r; +} + +static void vhost_vdpa_unmap(struct vhost_vdpa *v, u64 iova, u64 size) +{ + struct vhost_dev *dev = &v->vdev; + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + + vhost_vdpa_iotlb_unmap(v, iova, iova + size - 1); + + if (ops->dma_map) + ops->dma_unmap(vdpa, iova, size); + else if (ops->set_map) + ops->set_map(vdpa, dev->iotlb); + else + iommu_unmap(v->domain, iova, size); +} + +static int vhost_vdpa_process_iotlb_update(struct vhost_vdpa *v, + struct vhost_iotlb_msg *msg) +{ + struct vhost_dev *dev = &v->vdev; + struct vhost_iotlb *iotlb = dev->iotlb; + struct page **page_list; + unsigned long list_size = PAGE_SIZE / sizeof(struct page *); + unsigned int gup_flags = FOLL_LONGTERM; + unsigned long npages, cur_base, map_pfn, last_pfn = 0; + unsigned long locked, lock_limit, pinned, i; + u64 iova = msg->iova; + int ret = 0; + + if (vhost_iotlb_itree_first(iotlb, msg->iova, + msg->iova + msg->size - 1)) + return -EEXIST; + + page_list = (struct page **) __get_free_page(GFP_KERNEL); + if (!page_list) + return -ENOMEM; + + if (msg->perm & VHOST_ACCESS_WO) + gup_flags |= FOLL_WRITE; + + npages = PAGE_ALIGN(msg->size + (iova & ~PAGE_MASK)) >> PAGE_SHIFT; + if (!npages) + return -EINVAL; + + down_read(&dev->mm->mmap_sem); + + locked = atomic64_add_return(npages, &dev->mm->pinned_vm); + lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; + + if (locked > lock_limit) { + ret = -ENOMEM; + goto out; + } + + cur_base = msg->uaddr & PAGE_MASK; + iova &= PAGE_MASK; + + while (npages) { + pinned = min_t(unsigned long, npages, list_size); + ret = pin_user_pages(cur_base, pinned, + gup_flags, page_list, NULL); + if (ret != pinned) + goto out; + + if (!last_pfn) + map_pfn = page_to_pfn(page_list[0]); + + for (i = 0; i < ret; i++) { + unsigned long this_pfn = page_to_pfn(page_list[i]); + u64 csize; + + if (last_pfn && (this_pfn != last_pfn + 1)) { + /* Pin a contiguous chunk of memory */ + csize = (last_pfn - map_pfn + 1) << PAGE_SHIFT; + if (vhost_vdpa_map(v, iova, csize, + map_pfn << PAGE_SHIFT, + msg->perm)) + goto out; + map_pfn = this_pfn; + iova += csize; + } + + last_pfn = this_pfn; + } + + cur_base += ret << PAGE_SHIFT; + npages -= ret; + } + + /* Pin the rest chunk */ + ret = vhost_vdpa_map(v, iova, (last_pfn - map_pfn + 1) << PAGE_SHIFT, + map_pfn << PAGE_SHIFT, msg->perm); +out: + if (ret) { + vhost_vdpa_unmap(v, msg->iova, msg->size); + atomic64_sub(npages, &dev->mm->pinned_vm); + } + up_read(&dev->mm->mmap_sem); + free_page((unsigned long)page_list); + return ret; +} + +static int vhost_vdpa_process_iotlb_msg(struct vhost_dev *dev, + struct vhost_iotlb_msg *msg) +{ + struct vhost_vdpa *v = container_of(dev, struct vhost_vdpa, vdev); + int r = 0; + + r = vhost_dev_check_owner(dev); + if (r) + return r; + + switch (msg->type) { + case VHOST_IOTLB_UPDATE: + r = vhost_vdpa_process_iotlb_update(v, msg); + break; + case VHOST_IOTLB_INVALIDATE: + vhost_vdpa_unmap(v, msg->iova, msg->size); + break; + default: + r = -EINVAL; + break; + } + + return r; +} + +static ssize_t vhost_vdpa_chr_write_iter(struct kiocb *iocb, + struct iov_iter *from) +{ + struct file *file = iocb->ki_filp; + struct vhost_vdpa *v = file->private_data; + struct vhost_dev *dev = &v->vdev; + + return vhost_chr_write_iter(dev, from); +} + +static int vhost_vdpa_alloc_domain(struct vhost_vdpa *v) +{ + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + struct device *dma_dev = vdpa_get_dma_dev(vdpa); + struct bus_type *bus; + int ret; + + /* Device want to do DMA by itself */ + if (ops->set_map || ops->dma_map) + return 0; + + bus = dma_dev->bus; + if (!bus) + return -EFAULT; + + if (!iommu_capable(bus, IOMMU_CAP_CACHE_COHERENCY)) + return -ENOTSUPP; + + v->domain = iommu_domain_alloc(bus); + if (!v->domain) + return -EIO; + + ret = iommu_attach_device(v->domain, dma_dev); + if (ret) + goto err_attach; + + return 0; + +err_attach: + iommu_domain_free(v->domain); + return ret; +} + +static void vhost_vdpa_free_domain(struct vhost_vdpa *v) +{ + struct vdpa_device *vdpa = v->vdpa; + struct device *dma_dev = vdpa_get_dma_dev(vdpa); + + if (v->domain) { + iommu_detach_device(v->domain, dma_dev); + iommu_domain_free(v->domain); + } + + v->domain = NULL; +} + +static int vhost_vdpa_open(struct inode *inode, struct file *filep) +{ + struct vhost_vdpa *v; + struct vhost_dev *dev; + struct vhost_virtqueue **vqs; + int nvqs, i, r, opened; + + v = container_of(inode->i_cdev, struct vhost_vdpa, cdev); + if (!v) + return -ENODEV; + + opened = atomic_cmpxchg(&v->opened, 0, 1); + if (opened) + return -EBUSY; + + nvqs = v->nvqs; + vhost_vdpa_reset(v); + + vqs = kmalloc_array(nvqs, sizeof(*vqs), GFP_KERNEL); + if (!vqs) { + r = -ENOMEM; + goto err; + } + + dev = &v->vdev; + for (i = 0; i < nvqs; i++) { + vqs[i] = &v->vqs[i]; + vqs[i]->handle_kick = handle_vq_kick; + } + vhost_dev_init(dev, vqs, nvqs, 0, 0, 0, + vhost_vdpa_process_iotlb_msg); + + dev->iotlb = vhost_iotlb_alloc(0, 0); + if (!dev->iotlb) { + r = -ENOMEM; + goto err_init_iotlb; + } + + r = vhost_vdpa_alloc_domain(v); + if (r) + goto err_init_iotlb; + + filep->private_data = v; + + return 0; + +err_init_iotlb: + vhost_dev_cleanup(&v->vdev); +err: + atomic_dec(&v->opened); + return r; +} + +static int vhost_vdpa_release(struct inode *inode, struct file *filep) +{ + struct vhost_vdpa *v = filep->private_data; + struct vhost_dev *d = &v->vdev; + + mutex_lock(&d->mutex); + filep->private_data = NULL; + vhost_vdpa_reset(v); + vhost_dev_stop(&v->vdev); + vhost_vdpa_iotlb_free(v); + vhost_vdpa_free_domain(v); + vhost_dev_cleanup(&v->vdev); + kfree(v->vdev.vqs); + mutex_unlock(&d->mutex); + + atomic_dec(&v->opened); + complete(&v->completion); + + return 0; +} + +static const struct file_operations vhost_vdpa_fops = { + .owner = THIS_MODULE, + .open = vhost_vdpa_open, + .release = vhost_vdpa_release, + .write_iter = vhost_vdpa_chr_write_iter, + .unlocked_ioctl = vhost_vdpa_unlocked_ioctl, + .compat_ioctl = compat_ptr_ioctl, +}; + +static void vhost_vdpa_release_dev(struct device *device) +{ + struct vhost_vdpa *v = + container_of(device, struct vhost_vdpa, dev); + + ida_simple_remove(&vhost_vdpa_ida, v->minor); + kfree(v->vqs); + kfree(v); +} + +static int vhost_vdpa_probe(struct vdpa_device *vdpa) +{ + const struct vdpa_config_ops *ops = vdpa->config; + struct vhost_vdpa *v; + int minor, nvqs = VHOST_VDPA_VQ_MAX; + int r; + + /* Currently, we only accept the network devices. */ + if (ops->get_device_id(vdpa) != VIRTIO_ID_NET) + return -ENOTSUPP; + + v = kzalloc(sizeof(*v), GFP_KERNEL | __GFP_RETRY_MAYFAIL); + if (!v) + return -ENOMEM; + + minor = ida_simple_get(&vhost_vdpa_ida, 0, + VHOST_VDPA_DEV_MAX, GFP_KERNEL); + if (minor < 0) { + kfree(v); + return minor; + } + + atomic_set(&v->opened, 0); + v->minor = minor; + v->vdpa = vdpa; + v->nvqs = nvqs; + v->virtio_id = ops->get_device_id(vdpa); + + device_initialize(&v->dev); + v->dev.release = vhost_vdpa_release_dev; + v->dev.parent = &vdpa->dev; + v->dev.devt = MKDEV(MAJOR(vhost_vdpa_major), minor); + v->vqs = kmalloc_array(nvqs, sizeof(struct vhost_virtqueue), + GFP_KERNEL); + if (!v->vqs) { + r = -ENOMEM; + goto err; + } + + r = dev_set_name(&v->dev, "vhost-vdpa-%u", minor); + if (r) + goto err; + + cdev_init(&v->cdev, &vhost_vdpa_fops); + v->cdev.owner = THIS_MODULE; + + r = cdev_device_add(&v->cdev, &v->dev); + if (r) + goto err; + + init_completion(&v->completion); + vdpa_set_drvdata(vdpa, v); + + return 0; + +err: + put_device(&v->dev); + return r; +} + +static void vhost_vdpa_remove(struct vdpa_device *vdpa) +{ + struct vhost_vdpa *v = vdpa_get_drvdata(vdpa); + int opened; + + cdev_device_del(&v->cdev, &v->dev); + + do { + opened = atomic_cmpxchg(&v->opened, 0, 1); + if (!opened) + break; + wait_for_completion(&v->completion); + } while (1); + + put_device(&v->dev); +} + +static struct vdpa_driver vhost_vdpa_driver = { + .driver = { + .name = "vhost_vdpa", + }, + .probe = vhost_vdpa_probe, + .remove = vhost_vdpa_remove, +}; + +static int __init vhost_vdpa_init(void) +{ + int r; + + r = alloc_chrdev_region(&vhost_vdpa_major, 0, VHOST_VDPA_DEV_MAX, + "vhost-vdpa"); + if (r) + goto err_alloc_chrdev; + + r = vdpa_register_driver(&vhost_vdpa_driver); + if (r) + goto err_vdpa_register_driver; + + return 0; + +err_vdpa_register_driver: + unregister_chrdev_region(vhost_vdpa_major, VHOST_VDPA_DEV_MAX); +err_alloc_chrdev: + return r; +} +module_init(vhost_vdpa_init); + +static void __exit vhost_vdpa_exit(void) +{ + vdpa_unregister_driver(&vhost_vdpa_driver); + unregister_chrdev_region(vhost_vdpa_major, VHOST_VDPA_DEV_MAX); +} +module_exit(vhost_vdpa_exit); + +MODULE_VERSION("0.0.1"); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Intel Corporation"); +MODULE_DESCRIPTION("vDPA-based vhost backend for virtio"); diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h index 40d028eed645..9fe72e4b1373 100644 --- a/include/uapi/linux/vhost.h +++ b/include/uapi/linux/vhost.h @@ -116,4 +116,28 @@ #define VHOST_VSOCK_SET_GUEST_CID _IOW(VHOST_VIRTIO, 0x60, __u64) #define VHOST_VSOCK_SET_RUNNING _IOW(VHOST_VIRTIO, 0x61, int) +/* VHOST_VDPA specific defines */ + +/* Get the device id. The device ids follow the same definition of + * the device id defined in virtio-spec. + */ +#define VHOST_VDPA_GET_DEVICE_ID _IOR(VHOST_VIRTIO, 0x70, __u32) +/* Get and set the status. The status bits follow the same definition + * of the device status defined in virtio-spec. + */ +#define VHOST_VDPA_GET_STATUS _IOR(VHOST_VIRTIO, 0x71, __u8) +#define VHOST_VDPA_SET_STATUS _IOW(VHOST_VIRTIO, 0x72, __u8) +/* Get and set the device config. The device config follows the same + * definition of the device config defined in virtio-spec. + */ +#define VHOST_VDPA_GET_CONFIG _IOR(VHOST_VIRTIO, 0x73, \ + struct vhost_vdpa_config) +#define VHOST_VDPA_SET_CONFIG _IOW(VHOST_VIRTIO, 0x74, \ + struct vhost_vdpa_config) +/* Enable/disable the ring. */ +#define VHOST_VDPA_SET_VRING_ENABLE _IOW(VHOST_VIRTIO, 0x75, \ + struct vhost_vring_state) +/* Get the max ring size. */ +#define VHOST_VDPA_GET_VRING_NUM _IOR(VHOST_VIRTIO, 0x76, __u16) + #endif diff --git a/include/uapi/linux/vhost_types.h b/include/uapi/linux/vhost_types.h index c907290ff065..669457ce5c48 100644 --- a/include/uapi/linux/vhost_types.h +++ b/include/uapi/linux/vhost_types.h @@ -119,6 +119,14 @@ struct vhost_scsi_target { unsigned short reserved; }; +/* VHOST_VDPA specific definitions */ + +struct vhost_vdpa_config { + __u32 off; + __u32 len; + __u8 buf[0]; +}; + /* Feature bits */ /* Log all write descriptors. Can be changed while device is active. */ #define VHOST_F_LOG_ALL 26 From 2c53d0f64c06f4580588ef16f50f63a112117a02 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Thu, 26 Mar 2020 22:01:24 +0800 Subject: [PATCH 421/635] vdpasim: vDPA device simulator This patch implements a software vDPA networking device. The datapath is implemented through vringh and workqueue. The device has an on-chip IOMMU which translates IOVA to PA. For kernel virtio drivers, vDPA simulator driver provides dma_ops. For vhost driers, set_map() methods of vdpa_config_ops is implemented to accept mappings from vhost. Currently, vDPA device simulator will loopback TX traffic to RX. So the main use case for the device is vDPA feature testing, prototyping and development. Note, there's no management API implemented, a vDPA device will be registered once the module is probed. We need to handle this in the future development. Signed-off-by: Jason Wang Link: https://lore.kernel.org/r/20200326140125.19794-9-jasowang@redhat.com Signed-off-by: Michael S. Tsirkin --- drivers/virtio/vdpa/Kconfig | 19 + drivers/virtio/vdpa/Makefile | 1 + drivers/virtio/vdpa/vdpa_sim/Makefile | 2 + drivers/virtio/vdpa/vdpa_sim/vdpa_sim.c | 629 ++++++++++++++++++++++++ 4 files changed, 651 insertions(+) create mode 100644 drivers/virtio/vdpa/vdpa_sim/Makefile create mode 100644 drivers/virtio/vdpa/vdpa_sim/vdpa_sim.c diff --git a/drivers/virtio/vdpa/Kconfig b/drivers/virtio/vdpa/Kconfig index 351617723d12..ae204465964e 100644 --- a/drivers/virtio/vdpa/Kconfig +++ b/drivers/virtio/vdpa/Kconfig @@ -5,3 +5,22 @@ config VDPA Enable this module to support vDPA device that uses a datapath which complies with virtio specifications with vendor specific control path. + +menuconfig VDPA_MENU + bool "VDPA drivers" + default n + +if VDPA_MENU + +config VDPA_SIM + tristate "vDPA device simulator" + depends on RUNTIME_TESTING_MENU + select VDPA + select VHOST_RING + default n + help + vDPA networking device simulator which loop TX traffic back + to RX. This device is used for testing, prototyping and + development of vDPA. + +endif # VDPA_MENU diff --git a/drivers/virtio/vdpa/Makefile b/drivers/virtio/vdpa/Makefile index ee6a35e8a4fb..3814af8e097b 100644 --- a/drivers/virtio/vdpa/Makefile +++ b/drivers/virtio/vdpa/Makefile @@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_VDPA) += vdpa.o +obj-$(CONFIG_VDPA_SIM) += vdpa_sim/ diff --git a/drivers/virtio/vdpa/vdpa_sim/Makefile b/drivers/virtio/vdpa/vdpa_sim/Makefile new file mode 100644 index 000000000000..b40278f65e04 --- /dev/null +++ b/drivers/virtio/vdpa/vdpa_sim/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_VDPA_SIM) += vdpa_sim.o diff --git a/drivers/virtio/vdpa/vdpa_sim/vdpa_sim.c b/drivers/virtio/vdpa/vdpa_sim/vdpa_sim.c new file mode 100644 index 000000000000..6e8a0cf2fdeb --- /dev/null +++ b/drivers/virtio/vdpa/vdpa_sim/vdpa_sim.c @@ -0,0 +1,629 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * VDPA networking device simulator. + * + * Copyright (c) 2020, Red Hat Inc. All rights reserved. + * Author: Jason Wang + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DRV_VERSION "0.1" +#define DRV_AUTHOR "Jason Wang " +#define DRV_DESC "vDPA Device Simulator" +#define DRV_LICENSE "GPL v2" + +struct vdpasim_virtqueue { + struct vringh vring; + struct vringh_kiov iov; + unsigned short head; + bool ready; + u64 desc_addr; + u64 device_addr; + u64 driver_addr; + u32 num; + void *private; + irqreturn_t (*cb)(void *data); +}; + +#define VDPASIM_QUEUE_ALIGN PAGE_SIZE +#define VDPASIM_QUEUE_MAX 256 +#define VDPASIM_DEVICE_ID 0x1 +#define VDPASIM_VENDOR_ID 0 +#define VDPASIM_VQ_NUM 0x2 +#define VDPASIM_NAME "vdpasim-netdev" + +static u64 vdpasim_features = (1ULL << VIRTIO_F_ANY_LAYOUT) | + (1ULL << VIRTIO_F_VERSION_1) | + (1ULL << VIRTIO_F_IOMMU_PLATFORM); + +/* State of each vdpasim device */ +struct vdpasim { + struct vdpa_device vdpa; + struct vdpasim_virtqueue vqs[2]; + struct work_struct work; + /* spinlock to synchronize virtqueue state */ + spinlock_t lock; + struct virtio_net_config config; + struct vhost_iotlb *iommu; + void *buffer; + u32 status; + u32 generation; + u64 features; +}; + +static struct vdpasim *vdpasim_dev; + +static struct vdpasim *vdpa_to_sim(struct vdpa_device *vdpa) +{ + return container_of(vdpa, struct vdpasim, vdpa); +} + +static struct vdpasim *dev_to_sim(struct device *dev) +{ + struct vdpa_device *vdpa = dev_to_vdpa(dev); + + return vdpa_to_sim(vdpa); +} + +static void vdpasim_queue_ready(struct vdpasim *vdpasim, unsigned int idx) +{ + struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; + int ret; + + ret = vringh_init_iotlb(&vq->vring, vdpasim_features, + VDPASIM_QUEUE_MAX, false, + (struct vring_desc *)(uintptr_t)vq->desc_addr, + (struct vring_avail *) + (uintptr_t)vq->driver_addr, + (struct vring_used *) + (uintptr_t)vq->device_addr); +} + +static void vdpasim_vq_reset(struct vdpasim_virtqueue *vq) +{ + vq->ready = 0; + vq->desc_addr = 0; + vq->driver_addr = 0; + vq->device_addr = 0; + vq->cb = NULL; + vq->private = NULL; + vringh_init_iotlb(&vq->vring, vdpasim_features, VDPASIM_QUEUE_MAX, + false, NULL, NULL, NULL); +} + +static void vdpasim_reset(struct vdpasim *vdpasim) +{ + int i; + + for (i = 0; i < VDPASIM_VQ_NUM; i++) + vdpasim_vq_reset(&vdpasim->vqs[i]); + + vhost_iotlb_reset(vdpasim->iommu); + + vdpasim->features = 0; + vdpasim->status = 0; + ++vdpasim->generation; +} + +static void vdpasim_work(struct work_struct *work) +{ + struct vdpasim *vdpasim = container_of(work, struct + vdpasim, work); + struct vdpasim_virtqueue *txq = &vdpasim->vqs[1]; + struct vdpasim_virtqueue *rxq = &vdpasim->vqs[0]; + size_t read, write, total_write; + int err; + int pkts = 0; + + spin_lock(&vdpasim->lock); + + if (!(vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK)) + goto out; + + if (!txq->ready || !rxq->ready) + goto out; + + while (true) { + total_write = 0; + err = vringh_getdesc_iotlb(&txq->vring, &txq->iov, NULL, + &txq->head, GFP_ATOMIC); + if (err <= 0) + break; + + err = vringh_getdesc_iotlb(&rxq->vring, NULL, &rxq->iov, + &rxq->head, GFP_ATOMIC); + if (err <= 0) { + vringh_complete_iotlb(&txq->vring, txq->head, 0); + break; + } + + while (true) { + read = vringh_iov_pull_iotlb(&txq->vring, &txq->iov, + vdpasim->buffer, + PAGE_SIZE); + if (read <= 0) + break; + + write = vringh_iov_push_iotlb(&rxq->vring, &rxq->iov, + vdpasim->buffer, read); + if (write <= 0) + break; + + total_write += write; + } + + /* Make sure data is wrote before advancing index */ + smp_wmb(); + + vringh_complete_iotlb(&txq->vring, txq->head, 0); + vringh_complete_iotlb(&rxq->vring, rxq->head, total_write); + + /* Make sure used is visible before rasing the interrupt. */ + smp_wmb(); + + local_bh_disable(); + if (txq->cb) + txq->cb(txq->private); + if (rxq->cb) + rxq->cb(rxq->private); + local_bh_enable(); + + if (++pkts > 4) { + schedule_work(&vdpasim->work); + goto out; + } + } + +out: + spin_unlock(&vdpasim->lock); +} + +static int dir_to_perm(enum dma_data_direction dir) +{ + int perm = -EFAULT; + + switch (dir) { + case DMA_FROM_DEVICE: + perm = VHOST_MAP_WO; + break; + case DMA_TO_DEVICE: + perm = VHOST_MAP_RO; + break; + case DMA_BIDIRECTIONAL: + perm = VHOST_MAP_RW; + break; + default: + break; + } + + return perm; +} + +static dma_addr_t vdpasim_map_page(struct device *dev, struct page *page, + unsigned long offset, size_t size, + enum dma_data_direction dir, + unsigned long attrs) +{ + struct vdpasim *vdpasim = dev_to_sim(dev); + struct vhost_iotlb *iommu = vdpasim->iommu; + u64 pa = (page_to_pfn(page) << PAGE_SHIFT) + offset; + int ret, perm = dir_to_perm(dir); + + if (perm < 0) + return DMA_MAPPING_ERROR; + + /* For simplicity, use identical mapping to avoid e.g iova + * allocator. + */ + ret = vhost_iotlb_add_range(iommu, pa, pa + size - 1, + pa, dir_to_perm(dir)); + if (ret) + return DMA_MAPPING_ERROR; + + return (dma_addr_t)(pa); +} + +static void vdpasim_unmap_page(struct device *dev, dma_addr_t dma_addr, + size_t size, enum dma_data_direction dir, + unsigned long attrs) +{ + struct vdpasim *vdpasim = dev_to_sim(dev); + struct vhost_iotlb *iommu = vdpasim->iommu; + + vhost_iotlb_del_range(iommu, (u64)dma_addr, + (u64)dma_addr + size - 1); +} + +static void *vdpasim_alloc_coherent(struct device *dev, size_t size, + dma_addr_t *dma_addr, gfp_t flag, + unsigned long attrs) +{ + struct vdpasim *vdpasim = dev_to_sim(dev); + struct vhost_iotlb *iommu = vdpasim->iommu; + void *addr = kmalloc(size, flag); + int ret; + + if (!addr) + *dma_addr = DMA_MAPPING_ERROR; + else { + u64 pa = virt_to_phys(addr); + + ret = vhost_iotlb_add_range(iommu, (u64)pa, + (u64)pa + size - 1, + pa, VHOST_MAP_RW); + if (ret) { + *dma_addr = DMA_MAPPING_ERROR; + kfree(addr); + addr = NULL; + } else + *dma_addr = (dma_addr_t)pa; + } + + return addr; +} + +static void vdpasim_free_coherent(struct device *dev, size_t size, + void *vaddr, dma_addr_t dma_addr, + unsigned long attrs) +{ + struct vdpasim *vdpasim = dev_to_sim(dev); + struct vhost_iotlb *iommu = vdpasim->iommu; + + vhost_iotlb_del_range(iommu, (u64)dma_addr, + (u64)dma_addr + size - 1); + kfree(phys_to_virt((uintptr_t)dma_addr)); +} + +static const struct dma_map_ops vdpasim_dma_ops = { + .map_page = vdpasim_map_page, + .unmap_page = vdpasim_unmap_page, + .alloc = vdpasim_alloc_coherent, + .free = vdpasim_free_coherent, +}; + +static const struct vdpa_config_ops vdpasim_net_config_ops; + +static struct vdpasim *vdpasim_create(void) +{ + struct virtio_net_config *config; + struct vdpasim *vdpasim; + struct device *dev; + int ret = -ENOMEM; + + vdpasim = vdpa_alloc_device(struct vdpasim, vdpa, NULL, + &vdpasim_net_config_ops); + if (!vdpasim) + goto err_alloc; + + INIT_WORK(&vdpasim->work, vdpasim_work); + spin_lock_init(&vdpasim->lock); + + dev = &vdpasim->vdpa.dev; + dev->coherent_dma_mask = DMA_BIT_MASK(64); + set_dma_ops(dev, &vdpasim_dma_ops); + + vdpasim->iommu = vhost_iotlb_alloc(2048, 0); + if (!vdpasim->iommu) + goto err_iommu; + + vdpasim->buffer = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!vdpasim->buffer) + goto err_iommu; + + config = &vdpasim->config; + config->mtu = 1500; + config->status = VIRTIO_NET_S_LINK_UP; + eth_random_addr(config->mac); + + vringh_set_iotlb(&vdpasim->vqs[0].vring, vdpasim->iommu); + vringh_set_iotlb(&vdpasim->vqs[1].vring, vdpasim->iommu); + + vdpasim->vdpa.dma_dev = dev; + ret = vdpa_register_device(&vdpasim->vdpa); + if (ret) + goto err_iommu; + + return vdpasim; + +err_iommu: + put_device(dev); +err_alloc: + return ERR_PTR(ret); +} + +static int vdpasim_set_vq_address(struct vdpa_device *vdpa, u16 idx, + u64 desc_area, u64 driver_area, + u64 device_area) +{ + struct vdpasim *vdpasim = vdpa_to_sim(vdpa); + struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; + + vq->desc_addr = desc_area; + vq->driver_addr = driver_area; + vq->device_addr = device_area; + + return 0; +} + +static void vdpasim_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num) +{ + struct vdpasim *vdpasim = vdpa_to_sim(vdpa); + struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; + + vq->num = num; +} + +static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx) +{ + struct vdpasim *vdpasim = vdpa_to_sim(vdpa); + struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; + + if (vq->ready) + schedule_work(&vdpasim->work); +} + +static void vdpasim_set_vq_cb(struct vdpa_device *vdpa, u16 idx, + struct vdpa_callback *cb) +{ + struct vdpasim *vdpasim = vdpa_to_sim(vdpa); + struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; + + vq->cb = cb->callback; + vq->private = cb->private; +} + +static void vdpasim_set_vq_ready(struct vdpa_device *vdpa, u16 idx, bool ready) +{ + struct vdpasim *vdpasim = vdpa_to_sim(vdpa); + struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; + + spin_lock(&vdpasim->lock); + vq->ready = ready; + if (vq->ready) + vdpasim_queue_ready(vdpasim, idx); + spin_unlock(&vdpasim->lock); +} + +static bool vdpasim_get_vq_ready(struct vdpa_device *vdpa, u16 idx) +{ + struct vdpasim *vdpasim = vdpa_to_sim(vdpa); + struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; + + return vq->ready; +} + +static int vdpasim_set_vq_state(struct vdpa_device *vdpa, u16 idx, u64 state) +{ + struct vdpasim *vdpasim = vdpa_to_sim(vdpa); + struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; + struct vringh *vrh = &vq->vring; + + spin_lock(&vdpasim->lock); + vrh->last_avail_idx = state; + spin_unlock(&vdpasim->lock); + + return 0; +} + +static u64 vdpasim_get_vq_state(struct vdpa_device *vdpa, u16 idx) +{ + struct vdpasim *vdpasim = vdpa_to_sim(vdpa); + struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; + struct vringh *vrh = &vq->vring; + + return vrh->last_avail_idx; +} + +static u16 vdpasim_get_vq_align(struct vdpa_device *vdpa) +{ + return VDPASIM_QUEUE_ALIGN; +} + +static u64 vdpasim_get_features(struct vdpa_device *vdpa) +{ + return vdpasim_features; +} + +static int vdpasim_set_features(struct vdpa_device *vdpa, u64 features) +{ + struct vdpasim *vdpasim = vdpa_to_sim(vdpa); + + /* DMA mapping must be done by driver */ + if (!(features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))) + return -EINVAL; + + vdpasim->features = features & vdpasim_features; + + return 0; +} + +static void vdpasim_set_config_cb(struct vdpa_device *vdpa, + struct vdpa_callback *cb) +{ + /* We don't support config interrupt */ +} + +static u16 vdpasim_get_vq_num_max(struct vdpa_device *vdpa) +{ + return VDPASIM_QUEUE_MAX; +} + +static u32 vdpasim_get_device_id(struct vdpa_device *vdpa) +{ + return VDPASIM_DEVICE_ID; +} + +static u32 vdpasim_get_vendor_id(struct vdpa_device *vdpa) +{ + return VDPASIM_VENDOR_ID; +} + +static u8 vdpasim_get_status(struct vdpa_device *vdpa) +{ + struct vdpasim *vdpasim = vdpa_to_sim(vdpa); + u8 status; + + spin_lock(&vdpasim->lock); + status = vdpasim->status; + spin_unlock(&vdpasim->lock); + + return vdpasim->status; +} + +static void vdpasim_set_status(struct vdpa_device *vdpa, u8 status) +{ + struct vdpasim *vdpasim = vdpa_to_sim(vdpa); + + spin_lock(&vdpasim->lock); + vdpasim->status = status; + if (status == 0) + vdpasim_reset(vdpasim); + spin_unlock(&vdpasim->lock); +} + +static void vdpasim_get_config(struct vdpa_device *vdpa, unsigned int offset, + void *buf, unsigned int len) +{ + struct vdpasim *vdpasim = vdpa_to_sim(vdpa); + + if (offset + len < sizeof(struct virtio_net_config)) + memcpy(buf, &vdpasim->config + offset, len); +} + +static void vdpasim_set_config(struct vdpa_device *vdpa, unsigned int offset, + const void *buf, unsigned int len) +{ + /* No writable config supportted by vdpasim */ +} + +static u32 vdpasim_get_generation(struct vdpa_device *vdpa) +{ + struct vdpasim *vdpasim = vdpa_to_sim(vdpa); + + return vdpasim->generation; +} + +static int vdpasim_set_map(struct vdpa_device *vdpa, + struct vhost_iotlb *iotlb) +{ + struct vdpasim *vdpasim = vdpa_to_sim(vdpa); + struct vhost_iotlb_map *map; + u64 start = 0ULL, last = 0ULL - 1; + int ret; + + vhost_iotlb_reset(vdpasim->iommu); + + for (map = vhost_iotlb_itree_first(iotlb, start, last); map; + map = vhost_iotlb_itree_next(map, start, last)) { + ret = vhost_iotlb_add_range(vdpasim->iommu, map->start, + map->last, map->addr, map->perm); + if (ret) + goto err; + } + return 0; + +err: + vhost_iotlb_reset(vdpasim->iommu); + return ret; +} + +static int vdpasim_dma_map(struct vdpa_device *vdpa, u64 iova, u64 size, + u64 pa, u32 perm) +{ + struct vdpasim *vdpasim = vdpa_to_sim(vdpa); + + return vhost_iotlb_add_range(vdpasim->iommu, iova, + iova + size - 1, pa, perm); +} + +static int vdpasim_dma_unmap(struct vdpa_device *vdpa, u64 iova, u64 size) +{ + struct vdpasim *vdpasim = vdpa_to_sim(vdpa); + + vhost_iotlb_del_range(vdpasim->iommu, iova, iova + size - 1); + + return 0; +} + +static void vdpasim_free(struct vdpa_device *vdpa) +{ + struct vdpasim *vdpasim = vdpa_to_sim(vdpa); + + cancel_work_sync(&vdpasim->work); + kfree(vdpasim->buffer); + if (vdpasim->iommu) + vhost_iotlb_free(vdpasim->iommu); +} + +static const struct vdpa_config_ops vdpasim_net_config_ops = { + .set_vq_address = vdpasim_set_vq_address, + .set_vq_num = vdpasim_set_vq_num, + .kick_vq = vdpasim_kick_vq, + .set_vq_cb = vdpasim_set_vq_cb, + .set_vq_ready = vdpasim_set_vq_ready, + .get_vq_ready = vdpasim_get_vq_ready, + .set_vq_state = vdpasim_set_vq_state, + .get_vq_state = vdpasim_get_vq_state, + .get_vq_align = vdpasim_get_vq_align, + .get_features = vdpasim_get_features, + .set_features = vdpasim_set_features, + .set_config_cb = vdpasim_set_config_cb, + .get_vq_num_max = vdpasim_get_vq_num_max, + .get_device_id = vdpasim_get_device_id, + .get_vendor_id = vdpasim_get_vendor_id, + .get_status = vdpasim_get_status, + .set_status = vdpasim_set_status, + .get_config = vdpasim_get_config, + .set_config = vdpasim_set_config, + .get_generation = vdpasim_get_generation, + .set_map = vdpasim_set_map, + .dma_map = vdpasim_dma_map, + .dma_unmap = vdpasim_dma_unmap, + .free = vdpasim_free, +}; + +static int __init vdpasim_dev_init(void) +{ + vdpasim_dev = vdpasim_create(); + + if (!IS_ERR(vdpasim_dev)) + return 0; + + return PTR_ERR(vdpasim_dev); +} + +static void __exit vdpasim_dev_exit(void) +{ + struct vdpa_device *vdpa = &vdpasim_dev->vdpa; + + vdpa_unregister_device(vdpa); +} + +module_init(vdpasim_dev_init) +module_exit(vdpasim_dev_exit) + +MODULE_VERSION(DRV_VERSION); +MODULE_LICENSE(DRV_LICENSE); +MODULE_AUTHOR(DRV_AUTHOR); +MODULE_DESCRIPTION(DRV_DESC); From 5a2414bc454e89e0515b47500734a65aa40cf9fe Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Thu, 26 Mar 2020 22:01:25 +0800 Subject: [PATCH 422/635] virtio: Intel IFC VF driver for VDPA This commit introduced two layers to drive IFC VF: (1) ifcvf_base layer, which handles IFC VF NIC hardware operations and configurations. (2) ifcvf_main layer, which complies to VDPA bus framework, implemented device operations for VDPA bus, handles device probe, bus attaching, vring operations, etc. Signed-off-by: Zhu Lingshan Signed-off-by: Bie Tiwei Signed-off-by: Wang Xiao Signed-off-by: Jason Wang Link: https://lore.kernel.org/r/20200326140125.19794-10-jasowang@redhat.com Signed-off-by: Michael S. Tsirkin --- drivers/virtio/vdpa/Kconfig | 11 + drivers/virtio/vdpa/Makefile | 1 + drivers/virtio/vdpa/ifcvf/Makefile | 3 + drivers/virtio/vdpa/ifcvf/ifcvf_base.c | 389 ++++++++++++++++++++++ drivers/virtio/vdpa/ifcvf/ifcvf_base.h | 118 +++++++ drivers/virtio/vdpa/ifcvf/ifcvf_main.c | 435 +++++++++++++++++++++++++ 6 files changed, 957 insertions(+) create mode 100644 drivers/virtio/vdpa/ifcvf/Makefile create mode 100644 drivers/virtio/vdpa/ifcvf/ifcvf_base.c create mode 100644 drivers/virtio/vdpa/ifcvf/ifcvf_base.h create mode 100644 drivers/virtio/vdpa/ifcvf/ifcvf_main.c diff --git a/drivers/virtio/vdpa/Kconfig b/drivers/virtio/vdpa/Kconfig index ae204465964e..7db1460104b7 100644 --- a/drivers/virtio/vdpa/Kconfig +++ b/drivers/virtio/vdpa/Kconfig @@ -23,4 +23,15 @@ config VDPA_SIM to RX. This device is used for testing, prototyping and development of vDPA. +config IFCVF + tristate "Intel IFC VF VDPA driver" + depends on PCI_MSI + select VDPA + default n + help + This kernel module can drive Intel IFC VF NIC to offload + virtio dataplane traffic to hardware. + To compile this driver as a module, choose M here: the module will + be called ifcvf. + endif # VDPA_MENU diff --git a/drivers/virtio/vdpa/Makefile b/drivers/virtio/vdpa/Makefile index 3814af8e097b..8bbb686ca7a2 100644 --- a/drivers/virtio/vdpa/Makefile +++ b/drivers/virtio/vdpa/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_VDPA) += vdpa.o obj-$(CONFIG_VDPA_SIM) += vdpa_sim/ +obj-$(CONFIG_IFCVF) += ifcvf/ diff --git a/drivers/virtio/vdpa/ifcvf/Makefile b/drivers/virtio/vdpa/ifcvf/Makefile new file mode 100644 index 000000000000..d709915995ab --- /dev/null +++ b/drivers/virtio/vdpa/ifcvf/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_IFCVF) += ifcvf.o +ifcvf-$(CONFIG_IFCVF) += ifcvf_main.o ifcvf_base.o diff --git a/drivers/virtio/vdpa/ifcvf/ifcvf_base.c b/drivers/virtio/vdpa/ifcvf/ifcvf_base.c new file mode 100644 index 000000000000..b61b06ea26d3 --- /dev/null +++ b/drivers/virtio/vdpa/ifcvf/ifcvf_base.c @@ -0,0 +1,389 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Intel IFC VF NIC driver for virtio dataplane offloading + * + * Copyright (C) 2020 Intel Corporation. + * + * Author: Zhu Lingshan + * + */ + +#include "ifcvf_base.h" + +static inline u8 ifc_ioread8(u8 __iomem *addr) +{ + return ioread8(addr); +} +static inline u16 ifc_ioread16 (__le16 __iomem *addr) +{ + return ioread16(addr); +} + +static inline u32 ifc_ioread32(__le32 __iomem *addr) +{ + return ioread32(addr); +} + +static inline void ifc_iowrite8(u8 value, u8 __iomem *addr) +{ + iowrite8(value, addr); +} + +static inline void ifc_iowrite16(u16 value, __le16 __iomem *addr) +{ + iowrite16(value, addr); +} + +static inline void ifc_iowrite32(u32 value, __le32 __iomem *addr) +{ + iowrite32(value, addr); +} + +static void ifc_iowrite64_twopart(u64 val, + __le32 __iomem *lo, __le32 __iomem *hi) +{ + ifc_iowrite32((u32)val, lo); + ifc_iowrite32(val >> 32, hi); +} + +struct ifcvf_adapter *vf_to_adapter(struct ifcvf_hw *hw) +{ + return container_of(hw, struct ifcvf_adapter, vf); +} + +static void __iomem *get_cap_addr(struct ifcvf_hw *hw, + struct virtio_pci_cap *cap) +{ + struct ifcvf_adapter *ifcvf; + struct pci_dev *pdev; + u32 length, offset; + u8 bar; + + length = le32_to_cpu(cap->length); + offset = le32_to_cpu(cap->offset); + bar = cap->bar; + + ifcvf= vf_to_adapter(hw); + pdev = ifcvf->pdev; + + if (bar >= IFCVF_PCI_MAX_RESOURCE) { + IFCVF_DBG(pdev, + "Invalid bar number %u to get capabilities\n", bar); + return NULL; + } + + if (offset + length > pci_resource_len(pdev, bar)) { + IFCVF_DBG(pdev, + "offset(%u) + len(%u) overflows bar%u's capability\n", + offset, length, bar); + return NULL; + } + + return hw->base[bar] + offset; +} + +static int ifcvf_read_config_range(struct pci_dev *dev, + uint32_t *val, int size, int where) +{ + int ret, i; + + for (i = 0; i < size; i += 4) { + ret = pci_read_config_dword(dev, where + i, val + i / 4); + if (ret < 0) + return ret; + } + + return 0; +} + +int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *pdev) +{ + struct virtio_pci_cap cap; + u16 notify_off; + int ret; + u8 pos; + u32 i; + + ret = pci_read_config_byte(pdev, PCI_CAPABILITY_LIST, &pos); + if (ret < 0) { + IFCVF_ERR(pdev, "Failed to read PCI capability list\n"); + return -EIO; + } + + while (pos) { + ret = ifcvf_read_config_range(pdev, (u32 *)&cap, + sizeof(cap), pos); + if (ret < 0) { + IFCVF_ERR(pdev, + "Failed to get PCI capability at %x\n", pos); + break; + } + + if (cap.cap_vndr != PCI_CAP_ID_VNDR) + goto next; + + switch (cap.cfg_type) { + case VIRTIO_PCI_CAP_COMMON_CFG: + hw->common_cfg = get_cap_addr(hw, &cap); + IFCVF_DBG(pdev, "hw->common_cfg = %p\n", + hw->common_cfg); + break; + case VIRTIO_PCI_CAP_NOTIFY_CFG: + pci_read_config_dword(pdev, pos + sizeof(cap), + &hw->notify_off_multiplier); + hw->notify_bar = cap.bar; + hw->notify_base = get_cap_addr(hw, &cap); + IFCVF_DBG(pdev, "hw->notify_base = %p\n", + hw->notify_base); + break; + case VIRTIO_PCI_CAP_ISR_CFG: + hw->isr = get_cap_addr(hw, &cap); + IFCVF_DBG(pdev, "hw->isr = %p\n", hw->isr); + break; + case VIRTIO_PCI_CAP_DEVICE_CFG: + hw->net_cfg = get_cap_addr(hw, &cap); + IFCVF_DBG(pdev, "hw->net_cfg = %p\n", hw->net_cfg); + break; + } + +next: + pos = cap.cap_next; + } + + if (hw->common_cfg == NULL || hw->notify_base == NULL || + hw->isr == NULL || hw->net_cfg == NULL) { + IFCVF_ERR(pdev, "Incomplete PCI capabilities\n"); + return -EIO; + } + + for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) { + ifc_iowrite16(i, &hw->common_cfg->queue_select); + notify_off = ifc_ioread16(&hw->common_cfg->queue_notify_off); + hw->vring[i].notify_addr = hw->notify_base + + notify_off * hw->notify_off_multiplier; + } + + hw->lm_cfg = hw->base[IFCVF_LM_BAR]; + + IFCVF_DBG(pdev, + "PCI capability mapping: common cfg: %p, notify base: %p\n, isr cfg: %p, device cfg: %p, multiplier: %u\n", + hw->common_cfg, hw->notify_base, hw->isr, + hw->net_cfg, hw->notify_off_multiplier); + + return 0; +} + +u8 ifcvf_get_status(struct ifcvf_hw *hw) +{ + return ifc_ioread8(&hw->common_cfg->device_status); +} + +void ifcvf_set_status(struct ifcvf_hw *hw, u8 status) +{ + ifc_iowrite8(status, &hw->common_cfg->device_status); +} + +void ifcvf_reset(struct ifcvf_hw *hw) +{ + ifcvf_set_status(hw, 0); + /* flush set_status, make sure VF is stopped, reset */ + ifcvf_get_status(hw); +} + +static void ifcvf_add_status(struct ifcvf_hw *hw, u8 status) +{ + if (status != 0) + status |= ifcvf_get_status(hw); + + ifcvf_set_status(hw, status); + ifcvf_get_status(hw); +} + +u64 ifcvf_get_features(struct ifcvf_hw *hw) +{ + struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg; + u32 features_lo, features_hi; + + ifc_iowrite32(0, &cfg->device_feature_select); + features_lo = ifc_ioread32(&cfg->device_feature); + + ifc_iowrite32(1, &cfg->device_feature_select); + features_hi = ifc_ioread32(&cfg->device_feature); + + return ((u64)features_hi << 32) | features_lo; +} + +void ifcvf_read_net_config(struct ifcvf_hw *hw, u64 offset, + void *dst, int length) +{ + u8 old_gen, new_gen, *p; + int i; + + WARN_ON(offset + length > sizeof(struct virtio_net_config)); + do { + old_gen = ifc_ioread8(&hw->common_cfg->config_generation); + p = dst; + for (i = 0; i < length; i++) + *p++ = ifc_ioread8(hw->net_cfg + offset + i); + + new_gen = ifc_ioread8(&hw->common_cfg->config_generation); + } while (old_gen != new_gen); +} + +void ifcvf_write_net_config(struct ifcvf_hw *hw, u64 offset, + const void *src, int length) +{ + const u8 *p; + int i; + + p = src; + WARN_ON(offset + length > sizeof(struct virtio_net_config)); + for (i = 0; i < length; i++) + ifc_iowrite8(*p++, hw->net_cfg + offset + i); +} + +static void ifcvf_set_features(struct ifcvf_hw *hw, u64 features) +{ + struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg; + + ifc_iowrite32(0, &cfg->guest_feature_select); + ifc_iowrite32((u32)features, &cfg->guest_feature); + + ifc_iowrite32(1, &cfg->guest_feature_select); + ifc_iowrite32(features >> 32, &cfg->guest_feature); +} + +static int ifcvf_config_features(struct ifcvf_hw *hw) +{ + struct ifcvf_adapter *ifcvf; + + ifcvf = vf_to_adapter(hw); + ifcvf_set_features(hw, hw->req_features); + ifcvf_add_status(hw, VIRTIO_CONFIG_S_FEATURES_OK); + + if (!(ifcvf_get_status(hw) & VIRTIO_CONFIG_S_FEATURES_OK)) { + IFCVF_ERR(ifcvf->pdev, "Failed to set FEATURES_OK status\n"); + return -EIO; + } + + return 0; +} + +u64 ifcvf_get_vq_state(struct ifcvf_hw *hw, u16 qid) +{ + struct ifcvf_lm_cfg __iomem *ifcvf_lm; + void __iomem *avail_idx_addr; + u16 last_avail_idx; + u32 q_pair_id; + + ifcvf_lm = (struct ifcvf_lm_cfg __iomem *)hw->lm_cfg; + q_pair_id = qid / (IFCVF_MAX_QUEUE_PAIRS * 2); + avail_idx_addr = &ifcvf_lm->vring_lm_cfg[q_pair_id].idx_addr[qid % 2]; + last_avail_idx = ifc_ioread16(avail_idx_addr); + + return last_avail_idx; +} + +int ifcvf_set_vq_state(struct ifcvf_hw *hw, u16 qid, u64 num) +{ + struct ifcvf_lm_cfg __iomem *ifcvf_lm; + void __iomem *avail_idx_addr; + u32 q_pair_id; + + ifcvf_lm = (struct ifcvf_lm_cfg __iomem *)hw->lm_cfg; + q_pair_id = qid / (IFCVF_MAX_QUEUE_PAIRS * 2); + avail_idx_addr = &ifcvf_lm->vring_lm_cfg[q_pair_id].idx_addr[qid % 2]; + hw->vring[qid].last_avail_idx = num; + ifc_iowrite16(num, avail_idx_addr); + + return 0; +} + +static int ifcvf_hw_enable(struct ifcvf_hw *hw) +{ + struct ifcvf_lm_cfg __iomem *ifcvf_lm; + struct virtio_pci_common_cfg __iomem *cfg; + struct ifcvf_adapter *ifcvf; + u32 i; + + ifcvf_lm = (struct ifcvf_lm_cfg __iomem *)hw->lm_cfg; + ifcvf = vf_to_adapter(hw); + cfg = hw->common_cfg; + ifc_iowrite16(IFCVF_MSI_CONFIG_OFF, &cfg->msix_config); + + if (ifc_ioread16(&cfg->msix_config) == VIRTIO_MSI_NO_VECTOR) { + IFCVF_ERR(ifcvf->pdev, "No msix vector for device config\n"); + return -EINVAL; + } + + for (i = 0; i < hw->nr_vring; i++) { + if (!hw->vring[i].ready) + break; + + ifc_iowrite16(i, &cfg->queue_select); + ifc_iowrite64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo, + &cfg->queue_desc_hi); + ifc_iowrite64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo, + &cfg->queue_avail_hi); + ifc_iowrite64_twopart(hw->vring[i].used, &cfg->queue_used_lo, + &cfg->queue_used_hi); + ifc_iowrite16(hw->vring[i].size, &cfg->queue_size); + ifc_iowrite16(i + IFCVF_MSI_QUEUE_OFF, &cfg->queue_msix_vector); + + if (ifc_ioread16(&cfg->queue_msix_vector) == + VIRTIO_MSI_NO_VECTOR) { + IFCVF_ERR(ifcvf->pdev, + "No msix vector for queue %u\n", i); + return -EINVAL; + } + + ifcvf_set_vq_state(hw, i, hw->vring[i].last_avail_idx); + ifc_iowrite16(1, &cfg->queue_enable); + } + + return 0; +} + +static void ifcvf_hw_disable(struct ifcvf_hw *hw) +{ + struct virtio_pci_common_cfg __iomem *cfg; + u32 i; + + cfg = hw->common_cfg; + ifc_iowrite16(VIRTIO_MSI_NO_VECTOR, &cfg->msix_config); + + for (i = 0; i < hw->nr_vring; i++) { + ifc_iowrite16(i, &cfg->queue_select); + ifc_iowrite16(VIRTIO_MSI_NO_VECTOR, &cfg->queue_msix_vector); + } + + ifc_ioread16(&cfg->queue_msix_vector); +} + +int ifcvf_start_hw(struct ifcvf_hw *hw) +{ + ifcvf_reset(hw); + ifcvf_add_status(hw, VIRTIO_CONFIG_S_ACKNOWLEDGE); + ifcvf_add_status(hw, VIRTIO_CONFIG_S_DRIVER); + + if (ifcvf_config_features(hw) < 0) + return -EINVAL; + + if (ifcvf_hw_enable(hw) < 0) + return -EINVAL; + + ifcvf_add_status(hw, VIRTIO_CONFIG_S_DRIVER_OK); + + return 0; +} + +void ifcvf_stop_hw(struct ifcvf_hw *hw) +{ + ifcvf_hw_disable(hw); + ifcvf_reset(hw); +} + +void ifcvf_notify_queue(struct ifcvf_hw *hw, u16 qid) +{ + ifc_iowrite16(qid, hw->vring[qid].notify_addr); +} diff --git a/drivers/virtio/vdpa/ifcvf/ifcvf_base.h b/drivers/virtio/vdpa/ifcvf/ifcvf_base.h new file mode 100644 index 000000000000..e80307092351 --- /dev/null +++ b/drivers/virtio/vdpa/ifcvf/ifcvf_base.h @@ -0,0 +1,118 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Intel IFC VF NIC driver for virtio dataplane offloading + * + * Copyright (C) 2020 Intel Corporation. + * + * Author: Zhu Lingshan + * + */ + +#ifndef _IFCVF_H_ +#define _IFCVF_H_ + +#include +#include +#include +#include +#include +#include + +#define IFCVF_VENDOR_ID 0x1AF4 +#define IFCVF_DEVICE_ID 0x1041 +#define IFCVF_SUBSYS_VENDOR_ID 0x8086 +#define IFCVF_SUBSYS_DEVICE_ID 0x001A + +#define IFCVF_SUPPORTED_FEATURES \ + ((1ULL << VIRTIO_NET_F_MAC) | \ + (1ULL << VIRTIO_F_ANY_LAYOUT) | \ + (1ULL << VIRTIO_F_VERSION_1) | \ + (1ULL << VIRTIO_F_ORDER_PLATFORM) | \ + (1ULL << VIRTIO_F_IOMMU_PLATFORM) | \ + (1ULL << VIRTIO_NET_F_MRG_RXBUF)) + +/* Only one queue pair for now. */ +#define IFCVF_MAX_QUEUE_PAIRS 1 + +#define IFCVF_QUEUE_ALIGNMENT PAGE_SIZE +#define IFCVF_QUEUE_MAX 32768 +#define IFCVF_MSI_CONFIG_OFF 0 +#define IFCVF_MSI_QUEUE_OFF 1 +#define IFCVF_PCI_MAX_RESOURCE 6 + +#define IFCVF_LM_CFG_SIZE 0x40 +#define IFCVF_LM_RING_STATE_OFFSET 0x20 +#define IFCVF_LM_BAR 4 + +#define IFCVF_ERR(pdev, fmt, ...) dev_err(&pdev->dev, fmt, ##__VA_ARGS__) +#define IFCVF_DBG(pdev, fmt, ...) dev_dbg(&pdev->dev, fmt, ##__VA_ARGS__) +#define IFCVF_INFO(pdev, fmt, ...) dev_info(&pdev->dev, fmt, ##__VA_ARGS__) + +#define ifcvf_private_to_vf(adapter) \ + (&((struct ifcvf_adapter *)adapter)->vf) + +#define IFCVF_MAX_INTR (IFCVF_MAX_QUEUE_PAIRS * 2 + 1) + +struct vring_info { + u64 desc; + u64 avail; + u64 used; + u16 size; + u16 last_avail_idx; + bool ready; + void __iomem *notify_addr; + u32 irq; + struct vdpa_callback cb; + char msix_name[256]; +}; + +struct ifcvf_hw { + u8 __iomem *isr; + /* Live migration */ + u8 __iomem *lm_cfg; + u16 nr_vring; + /* Notification bar number */ + u8 notify_bar; + /* Notificaiton bar address */ + void __iomem *notify_base; + u32 notify_off_multiplier; + u64 req_features; + struct virtio_pci_common_cfg __iomem *common_cfg; + void __iomem *net_cfg; + struct vring_info vring[IFCVF_MAX_QUEUE_PAIRS * 2]; + void __iomem * const *base; +}; + +struct ifcvf_adapter { + struct vdpa_device vdpa; + struct pci_dev *pdev; + struct ifcvf_hw vf; +}; + +struct ifcvf_vring_lm_cfg { + u32 idx_addr[2]; + u8 reserved[IFCVF_LM_CFG_SIZE - 8]; +}; + +struct ifcvf_lm_cfg { + u8 reserved[IFCVF_LM_RING_STATE_OFFSET]; + struct ifcvf_vring_lm_cfg vring_lm_cfg[IFCVF_MAX_QUEUE_PAIRS]; +}; + +int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *dev); +int ifcvf_start_hw(struct ifcvf_hw *hw); +void ifcvf_stop_hw(struct ifcvf_hw *hw); +void ifcvf_notify_queue(struct ifcvf_hw *hw, u16 qid); +void ifcvf_read_net_config(struct ifcvf_hw *hw, u64 offset, + void *dst, int length); +void ifcvf_write_net_config(struct ifcvf_hw *hw, u64 offset, + const void *src, int length); +u8 ifcvf_get_status(struct ifcvf_hw *hw); +void ifcvf_set_status(struct ifcvf_hw *hw, u8 status); +void io_write64_twopart(u64 val, u32 *lo, u32 *hi); +void ifcvf_reset(struct ifcvf_hw *hw); +u64 ifcvf_get_features(struct ifcvf_hw *hw); +u64 ifcvf_get_vq_state(struct ifcvf_hw *hw, u16 qid); +int ifcvf_set_vq_state(struct ifcvf_hw *hw, u16 qid, u64 num); +struct ifcvf_adapter *vf_to_adapter(struct ifcvf_hw *hw); +#endif /* _IFCVF_H_ */ diff --git a/drivers/virtio/vdpa/ifcvf/ifcvf_main.c b/drivers/virtio/vdpa/ifcvf/ifcvf_main.c new file mode 100644 index 000000000000..8d54dc5b08d2 --- /dev/null +++ b/drivers/virtio/vdpa/ifcvf/ifcvf_main.c @@ -0,0 +1,435 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Intel IFC VF NIC driver for virtio dataplane offloading + * + * Copyright (C) 2020 Intel Corporation. + * + * Author: Zhu Lingshan + * + */ + +#include +#include +#include +#include +#include "ifcvf_base.h" + +#define VERSION_STRING "0.1" +#define DRIVER_AUTHOR "Intel Corporation" +#define IFCVF_DRIVER_NAME "ifcvf" + +static irqreturn_t ifcvf_intr_handler(int irq, void *arg) +{ + struct vring_info *vring = arg; + + if (vring->cb.callback) + return vring->cb.callback(vring->cb.private); + + return IRQ_HANDLED; +} + +static int ifcvf_start_datapath(void *private) +{ + struct ifcvf_hw *vf = ifcvf_private_to_vf(private); + struct ifcvf_adapter *ifcvf; + u8 status; + int ret; + + ifcvf = vf_to_adapter(vf); + vf->nr_vring = IFCVF_MAX_QUEUE_PAIRS * 2; + ret = ifcvf_start_hw(vf); + if (ret < 0) { + status = ifcvf_get_status(vf); + status |= VIRTIO_CONFIG_S_FAILED; + ifcvf_set_status(vf, status); + } + + return ret; +} + +static int ifcvf_stop_datapath(void *private) +{ + struct ifcvf_hw *vf = ifcvf_private_to_vf(private); + int i; + + for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) + vf->vring[i].cb.callback = NULL; + + ifcvf_stop_hw(vf); + + return 0; +} + +static void ifcvf_reset_vring(struct ifcvf_adapter *adapter) +{ + struct ifcvf_hw *vf = ifcvf_private_to_vf(adapter); + int i; + + for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) { + vf->vring[i].last_avail_idx = 0; + vf->vring[i].desc = 0; + vf->vring[i].avail = 0; + vf->vring[i].used = 0; + vf->vring[i].ready = 0; + vf->vring[i].cb.callback = NULL; + vf->vring[i].cb.private = NULL; + } + + ifcvf_reset(vf); +} + +static struct ifcvf_adapter *vdpa_to_adapter(struct vdpa_device *vdpa_dev) +{ + return container_of(vdpa_dev, struct ifcvf_adapter, vdpa); +} + +static struct ifcvf_hw *vdpa_to_vf(struct vdpa_device *vdpa_dev) +{ + struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev); + + return &adapter->vf; +} + +static u64 ifcvf_vdpa_get_features(struct vdpa_device *vdpa_dev) +{ + struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); + u64 features; + + features = ifcvf_get_features(vf) & IFCVF_SUPPORTED_FEATURES; + + return features; +} + +static int ifcvf_vdpa_set_features(struct vdpa_device *vdpa_dev, u64 features) +{ + struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); + + vf->req_features = features; + + return 0; +} + +static u8 ifcvf_vdpa_get_status(struct vdpa_device *vdpa_dev) +{ + struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); + + return ifcvf_get_status(vf); +} + +static void ifcvf_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status) +{ + struct ifcvf_adapter *adapter; + struct ifcvf_hw *vf; + + vf = vdpa_to_vf(vdpa_dev); + adapter = dev_get_drvdata(vdpa_dev->dev.parent); + + if (status == 0) { + ifcvf_stop_datapath(adapter); + ifcvf_reset_vring(adapter); + return; + } + + if (status & VIRTIO_CONFIG_S_DRIVER_OK) { + if (ifcvf_start_datapath(adapter) < 0) + IFCVF_ERR(adapter->pdev, + "Failed to set ifcvf vdpa status %u\n", + status); + } + + ifcvf_set_status(vf, status); +} + +static u16 ifcvf_vdpa_get_vq_num_max(struct vdpa_device *vdpa_dev) +{ + return IFCVF_QUEUE_MAX; +} + +static u64 ifcvf_vdpa_get_vq_state(struct vdpa_device *vdpa_dev, u16 qid) +{ + struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); + + return ifcvf_get_vq_state(vf, qid); +} + +static int ifcvf_vdpa_set_vq_state(struct vdpa_device *vdpa_dev, u16 qid, + u64 num) +{ + struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); + + return ifcvf_set_vq_state(vf, qid, num); +} + +static void ifcvf_vdpa_set_vq_cb(struct vdpa_device *vdpa_dev, u16 qid, + struct vdpa_callback *cb) +{ + struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); + + vf->vring[qid].cb = *cb; +} + +static void ifcvf_vdpa_set_vq_ready(struct vdpa_device *vdpa_dev, + u16 qid, bool ready) +{ + struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); + + vf->vring[qid].ready = ready; +} + +static bool ifcvf_vdpa_get_vq_ready(struct vdpa_device *vdpa_dev, u16 qid) +{ + struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); + + return vf->vring[qid].ready; +} + +static void ifcvf_vdpa_set_vq_num(struct vdpa_device *vdpa_dev, u16 qid, + u32 num) +{ + struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); + + vf->vring[qid].size = num; +} + +static int ifcvf_vdpa_set_vq_address(struct vdpa_device *vdpa_dev, u16 qid, + u64 desc_area, u64 driver_area, + u64 device_area) +{ + struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); + + vf->vring[qid].desc = desc_area; + vf->vring[qid].avail = driver_area; + vf->vring[qid].used = device_area; + + return 0; +} + +static void ifcvf_vdpa_kick_vq(struct vdpa_device *vdpa_dev, u16 qid) +{ + struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); + + ifcvf_notify_queue(vf, qid); +} + +static u32 ifcvf_vdpa_get_generation(struct vdpa_device *vdpa_dev) +{ + struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); + + return ioread8(&vf->common_cfg->config_generation); +} + +static u32 ifcvf_vdpa_get_device_id(struct vdpa_device *vdpa_dev) +{ + return VIRTIO_ID_NET; +} + +static u32 ifcvf_vdpa_get_vendor_id(struct vdpa_device *vdpa_dev) +{ + return IFCVF_SUBSYS_VENDOR_ID; +} + +static u16 ifcvf_vdpa_get_vq_align(struct vdpa_device *vdpa_dev) +{ + return IFCVF_QUEUE_ALIGNMENT; +} + +static void ifcvf_vdpa_get_config(struct vdpa_device *vdpa_dev, + unsigned int offset, + void *buf, unsigned int len) +{ + struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); + + WARN_ON(offset + len > sizeof(struct virtio_net_config)); + ifcvf_read_net_config(vf, offset, buf, len); +} + +static void ifcvf_vdpa_set_config(struct vdpa_device *vdpa_dev, + unsigned int offset, const void *buf, + unsigned int len) +{ + struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); + + WARN_ON(offset + len > sizeof(struct virtio_net_config)); + ifcvf_write_net_config(vf, offset, buf, len); +} + +static void ifcvf_vdpa_set_config_cb(struct vdpa_device *vdpa_dev, + struct vdpa_callback *cb) +{ + /* We don't support config interrupt */ +} + +/* + * IFCVF currently does't have on-chip IOMMU, so not + * implemented set_map()/dma_map()/dma_unmap() + */ +static const struct vdpa_config_ops ifc_vdpa_ops = { + .get_features = ifcvf_vdpa_get_features, + .set_features = ifcvf_vdpa_set_features, + .get_status = ifcvf_vdpa_get_status, + .set_status = ifcvf_vdpa_set_status, + .get_vq_num_max = ifcvf_vdpa_get_vq_num_max, + .get_vq_state = ifcvf_vdpa_get_vq_state, + .set_vq_state = ifcvf_vdpa_set_vq_state, + .set_vq_cb = ifcvf_vdpa_set_vq_cb, + .set_vq_ready = ifcvf_vdpa_set_vq_ready, + .get_vq_ready = ifcvf_vdpa_get_vq_ready, + .set_vq_num = ifcvf_vdpa_set_vq_num, + .set_vq_address = ifcvf_vdpa_set_vq_address, + .kick_vq = ifcvf_vdpa_kick_vq, + .get_generation = ifcvf_vdpa_get_generation, + .get_device_id = ifcvf_vdpa_get_device_id, + .get_vendor_id = ifcvf_vdpa_get_vendor_id, + .get_vq_align = ifcvf_vdpa_get_vq_align, + .get_config = ifcvf_vdpa_get_config, + .set_config = ifcvf_vdpa_set_config, + .set_config_cb = ifcvf_vdpa_set_config_cb, +}; + +static int ifcvf_request_irq(struct ifcvf_adapter *adapter) +{ + struct pci_dev *pdev = adapter->pdev; + struct ifcvf_hw *vf = &adapter->vf; + int vector, i, ret, irq; + + + for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) { + snprintf(vf->vring[i].msix_name, 256, "ifcvf[%s]-%d\n", + pci_name(pdev), i); + vector = i + IFCVF_MSI_QUEUE_OFF; + irq = pci_irq_vector(pdev, vector); + ret = devm_request_irq(&pdev->dev, irq, + ifcvf_intr_handler, 0, + vf->vring[i].msix_name, + &vf->vring[i]); + if (ret) { + IFCVF_ERR(pdev, + "Failed to request irq for vq %d\n", i); + return ret; + } + vf->vring[i].irq = irq; + } + + return 0; +} + +static void ifcvf_free_irq_vectors(void *data) +{ + pci_free_irq_vectors(data); +} + +static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct device *dev = &pdev->dev; + struct ifcvf_adapter *adapter; + struct ifcvf_hw *vf; + int ret; + + ret = pcim_enable_device(pdev); + if (ret) { + IFCVF_ERR(pdev, "Failed to enable device\n"); + return ret; + } + + ret = pcim_iomap_regions(pdev, BIT(0) | BIT(2) | BIT(4), + IFCVF_DRIVER_NAME); + if (ret) { + IFCVF_ERR(pdev, "Failed to request MMIO region\n"); + return ret; + } + + ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); + if (ret) { + IFCVF_ERR(pdev, "No usable DMA confiugration\n"); + return ret; + } + + ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); + if (ret) { + IFCVF_ERR(pdev, + "No usable coherent DMA confiugration\n"); + return ret; + } + + ret = pci_alloc_irq_vectors(pdev, IFCVF_MAX_INTR, + IFCVF_MAX_INTR, PCI_IRQ_MSIX); + if (ret < 0) { + IFCVF_ERR(pdev, "Failed to alloc irq vectors\n"); + return ret; + } + + ret = devm_add_action_or_reset(dev, ifcvf_free_irq_vectors, pdev); + if (ret) { + IFCVF_ERR(pdev, + "Failed for adding devres for freeing irq vectors\n"); + return ret; + } + + adapter = vdpa_alloc_device(struct ifcvf_adapter, vdpa, + dev, &ifc_vdpa_ops); + if (adapter == NULL) { + IFCVF_ERR(pdev, "Failed to allocate vDPA structure"); + return -ENOMEM; + } + + pci_set_master(pdev); + pci_set_drvdata(pdev, adapter); + + vf = &adapter->vf; + vf->base = pcim_iomap_table(pdev); + + adapter->pdev = pdev; + adapter->vdpa.dma_dev = &pdev->dev; + + ret = ifcvf_request_irq(adapter); + if (ret) { + IFCVF_ERR(pdev, "Failed to request MSI-X irq\n"); + goto err; + } + + ret = ifcvf_init_hw(vf, pdev); + if (ret) { + IFCVF_ERR(pdev, "Failed to init IFCVF hw\n"); + goto err; + } + + ret = vdpa_register_device(&adapter->vdpa); + if (ret) { + IFCVF_ERR(pdev, "Failed to register ifcvf to vdpa bus"); + goto err; + } + + return 0; + +err: + put_device(&adapter->vdpa.dev); + return ret; +} + +static void ifcvf_remove(struct pci_dev *pdev) +{ + struct ifcvf_adapter *adapter = pci_get_drvdata(pdev); + + vdpa_unregister_device(&adapter->vdpa); +} + +static struct pci_device_id ifcvf_pci_ids[] = { + { PCI_DEVICE_SUB(IFCVF_VENDOR_ID, + IFCVF_DEVICE_ID, + IFCVF_SUBSYS_VENDOR_ID, + IFCVF_SUBSYS_DEVICE_ID) }, + { 0 }, +}; +MODULE_DEVICE_TABLE(pci, ifcvf_pci_ids); + +static struct pci_driver ifcvf_driver = { + .name = IFCVF_DRIVER_NAME, + .id_table = ifcvf_pci_ids, + .probe = ifcvf_probe, + .remove = ifcvf_remove, +}; + +module_pci_driver(ifcvf_driver); + +MODULE_LICENSE("GPL v2"); +MODULE_VERSION(VERSION_STRING); From c9b9f5f8c0f3cdb893cb86c168cdaa3aa5ed7278 Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Tue, 31 Mar 2020 15:15:14 -0400 Subject: [PATCH 423/635] vdpa: move to drivers/vdpa We have both vhost and virtio drivers that depend on vdpa. It's easier to locate it at a top level directory otherwise we run into issues e.g. if vhost is built-in but virtio is modular. Let's just move it up a level. Reported-by: Randy Dunlap Signed-off-by: Michael S. Tsirkin --- MAINTAINERS | 1 + drivers/Kconfig | 2 ++ drivers/Makefile | 1 + drivers/{virtio => }/vdpa/Kconfig | 0 drivers/{virtio => }/vdpa/Makefile | 0 drivers/{virtio => }/vdpa/ifcvf/Makefile | 0 drivers/{virtio => }/vdpa/ifcvf/ifcvf_base.c | 0 drivers/{virtio => }/vdpa/ifcvf/ifcvf_base.h | 0 drivers/{virtio => }/vdpa/ifcvf/ifcvf_main.c | 0 drivers/{virtio => }/vdpa/vdpa.c | 0 drivers/{virtio => }/vdpa/vdpa_sim/Makefile | 0 drivers/{virtio => }/vdpa/vdpa_sim/vdpa_sim.c | 0 drivers/virtio/Kconfig | 2 -- drivers/virtio/Makefile | 1 - 14 files changed, 4 insertions(+), 3 deletions(-) rename drivers/{virtio => }/vdpa/Kconfig (100%) rename drivers/{virtio => }/vdpa/Makefile (100%) rename drivers/{virtio => }/vdpa/ifcvf/Makefile (100%) rename drivers/{virtio => }/vdpa/ifcvf/ifcvf_base.c (100%) rename drivers/{virtio => }/vdpa/ifcvf/ifcvf_base.h (100%) rename drivers/{virtio => }/vdpa/ifcvf/ifcvf_main.c (100%) rename drivers/{virtio => }/vdpa/vdpa.c (100%) rename drivers/{virtio => }/vdpa/vdpa_sim/Makefile (100%) rename drivers/{virtio => }/vdpa/vdpa_sim/vdpa_sim.c (100%) diff --git a/MAINTAINERS b/MAINTAINERS index 70c47bc55343..7cfa55c765fd 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -17695,6 +17695,7 @@ L: virtualization@lists.linux-foundation.org S: Maintained F: Documentation/devicetree/bindings/virtio/ F: drivers/virtio/ +F: drivers/vdpa/ F: tools/virtio/ F: drivers/net/virtio_net.c F: drivers/block/virtio_blk.c diff --git a/drivers/Kconfig b/drivers/Kconfig index 7a6d8b2b68b4..ac23d520e916 100644 --- a/drivers/Kconfig +++ b/drivers/Kconfig @@ -138,6 +138,8 @@ source "drivers/virt/Kconfig" source "drivers/virtio/Kconfig" +source "drivers/vdpa/Kconfig" + source "drivers/vhost/Kconfig" source "drivers/hv/Kconfig" diff --git a/drivers/Makefile b/drivers/Makefile index 31cf17dee252..21688f3b1588 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -42,6 +42,7 @@ obj-$(CONFIG_DMADEVICES) += dma/ obj-y += soc/ obj-$(CONFIG_VIRTIO) += virtio/ +obj-$(CONFIG_VDPA) += vdpa/ obj-$(CONFIG_XEN) += xen/ # regulators early, since some subsystems rely on them to initialize diff --git a/drivers/virtio/vdpa/Kconfig b/drivers/vdpa/Kconfig similarity index 100% rename from drivers/virtio/vdpa/Kconfig rename to drivers/vdpa/Kconfig diff --git a/drivers/virtio/vdpa/Makefile b/drivers/vdpa/Makefile similarity index 100% rename from drivers/virtio/vdpa/Makefile rename to drivers/vdpa/Makefile diff --git a/drivers/virtio/vdpa/ifcvf/Makefile b/drivers/vdpa/ifcvf/Makefile similarity index 100% rename from drivers/virtio/vdpa/ifcvf/Makefile rename to drivers/vdpa/ifcvf/Makefile diff --git a/drivers/virtio/vdpa/ifcvf/ifcvf_base.c b/drivers/vdpa/ifcvf/ifcvf_base.c similarity index 100% rename from drivers/virtio/vdpa/ifcvf/ifcvf_base.c rename to drivers/vdpa/ifcvf/ifcvf_base.c diff --git a/drivers/virtio/vdpa/ifcvf/ifcvf_base.h b/drivers/vdpa/ifcvf/ifcvf_base.h similarity index 100% rename from drivers/virtio/vdpa/ifcvf/ifcvf_base.h rename to drivers/vdpa/ifcvf/ifcvf_base.h diff --git a/drivers/virtio/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c similarity index 100% rename from drivers/virtio/vdpa/ifcvf/ifcvf_main.c rename to drivers/vdpa/ifcvf/ifcvf_main.c diff --git a/drivers/virtio/vdpa/vdpa.c b/drivers/vdpa/vdpa.c similarity index 100% rename from drivers/virtio/vdpa/vdpa.c rename to drivers/vdpa/vdpa.c diff --git a/drivers/virtio/vdpa/vdpa_sim/Makefile b/drivers/vdpa/vdpa_sim/Makefile similarity index 100% rename from drivers/virtio/vdpa/vdpa_sim/Makefile rename to drivers/vdpa/vdpa_sim/Makefile diff --git a/drivers/virtio/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c similarity index 100% rename from drivers/virtio/vdpa/vdpa_sim/vdpa_sim.c rename to drivers/vdpa/vdpa_sim/vdpa_sim.c diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig index 99e424570644..2aadf398d8cc 100644 --- a/drivers/virtio/Kconfig +++ b/drivers/virtio/Kconfig @@ -109,5 +109,3 @@ config VIRTIO_MMIO_CMDLINE_DEVICES If unsure, say 'N'. endif # VIRTIO_MENU - -source "drivers/virtio/vdpa/Kconfig" diff --git a/drivers/virtio/Makefile b/drivers/virtio/Makefile index 3407ac03fe60..29a1386ecc03 100644 --- a/drivers/virtio/Makefile +++ b/drivers/virtio/Makefile @@ -7,4 +7,3 @@ virtio_pci-$(CONFIG_VIRTIO_PCI_LEGACY) += virtio_pci_legacy.o obj-$(CONFIG_VIRTIO_BALLOON) += virtio_balloon.o obj-$(CONFIG_VIRTIO_INPUT) += virtio_input.o obj-$(CONFIG_VIRTIO_VDPA) += virtio_vdpa.o -obj-$(CONFIG_VDPA) += vdpa/ From 457df33e035a2cffc6561992f3f25a6c61605c46 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 2 Apr 2020 09:08:53 -0700 Subject: [PATCH 424/635] iomap: Handle memory allocation failure in readahead bio_alloc() can fail when we use GFP_NORETRY. If it does, allocate a bio large enough for a single page like mpage_readpages() does. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/iomap/buffered-io.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index f080f542911b..56f2cca765be 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -302,6 +302,7 @@ iomap_readpage_actor(struct inode *inode, loff_t pos, loff_t length, void *data, if (!ctx->bio || !is_contig || bio_full(ctx->bio, plen)) { gfp_t gfp = mapping_gfp_constraint(page->mapping, GFP_KERNEL); + gfp_t orig_gfp = gfp; int nr_vecs = (length + PAGE_SIZE - 1) >> PAGE_SHIFT; if (ctx->bio) @@ -310,6 +311,13 @@ iomap_readpage_actor(struct inode *inode, loff_t pos, loff_t length, void *data, if (ctx->is_readahead) /* same as readahead_gfp_mask */ gfp |= __GFP_NORETRY | __GFP_NOWARN; ctx->bio = bio_alloc(gfp, min(BIO_MAX_PAGES, nr_vecs)); + /* + * If the bio_alloc fails, try it again for a single page to + * avoid having to deal with partial page reads. This emulates + * what do_mpage_readpage does. + */ + if (!ctx->bio) + ctx->bio = bio_alloc(orig_gfp, 1); ctx->bio->bi_opf = REQ_OP_READ; if (ctx->is_readahead) ctx->bio->bi_opf |= REQ_RAHEAD; From 529af90576cfa44aa107e9876e2ebaa053983986 Mon Sep 17 00:00:00 2001 From: Scott Mayhew Date: Thu, 2 Apr 2020 17:20:44 -0400 Subject: [PATCH 425/635] NFS: Fix a few constant_table array definitions nfs_vers_tokens, nfs_xprt_protocol_tokens, and nfs_secflavor_tokens were all missing an empty item at the end of the array, allowing lookup_constant() to potentially walk off the end and trigger and oops. Reported-by: Olga Kornievskaia Signed-off-by: Scott Mayhew Fixes: e38bb238ed8c ("NFS: Convert mount option parsing to use functionality from fs_parser.h") Cc: stable@vger.kernel.org # v5.6 Signed-off-by: Trond Myklebust --- fs/nfs/fs_context.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fs/nfs/fs_context.c b/fs/nfs/fs_context.c index 566dd59570e6..ccc88be88d6a 100644 --- a/fs/nfs/fs_context.c +++ b/fs/nfs/fs_context.c @@ -190,6 +190,7 @@ static const struct constant_table nfs_vers_tokens[] = { { "4.0", Opt_vers_4_0 }, { "4.1", Opt_vers_4_1 }, { "4.2", Opt_vers_4_2 }, + {} }; enum { @@ -202,13 +203,14 @@ enum { nr__Opt_xprt }; -static const struct constant_table nfs_xprt_protocol_tokens[nr__Opt_xprt] = { +static const struct constant_table nfs_xprt_protocol_tokens[] = { { "rdma", Opt_xprt_rdma }, { "rdma6", Opt_xprt_rdma6 }, { "tcp", Opt_xprt_tcp }, { "tcp6", Opt_xprt_tcp6 }, { "udp", Opt_xprt_udp }, { "udp6", Opt_xprt_udp6 }, + {} }; enum { @@ -239,6 +241,7 @@ static const struct constant_table nfs_secflavor_tokens[] = { { "spkm3i", Opt_sec_spkmi }, { "spkm3p", Opt_sec_spkmp }, { "sys", Opt_sec_sys }, + {} }; /* From 75da98586af75eb80664714a67a9895bf0a5517e Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Thu, 2 Apr 2020 10:34:36 -0400 Subject: [PATCH 426/635] NFS: finish_automount() requires us to hold 2 refs to the mount record We must not return from nfs_d_automount() without holding 2 references to the mount record. Doing so, will trigger the BUG() in finish_automount(). Also ensure that we don't try to reschedule the automount timer with a negative or zero timeout value. Fixes: 22a1ae9a93fb ("NFS: If nfs_mountpoint_expiry_timeout < 0, do not expire submounts") Cc: stable@vger.kernel.org # v5.5+ Signed-off-by: Trond Myklebust --- fs/nfs/namespace.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/fs/nfs/namespace.c b/fs/nfs/namespace.c index da67820462f2..fe19ae280262 100644 --- a/fs/nfs/namespace.c +++ b/fs/nfs/namespace.c @@ -145,6 +145,7 @@ struct vfsmount *nfs_d_automount(struct path *path) struct vfsmount *mnt = ERR_PTR(-ENOMEM); struct nfs_server *server = NFS_SERVER(d_inode(path->dentry)); struct nfs_client *client = server->nfs_client; + int timeout = READ_ONCE(nfs_mountpoint_expiry_timeout); int ret; if (IS_ROOT(path->dentry)) @@ -190,12 +191,12 @@ struct vfsmount *nfs_d_automount(struct path *path) if (IS_ERR(mnt)) goto out_fc; - if (nfs_mountpoint_expiry_timeout < 0) + mntget(mnt); /* prevent immediate expiration */ + if (timeout <= 0) goto out_fc; - mntget(mnt); /* prevent immediate expiration */ mnt_set_expiry(mnt, &nfs_automount_list); - schedule_delayed_work(&nfs_automount_task, nfs_mountpoint_expiry_timeout); + schedule_delayed_work(&nfs_automount_task, timeout); out_fc: put_fs_context(fc); @@ -233,10 +234,11 @@ const struct inode_operations nfs_referral_inode_operations = { static void nfs_expire_automounts(struct work_struct *work) { struct list_head *list = &nfs_automount_list; + int timeout = READ_ONCE(nfs_mountpoint_expiry_timeout); mark_mounts_for_expiry(list); - if (!list_empty(list)) - schedule_delayed_work(&nfs_automount_task, nfs_mountpoint_expiry_timeout); + if (!list_empty(list) && timeout > 0) + schedule_delayed_work(&nfs_automount_task, timeout); } void nfs_release_automount_timer(void) From f30a6ea0f3a582cea72b0373612d4b38f383fbd9 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Thu, 2 Apr 2020 12:37:25 -0400 Subject: [PATCH 427/635] NFS: Add a module parameter to set nfs_mountpoint_expiry_timeout Setting nfs_mountpoint_expiry_timeout() to a negative value stops mountpoint expiration, while setting it to a positive value restarts the scheduler. Signed-off-by: Trond Myklebust --- fs/nfs/namespace.c | 50 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/fs/nfs/namespace.c b/fs/nfs/namespace.c index fe19ae280262..6b063227e34e 100644 --- a/fs/nfs/namespace.c +++ b/fs/nfs/namespace.c @@ -311,3 +311,53 @@ int nfs_submount(struct fs_context *fc, struct nfs_server *server) return nfs_do_submount(fc); } EXPORT_SYMBOL_GPL(nfs_submount); + +static int param_set_nfs_timeout(const char *val, const struct kernel_param *kp) +{ + long num; + int ret; + + if (!val) + return -EINVAL; + ret = kstrtol(val, 0, &num); + if (ret) + return -EINVAL; + if (num > 0) { + if (num >= INT_MAX / HZ) + num = INT_MAX; + else + num *= HZ; + *((int *)kp->arg) = num; + if (!list_empty(&nfs_automount_list)) + mod_delayed_work(system_wq, &nfs_automount_task, num); + } else { + *((int *)kp->arg) = -1*HZ; + cancel_delayed_work(&nfs_automount_task); + } + return 0; +} + +static int param_get_nfs_timeout(char *buffer, const struct kernel_param *kp) +{ + long num = *((int *)kp->arg); + + if (num > 0) { + if (num >= INT_MAX - (HZ - 1)) + num = INT_MAX / HZ; + else + num = (num + (HZ - 1)) / HZ; + } else + num = -1; + return scnprintf(buffer, PAGE_SIZE, "%li\n", num); +} + +static const struct kernel_param_ops param_ops_nfs_timeout = { + .set = param_set_nfs_timeout, + .get = param_get_nfs_timeout, +}; +#define param_check_nfs_timeout(name, p) __param_check(name, p, int); + +module_param(nfs_mountpoint_expiry_timeout, nfs_timeout, 0644); +MODULE_PARM_DESC(nfs_mountpoint_expiry_timeout, + "Set the NFS automounted mountpoint timeout value (seconds)." + "Values <= 0 turn expiration off."); From 5d64efe79703c4de1f017af9f7643ead1dc85432 Mon Sep 17 00:00:00 2001 From: Vivek Goyal Date: Fri, 28 Feb 2020 11:34:51 -0500 Subject: [PATCH 428/635] pmem: Add functions for reading/writing page to/from pmem This splits pmem_do_bvec() into pmem_do_read() and pmem_do_write(). pmem_do_write() will be used by pmem zero_page_range() as well. Hence sharing the same code. Suggested-by: Christoph Hellwig Signed-off-by: Vivek Goyal Reviewed-by: Christoph Hellwig Reviewed-by: Pankaj Gupta Link: https://lore.kernel.org/r/20200228163456.1587-2-vgoyal@redhat.com Signed-off-by: Dan Williams --- drivers/nvdimm/pmem.c | 86 +++++++++++++++++++++++++------------------ 1 file changed, 50 insertions(+), 36 deletions(-) diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index 4eae441f86c9..075b11682192 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -136,9 +136,25 @@ static blk_status_t read_pmem(struct page *page, unsigned int off, return BLK_STS_OK; } -static blk_status_t pmem_do_bvec(struct pmem_device *pmem, struct page *page, - unsigned int len, unsigned int off, unsigned int op, - sector_t sector) +static blk_status_t pmem_do_read(struct pmem_device *pmem, + struct page *page, unsigned int page_off, + sector_t sector, unsigned int len) +{ + blk_status_t rc; + phys_addr_t pmem_off = sector * 512 + pmem->data_offset; + void *pmem_addr = pmem->virt_addr + pmem_off; + + if (unlikely(is_bad_pmem(&pmem->bb, sector, len))) + return BLK_STS_IOERR; + + rc = read_pmem(page, page_off, pmem_addr, len); + flush_dcache_page(page); + return rc; +} + +static blk_status_t pmem_do_write(struct pmem_device *pmem, + struct page *page, unsigned int page_off, + sector_t sector, unsigned int len) { blk_status_t rc = BLK_STS_OK; bool bad_pmem = false; @@ -148,34 +164,25 @@ static blk_status_t pmem_do_bvec(struct pmem_device *pmem, struct page *page, if (unlikely(is_bad_pmem(&pmem->bb, sector, len))) bad_pmem = true; - if (!op_is_write(op)) { - if (unlikely(bad_pmem)) - rc = BLK_STS_IOERR; - else { - rc = read_pmem(page, off, pmem_addr, len); - flush_dcache_page(page); - } - } else { - /* - * Note that we write the data both before and after - * clearing poison. The write before clear poison - * handles situations where the latest written data is - * preserved and the clear poison operation simply marks - * the address range as valid without changing the data. - * In this case application software can assume that an - * interrupted write will either return the new good - * data or an error. - * - * However, if pmem_clear_poison() leaves the data in an - * indeterminate state we need to perform the write - * after clear poison. - */ - flush_dcache_page(page); - write_pmem(pmem_addr, page, off, len); - if (unlikely(bad_pmem)) { - rc = pmem_clear_poison(pmem, pmem_off, len); - write_pmem(pmem_addr, page, off, len); - } + /* + * Note that we write the data both before and after + * clearing poison. The write before clear poison + * handles situations where the latest written data is + * preserved and the clear poison operation simply marks + * the address range as valid without changing the data. + * In this case application software can assume that an + * interrupted write will either return the new good + * data or an error. + * + * However, if pmem_clear_poison() leaves the data in an + * indeterminate state we need to perform the write + * after clear poison. + */ + flush_dcache_page(page); + write_pmem(pmem_addr, page, page_off, len); + if (unlikely(bad_pmem)) { + rc = pmem_clear_poison(pmem, pmem_off, len); + write_pmem(pmem_addr, page, page_off, len); } return rc; @@ -197,8 +204,12 @@ static blk_qc_t pmem_make_request(struct request_queue *q, struct bio *bio) do_acct = nd_iostat_start(bio, &start); bio_for_each_segment(bvec, bio, iter) { - rc = pmem_do_bvec(pmem, bvec.bv_page, bvec.bv_len, - bvec.bv_offset, bio_op(bio), iter.bi_sector); + if (op_is_write(bio_op(bio))) + rc = pmem_do_write(pmem, bvec.bv_page, bvec.bv_offset, + iter.bi_sector, bvec.bv_len); + else + rc = pmem_do_read(pmem, bvec.bv_page, bvec.bv_offset, + iter.bi_sector, bvec.bv_len); if (rc) { bio->bi_status = rc; break; @@ -223,9 +234,12 @@ static int pmem_rw_page(struct block_device *bdev, sector_t sector, struct pmem_device *pmem = bdev->bd_queue->queuedata; blk_status_t rc; - rc = pmem_do_bvec(pmem, page, hpage_nr_pages(page) * PAGE_SIZE, - 0, op, sector); - + if (op_is_write(op)) + rc = pmem_do_write(pmem, page, 0, sector, + hpage_nr_pages(page) * PAGE_SIZE); + else + rc = pmem_do_read(pmem, page, 0, sector, + hpage_nr_pages(page) * PAGE_SIZE); /* * The ->rw_page interface is subtle and tricky. The core * retries on any error, so we can only invoke page_endio() in From f605a263e0690177ecc180417eacf2b5507dd177 Mon Sep 17 00:00:00 2001 From: Vivek Goyal Date: Fri, 28 Feb 2020 11:34:52 -0500 Subject: [PATCH 429/635] dax, pmem: Add a dax operation zero_page_range Add a dax operation zero_page_range, to zero a page. This will also clear any known poison in the page being zeroed. As of now, zeroing of one page is allowed in a single call. There are no callers which are trying to zero more than a page in a single call. Once we grow the callers which zero more than a page in single call, we can add that support. Primary reason for not doing that yet is that this will add little complexity in dm implementation where a range might be spanning multiple underlying targets and one will have to split the range into multiple sub ranges and call zero_page_range() on individual targets. Suggested-by: Christoph Hellwig Signed-off-by: Vivek Goyal Reviewed-by: Pankaj Gupta Link: https://lore.kernel.org/r/20200228163456.1587-3-vgoyal@redhat.com Signed-off-by: Dan Williams --- drivers/dax/super.c | 20 ++++++++++++++++++++ drivers/nvdimm/pmem.c | 11 +++++++++++ include/linux/dax.h | 4 ++++ 3 files changed, 35 insertions(+) diff --git a/drivers/dax/super.c b/drivers/dax/super.c index 0aa4b6bc5101..e498daf3c0d7 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -344,6 +344,26 @@ size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr, } EXPORT_SYMBOL_GPL(dax_copy_to_iter); +int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff, + size_t nr_pages) +{ + if (!dax_alive(dax_dev)) + return -ENXIO; + + if (!dax_dev->ops->zero_page_range) + return -EOPNOTSUPP; + /* + * There are no callers that want to zero more than one page as of now. + * Once users are there, this check can be removed after the + * device mapper code has been updated to split ranges across targets. + */ + if (nr_pages != 1) + return -EIO; + + return dax_dev->ops->zero_page_range(dax_dev, pgoff, nr_pages); +} +EXPORT_SYMBOL_GPL(dax_zero_page_range); + #ifdef CONFIG_ARCH_HAS_PMEM_API void arch_wb_cache_pmem(void *addr, size_t size); void dax_flush(struct dax_device *dax_dev, void *addr, size_t size) diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index 075b11682192..5b774ddd0efb 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -282,6 +282,16 @@ static const struct block_device_operations pmem_fops = { .revalidate_disk = nvdimm_revalidate_disk, }; +static int pmem_dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff, + size_t nr_pages) +{ + struct pmem_device *pmem = dax_get_private(dax_dev); + + return blk_status_to_errno(pmem_do_write(pmem, ZERO_PAGE(0), 0, + PFN_PHYS(pgoff) >> SECTOR_SHIFT, + PAGE_SIZE)); +} + static long pmem_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages, void **kaddr, pfn_t *pfn) { @@ -313,6 +323,7 @@ static const struct dax_operations pmem_dax_ops = { .dax_supported = generic_fsdax_supported, .copy_from_iter = pmem_copy_from_iter, .copy_to_iter = pmem_copy_to_iter, + .zero_page_range = pmem_dax_zero_page_range, }; static const struct attribute_group *pmem_attribute_groups[] = { diff --git a/include/linux/dax.h b/include/linux/dax.h index 328c2dbb4409..71735c430c05 100644 --- a/include/linux/dax.h +++ b/include/linux/dax.h @@ -34,6 +34,8 @@ struct dax_operations { /* copy_to_iter: required operation for fs-dax direct-i/o */ size_t (*copy_to_iter)(struct dax_device *, pgoff_t, void *, size_t, struct iov_iter *); + /* zero_page_range: required operation. Zero page range */ + int (*zero_page_range)(struct dax_device *, pgoff_t, size_t); }; extern struct attribute_group dax_attribute_group; @@ -199,6 +201,8 @@ size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i); size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i); +int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff, + size_t nr_pages); void dax_flush(struct dax_device *dax_dev, void *addr, size_t size); ssize_t dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter, From 79fa974ff6bc3f7711f409c88a9552ab97b9ecb5 Mon Sep 17 00:00:00 2001 From: Vivek Goyal Date: Fri, 28 Feb 2020 11:34:53 -0500 Subject: [PATCH 430/635] s390,dcssblk,dax: Add dax zero_page_range operation to dcssblk driver Add dax operation zero_page_range for dcssblk driver. Suggested-by: Christoph Hellwig Signed-off-by: Vivek Goyal Reviewed-by: Gerald Schaefer CC: linux-s390@vger.kernel.org Link: https://lore.kernel.org/r/20200228163456.1587-4-vgoyal@redhat.com Signed-off-by: Dan Williams --- drivers/s390/block/dcssblk.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/drivers/s390/block/dcssblk.c b/drivers/s390/block/dcssblk.c index 63502ca537eb..ab3e2898816c 100644 --- a/drivers/s390/block/dcssblk.c +++ b/drivers/s390/block/dcssblk.c @@ -57,11 +57,26 @@ static size_t dcssblk_dax_copy_to_iter(struct dax_device *dax_dev, return copy_to_iter(addr, bytes, i); } +static int dcssblk_dax_zero_page_range(struct dax_device *dax_dev, + pgoff_t pgoff, size_t nr_pages) +{ + long rc; + void *kaddr; + + rc = dax_direct_access(dax_dev, pgoff, nr_pages, &kaddr, NULL); + if (rc < 0) + return rc; + memset(kaddr, 0, nr_pages << PAGE_SHIFT); + dax_flush(dax_dev, kaddr, nr_pages << PAGE_SHIFT); + return 0; +} + static const struct dax_operations dcssblk_dax_ops = { .direct_access = dcssblk_dax_direct_access, .dax_supported = generic_fsdax_supported, .copy_from_iter = dcssblk_dax_copy_from_iter, .copy_to_iter = dcssblk_dax_copy_to_iter, + .zero_page_range = dcssblk_dax_zero_page_range, }; struct dcssblk_dev_info { From cdf6cdcd3b99a99ea9ecc1b05d1d040d5a69a134 Mon Sep 17 00:00:00 2001 From: Vivek Goyal Date: Fri, 28 Feb 2020 11:34:54 -0500 Subject: [PATCH 431/635] dm,dax: Add dax zero_page_range operation This patch adds support for dax zero_page_range operation to dm targets. Signed-off-by: Vivek Goyal Acked-by: Mike Snitzer Link: https://lore.kernel.org/r/20200228163456.1587-5-vgoyal@redhat.com Signed-off-by: Dan Williams --- drivers/md/dm-linear.c | 18 ++++++++++++++++++ drivers/md/dm-log-writes.c | 17 +++++++++++++++++ drivers/md/dm-stripe.c | 23 +++++++++++++++++++++++ drivers/md/dm.c | 30 ++++++++++++++++++++++++++++++ include/linux/device-mapper.h | 3 +++ 5 files changed, 91 insertions(+) diff --git a/drivers/md/dm-linear.c b/drivers/md/dm-linear.c index 8d07fdf63a47..e1db43446327 100644 --- a/drivers/md/dm-linear.c +++ b/drivers/md/dm-linear.c @@ -201,10 +201,27 @@ static size_t linear_dax_copy_to_iter(struct dm_target *ti, pgoff_t pgoff, return dax_copy_to_iter(dax_dev, pgoff, addr, bytes, i); } +static int linear_dax_zero_page_range(struct dm_target *ti, pgoff_t pgoff, + size_t nr_pages) +{ + int ret; + struct linear_c *lc = ti->private; + struct block_device *bdev = lc->dev->bdev; + struct dax_device *dax_dev = lc->dev->dax_dev; + sector_t dev_sector, sector = pgoff * PAGE_SECTORS; + + dev_sector = linear_map_sector(ti, sector); + ret = bdev_dax_pgoff(bdev, dev_sector, nr_pages << PAGE_SHIFT, &pgoff); + if (ret) + return ret; + return dax_zero_page_range(dax_dev, pgoff, nr_pages); +} + #else #define linear_dax_direct_access NULL #define linear_dax_copy_from_iter NULL #define linear_dax_copy_to_iter NULL +#define linear_dax_zero_page_range NULL #endif static struct target_type linear_target = { @@ -226,6 +243,7 @@ static struct target_type linear_target = { .direct_access = linear_dax_direct_access, .dax_copy_from_iter = linear_dax_copy_from_iter, .dax_copy_to_iter = linear_dax_copy_to_iter, + .dax_zero_page_range = linear_dax_zero_page_range, }; int __init dm_linear_init(void) diff --git a/drivers/md/dm-log-writes.c b/drivers/md/dm-log-writes.c index 99721c76225d..8ea20b56b4d6 100644 --- a/drivers/md/dm-log-writes.c +++ b/drivers/md/dm-log-writes.c @@ -994,10 +994,26 @@ static size_t log_writes_dax_copy_to_iter(struct dm_target *ti, return dax_copy_to_iter(lc->dev->dax_dev, pgoff, addr, bytes, i); } +static int log_writes_dax_zero_page_range(struct dm_target *ti, pgoff_t pgoff, + size_t nr_pages) +{ + int ret; + struct log_writes_c *lc = ti->private; + sector_t sector = pgoff * PAGE_SECTORS; + + ret = bdev_dax_pgoff(lc->dev->bdev, sector, nr_pages << PAGE_SHIFT, + &pgoff); + if (ret) + return ret; + return dax_zero_page_range(lc->dev->dax_dev, pgoff, + nr_pages << PAGE_SHIFT); +} + #else #define log_writes_dax_direct_access NULL #define log_writes_dax_copy_from_iter NULL #define log_writes_dax_copy_to_iter NULL +#define log_writes_dax_zero_page_range NULL #endif static struct target_type log_writes_target = { @@ -1016,6 +1032,7 @@ static struct target_type log_writes_target = { .direct_access = log_writes_dax_direct_access, .dax_copy_from_iter = log_writes_dax_copy_from_iter, .dax_copy_to_iter = log_writes_dax_copy_to_iter, + .dax_zero_page_range = log_writes_dax_zero_page_range, }; static int __init dm_log_writes_init(void) diff --git a/drivers/md/dm-stripe.c b/drivers/md/dm-stripe.c index 63bbcc20f49a..fa813c0f993d 100644 --- a/drivers/md/dm-stripe.c +++ b/drivers/md/dm-stripe.c @@ -360,10 +360,32 @@ static size_t stripe_dax_copy_to_iter(struct dm_target *ti, pgoff_t pgoff, return dax_copy_to_iter(dax_dev, pgoff, addr, bytes, i); } +static int stripe_dax_zero_page_range(struct dm_target *ti, pgoff_t pgoff, + size_t nr_pages) +{ + int ret; + sector_t dev_sector, sector = pgoff * PAGE_SECTORS; + struct stripe_c *sc = ti->private; + struct dax_device *dax_dev; + struct block_device *bdev; + uint32_t stripe; + + stripe_map_sector(sc, sector, &stripe, &dev_sector); + dev_sector += sc->stripe[stripe].physical_start; + dax_dev = sc->stripe[stripe].dev->dax_dev; + bdev = sc->stripe[stripe].dev->bdev; + + ret = bdev_dax_pgoff(bdev, dev_sector, nr_pages << PAGE_SHIFT, &pgoff); + if (ret) + return ret; + return dax_zero_page_range(dax_dev, pgoff, nr_pages); +} + #else #define stripe_dax_direct_access NULL #define stripe_dax_copy_from_iter NULL #define stripe_dax_copy_to_iter NULL +#define stripe_dax_zero_page_range NULL #endif /* @@ -486,6 +508,7 @@ static struct target_type stripe_target = { .direct_access = stripe_dax_direct_access, .dax_copy_from_iter = stripe_dax_copy_from_iter, .dax_copy_to_iter = stripe_dax_copy_to_iter, + .dax_zero_page_range = stripe_dax_zero_page_range, }; int __init dm_stripe_init(void) diff --git a/drivers/md/dm.c b/drivers/md/dm.c index b89f07ee2eff..aa72d9e757c1 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -1198,6 +1198,35 @@ static size_t dm_dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, return ret; } +static int dm_dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff, + size_t nr_pages) +{ + struct mapped_device *md = dax_get_private(dax_dev); + sector_t sector = pgoff * PAGE_SECTORS; + struct dm_target *ti; + int ret = -EIO; + int srcu_idx; + + ti = dm_dax_get_live_target(md, sector, &srcu_idx); + + if (!ti) + goto out; + if (WARN_ON(!ti->type->dax_zero_page_range)) { + /* + * ->zero_page_range() is mandatory dax operation. If we are + * here, something is wrong. + */ + dm_put_live_table(md, srcu_idx); + goto out; + } + ret = ti->type->dax_zero_page_range(ti, pgoff, nr_pages); + + out: + dm_put_live_table(md, srcu_idx); + + return ret; +} + /* * A target may call dm_accept_partial_bio only from the map routine. It is * allowed for all bio types except REQ_PREFLUSH, REQ_OP_ZONE_RESET, @@ -3199,6 +3228,7 @@ static const struct dax_operations dm_dax_ops = { .dax_supported = dm_dax_supported, .copy_from_iter = dm_dax_copy_from_iter, .copy_to_iter = dm_dax_copy_to_iter, + .zero_page_range = dm_dax_zero_page_range, }; /* diff --git a/include/linux/device-mapper.h b/include/linux/device-mapper.h index 475668c69dbc..af48d9da3916 100644 --- a/include/linux/device-mapper.h +++ b/include/linux/device-mapper.h @@ -141,6 +141,8 @@ typedef long (*dm_dax_direct_access_fn) (struct dm_target *ti, pgoff_t pgoff, long nr_pages, void **kaddr, pfn_t *pfn); typedef size_t (*dm_dax_copy_iter_fn)(struct dm_target *ti, pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i); +typedef int (*dm_dax_zero_page_range_fn)(struct dm_target *ti, pgoff_t pgoff, + size_t nr_pages); #define PAGE_SECTORS (PAGE_SIZE / 512) void dm_error(const char *message); @@ -195,6 +197,7 @@ struct target_type { dm_dax_direct_access_fn direct_access; dm_dax_copy_iter_fn dax_copy_from_iter; dm_dax_copy_iter_fn dax_copy_to_iter; + dm_dax_zero_page_range_fn dax_zero_page_range; /* For internal device-mapper use. */ struct list_head list; From 0a23f9ffa5ac20d3c95a6e850f4ee68e4034f781 Mon Sep 17 00:00:00 2001 From: Vivek Goyal Date: Fri, 28 Feb 2020 11:34:55 -0500 Subject: [PATCH 432/635] dax: Use new dax zero page method for zeroing a page Use new dax native zero page method for zeroing page if I/O is page aligned. Otherwise fall back to direct_access() + memcpy(). This gets rid of one of the depenendency on block device in dax path. Signed-off-by: Vivek Goyal Link: https://lore.kernel.org/r/20200228163456.1587-6-vgoyal@redhat.com Signed-off-by: Dan Williams --- fs/dax.c | 53 +++++++++++++++++++++++------------------------------ 1 file changed, 23 insertions(+), 30 deletions(-) diff --git a/fs/dax.c b/fs/dax.c index 35da144375a0..98ba3756163a 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -1038,47 +1038,40 @@ static vm_fault_t dax_load_hole(struct xa_state *xas, return ret; } -static bool dax_range_is_aligned(struct block_device *bdev, - unsigned int offset, unsigned int length) -{ - unsigned short sector_size = bdev_logical_block_size(bdev); - - if (!IS_ALIGNED(offset, sector_size)) - return false; - if (!IS_ALIGNED(length, sector_size)) - return false; - - return true; -} - int __dax_zero_page_range(struct block_device *bdev, struct dax_device *dax_dev, sector_t sector, unsigned int offset, unsigned int size) { - if (dax_range_is_aligned(bdev, offset, size)) { - sector_t start_sector = sector + (offset >> 9); + pgoff_t pgoff; + long rc, id; + void *kaddr; + bool page_aligned = false; - return blkdev_issue_zeroout(bdev, start_sector, - size >> 9, GFP_NOFS, 0); - } else { - pgoff_t pgoff; - long rc, id; - void *kaddr; - rc = bdev_dax_pgoff(bdev, sector, PAGE_SIZE, &pgoff); - if (rc) - return rc; + if (IS_ALIGNED(sector << SECTOR_SHIFT, PAGE_SIZE) && + IS_ALIGNED(size, PAGE_SIZE)) + page_aligned = true; - id = dax_read_lock(); + rc = bdev_dax_pgoff(bdev, sector, PAGE_SIZE, &pgoff); + if (rc) + return rc; + + id = dax_read_lock(); + + if (page_aligned) + rc = dax_zero_page_range(dax_dev, pgoff, size >> PAGE_SHIFT); + else rc = dax_direct_access(dax_dev, pgoff, 1, &kaddr, NULL); - if (rc < 0) { - dax_read_unlock(id); - return rc; - } + if (rc < 0) { + dax_read_unlock(id); + return rc; + } + + if (!page_aligned) { memset(kaddr + offset, 0, size); dax_flush(dax_dev, kaddr + offset, size); - dax_read_unlock(id); } + dax_read_unlock(id); return 0; } EXPORT_SYMBOL_GPL(__dax_zero_page_range); From 4f3b4f161d7a070d2181dbcf7fbd97c7631d5c24 Mon Sep 17 00:00:00 2001 From: Vivek Goyal Date: Fri, 28 Feb 2020 11:34:56 -0500 Subject: [PATCH 433/635] dax,iomap: Add helper dax_iomap_zero() to zero a range Add a helper dax_ioamp_zero() to zero a range. This patch basically merges __dax_zero_page_range() and iomap_dax_zero(). Suggested-by: Christoph Hellwig Signed-off-by: Vivek Goyal Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20200228163456.1587-7-vgoyal@redhat.com Signed-off-by: Dan Williams --- fs/dax.c | 16 ++++++++-------- fs/iomap/buffered-io.c | 9 +-------- include/linux/dax.h | 17 +++-------------- 3 files changed, 12 insertions(+), 30 deletions(-) diff --git a/fs/dax.c b/fs/dax.c index 98ba3756163a..11b16729b86f 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -1038,10 +1038,10 @@ static vm_fault_t dax_load_hole(struct xa_state *xas, return ret; } -int __dax_zero_page_range(struct block_device *bdev, - struct dax_device *dax_dev, sector_t sector, - unsigned int offset, unsigned int size) +int dax_iomap_zero(loff_t pos, unsigned offset, unsigned size, + struct iomap *iomap) { + sector_t sector = iomap_sector(iomap, pos & PAGE_MASK); pgoff_t pgoff; long rc, id; void *kaddr; @@ -1052,16 +1052,17 @@ int __dax_zero_page_range(struct block_device *bdev, IS_ALIGNED(size, PAGE_SIZE)) page_aligned = true; - rc = bdev_dax_pgoff(bdev, sector, PAGE_SIZE, &pgoff); + rc = bdev_dax_pgoff(iomap->bdev, sector, PAGE_SIZE, &pgoff); if (rc) return rc; id = dax_read_lock(); if (page_aligned) - rc = dax_zero_page_range(dax_dev, pgoff, size >> PAGE_SHIFT); + rc = dax_zero_page_range(iomap->dax_dev, pgoff, + size >> PAGE_SHIFT); else - rc = dax_direct_access(dax_dev, pgoff, 1, &kaddr, NULL); + rc = dax_direct_access(iomap->dax_dev, pgoff, 1, &kaddr, NULL); if (rc < 0) { dax_read_unlock(id); return rc; @@ -1069,12 +1070,11 @@ int __dax_zero_page_range(struct block_device *bdev, if (!page_aligned) { memset(kaddr + offset, 0, size); - dax_flush(dax_dev, kaddr + offset, size); + dax_flush(iomap->dax_dev, kaddr + offset, size); } dax_read_unlock(id); return 0; } -EXPORT_SYMBOL_GPL(__dax_zero_page_range); static loff_t dax_iomap_actor(struct inode *inode, loff_t pos, loff_t length, void *data, diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index 7c84c4c027c4..6f750da545e5 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -974,13 +974,6 @@ static int iomap_zero(struct inode *inode, loff_t pos, unsigned offset, return iomap_write_end(inode, pos, bytes, bytes, page, iomap, srcmap); } -static int iomap_dax_zero(loff_t pos, unsigned offset, unsigned bytes, - struct iomap *iomap) -{ - return __dax_zero_page_range(iomap->bdev, iomap->dax_dev, - iomap_sector(iomap, pos & PAGE_MASK), offset, bytes); -} - static loff_t iomap_zero_range_actor(struct inode *inode, loff_t pos, loff_t count, void *data, struct iomap *iomap, struct iomap *srcmap) @@ -1000,7 +993,7 @@ iomap_zero_range_actor(struct inode *inode, loff_t pos, loff_t count, bytes = min_t(loff_t, PAGE_SIZE - offset, count); if (IS_DAX(inode)) - status = iomap_dax_zero(pos, offset, bytes, iomap); + status = dax_iomap_zero(pos, offset, bytes, iomap); else status = iomap_zero(inode, pos, offset, bytes, iomap, srcmap); diff --git a/include/linux/dax.h b/include/linux/dax.h index 71735c430c05..d7af5d243f24 100644 --- a/include/linux/dax.h +++ b/include/linux/dax.h @@ -13,6 +13,7 @@ typedef unsigned long dax_entry_t; struct iomap_ops; +struct iomap; struct dax_device; struct dax_operations { /* @@ -214,20 +215,8 @@ vm_fault_t dax_finish_sync_fault(struct vm_fault *vmf, int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index); int dax_invalidate_mapping_entry_sync(struct address_space *mapping, pgoff_t index); - -#ifdef CONFIG_FS_DAX -int __dax_zero_page_range(struct block_device *bdev, - struct dax_device *dax_dev, sector_t sector, - unsigned int offset, unsigned int length); -#else -static inline int __dax_zero_page_range(struct block_device *bdev, - struct dax_device *dax_dev, sector_t sector, - unsigned int offset, unsigned int length) -{ - return -ENXIO; -} -#endif - +int dax_iomap_zero(loff_t pos, unsigned offset, unsigned size, + struct iomap *iomap); static inline bool dax_mapping(struct address_space *mapping) { return mapping->host && IS_DAX(mapping->host); From 4e4ced93794acb42adb19484132966defba8f3a6 Mon Sep 17 00:00:00 2001 From: Vivek Goyal Date: Wed, 1 Apr 2020 12:11:25 -0400 Subject: [PATCH 434/635] dax: Move mandatory ->zero_page_range() check in alloc_dax() zero_page_range() dax operation is mandatory for dax devices. Right now that check happens in dax_zero_page_range() function. Dan thinks that's too late and its better to do the check earlier in alloc_dax(). I also modified alloc_dax() to return pointer with error code in it in case of failure. Right now it returns NULL and caller assumes failure happened due to -ENOMEM. But with this ->zero_page_range() check, I need to return -EINVAL instead. Signed-off-by: Vivek Goyal Link: https://lore.kernel.org/r/20200401161125.GB9398@redhat.com Signed-off-by: Dan Williams --- drivers/dax/bus.c | 4 +++- drivers/dax/super.c | 14 +++++++++----- drivers/md/dm.c | 2 +- drivers/nvdimm/pmem.c | 4 ++-- drivers/s390/block/dcssblk.c | 5 +++-- 5 files changed, 18 insertions(+), 11 deletions(-) diff --git a/drivers/dax/bus.c b/drivers/dax/bus.c index 46e46047a1f7..df238c8b6ef2 100644 --- a/drivers/dax/bus.c +++ b/drivers/dax/bus.c @@ -421,8 +421,10 @@ struct dev_dax *__devm_create_dev_dax(struct dax_region *dax_region, int id, * device outside of mmap of the resulting character device. */ dax_dev = alloc_dax(dev_dax, NULL, NULL, DAXDEV_F_SYNC); - if (!dax_dev) + if (IS_ERR(dax_dev)) { + rc = PTR_ERR(dax_dev); goto err; + } /* a device_dax instance is dead while the driver is not attached */ kill_dax(dax_dev); diff --git a/drivers/dax/super.c b/drivers/dax/super.c index e498daf3c0d7..8e32345be0f7 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -349,9 +349,6 @@ int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff, { if (!dax_alive(dax_dev)) return -ENXIO; - - if (!dax_dev->ops->zero_page_range) - return -EOPNOTSUPP; /* * There are no callers that want to zero more than one page as of now. * Once users are there, this check can be removed after the @@ -571,9 +568,16 @@ struct dax_device *alloc_dax(void *private, const char *__host, dev_t devt; int minor; + if (ops && !ops->zero_page_range) { + pr_debug("%s: error: device does not provide dax" + " operation zero_page_range()\n", + __host ? __host : "Unknown"); + return ERR_PTR(-EINVAL); + } + host = kstrdup(__host, GFP_KERNEL); if (__host && !host) - return NULL; + return ERR_PTR(-ENOMEM); minor = ida_simple_get(&dax_minor_ida, 0, MINORMASK+1, GFP_KERNEL); if (minor < 0) @@ -596,7 +600,7 @@ struct dax_device *alloc_dax(void *private, const char *__host, ida_simple_remove(&dax_minor_ida, minor); err_minor: kfree(host); - return NULL; + return ERR_PTR(-ENOMEM); } EXPORT_SYMBOL_GPL(alloc_dax); diff --git a/drivers/md/dm.c b/drivers/md/dm.c index aa72d9e757c1..6504f0a358e5 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -2005,7 +2005,7 @@ static struct mapped_device *alloc_dev(int minor) if (IS_ENABLED(CONFIG_DAX_DRIVER)) { md->dax_dev = alloc_dax(md, md->disk->disk_name, &dm_dax_ops, 0); - if (!md->dax_dev) + if (IS_ERR(md->dax_dev)) goto bad; } diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index 5b774ddd0efb..715cb0696525 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -487,9 +487,9 @@ static int pmem_attach_disk(struct device *dev, if (is_nvdimm_sync(nd_region)) flags = DAXDEV_F_SYNC; dax_dev = alloc_dax(pmem, disk->disk_name, &pmem_dax_ops, flags); - if (!dax_dev) { + if (IS_ERR(dax_dev)) { put_disk(disk); - return -ENOMEM; + return PTR_ERR(dax_dev); } dax_write_cache(dax_dev, nvdimm_has_cache(nd_region)); pmem->dax_dev = dax_dev; diff --git a/drivers/s390/block/dcssblk.c b/drivers/s390/block/dcssblk.c index ab3e2898816c..9b05cf1a25c4 100644 --- a/drivers/s390/block/dcssblk.c +++ b/drivers/s390/block/dcssblk.c @@ -695,8 +695,9 @@ dcssblk_add_store(struct device *dev, struct device_attribute *attr, const char dev_info->dax_dev = alloc_dax(dev_info, dev_info->gd->disk_name, &dcssblk_dax_ops, DAXDEV_F_SYNC); - if (!dev_info->dax_dev) { - rc = -ENOMEM; + if (IS_ERR(dev_info->dax_dev)) { + rc = PTR_ERR(dev_info->dax_dev); + dev_info->dax_dev = NULL; goto put_dev; } From b5f13031cd8da1b8f1e277d03a773dc46f7cff11 Mon Sep 17 00:00:00 2001 From: YueHaibing Date: Mon, 30 Mar 2020 16:36:43 +0800 Subject: [PATCH 435/635] crypto: hisilicon - Fix build error When UACCE is m, CRYPTO_DEV_HISI_QM cannot be built-in. But CRYPTO_DEV_HISI_QM is selected by CRYPTO_DEV_HISI_SEC2 and CRYPTO_DEV_HISI_HPRE unconditionally, which may leads this: drivers/crypto/hisilicon/qm.o: In function 'qm_alloc_uacce': drivers/crypto/hisilicon/qm.c:1579: undefined reference to 'uacce_alloc' Add Kconfig dependency to enforce usable configurations. Fixes: 47c16b449921 ("crypto: hisilicon - qm depends on UACCE") Signed-off-by: YueHaibing Reviewed-by: Arnd Bergmann Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/Kconfig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/crypto/hisilicon/Kconfig b/drivers/crypto/hisilicon/Kconfig index 095850d01dcc..f09c6cf7823e 100644 --- a/drivers/crypto/hisilicon/Kconfig +++ b/drivers/crypto/hisilicon/Kconfig @@ -27,6 +27,7 @@ config CRYPTO_DEV_HISI_SEC2 select CRYPTO_SHA256 select CRYPTO_SHA512 depends on PCI && PCI_MSI + depends on UACCE || UACCE=n depends on ARM64 || (COMPILE_TEST && 64BIT) help Support for HiSilicon SEC Engine of version 2 in crypto subsystem. @@ -58,6 +59,7 @@ config CRYPTO_DEV_HISI_ZIP config CRYPTO_DEV_HISI_HPRE tristate "Support for HISI HPRE accelerator" depends on PCI && PCI_MSI + depends on UACCE || UACCE=n depends on ARM64 || (COMPILE_TEST && 64BIT) select CRYPTO_DEV_HISI_QM select CRYPTO_DH From 755bddd1e4eaf9178758bd554c60aaab46fc42ba Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Thu, 2 Apr 2020 00:10:12 +0100 Subject: [PATCH 436/635] crypto: marvell/octeontx - fix double free of ptr Currently in the case where eq->src != req->ds, the allocation of ptr is kfree'd at the end of the code block. However later on in the case where enc is not null any of the error return paths that return via the error handling return path end up performing an erroneous second kfree of ptr. Fix this by adding an error exit label error_free and only jump to this when ptr needs kfree'ing thus avoiding the double free issue. Addresses-Coverity: ("Double free") Fixes: 10b4f09491bf ("crypto: marvell - add the Virtual Function driver for CPT") Signed-off-by: Colin Ian King Signed-off-by: Herbert Xu --- drivers/crypto/marvell/octeontx/otx_cptvf_algs.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c b/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c index 946fb62949b2..06202bcffb33 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c +++ b/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c @@ -1161,13 +1161,13 @@ static inline u32 create_aead_null_output_list(struct aead_request *req, inputlen); if (status != inputlen) { status = -EINVAL; - goto error; + goto error_free; } status = sg_copy_from_buffer(req->dst, sg_nents(req->dst), ptr, inputlen); if (status != inputlen) { status = -EINVAL; - goto error; + goto error_free; } kfree(ptr); } @@ -1209,8 +1209,10 @@ static inline u32 create_aead_null_output_list(struct aead_request *req, req_info->outcnt = argcnt; return 0; -error: + +error_free: kfree(ptr); +error: return status; } From 46a010dd6896045484c7749b3d30b685c649eb18 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 24 Mar 2020 10:41:51 +0100 Subject: [PATCH 437/635] kVM SVM: Move SVM related files to own sub-directory Move svm.c and pmu_amd.c into their own arch/x86/kvm/svm/ subdirectory. Signed-off-by: Joerg Roedel Message-Id: <20200324094154.32352-2-joro@8bytes.org> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/Makefile | 2 +- arch/x86/kvm/{pmu_amd.c => svm/pmu.c} | 0 arch/x86/kvm/{ => svm}/svm.c | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename arch/x86/kvm/{pmu_amd.c => svm/pmu.c} (100%) rename arch/x86/kvm/{ => svm}/svm.c (100%) diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index e553f0fdd87d..0ef050982c37 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -14,7 +14,7 @@ kvm-y += x86.o emulate.o i8259.o irq.o lapic.o \ hyperv.o debugfs.o mmu/mmu.o mmu/page_track.o kvm-intel-y += vmx/vmx.o vmx/vmenter.o vmx/pmu_intel.o vmx/vmcs12.o vmx/evmcs.o vmx/nested.o -kvm-amd-y += svm.o pmu_amd.o +kvm-amd-y += svm/svm.o svm/pmu.o obj-$(CONFIG_KVM) += kvm.o obj-$(CONFIG_KVM_INTEL) += kvm-intel.o diff --git a/arch/x86/kvm/pmu_amd.c b/arch/x86/kvm/svm/pmu.c similarity index 100% rename from arch/x86/kvm/pmu_amd.c rename to arch/x86/kvm/svm/pmu.c diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm/svm.c similarity index 100% rename from arch/x86/kvm/svm.c rename to arch/x86/kvm/svm/svm.c From 883b0a91f41ab705daa04c24e59d708e457a0bed Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 24 Mar 2020 10:41:52 +0100 Subject: [PATCH 438/635] KVM: SVM: Move Nested SVM Implementation to nested.c Split out the code for the nested SVM implementation and move it to a separate file. Signed-off-by: Joerg Roedel Message-Id: <20200324094154.32352-3-joro@8bytes.org> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/Makefile | 2 +- arch/x86/kvm/svm/nested.c | 823 ++++++++++++++++++++++++++ arch/x86/kvm/svm/svm.c | 1155 +------------------------------------ arch/x86/kvm/svm/svm.h | 381 ++++++++++++ 4 files changed, 1216 insertions(+), 1145 deletions(-) create mode 100644 arch/x86/kvm/svm/nested.c create mode 100644 arch/x86/kvm/svm/svm.h diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index 0ef050982c37..0acec73ef236 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -14,7 +14,7 @@ kvm-y += x86.o emulate.o i8259.o irq.o lapic.o \ hyperv.o debugfs.o mmu/mmu.o mmu/page_track.o kvm-intel-y += vmx/vmx.o vmx/vmenter.o vmx/pmu_intel.o vmx/vmcs12.o vmx/evmcs.o vmx/nested.o -kvm-amd-y += svm/svm.o svm/pmu.o +kvm-amd-y += svm/svm.o svm/pmu.o svm/nested.o obj-$(CONFIG_KVM) += kvm.o obj-$(CONFIG_KVM_INTEL) += kvm-intel.o diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c new file mode 100644 index 000000000000..90a1ca939627 --- /dev/null +++ b/arch/x86/kvm/svm/nested.c @@ -0,0 +1,823 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Kernel-based Virtual Machine driver for Linux + * + * AMD SVM support + * + * Copyright (C) 2006 Qumranet, Inc. + * Copyright 2010 Red Hat, Inc. and/or its affiliates. + * + * Authors: + * Yaniv Kamay + * Avi Kivity + */ + +#define pr_fmt(fmt) "SVM: " fmt + +#include +#include +#include + +#include + +#include "kvm_emulate.h" +#include "trace.h" +#include "mmu.h" +#include "x86.h" +#include "svm.h" + +static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu, + struct x86_exception *fault) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + if (svm->vmcb->control.exit_code != SVM_EXIT_NPF) { + /* + * TODO: track the cause of the nested page fault, and + * correctly fill in the high bits of exit_info_1. + */ + svm->vmcb->control.exit_code = SVM_EXIT_NPF; + svm->vmcb->control.exit_code_hi = 0; + svm->vmcb->control.exit_info_1 = (1ULL << 32); + svm->vmcb->control.exit_info_2 = fault->address; + } + + svm->vmcb->control.exit_info_1 &= ~0xffffffffULL; + svm->vmcb->control.exit_info_1 |= fault->error_code; + + /* + * The present bit is always zero for page structure faults on real + * hardware. + */ + if (svm->vmcb->control.exit_info_1 & (2ULL << 32)) + svm->vmcb->control.exit_info_1 &= ~1; + + nested_svm_vmexit(svm); +} + +static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index) +{ + struct vcpu_svm *svm = to_svm(vcpu); + u64 cr3 = svm->nested.nested_cr3; + u64 pdpte; + int ret; + + ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(__sme_clr(cr3)), &pdpte, + offset_in_page(cr3) + index * 8, 8); + if (ret) + return 0; + return pdpte; +} + +static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + return svm->nested.nested_cr3; +} + +static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu) +{ + WARN_ON(mmu_is_nested(vcpu)); + + vcpu->arch.mmu = &vcpu->arch.guest_mmu; + kvm_init_shadow_mmu(vcpu); + vcpu->arch.mmu->get_guest_pgd = nested_svm_get_tdp_cr3; + vcpu->arch.mmu->get_pdptr = nested_svm_get_tdp_pdptr; + vcpu->arch.mmu->inject_page_fault = nested_svm_inject_npf_exit; + vcpu->arch.mmu->shadow_root_level = kvm_x86_ops.get_tdp_level(vcpu); + reset_shadow_zero_bits_mask(vcpu, vcpu->arch.mmu); + vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu; +} + +static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu) +{ + vcpu->arch.mmu = &vcpu->arch.root_mmu; + vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; +} + +void recalc_intercepts(struct vcpu_svm *svm) +{ + struct vmcb_control_area *c, *h; + struct nested_state *g; + + mark_dirty(svm->vmcb, VMCB_INTERCEPTS); + + if (!is_guest_mode(&svm->vcpu)) + return; + + c = &svm->vmcb->control; + h = &svm->nested.hsave->control; + g = &svm->nested; + + c->intercept_cr = h->intercept_cr; + c->intercept_dr = h->intercept_dr; + c->intercept_exceptions = h->intercept_exceptions; + c->intercept = h->intercept; + + if (svm->vcpu.arch.hflags & HF_VINTR_MASK) { + /* We only want the cr8 intercept bits of L1 */ + c->intercept_cr &= ~(1U << INTERCEPT_CR8_READ); + c->intercept_cr &= ~(1U << INTERCEPT_CR8_WRITE); + + /* + * Once running L2 with HF_VINTR_MASK, EFLAGS.IF does not + * affect any interrupt we may want to inject; therefore, + * interrupt window vmexits are irrelevant to L0. + */ + c->intercept &= ~(1ULL << INTERCEPT_VINTR); + } + + /* We don't want to see VMMCALLs from a nested guest */ + c->intercept &= ~(1ULL << INTERCEPT_VMMCALL); + + c->intercept_cr |= g->intercept_cr; + c->intercept_dr |= g->intercept_dr; + c->intercept_exceptions |= g->intercept_exceptions; + c->intercept |= g->intercept; +} + +static void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *from_vmcb) +{ + struct vmcb_control_area *dst = &dst_vmcb->control; + struct vmcb_control_area *from = &from_vmcb->control; + + dst->intercept_cr = from->intercept_cr; + dst->intercept_dr = from->intercept_dr; + dst->intercept_exceptions = from->intercept_exceptions; + dst->intercept = from->intercept; + dst->iopm_base_pa = from->iopm_base_pa; + dst->msrpm_base_pa = from->msrpm_base_pa; + dst->tsc_offset = from->tsc_offset; + dst->asid = from->asid; + dst->tlb_ctl = from->tlb_ctl; + dst->int_ctl = from->int_ctl; + dst->int_vector = from->int_vector; + dst->int_state = from->int_state; + dst->exit_code = from->exit_code; + dst->exit_code_hi = from->exit_code_hi; + dst->exit_info_1 = from->exit_info_1; + dst->exit_info_2 = from->exit_info_2; + dst->exit_int_info = from->exit_int_info; + dst->exit_int_info_err = from->exit_int_info_err; + dst->nested_ctl = from->nested_ctl; + dst->event_inj = from->event_inj; + dst->event_inj_err = from->event_inj_err; + dst->nested_cr3 = from->nested_cr3; + dst->virt_ext = from->virt_ext; + dst->pause_filter_count = from->pause_filter_count; + dst->pause_filter_thresh = from->pause_filter_thresh; +} + +static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm) +{ + /* + * This function merges the msr permission bitmaps of kvm and the + * nested vmcb. It is optimized in that it only merges the parts where + * the kvm msr permission bitmap may contain zero bits + */ + int i; + + if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT))) + return true; + + for (i = 0; i < MSRPM_OFFSETS; i++) { + u32 value, p; + u64 offset; + + if (msrpm_offsets[i] == 0xffffffff) + break; + + p = msrpm_offsets[i]; + offset = svm->nested.vmcb_msrpm + (p * 4); + + if (kvm_vcpu_read_guest(&svm->vcpu, offset, &value, 4)) + return false; + + svm->nested.msrpm[p] = svm->msrpm[p] | value; + } + + svm->vmcb->control.msrpm_base_pa = __sme_set(__pa(svm->nested.msrpm)); + + return true; +} + +static bool nested_vmcb_checks(struct vmcb *vmcb) +{ + if ((vmcb->save.efer & EFER_SVME) == 0) + return false; + + if ((vmcb->control.intercept & (1ULL << INTERCEPT_VMRUN)) == 0) + return false; + + if (vmcb->control.asid == 0) + return false; + + if ((vmcb->control.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) && + !npt_enabled) + return false; + + return true; +} + +void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, + struct vmcb *nested_vmcb, struct kvm_host_map *map) +{ + bool evaluate_pending_interrupts = + is_intercept(svm, INTERCEPT_VINTR) || + is_intercept(svm, INTERCEPT_IRET); + + if (kvm_get_rflags(&svm->vcpu) & X86_EFLAGS_IF) + svm->vcpu.arch.hflags |= HF_HIF_MASK; + else + svm->vcpu.arch.hflags &= ~HF_HIF_MASK; + + if (nested_vmcb->control.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) { + svm->nested.nested_cr3 = nested_vmcb->control.nested_cr3; + nested_svm_init_mmu_context(&svm->vcpu); + } + + /* Load the nested guest state */ + svm->vmcb->save.es = nested_vmcb->save.es; + svm->vmcb->save.cs = nested_vmcb->save.cs; + svm->vmcb->save.ss = nested_vmcb->save.ss; + svm->vmcb->save.ds = nested_vmcb->save.ds; + svm->vmcb->save.gdtr = nested_vmcb->save.gdtr; + svm->vmcb->save.idtr = nested_vmcb->save.idtr; + kvm_set_rflags(&svm->vcpu, nested_vmcb->save.rflags); + svm_set_efer(&svm->vcpu, nested_vmcb->save.efer); + svm_set_cr0(&svm->vcpu, nested_vmcb->save.cr0); + svm_set_cr4(&svm->vcpu, nested_vmcb->save.cr4); + if (npt_enabled) { + svm->vmcb->save.cr3 = nested_vmcb->save.cr3; + svm->vcpu.arch.cr3 = nested_vmcb->save.cr3; + } else + (void)kvm_set_cr3(&svm->vcpu, nested_vmcb->save.cr3); + + /* Guest paging mode is active - reset mmu */ + kvm_mmu_reset_context(&svm->vcpu); + + svm->vmcb->save.cr2 = svm->vcpu.arch.cr2 = nested_vmcb->save.cr2; + kvm_rax_write(&svm->vcpu, nested_vmcb->save.rax); + kvm_rsp_write(&svm->vcpu, nested_vmcb->save.rsp); + kvm_rip_write(&svm->vcpu, nested_vmcb->save.rip); + + /* In case we don't even reach vcpu_run, the fields are not updated */ + svm->vmcb->save.rax = nested_vmcb->save.rax; + svm->vmcb->save.rsp = nested_vmcb->save.rsp; + svm->vmcb->save.rip = nested_vmcb->save.rip; + svm->vmcb->save.dr7 = nested_vmcb->save.dr7; + svm->vmcb->save.dr6 = nested_vmcb->save.dr6; + svm->vmcb->save.cpl = nested_vmcb->save.cpl; + + svm->nested.vmcb_msrpm = nested_vmcb->control.msrpm_base_pa & ~0x0fffULL; + svm->nested.vmcb_iopm = nested_vmcb->control.iopm_base_pa & ~0x0fffULL; + + /* cache intercepts */ + svm->nested.intercept_cr = nested_vmcb->control.intercept_cr; + svm->nested.intercept_dr = nested_vmcb->control.intercept_dr; + svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions; + svm->nested.intercept = nested_vmcb->control.intercept; + + svm_flush_tlb(&svm->vcpu, true); + svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK; + if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK) + svm->vcpu.arch.hflags |= HF_VINTR_MASK; + else + svm->vcpu.arch.hflags &= ~HF_VINTR_MASK; + + svm->vcpu.arch.tsc_offset += nested_vmcb->control.tsc_offset; + svm->vmcb->control.tsc_offset = svm->vcpu.arch.tsc_offset; + + svm->vmcb->control.virt_ext = nested_vmcb->control.virt_ext; + svm->vmcb->control.int_vector = nested_vmcb->control.int_vector; + svm->vmcb->control.int_state = nested_vmcb->control.int_state; + svm->vmcb->control.event_inj = nested_vmcb->control.event_inj; + svm->vmcb->control.event_inj_err = nested_vmcb->control.event_inj_err; + + svm->vmcb->control.pause_filter_count = + nested_vmcb->control.pause_filter_count; + svm->vmcb->control.pause_filter_thresh = + nested_vmcb->control.pause_filter_thresh; + + kvm_vcpu_unmap(&svm->vcpu, map, true); + + /* Enter Guest-Mode */ + enter_guest_mode(&svm->vcpu); + + /* + * Merge guest and host intercepts - must be called with vcpu in + * guest-mode to take affect here + */ + recalc_intercepts(svm); + + svm->nested.vmcb = vmcb_gpa; + + /* + * If L1 had a pending IRQ/NMI before executing VMRUN, + * which wasn't delivered because it was disallowed (e.g. + * interrupts disabled), L0 needs to evaluate if this pending + * event should cause an exit from L2 to L1 or be delivered + * directly to L2. + * + * Usually this would be handled by the processor noticing an + * IRQ/NMI window request. However, VMRUN can unblock interrupts + * by implicitly setting GIF, so force L0 to perform pending event + * evaluation by requesting a KVM_REQ_EVENT. + */ + enable_gif(svm); + if (unlikely(evaluate_pending_interrupts)) + kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); + + mark_all_dirty(svm->vmcb); +} + +int nested_svm_vmrun(struct vcpu_svm *svm) +{ + int ret; + struct vmcb *nested_vmcb; + struct vmcb *hsave = svm->nested.hsave; + struct vmcb *vmcb = svm->vmcb; + struct kvm_host_map map; + u64 vmcb_gpa; + + vmcb_gpa = svm->vmcb->save.rax; + + ret = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(vmcb_gpa), &map); + if (ret == -EINVAL) { + kvm_inject_gp(&svm->vcpu, 0); + return 1; + } else if (ret) { + return kvm_skip_emulated_instruction(&svm->vcpu); + } + + ret = kvm_skip_emulated_instruction(&svm->vcpu); + + nested_vmcb = map.hva; + + if (!nested_vmcb_checks(nested_vmcb)) { + nested_vmcb->control.exit_code = SVM_EXIT_ERR; + nested_vmcb->control.exit_code_hi = 0; + nested_vmcb->control.exit_info_1 = 0; + nested_vmcb->control.exit_info_2 = 0; + + kvm_vcpu_unmap(&svm->vcpu, &map, true); + + return ret; + } + + trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb_gpa, + nested_vmcb->save.rip, + nested_vmcb->control.int_ctl, + nested_vmcb->control.event_inj, + nested_vmcb->control.nested_ctl); + + trace_kvm_nested_intercepts(nested_vmcb->control.intercept_cr & 0xffff, + nested_vmcb->control.intercept_cr >> 16, + nested_vmcb->control.intercept_exceptions, + nested_vmcb->control.intercept); + + /* Clear internal status */ + kvm_clear_exception_queue(&svm->vcpu); + kvm_clear_interrupt_queue(&svm->vcpu); + + /* + * Save the old vmcb, so we don't need to pick what we save, but can + * restore everything when a VMEXIT occurs + */ + hsave->save.es = vmcb->save.es; + hsave->save.cs = vmcb->save.cs; + hsave->save.ss = vmcb->save.ss; + hsave->save.ds = vmcb->save.ds; + hsave->save.gdtr = vmcb->save.gdtr; + hsave->save.idtr = vmcb->save.idtr; + hsave->save.efer = svm->vcpu.arch.efer; + hsave->save.cr0 = kvm_read_cr0(&svm->vcpu); + hsave->save.cr4 = svm->vcpu.arch.cr4; + hsave->save.rflags = kvm_get_rflags(&svm->vcpu); + hsave->save.rip = kvm_rip_read(&svm->vcpu); + hsave->save.rsp = vmcb->save.rsp; + hsave->save.rax = vmcb->save.rax; + if (npt_enabled) + hsave->save.cr3 = vmcb->save.cr3; + else + hsave->save.cr3 = kvm_read_cr3(&svm->vcpu); + + copy_vmcb_control_area(hsave, vmcb); + + enter_svm_guest_mode(svm, vmcb_gpa, nested_vmcb, &map); + + if (!nested_svm_vmrun_msrpm(svm)) { + svm->vmcb->control.exit_code = SVM_EXIT_ERR; + svm->vmcb->control.exit_code_hi = 0; + svm->vmcb->control.exit_info_1 = 0; + svm->vmcb->control.exit_info_2 = 0; + + nested_svm_vmexit(svm); + } + + return ret; +} + +void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb) +{ + to_vmcb->save.fs = from_vmcb->save.fs; + to_vmcb->save.gs = from_vmcb->save.gs; + to_vmcb->save.tr = from_vmcb->save.tr; + to_vmcb->save.ldtr = from_vmcb->save.ldtr; + to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base; + to_vmcb->save.star = from_vmcb->save.star; + to_vmcb->save.lstar = from_vmcb->save.lstar; + to_vmcb->save.cstar = from_vmcb->save.cstar; + to_vmcb->save.sfmask = from_vmcb->save.sfmask; + to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs; + to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp; + to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip; +} + +int nested_svm_vmexit(struct vcpu_svm *svm) +{ + int rc; + struct vmcb *nested_vmcb; + struct vmcb *hsave = svm->nested.hsave; + struct vmcb *vmcb = svm->vmcb; + struct kvm_host_map map; + + trace_kvm_nested_vmexit_inject(vmcb->control.exit_code, + vmcb->control.exit_info_1, + vmcb->control.exit_info_2, + vmcb->control.exit_int_info, + vmcb->control.exit_int_info_err, + KVM_ISA_SVM); + + rc = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(svm->nested.vmcb), &map); + if (rc) { + if (rc == -EINVAL) + kvm_inject_gp(&svm->vcpu, 0); + return 1; + } + + nested_vmcb = map.hva; + + /* Exit Guest-Mode */ + leave_guest_mode(&svm->vcpu); + svm->nested.vmcb = 0; + + /* Give the current vmcb to the guest */ + disable_gif(svm); + + nested_vmcb->save.es = vmcb->save.es; + nested_vmcb->save.cs = vmcb->save.cs; + nested_vmcb->save.ss = vmcb->save.ss; + nested_vmcb->save.ds = vmcb->save.ds; + nested_vmcb->save.gdtr = vmcb->save.gdtr; + nested_vmcb->save.idtr = vmcb->save.idtr; + nested_vmcb->save.efer = svm->vcpu.arch.efer; + nested_vmcb->save.cr0 = kvm_read_cr0(&svm->vcpu); + nested_vmcb->save.cr3 = kvm_read_cr3(&svm->vcpu); + nested_vmcb->save.cr2 = vmcb->save.cr2; + nested_vmcb->save.cr4 = svm->vcpu.arch.cr4; + nested_vmcb->save.rflags = kvm_get_rflags(&svm->vcpu); + nested_vmcb->save.rip = vmcb->save.rip; + nested_vmcb->save.rsp = vmcb->save.rsp; + nested_vmcb->save.rax = vmcb->save.rax; + nested_vmcb->save.dr7 = vmcb->save.dr7; + nested_vmcb->save.dr6 = vmcb->save.dr6; + nested_vmcb->save.cpl = vmcb->save.cpl; + + nested_vmcb->control.int_ctl = vmcb->control.int_ctl; + nested_vmcb->control.int_vector = vmcb->control.int_vector; + nested_vmcb->control.int_state = vmcb->control.int_state; + nested_vmcb->control.exit_code = vmcb->control.exit_code; + nested_vmcb->control.exit_code_hi = vmcb->control.exit_code_hi; + nested_vmcb->control.exit_info_1 = vmcb->control.exit_info_1; + nested_vmcb->control.exit_info_2 = vmcb->control.exit_info_2; + nested_vmcb->control.exit_int_info = vmcb->control.exit_int_info; + nested_vmcb->control.exit_int_info_err = vmcb->control.exit_int_info_err; + + if (svm->nrips_enabled) + nested_vmcb->control.next_rip = vmcb->control.next_rip; + + /* + * If we emulate a VMRUN/#VMEXIT in the same host #vmexit cycle we have + * to make sure that we do not lose injected events. So check event_inj + * here and copy it to exit_int_info if it is valid. + * Exit_int_info and event_inj can't be both valid because the case + * below only happens on a VMRUN instruction intercept which has + * no valid exit_int_info set. + */ + if (vmcb->control.event_inj & SVM_EVTINJ_VALID) { + struct vmcb_control_area *nc = &nested_vmcb->control; + + nc->exit_int_info = vmcb->control.event_inj; + nc->exit_int_info_err = vmcb->control.event_inj_err; + } + + nested_vmcb->control.tlb_ctl = 0; + nested_vmcb->control.event_inj = 0; + nested_vmcb->control.event_inj_err = 0; + + nested_vmcb->control.pause_filter_count = + svm->vmcb->control.pause_filter_count; + nested_vmcb->control.pause_filter_thresh = + svm->vmcb->control.pause_filter_thresh; + + /* We always set V_INTR_MASKING and remember the old value in hflags */ + if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK)) + nested_vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK; + + /* Restore the original control entries */ + copy_vmcb_control_area(vmcb, hsave); + + svm->vcpu.arch.tsc_offset = svm->vmcb->control.tsc_offset; + kvm_clear_exception_queue(&svm->vcpu); + kvm_clear_interrupt_queue(&svm->vcpu); + + svm->nested.nested_cr3 = 0; + + /* Restore selected save entries */ + svm->vmcb->save.es = hsave->save.es; + svm->vmcb->save.cs = hsave->save.cs; + svm->vmcb->save.ss = hsave->save.ss; + svm->vmcb->save.ds = hsave->save.ds; + svm->vmcb->save.gdtr = hsave->save.gdtr; + svm->vmcb->save.idtr = hsave->save.idtr; + kvm_set_rflags(&svm->vcpu, hsave->save.rflags); + svm_set_efer(&svm->vcpu, hsave->save.efer); + svm_set_cr0(&svm->vcpu, hsave->save.cr0 | X86_CR0_PE); + svm_set_cr4(&svm->vcpu, hsave->save.cr4); + if (npt_enabled) { + svm->vmcb->save.cr3 = hsave->save.cr3; + svm->vcpu.arch.cr3 = hsave->save.cr3; + } else { + (void)kvm_set_cr3(&svm->vcpu, hsave->save.cr3); + } + kvm_rax_write(&svm->vcpu, hsave->save.rax); + kvm_rsp_write(&svm->vcpu, hsave->save.rsp); + kvm_rip_write(&svm->vcpu, hsave->save.rip); + svm->vmcb->save.dr7 = 0; + svm->vmcb->save.cpl = 0; + svm->vmcb->control.exit_int_info = 0; + + mark_all_dirty(svm->vmcb); + + kvm_vcpu_unmap(&svm->vcpu, &map, true); + + nested_svm_uninit_mmu_context(&svm->vcpu); + kvm_mmu_reset_context(&svm->vcpu); + kvm_mmu_load(&svm->vcpu); + + /* + * Drop what we picked up for L2 via svm_complete_interrupts() so it + * doesn't end up in L1. + */ + svm->vcpu.arch.nmi_injected = false; + kvm_clear_exception_queue(&svm->vcpu); + kvm_clear_interrupt_queue(&svm->vcpu); + + return 0; +} + +static int nested_svm_exit_handled_msr(struct vcpu_svm *svm) +{ + u32 offset, msr, value; + int write, mask; + + if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT))) + return NESTED_EXIT_HOST; + + msr = svm->vcpu.arch.regs[VCPU_REGS_RCX]; + offset = svm_msrpm_offset(msr); + write = svm->vmcb->control.exit_info_1 & 1; + mask = 1 << ((2 * (msr & 0xf)) + write); + + if (offset == MSR_INVALID) + return NESTED_EXIT_DONE; + + /* Offset is in 32 bit units but need in 8 bit units */ + offset *= 4; + + if (kvm_vcpu_read_guest(&svm->vcpu, svm->nested.vmcb_msrpm + offset, &value, 4)) + return NESTED_EXIT_DONE; + + return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST; +} + +/* DB exceptions for our internal use must not cause vmexit */ +static int nested_svm_intercept_db(struct vcpu_svm *svm) +{ + unsigned long dr6; + + /* if we're not singlestepping, it's not ours */ + if (!svm->nmi_singlestep) + return NESTED_EXIT_DONE; + + /* if it's not a singlestep exception, it's not ours */ + if (kvm_get_dr(&svm->vcpu, 6, &dr6)) + return NESTED_EXIT_DONE; + if (!(dr6 & DR6_BS)) + return NESTED_EXIT_DONE; + + /* if the guest is singlestepping, it should get the vmexit */ + if (svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF) { + disable_nmi_singlestep(svm); + return NESTED_EXIT_DONE; + } + + /* it's ours, the nested hypervisor must not see this one */ + return NESTED_EXIT_HOST; +} + +static int nested_svm_intercept_ioio(struct vcpu_svm *svm) +{ + unsigned port, size, iopm_len; + u16 val, mask; + u8 start_bit; + u64 gpa; + + if (!(svm->nested.intercept & (1ULL << INTERCEPT_IOIO_PROT))) + return NESTED_EXIT_HOST; + + port = svm->vmcb->control.exit_info_1 >> 16; + size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >> + SVM_IOIO_SIZE_SHIFT; + gpa = svm->nested.vmcb_iopm + (port / 8); + start_bit = port % 8; + iopm_len = (start_bit + size > 8) ? 2 : 1; + mask = (0xf >> (4 - size)) << start_bit; + val = 0; + + if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len)) + return NESTED_EXIT_DONE; + + return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST; +} + +static int nested_svm_intercept(struct vcpu_svm *svm) +{ + u32 exit_code = svm->vmcb->control.exit_code; + int vmexit = NESTED_EXIT_HOST; + + switch (exit_code) { + case SVM_EXIT_MSR: + vmexit = nested_svm_exit_handled_msr(svm); + break; + case SVM_EXIT_IOIO: + vmexit = nested_svm_intercept_ioio(svm); + break; + case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: { + u32 bit = 1U << (exit_code - SVM_EXIT_READ_CR0); + if (svm->nested.intercept_cr & bit) + vmexit = NESTED_EXIT_DONE; + break; + } + case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: { + u32 bit = 1U << (exit_code - SVM_EXIT_READ_DR0); + if (svm->nested.intercept_dr & bit) + vmexit = NESTED_EXIT_DONE; + break; + } + case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: { + u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE); + if (svm->nested.intercept_exceptions & excp_bits) { + if (exit_code == SVM_EXIT_EXCP_BASE + DB_VECTOR) + vmexit = nested_svm_intercept_db(svm); + else + vmexit = NESTED_EXIT_DONE; + } + /* async page fault always cause vmexit */ + else if ((exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) && + svm->vcpu.arch.exception.nested_apf != 0) + vmexit = NESTED_EXIT_DONE; + break; + } + case SVM_EXIT_ERR: { + vmexit = NESTED_EXIT_DONE; + break; + } + default: { + u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR); + if (svm->nested.intercept & exit_bits) + vmexit = NESTED_EXIT_DONE; + } + } + + return vmexit; +} + +int nested_svm_exit_handled(struct vcpu_svm *svm) +{ + int vmexit; + + vmexit = nested_svm_intercept(svm); + + if (vmexit == NESTED_EXIT_DONE) + nested_svm_vmexit(svm); + + return vmexit; +} + +int nested_svm_check_permissions(struct vcpu_svm *svm) +{ + if (!(svm->vcpu.arch.efer & EFER_SVME) || + !is_paging(&svm->vcpu)) { + kvm_queue_exception(&svm->vcpu, UD_VECTOR); + return 1; + } + + if (svm->vmcb->save.cpl) { + kvm_inject_gp(&svm->vcpu, 0); + return 1; + } + + return 0; +} + +int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr, + bool has_error_code, u32 error_code) +{ + int vmexit; + + if (!is_guest_mode(&svm->vcpu)) + return 0; + + vmexit = nested_svm_intercept(svm); + if (vmexit != NESTED_EXIT_DONE) + return 0; + + svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr; + svm->vmcb->control.exit_code_hi = 0; + svm->vmcb->control.exit_info_1 = error_code; + + /* + * EXITINFO2 is undefined for all exception intercepts other + * than #PF. + */ + if (svm->vcpu.arch.exception.nested_apf) + svm->vmcb->control.exit_info_2 = svm->vcpu.arch.apf.nested_apf_token; + else if (svm->vcpu.arch.exception.has_payload) + svm->vmcb->control.exit_info_2 = svm->vcpu.arch.exception.payload; + else + svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2; + + svm->nested.exit_required = true; + return vmexit; +} + +static void nested_svm_intr(struct vcpu_svm *svm) +{ + svm->vmcb->control.exit_code = SVM_EXIT_INTR; + svm->vmcb->control.exit_info_1 = 0; + svm->vmcb->control.exit_info_2 = 0; + + /* nested_svm_vmexit this gets called afterwards from handle_exit */ + svm->nested.exit_required = true; + trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip); +} + +static bool nested_exit_on_intr(struct vcpu_svm *svm) +{ + return (svm->nested.intercept & 1ULL); +} + +int svm_check_nested_events(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + bool block_nested_events = + kvm_event_needs_reinjection(vcpu) || svm->nested.exit_required; + + if (kvm_cpu_has_interrupt(vcpu) && nested_exit_on_intr(svm)) { + if (block_nested_events) + return -EBUSY; + nested_svm_intr(svm); + return 0; + } + + return 0; +} + +int nested_svm_exit_special(struct vcpu_svm *svm) +{ + u32 exit_code = svm->vmcb->control.exit_code; + + switch (exit_code) { + case SVM_EXIT_INTR: + case SVM_EXIT_NMI: + case SVM_EXIT_EXCP_BASE + MC_VECTOR: + return NESTED_EXIT_HOST; + case SVM_EXIT_NPF: + /* For now we are always handling NPFs when using them */ + if (npt_enabled) + return NESTED_EXIT_HOST; + break; + case SVM_EXIT_EXCP_BASE + PF_VECTOR: + /* When we're shadowing, trap PFs, but not async PF */ + if (!npt_enabled && svm->vcpu.arch.apf.host_apf_reason == 0) + return NESTED_EXIT_HOST; + break; + default: + break; + } + + return NESTED_EXIT_CONTINUE; +} diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 851e9cc79930..a24e2de4acda 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -53,6 +53,8 @@ #include #include "trace.h" +#include "svm.h" + #define __ex(x) __kvm_handle_fault_on_reboot(x) MODULE_AUTHOR("Qumranet"); @@ -82,10 +84,6 @@ MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id); #define SVM_AVIC_DOORBELL 0xc001011b -#define NESTED_EXIT_HOST 0 /* Exit handled on host level */ -#define NESTED_EXIT_DONE 1 /* Exit caused nested vmexit */ -#define NESTED_EXIT_CONTINUE 2 /* Further checks needed */ - #define DEBUGCTL_RESERVED_BITS (~(0x3fULL)) #define TSC_RATIO_RSVD 0xffffff0000000000ULL @@ -119,68 +117,7 @@ MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id); static bool erratum_383_found __read_mostly; -static const u32 host_save_user_msrs[] = { -#ifdef CONFIG_X86_64 - MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE, - MSR_FS_BASE, -#endif - MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP, - MSR_TSC_AUX, -}; - -#define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs) - -struct kvm_sev_info { - bool active; /* SEV enabled guest */ - unsigned int asid; /* ASID used for this guest */ - unsigned int handle; /* SEV firmware handle */ - int fd; /* SEV device fd */ - unsigned long pages_locked; /* Number of pages locked */ - struct list_head regions_list; /* List of registered regions */ -}; - -struct kvm_svm { - struct kvm kvm; - - /* Struct members for AVIC */ - u32 avic_vm_id; - struct page *avic_logical_id_table_page; - struct page *avic_physical_id_table_page; - struct hlist_node hnode; - - struct kvm_sev_info sev_info; -}; - -struct kvm_vcpu; - -struct nested_state { - struct vmcb *hsave; - u64 hsave_msr; - u64 vm_cr_msr; - u64 vmcb; - - /* These are the merged vectors */ - u32 *msrpm; - - /* gpa pointers to the real vectors */ - u64 vmcb_msrpm; - u64 vmcb_iopm; - - /* A VMEXIT is required but not yet emulated */ - bool exit_required; - - /* cache for intercepts of the guest */ - u32 intercept_cr; - u32 intercept_dr; - u32 intercept_exceptions; - u64 intercept; - - /* Nested Paging related state */ - u64 nested_cr3; -}; - -#define MSRPM_OFFSETS 16 -static u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly; +u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly; /* * Set osvw_len to higher value when updated Revision Guides @@ -188,70 +125,6 @@ static u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly; */ static uint64_t osvw_len = 4, osvw_status; -struct vcpu_svm { - struct kvm_vcpu vcpu; - struct vmcb *vmcb; - unsigned long vmcb_pa; - struct svm_cpu_data *svm_data; - uint64_t asid_generation; - uint64_t sysenter_esp; - uint64_t sysenter_eip; - uint64_t tsc_aux; - - u64 msr_decfg; - - u64 next_rip; - - u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS]; - struct { - u16 fs; - u16 gs; - u16 ldt; - u64 gs_base; - } host; - - u64 spec_ctrl; - /* - * Contains guest-controlled bits of VIRT_SPEC_CTRL, which will be - * translated into the appropriate L2_CFG bits on the host to - * perform speculative control. - */ - u64 virt_spec_ctrl; - - u32 *msrpm; - - ulong nmi_iret_rip; - - struct nested_state nested; - - bool nmi_singlestep; - u64 nmi_singlestep_guest_rflags; - - unsigned int3_injected; - unsigned long int3_rip; - - /* cached guest cpuid flags for faster access */ - bool nrips_enabled : 1; - - u32 ldr_reg; - u32 dfr_reg; - struct page *avic_backing_page; - u64 *avic_physical_id_cache; - bool avic_is_running; - - /* - * Per-vcpu list of struct amd_svm_iommu_ir: - * This is used mainly to store interrupt remapping information used - * when update the vcpu affinity. This avoids the need to scan for - * IRTE and try to match ga_tag in the IOMMU driver. - */ - struct list_head ir_list; - spinlock_t ir_list_lock; - - /* which host CPU was used for running this vcpu */ - unsigned int last_cpu; -}; - /* * This is a wrapper of struct amd_iommu_ir_data. */ @@ -272,8 +145,6 @@ struct amd_svm_iommu_ir { static DEFINE_PER_CPU(u64, current_tsc_ratio); #define TSC_RATIO_DEFAULT 0x0100000000ULL -#define MSR_INVALID 0xffffffffU - static const struct svm_direct_access_msrs { u32 index; /* Index of the MSR */ bool always; /* True if intercept is always on */ @@ -299,9 +170,9 @@ static const struct svm_direct_access_msrs { /* enable NPT for AMD64 and X86 with PAE */ #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE) -static bool npt_enabled = true; +bool npt_enabled = true; #else -static bool npt_enabled; +bool npt_enabled; #endif /* @@ -387,41 +258,10 @@ module_param(dump_invalid_vmcb, bool, 0644); static u8 rsm_ins_bytes[] = "\x0f\xaa"; -static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0); -static void svm_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa); static void svm_complete_interrupts(struct vcpu_svm *svm); static void svm_toggle_avic_for_irq_window(struct kvm_vcpu *vcpu, bool activate); static inline void avic_post_state_restore(struct kvm_vcpu *vcpu); -static int nested_svm_exit_handled(struct vcpu_svm *svm); -static int nested_svm_intercept(struct vcpu_svm *svm); -static int nested_svm_vmexit(struct vcpu_svm *svm); -static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr, - bool has_error_code, u32 error_code); - -enum { - VMCB_INTERCEPTS, /* Intercept vectors, TSC offset, - pause filter count */ - VMCB_PERM_MAP, /* IOPM Base and MSRPM Base */ - VMCB_ASID, /* ASID */ - VMCB_INTR, /* int_ctl, int_vector */ - VMCB_NPT, /* npt_en, nCR3, gPAT */ - VMCB_CR, /* CR0, CR3, CR4, EFER */ - VMCB_DR, /* DR6, DR7 */ - VMCB_DT, /* GDT, IDT */ - VMCB_SEG, /* CS, DS, SS, ES, CPL */ - VMCB_CR2, /* CR2 only */ - VMCB_LBR, /* DBGCTL, BR_FROM, BR_TO, LAST_EX_FROM, LAST_EX_TO */ - VMCB_AVIC, /* AVIC APIC_BAR, AVIC APIC_BACKING_PAGE, - * AVIC PHYSICAL_TABLE pointer, - * AVIC LOGICAL_TABLE pointer - */ - VMCB_DIRTY_MAX, -}; - -/* TPR and CR2 are always written before VMRUN */ -#define VMCB_ALWAYS_DIRTY_MASK ((1U << VMCB_INTR) | (1U << VMCB_CR2)) - #define VMCB_AVIC_APIC_BAR_MASK 0xFFFFFFFFFF000ULL static int sev_flush_asids(void); @@ -470,27 +310,6 @@ static inline int sev_get_asid(struct kvm *kvm) return sev->asid; } -static inline void mark_all_dirty(struct vmcb *vmcb) -{ - vmcb->control.clean = 0; -} - -static inline void mark_all_clean(struct vmcb *vmcb) -{ - vmcb->control.clean = ((1 << VMCB_DIRTY_MAX) - 1) - & ~VMCB_ALWAYS_DIRTY_MASK; -} - -static inline void mark_dirty(struct vmcb *vmcb, int bit) -{ - vmcb->control.clean &= ~(1 << bit); -} - -static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu) -{ - return container_of(vcpu, struct vcpu_svm, vcpu); -} - static inline void avic_update_vapic_bar(struct vcpu_svm *svm, u64 data) { svm->vmcb->control.avic_vapic_bar = data & VMCB_AVIC_APIC_BAR_MASK; @@ -508,183 +327,6 @@ static inline bool avic_vcpu_is_running(struct kvm_vcpu *vcpu) return (READ_ONCE(*entry) & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK); } -static void recalc_intercepts(struct vcpu_svm *svm) -{ - struct vmcb_control_area *c, *h; - struct nested_state *g; - - mark_dirty(svm->vmcb, VMCB_INTERCEPTS); - - if (!is_guest_mode(&svm->vcpu)) - return; - - c = &svm->vmcb->control; - h = &svm->nested.hsave->control; - g = &svm->nested; - - c->intercept_cr = h->intercept_cr; - c->intercept_dr = h->intercept_dr; - c->intercept_exceptions = h->intercept_exceptions; - c->intercept = h->intercept; - - if (svm->vcpu.arch.hflags & HF_VINTR_MASK) { - /* We only want the cr8 intercept bits of L1 */ - c->intercept_cr &= ~(1U << INTERCEPT_CR8_READ); - c->intercept_cr &= ~(1U << INTERCEPT_CR8_WRITE); - - /* - * Once running L2 with HF_VINTR_MASK, EFLAGS.IF does not - * affect any interrupt we may want to inject; therefore, - * interrupt window vmexits are irrelevant to L0. - */ - c->intercept &= ~(1ULL << INTERCEPT_VINTR); - } - - /* We don't want to see VMMCALLs from a nested guest */ - c->intercept &= ~(1ULL << INTERCEPT_VMMCALL); - - c->intercept_cr |= g->intercept_cr; - c->intercept_dr |= g->intercept_dr; - c->intercept_exceptions |= g->intercept_exceptions; - c->intercept |= g->intercept; -} - -static inline struct vmcb *get_host_vmcb(struct vcpu_svm *svm) -{ - if (is_guest_mode(&svm->vcpu)) - return svm->nested.hsave; - else - return svm->vmcb; -} - -static inline void set_cr_intercept(struct vcpu_svm *svm, int bit) -{ - struct vmcb *vmcb = get_host_vmcb(svm); - - vmcb->control.intercept_cr |= (1U << bit); - - recalc_intercepts(svm); -} - -static inline void clr_cr_intercept(struct vcpu_svm *svm, int bit) -{ - struct vmcb *vmcb = get_host_vmcb(svm); - - vmcb->control.intercept_cr &= ~(1U << bit); - - recalc_intercepts(svm); -} - -static inline bool is_cr_intercept(struct vcpu_svm *svm, int bit) -{ - struct vmcb *vmcb = get_host_vmcb(svm); - - return vmcb->control.intercept_cr & (1U << bit); -} - -static inline void set_dr_intercepts(struct vcpu_svm *svm) -{ - struct vmcb *vmcb = get_host_vmcb(svm); - - vmcb->control.intercept_dr = (1 << INTERCEPT_DR0_READ) - | (1 << INTERCEPT_DR1_READ) - | (1 << INTERCEPT_DR2_READ) - | (1 << INTERCEPT_DR3_READ) - | (1 << INTERCEPT_DR4_READ) - | (1 << INTERCEPT_DR5_READ) - | (1 << INTERCEPT_DR6_READ) - | (1 << INTERCEPT_DR7_READ) - | (1 << INTERCEPT_DR0_WRITE) - | (1 << INTERCEPT_DR1_WRITE) - | (1 << INTERCEPT_DR2_WRITE) - | (1 << INTERCEPT_DR3_WRITE) - | (1 << INTERCEPT_DR4_WRITE) - | (1 << INTERCEPT_DR5_WRITE) - | (1 << INTERCEPT_DR6_WRITE) - | (1 << INTERCEPT_DR7_WRITE); - - recalc_intercepts(svm); -} - -static inline void clr_dr_intercepts(struct vcpu_svm *svm) -{ - struct vmcb *vmcb = get_host_vmcb(svm); - - vmcb->control.intercept_dr = 0; - - recalc_intercepts(svm); -} - -static inline void set_exception_intercept(struct vcpu_svm *svm, int bit) -{ - struct vmcb *vmcb = get_host_vmcb(svm); - - vmcb->control.intercept_exceptions |= (1U << bit); - - recalc_intercepts(svm); -} - -static inline void clr_exception_intercept(struct vcpu_svm *svm, int bit) -{ - struct vmcb *vmcb = get_host_vmcb(svm); - - vmcb->control.intercept_exceptions &= ~(1U << bit); - - recalc_intercepts(svm); -} - -static inline void set_intercept(struct vcpu_svm *svm, int bit) -{ - struct vmcb *vmcb = get_host_vmcb(svm); - - vmcb->control.intercept |= (1ULL << bit); - - recalc_intercepts(svm); -} - -static inline void clr_intercept(struct vcpu_svm *svm, int bit) -{ - struct vmcb *vmcb = get_host_vmcb(svm); - - vmcb->control.intercept &= ~(1ULL << bit); - - recalc_intercepts(svm); -} - -static inline bool is_intercept(struct vcpu_svm *svm, int bit) -{ - return (svm->vmcb->control.intercept & (1ULL << bit)) != 0; -} - -static inline bool vgif_enabled(struct vcpu_svm *svm) -{ - return !!(svm->vmcb->control.int_ctl & V_GIF_ENABLE_MASK); -} - -static inline void enable_gif(struct vcpu_svm *svm) -{ - if (vgif_enabled(svm)) - svm->vmcb->control.int_ctl |= V_GIF_MASK; - else - svm->vcpu.arch.hflags |= HF_GIF_MASK; -} - -static inline void disable_gif(struct vcpu_svm *svm) -{ - if (vgif_enabled(svm)) - svm->vmcb->control.int_ctl &= ~V_GIF_MASK; - else - svm->vcpu.arch.hflags &= ~HF_GIF_MASK; -} - -static inline bool gif_set(struct vcpu_svm *svm) -{ - if (vgif_enabled(svm)) - return !!(svm->vmcb->control.int_ctl & V_GIF_MASK); - else - return !!(svm->vcpu.arch.hflags & HF_GIF_MASK); -} - static unsigned long iopm_base; struct kvm_ldttss_desc { @@ -720,7 +362,7 @@ static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000}; #define MSRS_RANGE_SIZE 2048 #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2) -static u32 svm_msrpm_offset(u32 msr) +u32 svm_msrpm_offset(u32 msr) { u32 offset; int i; @@ -767,7 +409,7 @@ static int get_npt_level(struct kvm_vcpu *vcpu) #endif } -static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer) +void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer) { vcpu->arch.efer = efer; @@ -1198,7 +840,7 @@ static void svm_disable_lbrv(struct vcpu_svm *svm) set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 0, 0); } -static void disable_nmi_singlestep(struct vcpu_svm *svm) +void disable_nmi_singlestep(struct vcpu_svm *svm) { svm->nmi_singlestep = false; @@ -2652,7 +2294,7 @@ static void update_cr0_intercept(struct vcpu_svm *svm) } } -static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) +void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) { struct vcpu_svm *svm = to_svm(vcpu); @@ -2686,7 +2328,7 @@ static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) update_cr0_intercept(svm); } -static int svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) +int svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) { unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE; unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4; @@ -3022,776 +2664,6 @@ static int vmmcall_interception(struct vcpu_svm *svm) return kvm_emulate_hypercall(&svm->vcpu); } -static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu) -{ - struct vcpu_svm *svm = to_svm(vcpu); - - return svm->nested.nested_cr3; -} - -static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index) -{ - struct vcpu_svm *svm = to_svm(vcpu); - u64 cr3 = svm->nested.nested_cr3; - u64 pdpte; - int ret; - - ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(__sme_clr(cr3)), &pdpte, - offset_in_page(cr3) + index * 8, 8); - if (ret) - return 0; - return pdpte; -} - -static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu, - struct x86_exception *fault) -{ - struct vcpu_svm *svm = to_svm(vcpu); - - if (svm->vmcb->control.exit_code != SVM_EXIT_NPF) { - /* - * TODO: track the cause of the nested page fault, and - * correctly fill in the high bits of exit_info_1. - */ - svm->vmcb->control.exit_code = SVM_EXIT_NPF; - svm->vmcb->control.exit_code_hi = 0; - svm->vmcb->control.exit_info_1 = (1ULL << 32); - svm->vmcb->control.exit_info_2 = fault->address; - } - - svm->vmcb->control.exit_info_1 &= ~0xffffffffULL; - svm->vmcb->control.exit_info_1 |= fault->error_code; - - /* - * The present bit is always zero for page structure faults on real - * hardware. - */ - if (svm->vmcb->control.exit_info_1 & (2ULL << 32)) - svm->vmcb->control.exit_info_1 &= ~1; - - nested_svm_vmexit(svm); -} - -static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu) -{ - WARN_ON(mmu_is_nested(vcpu)); - - vcpu->arch.mmu = &vcpu->arch.guest_mmu; - kvm_init_shadow_mmu(vcpu); - vcpu->arch.mmu->get_guest_pgd = nested_svm_get_tdp_cr3; - vcpu->arch.mmu->get_pdptr = nested_svm_get_tdp_pdptr; - vcpu->arch.mmu->inject_page_fault = nested_svm_inject_npf_exit; - vcpu->arch.mmu->shadow_root_level = get_npt_level(vcpu); - reset_shadow_zero_bits_mask(vcpu, vcpu->arch.mmu); - vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu; -} - -static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu) -{ - vcpu->arch.mmu = &vcpu->arch.root_mmu; - vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; -} - -static int nested_svm_check_permissions(struct vcpu_svm *svm) -{ - if (!(svm->vcpu.arch.efer & EFER_SVME) || - !is_paging(&svm->vcpu)) { - kvm_queue_exception(&svm->vcpu, UD_VECTOR); - return 1; - } - - if (svm->vmcb->save.cpl) { - kvm_inject_gp(&svm->vcpu, 0); - return 1; - } - - return 0; -} - -static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr, - bool has_error_code, u32 error_code) -{ - int vmexit; - - if (!is_guest_mode(&svm->vcpu)) - return 0; - - vmexit = nested_svm_intercept(svm); - if (vmexit != NESTED_EXIT_DONE) - return 0; - - svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr; - svm->vmcb->control.exit_code_hi = 0; - svm->vmcb->control.exit_info_1 = error_code; - - /* - * EXITINFO2 is undefined for all exception intercepts other - * than #PF. - */ - if (svm->vcpu.arch.exception.nested_apf) - svm->vmcb->control.exit_info_2 = svm->vcpu.arch.apf.nested_apf_token; - else if (svm->vcpu.arch.exception.has_payload) - svm->vmcb->control.exit_info_2 = svm->vcpu.arch.exception.payload; - else - svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2; - - svm->nested.exit_required = true; - return vmexit; -} - -static void nested_svm_intr(struct vcpu_svm *svm) -{ - svm->vmcb->control.exit_code = SVM_EXIT_INTR; - svm->vmcb->control.exit_info_1 = 0; - svm->vmcb->control.exit_info_2 = 0; - - /* nested_svm_vmexit this gets called afterwards from handle_exit */ - svm->nested.exit_required = true; - trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip); -} - -static bool nested_exit_on_intr(struct vcpu_svm *svm) -{ - return (svm->nested.intercept & 1ULL); -} - -static int svm_check_nested_events(struct kvm_vcpu *vcpu) -{ - struct vcpu_svm *svm = to_svm(vcpu); - bool block_nested_events = - kvm_event_needs_reinjection(vcpu) || svm->nested.exit_required; - - if (kvm_cpu_has_interrupt(vcpu) && nested_exit_on_intr(svm)) { - if (block_nested_events) - return -EBUSY; - nested_svm_intr(svm); - return 0; - } - - return 0; -} - -/* This function returns true if it is save to enable the nmi window */ -static inline bool nested_svm_nmi(struct vcpu_svm *svm) -{ - if (!is_guest_mode(&svm->vcpu)) - return true; - - if (!(svm->nested.intercept & (1ULL << INTERCEPT_NMI))) - return true; - - svm->vmcb->control.exit_code = SVM_EXIT_NMI; - svm->nested.exit_required = true; - - return false; -} - -static int nested_svm_intercept_ioio(struct vcpu_svm *svm) -{ - unsigned port, size, iopm_len; - u16 val, mask; - u8 start_bit; - u64 gpa; - - if (!(svm->nested.intercept & (1ULL << INTERCEPT_IOIO_PROT))) - return NESTED_EXIT_HOST; - - port = svm->vmcb->control.exit_info_1 >> 16; - size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >> - SVM_IOIO_SIZE_SHIFT; - gpa = svm->nested.vmcb_iopm + (port / 8); - start_bit = port % 8; - iopm_len = (start_bit + size > 8) ? 2 : 1; - mask = (0xf >> (4 - size)) << start_bit; - val = 0; - - if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len)) - return NESTED_EXIT_DONE; - - return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST; -} - -static int nested_svm_exit_handled_msr(struct vcpu_svm *svm) -{ - u32 offset, msr, value; - int write, mask; - - if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT))) - return NESTED_EXIT_HOST; - - msr = svm->vcpu.arch.regs[VCPU_REGS_RCX]; - offset = svm_msrpm_offset(msr); - write = svm->vmcb->control.exit_info_1 & 1; - mask = 1 << ((2 * (msr & 0xf)) + write); - - if (offset == MSR_INVALID) - return NESTED_EXIT_DONE; - - /* Offset is in 32 bit units but need in 8 bit units */ - offset *= 4; - - if (kvm_vcpu_read_guest(&svm->vcpu, svm->nested.vmcb_msrpm + offset, &value, 4)) - return NESTED_EXIT_DONE; - - return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST; -} - -/* DB exceptions for our internal use must not cause vmexit */ -static int nested_svm_intercept_db(struct vcpu_svm *svm) -{ - unsigned long dr6; - - /* if we're not singlestepping, it's not ours */ - if (!svm->nmi_singlestep) - return NESTED_EXIT_DONE; - - /* if it's not a singlestep exception, it's not ours */ - if (kvm_get_dr(&svm->vcpu, 6, &dr6)) - return NESTED_EXIT_DONE; - if (!(dr6 & DR6_BS)) - return NESTED_EXIT_DONE; - - /* if the guest is singlestepping, it should get the vmexit */ - if (svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF) { - disable_nmi_singlestep(svm); - return NESTED_EXIT_DONE; - } - - /* it's ours, the nested hypervisor must not see this one */ - return NESTED_EXIT_HOST; -} - -static int nested_svm_exit_special(struct vcpu_svm *svm) -{ - u32 exit_code = svm->vmcb->control.exit_code; - - switch (exit_code) { - case SVM_EXIT_INTR: - case SVM_EXIT_NMI: - case SVM_EXIT_EXCP_BASE + MC_VECTOR: - return NESTED_EXIT_HOST; - case SVM_EXIT_NPF: - /* For now we are always handling NPFs when using them */ - if (npt_enabled) - return NESTED_EXIT_HOST; - break; - case SVM_EXIT_EXCP_BASE + PF_VECTOR: - /* When we're shadowing, trap PFs, but not async PF */ - if (!npt_enabled && svm->vcpu.arch.apf.host_apf_reason == 0) - return NESTED_EXIT_HOST; - break; - default: - break; - } - - return NESTED_EXIT_CONTINUE; -} - -static int nested_svm_intercept(struct vcpu_svm *svm) -{ - u32 exit_code = svm->vmcb->control.exit_code; - int vmexit = NESTED_EXIT_HOST; - - switch (exit_code) { - case SVM_EXIT_MSR: - vmexit = nested_svm_exit_handled_msr(svm); - break; - case SVM_EXIT_IOIO: - vmexit = nested_svm_intercept_ioio(svm); - break; - case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: { - u32 bit = 1U << (exit_code - SVM_EXIT_READ_CR0); - if (svm->nested.intercept_cr & bit) - vmexit = NESTED_EXIT_DONE; - break; - } - case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: { - u32 bit = 1U << (exit_code - SVM_EXIT_READ_DR0); - if (svm->nested.intercept_dr & bit) - vmexit = NESTED_EXIT_DONE; - break; - } - case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: { - u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE); - if (svm->nested.intercept_exceptions & excp_bits) { - if (exit_code == SVM_EXIT_EXCP_BASE + DB_VECTOR) - vmexit = nested_svm_intercept_db(svm); - else - vmexit = NESTED_EXIT_DONE; - } - /* async page fault always cause vmexit */ - else if ((exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) && - svm->vcpu.arch.exception.nested_apf != 0) - vmexit = NESTED_EXIT_DONE; - break; - } - case SVM_EXIT_ERR: { - vmexit = NESTED_EXIT_DONE; - break; - } - default: { - u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR); - if (svm->nested.intercept & exit_bits) - vmexit = NESTED_EXIT_DONE; - } - } - - return vmexit; -} - -static int nested_svm_exit_handled(struct vcpu_svm *svm) -{ - int vmexit; - - vmexit = nested_svm_intercept(svm); - - if (vmexit == NESTED_EXIT_DONE) - nested_svm_vmexit(svm); - - return vmexit; -} - -static inline void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *from_vmcb) -{ - struct vmcb_control_area *dst = &dst_vmcb->control; - struct vmcb_control_area *from = &from_vmcb->control; - - dst->intercept_cr = from->intercept_cr; - dst->intercept_dr = from->intercept_dr; - dst->intercept_exceptions = from->intercept_exceptions; - dst->intercept = from->intercept; - dst->iopm_base_pa = from->iopm_base_pa; - dst->msrpm_base_pa = from->msrpm_base_pa; - dst->tsc_offset = from->tsc_offset; - dst->asid = from->asid; - dst->tlb_ctl = from->tlb_ctl; - dst->int_ctl = from->int_ctl; - dst->int_vector = from->int_vector; - dst->int_state = from->int_state; - dst->exit_code = from->exit_code; - dst->exit_code_hi = from->exit_code_hi; - dst->exit_info_1 = from->exit_info_1; - dst->exit_info_2 = from->exit_info_2; - dst->exit_int_info = from->exit_int_info; - dst->exit_int_info_err = from->exit_int_info_err; - dst->nested_ctl = from->nested_ctl; - dst->event_inj = from->event_inj; - dst->event_inj_err = from->event_inj_err; - dst->nested_cr3 = from->nested_cr3; - dst->virt_ext = from->virt_ext; - dst->pause_filter_count = from->pause_filter_count; - dst->pause_filter_thresh = from->pause_filter_thresh; -} - -static int nested_svm_vmexit(struct vcpu_svm *svm) -{ - int rc; - struct vmcb *nested_vmcb; - struct vmcb *hsave = svm->nested.hsave; - struct vmcb *vmcb = svm->vmcb; - struct kvm_host_map map; - - trace_kvm_nested_vmexit_inject(vmcb->control.exit_code, - vmcb->control.exit_info_1, - vmcb->control.exit_info_2, - vmcb->control.exit_int_info, - vmcb->control.exit_int_info_err, - KVM_ISA_SVM); - - rc = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(svm->nested.vmcb), &map); - if (rc) { - if (rc == -EINVAL) - kvm_inject_gp(&svm->vcpu, 0); - return 1; - } - - nested_vmcb = map.hva; - - /* Exit Guest-Mode */ - leave_guest_mode(&svm->vcpu); - svm->nested.vmcb = 0; - - /* Give the current vmcb to the guest */ - disable_gif(svm); - - nested_vmcb->save.es = vmcb->save.es; - nested_vmcb->save.cs = vmcb->save.cs; - nested_vmcb->save.ss = vmcb->save.ss; - nested_vmcb->save.ds = vmcb->save.ds; - nested_vmcb->save.gdtr = vmcb->save.gdtr; - nested_vmcb->save.idtr = vmcb->save.idtr; - nested_vmcb->save.efer = svm->vcpu.arch.efer; - nested_vmcb->save.cr0 = kvm_read_cr0(&svm->vcpu); - nested_vmcb->save.cr3 = kvm_read_cr3(&svm->vcpu); - nested_vmcb->save.cr2 = vmcb->save.cr2; - nested_vmcb->save.cr4 = svm->vcpu.arch.cr4; - nested_vmcb->save.rflags = kvm_get_rflags(&svm->vcpu); - nested_vmcb->save.rip = vmcb->save.rip; - nested_vmcb->save.rsp = vmcb->save.rsp; - nested_vmcb->save.rax = vmcb->save.rax; - nested_vmcb->save.dr7 = vmcb->save.dr7; - nested_vmcb->save.dr6 = vmcb->save.dr6; - nested_vmcb->save.cpl = vmcb->save.cpl; - - nested_vmcb->control.int_ctl = vmcb->control.int_ctl; - nested_vmcb->control.int_vector = vmcb->control.int_vector; - nested_vmcb->control.int_state = vmcb->control.int_state; - nested_vmcb->control.exit_code = vmcb->control.exit_code; - nested_vmcb->control.exit_code_hi = vmcb->control.exit_code_hi; - nested_vmcb->control.exit_info_1 = vmcb->control.exit_info_1; - nested_vmcb->control.exit_info_2 = vmcb->control.exit_info_2; - nested_vmcb->control.exit_int_info = vmcb->control.exit_int_info; - nested_vmcb->control.exit_int_info_err = vmcb->control.exit_int_info_err; - - if (svm->nrips_enabled) - nested_vmcb->control.next_rip = vmcb->control.next_rip; - - /* - * If we emulate a VMRUN/#VMEXIT in the same host #vmexit cycle we have - * to make sure that we do not lose injected events. So check event_inj - * here and copy it to exit_int_info if it is valid. - * Exit_int_info and event_inj can't be both valid because the case - * below only happens on a VMRUN instruction intercept which has - * no valid exit_int_info set. - */ - if (vmcb->control.event_inj & SVM_EVTINJ_VALID) { - struct vmcb_control_area *nc = &nested_vmcb->control; - - nc->exit_int_info = vmcb->control.event_inj; - nc->exit_int_info_err = vmcb->control.event_inj_err; - } - - nested_vmcb->control.tlb_ctl = 0; - nested_vmcb->control.event_inj = 0; - nested_vmcb->control.event_inj_err = 0; - - nested_vmcb->control.pause_filter_count = - svm->vmcb->control.pause_filter_count; - nested_vmcb->control.pause_filter_thresh = - svm->vmcb->control.pause_filter_thresh; - - /* We always set V_INTR_MASKING and remember the old value in hflags */ - if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK)) - nested_vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK; - - /* Restore the original control entries */ - copy_vmcb_control_area(vmcb, hsave); - - svm->vcpu.arch.tsc_offset = svm->vmcb->control.tsc_offset; - kvm_clear_exception_queue(&svm->vcpu); - kvm_clear_interrupt_queue(&svm->vcpu); - - svm->nested.nested_cr3 = 0; - - /* Restore selected save entries */ - svm->vmcb->save.es = hsave->save.es; - svm->vmcb->save.cs = hsave->save.cs; - svm->vmcb->save.ss = hsave->save.ss; - svm->vmcb->save.ds = hsave->save.ds; - svm->vmcb->save.gdtr = hsave->save.gdtr; - svm->vmcb->save.idtr = hsave->save.idtr; - kvm_set_rflags(&svm->vcpu, hsave->save.rflags); - svm_set_efer(&svm->vcpu, hsave->save.efer); - svm_set_cr0(&svm->vcpu, hsave->save.cr0 | X86_CR0_PE); - svm_set_cr4(&svm->vcpu, hsave->save.cr4); - if (npt_enabled) { - svm->vmcb->save.cr3 = hsave->save.cr3; - svm->vcpu.arch.cr3 = hsave->save.cr3; - } else { - (void)kvm_set_cr3(&svm->vcpu, hsave->save.cr3); - } - kvm_rax_write(&svm->vcpu, hsave->save.rax); - kvm_rsp_write(&svm->vcpu, hsave->save.rsp); - kvm_rip_write(&svm->vcpu, hsave->save.rip); - svm->vmcb->save.dr7 = 0; - svm->vmcb->save.cpl = 0; - svm->vmcb->control.exit_int_info = 0; - - mark_all_dirty(svm->vmcb); - - kvm_vcpu_unmap(&svm->vcpu, &map, true); - - nested_svm_uninit_mmu_context(&svm->vcpu); - kvm_mmu_reset_context(&svm->vcpu); - kvm_mmu_load(&svm->vcpu); - - /* - * Drop what we picked up for L2 via svm_complete_interrupts() so it - * doesn't end up in L1. - */ - svm->vcpu.arch.nmi_injected = false; - kvm_clear_exception_queue(&svm->vcpu); - kvm_clear_interrupt_queue(&svm->vcpu); - - return 0; -} - -static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm) -{ - /* - * This function merges the msr permission bitmaps of kvm and the - * nested vmcb. It is optimized in that it only merges the parts where - * the kvm msr permission bitmap may contain zero bits - */ - int i; - - if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT))) - return true; - - for (i = 0; i < MSRPM_OFFSETS; i++) { - u32 value, p; - u64 offset; - - if (msrpm_offsets[i] == 0xffffffff) - break; - - p = msrpm_offsets[i]; - offset = svm->nested.vmcb_msrpm + (p * 4); - - if (kvm_vcpu_read_guest(&svm->vcpu, offset, &value, 4)) - return false; - - svm->nested.msrpm[p] = svm->msrpm[p] | value; - } - - svm->vmcb->control.msrpm_base_pa = __sme_set(__pa(svm->nested.msrpm)); - - return true; -} - -static bool nested_vmcb_checks(struct vmcb *vmcb) -{ - if ((vmcb->save.efer & EFER_SVME) == 0) - return false; - - if ((vmcb->control.intercept & (1ULL << INTERCEPT_VMRUN)) == 0) - return false; - - if (vmcb->control.asid == 0) - return false; - - if ((vmcb->control.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) && - !npt_enabled) - return false; - - return true; -} - -static void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, - struct vmcb *nested_vmcb, struct kvm_host_map *map) -{ - bool evaluate_pending_interrupts = - is_intercept(svm, INTERCEPT_VINTR) || - is_intercept(svm, INTERCEPT_IRET); - - if (kvm_get_rflags(&svm->vcpu) & X86_EFLAGS_IF) - svm->vcpu.arch.hflags |= HF_HIF_MASK; - else - svm->vcpu.arch.hflags &= ~HF_HIF_MASK; - - if (nested_vmcb->control.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) { - svm->nested.nested_cr3 = nested_vmcb->control.nested_cr3; - nested_svm_init_mmu_context(&svm->vcpu); - } - - /* Load the nested guest state */ - svm->vmcb->save.es = nested_vmcb->save.es; - svm->vmcb->save.cs = nested_vmcb->save.cs; - svm->vmcb->save.ss = nested_vmcb->save.ss; - svm->vmcb->save.ds = nested_vmcb->save.ds; - svm->vmcb->save.gdtr = nested_vmcb->save.gdtr; - svm->vmcb->save.idtr = nested_vmcb->save.idtr; - kvm_set_rflags(&svm->vcpu, nested_vmcb->save.rflags); - svm_set_efer(&svm->vcpu, nested_vmcb->save.efer); - svm_set_cr0(&svm->vcpu, nested_vmcb->save.cr0); - svm_set_cr4(&svm->vcpu, nested_vmcb->save.cr4); - if (npt_enabled) { - svm->vmcb->save.cr3 = nested_vmcb->save.cr3; - svm->vcpu.arch.cr3 = nested_vmcb->save.cr3; - } else - (void)kvm_set_cr3(&svm->vcpu, nested_vmcb->save.cr3); - - /* Guest paging mode is active - reset mmu */ - kvm_mmu_reset_context(&svm->vcpu); - - svm->vmcb->save.cr2 = svm->vcpu.arch.cr2 = nested_vmcb->save.cr2; - kvm_rax_write(&svm->vcpu, nested_vmcb->save.rax); - kvm_rsp_write(&svm->vcpu, nested_vmcb->save.rsp); - kvm_rip_write(&svm->vcpu, nested_vmcb->save.rip); - - /* In case we don't even reach vcpu_run, the fields are not updated */ - svm->vmcb->save.rax = nested_vmcb->save.rax; - svm->vmcb->save.rsp = nested_vmcb->save.rsp; - svm->vmcb->save.rip = nested_vmcb->save.rip; - svm->vmcb->save.dr7 = nested_vmcb->save.dr7; - svm->vmcb->save.dr6 = nested_vmcb->save.dr6; - svm->vmcb->save.cpl = nested_vmcb->save.cpl; - - svm->nested.vmcb_msrpm = nested_vmcb->control.msrpm_base_pa & ~0x0fffULL; - svm->nested.vmcb_iopm = nested_vmcb->control.iopm_base_pa & ~0x0fffULL; - - /* cache intercepts */ - svm->nested.intercept_cr = nested_vmcb->control.intercept_cr; - svm->nested.intercept_dr = nested_vmcb->control.intercept_dr; - svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions; - svm->nested.intercept = nested_vmcb->control.intercept; - - svm_flush_tlb(&svm->vcpu, true); - svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK; - if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK) - svm->vcpu.arch.hflags |= HF_VINTR_MASK; - else - svm->vcpu.arch.hflags &= ~HF_VINTR_MASK; - - svm->vcpu.arch.tsc_offset += nested_vmcb->control.tsc_offset; - svm->vmcb->control.tsc_offset = svm->vcpu.arch.tsc_offset; - - svm->vmcb->control.virt_ext = nested_vmcb->control.virt_ext; - svm->vmcb->control.int_vector = nested_vmcb->control.int_vector; - svm->vmcb->control.int_state = nested_vmcb->control.int_state; - svm->vmcb->control.event_inj = nested_vmcb->control.event_inj; - svm->vmcb->control.event_inj_err = nested_vmcb->control.event_inj_err; - - svm->vmcb->control.pause_filter_count = - nested_vmcb->control.pause_filter_count; - svm->vmcb->control.pause_filter_thresh = - nested_vmcb->control.pause_filter_thresh; - - kvm_vcpu_unmap(&svm->vcpu, map, true); - - /* Enter Guest-Mode */ - enter_guest_mode(&svm->vcpu); - - /* - * Merge guest and host intercepts - must be called with vcpu in - * guest-mode to take affect here - */ - recalc_intercepts(svm); - - svm->nested.vmcb = vmcb_gpa; - - /* - * If L1 had a pending IRQ/NMI before executing VMRUN, - * which wasn't delivered because it was disallowed (e.g. - * interrupts disabled), L0 needs to evaluate if this pending - * event should cause an exit from L2 to L1 or be delivered - * directly to L2. - * - * Usually this would be handled by the processor noticing an - * IRQ/NMI window request. However, VMRUN can unblock interrupts - * by implicitly setting GIF, so force L0 to perform pending event - * evaluation by requesting a KVM_REQ_EVENT. - */ - enable_gif(svm); - if (unlikely(evaluate_pending_interrupts)) - kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); - - mark_all_dirty(svm->vmcb); -} - -static int nested_svm_vmrun(struct vcpu_svm *svm) -{ - int ret; - struct vmcb *nested_vmcb; - struct vmcb *hsave = svm->nested.hsave; - struct vmcb *vmcb = svm->vmcb; - struct kvm_host_map map; - u64 vmcb_gpa; - - vmcb_gpa = svm->vmcb->save.rax; - - ret = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(vmcb_gpa), &map); - if (ret == -EINVAL) { - kvm_inject_gp(&svm->vcpu, 0); - return 1; - } else if (ret) { - return kvm_skip_emulated_instruction(&svm->vcpu); - } - - ret = kvm_skip_emulated_instruction(&svm->vcpu); - - nested_vmcb = map.hva; - - if (!nested_vmcb_checks(nested_vmcb)) { - nested_vmcb->control.exit_code = SVM_EXIT_ERR; - nested_vmcb->control.exit_code_hi = 0; - nested_vmcb->control.exit_info_1 = 0; - nested_vmcb->control.exit_info_2 = 0; - - kvm_vcpu_unmap(&svm->vcpu, &map, true); - - return ret; - } - - trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb_gpa, - nested_vmcb->save.rip, - nested_vmcb->control.int_ctl, - nested_vmcb->control.event_inj, - nested_vmcb->control.nested_ctl); - - trace_kvm_nested_intercepts(nested_vmcb->control.intercept_cr & 0xffff, - nested_vmcb->control.intercept_cr >> 16, - nested_vmcb->control.intercept_exceptions, - nested_vmcb->control.intercept); - - /* Clear internal status */ - kvm_clear_exception_queue(&svm->vcpu); - kvm_clear_interrupt_queue(&svm->vcpu); - - /* - * Save the old vmcb, so we don't need to pick what we save, but can - * restore everything when a VMEXIT occurs - */ - hsave->save.es = vmcb->save.es; - hsave->save.cs = vmcb->save.cs; - hsave->save.ss = vmcb->save.ss; - hsave->save.ds = vmcb->save.ds; - hsave->save.gdtr = vmcb->save.gdtr; - hsave->save.idtr = vmcb->save.idtr; - hsave->save.efer = svm->vcpu.arch.efer; - hsave->save.cr0 = kvm_read_cr0(&svm->vcpu); - hsave->save.cr4 = svm->vcpu.arch.cr4; - hsave->save.rflags = kvm_get_rflags(&svm->vcpu); - hsave->save.rip = kvm_rip_read(&svm->vcpu); - hsave->save.rsp = vmcb->save.rsp; - hsave->save.rax = vmcb->save.rax; - if (npt_enabled) - hsave->save.cr3 = vmcb->save.cr3; - else - hsave->save.cr3 = kvm_read_cr3(&svm->vcpu); - - copy_vmcb_control_area(hsave, vmcb); - - enter_svm_guest_mode(svm, vmcb_gpa, nested_vmcb, &map); - - if (!nested_svm_vmrun_msrpm(svm)) { - svm->vmcb->control.exit_code = SVM_EXIT_ERR; - svm->vmcb->control.exit_code_hi = 0; - svm->vmcb->control.exit_info_1 = 0; - svm->vmcb->control.exit_info_2 = 0; - - nested_svm_vmexit(svm); - } - - return ret; -} - -static void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb) -{ - to_vmcb->save.fs = from_vmcb->save.fs; - to_vmcb->save.gs = from_vmcb->save.gs; - to_vmcb->save.tr = from_vmcb->save.tr; - to_vmcb->save.ldtr = from_vmcb->save.ldtr; - to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base; - to_vmcb->save.star = from_vmcb->save.star; - to_vmcb->save.lstar = from_vmcb->save.lstar; - to_vmcb->save.cstar = from_vmcb->save.cstar; - to_vmcb->save.sfmask = from_vmcb->save.sfmask; - to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs; - to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp; - to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip; -} - static int vmload_interception(struct vcpu_svm *svm) { struct vmcb *nested_vmcb; @@ -5186,11 +4058,6 @@ static void svm_set_irq(struct kvm_vcpu *vcpu) SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR; } -static inline bool svm_nested_virtualize_tpr(struct kvm_vcpu *vcpu) -{ - return is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK); -} - static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) { struct vcpu_svm *svm = to_svm(vcpu); @@ -5632,7 +4499,7 @@ static int svm_set_identity_map_addr(struct kvm *kvm, u64 ident_addr) return 0; } -static void svm_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa) +void svm_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa) { struct vcpu_svm *svm = to_svm(vcpu); diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h new file mode 100644 index 000000000000..f4c446d7a31e --- /dev/null +++ b/arch/x86/kvm/svm/svm.h @@ -0,0 +1,381 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Kernel-based Virtual Machine driver for Linux + * + * AMD SVM support + * + * Copyright (C) 2006 Qumranet, Inc. + * Copyright 2010 Red Hat, Inc. and/or its affiliates. + * + * Authors: + * Yaniv Kamay + * Avi Kivity + */ + +#ifndef __SVM_SVM_H +#define __SVM_SVM_H + +#include +#include + +#include + +static const u32 host_save_user_msrs[] = { +#ifdef CONFIG_X86_64 + MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE, + MSR_FS_BASE, +#endif + MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP, + MSR_TSC_AUX, +}; + +#define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs) + +#define MSRPM_OFFSETS 16 +extern u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly; +extern bool npt_enabled; + +enum { + VMCB_INTERCEPTS, /* Intercept vectors, TSC offset, + pause filter count */ + VMCB_PERM_MAP, /* IOPM Base and MSRPM Base */ + VMCB_ASID, /* ASID */ + VMCB_INTR, /* int_ctl, int_vector */ + VMCB_NPT, /* npt_en, nCR3, gPAT */ + VMCB_CR, /* CR0, CR3, CR4, EFER */ + VMCB_DR, /* DR6, DR7 */ + VMCB_DT, /* GDT, IDT */ + VMCB_SEG, /* CS, DS, SS, ES, CPL */ + VMCB_CR2, /* CR2 only */ + VMCB_LBR, /* DBGCTL, BR_FROM, BR_TO, LAST_EX_FROM, LAST_EX_TO */ + VMCB_AVIC, /* AVIC APIC_BAR, AVIC APIC_BACKING_PAGE, + * AVIC PHYSICAL_TABLE pointer, + * AVIC LOGICAL_TABLE pointer + */ + VMCB_DIRTY_MAX, +}; + +/* TPR and CR2 are always written before VMRUN */ +#define VMCB_ALWAYS_DIRTY_MASK ((1U << VMCB_INTR) | (1U << VMCB_CR2)) + +struct kvm_sev_info { + bool active; /* SEV enabled guest */ + unsigned int asid; /* ASID used for this guest */ + unsigned int handle; /* SEV firmware handle */ + int fd; /* SEV device fd */ + unsigned long pages_locked; /* Number of pages locked */ + struct list_head regions_list; /* List of registered regions */ +}; + +struct kvm_svm { + struct kvm kvm; + + /* Struct members for AVIC */ + u32 avic_vm_id; + struct page *avic_logical_id_table_page; + struct page *avic_physical_id_table_page; + struct hlist_node hnode; + + struct kvm_sev_info sev_info; +}; + +struct kvm_vcpu; + +struct nested_state { + struct vmcb *hsave; + u64 hsave_msr; + u64 vm_cr_msr; + u64 vmcb; + + /* These are the merged vectors */ + u32 *msrpm; + + /* gpa pointers to the real vectors */ + u64 vmcb_msrpm; + u64 vmcb_iopm; + + /* A VMEXIT is required but not yet emulated */ + bool exit_required; + + /* cache for intercepts of the guest */ + u32 intercept_cr; + u32 intercept_dr; + u32 intercept_exceptions; + u64 intercept; + + /* Nested Paging related state */ + u64 nested_cr3; +}; + +struct vcpu_svm { + struct kvm_vcpu vcpu; + struct vmcb *vmcb; + unsigned long vmcb_pa; + struct svm_cpu_data *svm_data; + uint64_t asid_generation; + uint64_t sysenter_esp; + uint64_t sysenter_eip; + uint64_t tsc_aux; + + u64 msr_decfg; + + u64 next_rip; + + u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS]; + struct { + u16 fs; + u16 gs; + u16 ldt; + u64 gs_base; + } host; + + u64 spec_ctrl; + /* + * Contains guest-controlled bits of VIRT_SPEC_CTRL, which will be + * translated into the appropriate L2_CFG bits on the host to + * perform speculative control. + */ + u64 virt_spec_ctrl; + + u32 *msrpm; + + ulong nmi_iret_rip; + + struct nested_state nested; + + bool nmi_singlestep; + u64 nmi_singlestep_guest_rflags; + + unsigned int3_injected; + unsigned long int3_rip; + + /* cached guest cpuid flags for faster access */ + bool nrips_enabled : 1; + + u32 ldr_reg; + u32 dfr_reg; + struct page *avic_backing_page; + u64 *avic_physical_id_cache; + bool avic_is_running; + + /* + * Per-vcpu list of struct amd_svm_iommu_ir: + * This is used mainly to store interrupt remapping information used + * when update the vcpu affinity. This avoids the need to scan for + * IRTE and try to match ga_tag in the IOMMU driver. + */ + struct list_head ir_list; + spinlock_t ir_list_lock; + + /* which host CPU was used for running this vcpu */ + unsigned int last_cpu; +}; + +void recalc_intercepts(struct vcpu_svm *svm); + +static inline void mark_all_dirty(struct vmcb *vmcb) +{ + vmcb->control.clean = 0; +} + +static inline void mark_all_clean(struct vmcb *vmcb) +{ + vmcb->control.clean = ((1 << VMCB_DIRTY_MAX) - 1) + & ~VMCB_ALWAYS_DIRTY_MASK; +} + +static inline void mark_dirty(struct vmcb *vmcb, int bit) +{ + vmcb->control.clean &= ~(1 << bit); +} + +static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu) +{ + return container_of(vcpu, struct vcpu_svm, vcpu); +} + +static inline struct vmcb *get_host_vmcb(struct vcpu_svm *svm) +{ + if (is_guest_mode(&svm->vcpu)) + return svm->nested.hsave; + else + return svm->vmcb; +} + +static inline void set_cr_intercept(struct vcpu_svm *svm, int bit) +{ + struct vmcb *vmcb = get_host_vmcb(svm); + + vmcb->control.intercept_cr |= (1U << bit); + + recalc_intercepts(svm); +} + +static inline void clr_cr_intercept(struct vcpu_svm *svm, int bit) +{ + struct vmcb *vmcb = get_host_vmcb(svm); + + vmcb->control.intercept_cr &= ~(1U << bit); + + recalc_intercepts(svm); +} + +static inline bool is_cr_intercept(struct vcpu_svm *svm, int bit) +{ + struct vmcb *vmcb = get_host_vmcb(svm); + + return vmcb->control.intercept_cr & (1U << bit); +} + +static inline void set_dr_intercepts(struct vcpu_svm *svm) +{ + struct vmcb *vmcb = get_host_vmcb(svm); + + vmcb->control.intercept_dr = (1 << INTERCEPT_DR0_READ) + | (1 << INTERCEPT_DR1_READ) + | (1 << INTERCEPT_DR2_READ) + | (1 << INTERCEPT_DR3_READ) + | (1 << INTERCEPT_DR4_READ) + | (1 << INTERCEPT_DR5_READ) + | (1 << INTERCEPT_DR6_READ) + | (1 << INTERCEPT_DR7_READ) + | (1 << INTERCEPT_DR0_WRITE) + | (1 << INTERCEPT_DR1_WRITE) + | (1 << INTERCEPT_DR2_WRITE) + | (1 << INTERCEPT_DR3_WRITE) + | (1 << INTERCEPT_DR4_WRITE) + | (1 << INTERCEPT_DR5_WRITE) + | (1 << INTERCEPT_DR6_WRITE) + | (1 << INTERCEPT_DR7_WRITE); + + recalc_intercepts(svm); +} + +static inline void clr_dr_intercepts(struct vcpu_svm *svm) +{ + struct vmcb *vmcb = get_host_vmcb(svm); + + vmcb->control.intercept_dr = 0; + + recalc_intercepts(svm); +} + +static inline void set_exception_intercept(struct vcpu_svm *svm, int bit) +{ + struct vmcb *vmcb = get_host_vmcb(svm); + + vmcb->control.intercept_exceptions |= (1U << bit); + + recalc_intercepts(svm); +} + +static inline void clr_exception_intercept(struct vcpu_svm *svm, int bit) +{ + struct vmcb *vmcb = get_host_vmcb(svm); + + vmcb->control.intercept_exceptions &= ~(1U << bit); + + recalc_intercepts(svm); +} + +static inline void set_intercept(struct vcpu_svm *svm, int bit) +{ + struct vmcb *vmcb = get_host_vmcb(svm); + + vmcb->control.intercept |= (1ULL << bit); + + recalc_intercepts(svm); +} + +static inline void clr_intercept(struct vcpu_svm *svm, int bit) +{ + struct vmcb *vmcb = get_host_vmcb(svm); + + vmcb->control.intercept &= ~(1ULL << bit); + + recalc_intercepts(svm); +} + +static inline bool is_intercept(struct vcpu_svm *svm, int bit) +{ + return (svm->vmcb->control.intercept & (1ULL << bit)) != 0; +} + +static inline bool vgif_enabled(struct vcpu_svm *svm) +{ + return !!(svm->vmcb->control.int_ctl & V_GIF_ENABLE_MASK); +} + +static inline void enable_gif(struct vcpu_svm *svm) +{ + if (vgif_enabled(svm)) + svm->vmcb->control.int_ctl |= V_GIF_MASK; + else + svm->vcpu.arch.hflags |= HF_GIF_MASK; +} + +static inline void disable_gif(struct vcpu_svm *svm) +{ + if (vgif_enabled(svm)) + svm->vmcb->control.int_ctl &= ~V_GIF_MASK; + else + svm->vcpu.arch.hflags &= ~HF_GIF_MASK; +} + +static inline bool gif_set(struct vcpu_svm *svm) +{ + if (vgif_enabled(svm)) + return !!(svm->vmcb->control.int_ctl & V_GIF_MASK); + else + return !!(svm->vcpu.arch.hflags & HF_GIF_MASK); +} + +/* svm.c */ +#define MSR_INVALID 0xffffffffU + +u32 svm_msrpm_offset(u32 msr); +void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer); +void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0); +int svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4); +void svm_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa); +void disable_nmi_singlestep(struct vcpu_svm *svm); + +/* nested.c */ + +#define NESTED_EXIT_HOST 0 /* Exit handled on host level */ +#define NESTED_EXIT_DONE 1 /* Exit caused nested vmexit */ +#define NESTED_EXIT_CONTINUE 2 /* Further checks needed */ + +/* This function returns true if it is save to enable the nmi window */ +static inline bool nested_svm_nmi(struct vcpu_svm *svm) +{ + if (!is_guest_mode(&svm->vcpu)) + return true; + + if (!(svm->nested.intercept & (1ULL << INTERCEPT_NMI))) + return true; + + svm->vmcb->control.exit_code = SVM_EXIT_NMI; + svm->nested.exit_required = true; + + return false; +} + +static inline bool svm_nested_virtualize_tpr(struct kvm_vcpu *vcpu) +{ + return is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK); +} + +void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, + struct vmcb *nested_vmcb, struct kvm_host_map *map); +int nested_svm_vmrun(struct vcpu_svm *svm); +void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb); +int nested_svm_vmexit(struct vcpu_svm *svm); +int nested_svm_exit_handled(struct vcpu_svm *svm); +int nested_svm_check_permissions(struct vcpu_svm *svm); +int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr, + bool has_error_code, u32 error_code); +int svm_check_nested_events(struct kvm_vcpu *vcpu); +int nested_svm_exit_special(struct vcpu_svm *svm); + +#endif From ef0f64960d012cbab8f55f305ef36bb6de4e1a9b Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 31 Mar 2020 12:17:38 -0400 Subject: [PATCH 439/635] KVM: SVM: Move AVIC code to separate file Move the AVIC related functions from svm.c to the new avic.c file. Signed-off-by: Joerg Roedel Message-Id: <20200324094154.32352-4-joro@8bytes.org> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/Makefile | 2 +- arch/x86/kvm/svm/avic.c | 1027 ++++++++++++++++++++++++++++++++++++++ arch/x86/kvm/svm/svm.c | 1050 +-------------------------------------- arch/x86/kvm/svm/svm.h | 62 +++ 4 files changed, 1091 insertions(+), 1050 deletions(-) create mode 100644 arch/x86/kvm/svm/avic.c diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index 0acec73ef236..31aca07d8f12 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -14,7 +14,7 @@ kvm-y += x86.o emulate.o i8259.o irq.o lapic.o \ hyperv.o debugfs.o mmu/mmu.o mmu/page_track.o kvm-intel-y += vmx/vmx.o vmx/vmenter.o vmx/pmu_intel.o vmx/vmcs12.o vmx/evmcs.o vmx/nested.o -kvm-amd-y += svm/svm.o svm/pmu.o svm/nested.o +kvm-amd-y += svm/svm.o svm/pmu.o svm/nested.o svm/avic.o obj-$(CONFIG_KVM) += kvm.o obj-$(CONFIG_KVM_INTEL) += kvm-intel.o diff --git a/arch/x86/kvm/svm/avic.c b/arch/x86/kvm/svm/avic.c new file mode 100644 index 000000000000..e80daa98682f --- /dev/null +++ b/arch/x86/kvm/svm/avic.c @@ -0,0 +1,1027 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Kernel-based Virtual Machine driver for Linux + * + * AMD SVM support + * + * Copyright (C) 2006 Qumranet, Inc. + * Copyright 2010 Red Hat, Inc. and/or its affiliates. + * + * Authors: + * Yaniv Kamay + * Avi Kivity + */ + +#define pr_fmt(fmt) "SVM: " fmt + +#include +#include +#include +#include + +#include + +#include "trace.h" +#include "lapic.h" +#include "x86.h" +#include "irq.h" +#include "svm.h" + +/* enable / disable AVIC */ +int avic; +#ifdef CONFIG_X86_LOCAL_APIC +module_param(avic, int, S_IRUGO); +#endif + +#define SVM_AVIC_DOORBELL 0xc001011b + +#define AVIC_HPA_MASK ~((0xFFFULL << 52) | 0xFFF) + +/* + * 0xff is broadcast, so the max index allowed for physical APIC ID + * table is 0xfe. APIC IDs above 0xff are reserved. + */ +#define AVIC_MAX_PHYSICAL_ID_COUNT 255 + +#define AVIC_UNACCEL_ACCESS_WRITE_MASK 1 +#define AVIC_UNACCEL_ACCESS_OFFSET_MASK 0xFF0 +#define AVIC_UNACCEL_ACCESS_VECTOR_MASK 0xFFFFFFFF + +/* AVIC GATAG is encoded using VM and VCPU IDs */ +#define AVIC_VCPU_ID_BITS 8 +#define AVIC_VCPU_ID_MASK ((1 << AVIC_VCPU_ID_BITS) - 1) + +#define AVIC_VM_ID_BITS 24 +#define AVIC_VM_ID_NR (1 << AVIC_VM_ID_BITS) +#define AVIC_VM_ID_MASK ((1 << AVIC_VM_ID_BITS) - 1) + +#define AVIC_GATAG(x, y) (((x & AVIC_VM_ID_MASK) << AVIC_VCPU_ID_BITS) | \ + (y & AVIC_VCPU_ID_MASK)) +#define AVIC_GATAG_TO_VMID(x) ((x >> AVIC_VCPU_ID_BITS) & AVIC_VM_ID_MASK) +#define AVIC_GATAG_TO_VCPUID(x) (x & AVIC_VCPU_ID_MASK) + +/* Note: + * This hash table is used to map VM_ID to a struct kvm_svm, + * when handling AMD IOMMU GALOG notification to schedule in + * a particular vCPU. + */ +#define SVM_VM_DATA_HASH_BITS 8 +static DEFINE_HASHTABLE(svm_vm_data_hash, SVM_VM_DATA_HASH_BITS); +static u32 next_vm_id = 0; +static bool next_vm_id_wrapped = 0; +static DEFINE_SPINLOCK(svm_vm_data_hash_lock); + +/* + * This is a wrapper of struct amd_iommu_ir_data. + */ +struct amd_svm_iommu_ir { + struct list_head node; /* Used by SVM for per-vcpu ir_list */ + void *data; /* Storing pointer to struct amd_ir_data */ +}; + +enum avic_ipi_failure_cause { + AVIC_IPI_FAILURE_INVALID_INT_TYPE, + AVIC_IPI_FAILURE_TARGET_NOT_RUNNING, + AVIC_IPI_FAILURE_INVALID_TARGET, + AVIC_IPI_FAILURE_INVALID_BACKING_PAGE, +}; + +/* Note: + * This function is called from IOMMU driver to notify + * SVM to schedule in a particular vCPU of a particular VM. + */ +int avic_ga_log_notifier(u32 ga_tag) +{ + unsigned long flags; + struct kvm_svm *kvm_svm; + struct kvm_vcpu *vcpu = NULL; + u32 vm_id = AVIC_GATAG_TO_VMID(ga_tag); + u32 vcpu_id = AVIC_GATAG_TO_VCPUID(ga_tag); + + pr_debug("SVM: %s: vm_id=%#x, vcpu_id=%#x\n", __func__, vm_id, vcpu_id); + trace_kvm_avic_ga_log(vm_id, vcpu_id); + + spin_lock_irqsave(&svm_vm_data_hash_lock, flags); + hash_for_each_possible(svm_vm_data_hash, kvm_svm, hnode, vm_id) { + if (kvm_svm->avic_vm_id != vm_id) + continue; + vcpu = kvm_get_vcpu_by_id(&kvm_svm->kvm, vcpu_id); + break; + } + spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); + + /* Note: + * At this point, the IOMMU should have already set the pending + * bit in the vAPIC backing page. So, we just need to schedule + * in the vcpu. + */ + if (vcpu) + kvm_vcpu_wake_up(vcpu); + + return 0; +} + +void avic_vm_destroy(struct kvm *kvm) +{ + unsigned long flags; + struct kvm_svm *kvm_svm = to_kvm_svm(kvm); + + if (!avic) + return; + + if (kvm_svm->avic_logical_id_table_page) + __free_page(kvm_svm->avic_logical_id_table_page); + if (kvm_svm->avic_physical_id_table_page) + __free_page(kvm_svm->avic_physical_id_table_page); + + spin_lock_irqsave(&svm_vm_data_hash_lock, flags); + hash_del(&kvm_svm->hnode); + spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); +} + +int avic_vm_init(struct kvm *kvm) +{ + unsigned long flags; + int err = -ENOMEM; + struct kvm_svm *kvm_svm = to_kvm_svm(kvm); + struct kvm_svm *k2; + struct page *p_page; + struct page *l_page; + u32 vm_id; + + if (!avic) + return 0; + + /* Allocating physical APIC ID table (4KB) */ + p_page = alloc_page(GFP_KERNEL_ACCOUNT); + if (!p_page) + goto free_avic; + + kvm_svm->avic_physical_id_table_page = p_page; + clear_page(page_address(p_page)); + + /* Allocating logical APIC ID table (4KB) */ + l_page = alloc_page(GFP_KERNEL_ACCOUNT); + if (!l_page) + goto free_avic; + + kvm_svm->avic_logical_id_table_page = l_page; + clear_page(page_address(l_page)); + + spin_lock_irqsave(&svm_vm_data_hash_lock, flags); + again: + vm_id = next_vm_id = (next_vm_id + 1) & AVIC_VM_ID_MASK; + if (vm_id == 0) { /* id is 1-based, zero is not okay */ + next_vm_id_wrapped = 1; + goto again; + } + /* Is it still in use? Only possible if wrapped at least once */ + if (next_vm_id_wrapped) { + hash_for_each_possible(svm_vm_data_hash, k2, hnode, vm_id) { + if (k2->avic_vm_id == vm_id) + goto again; + } + } + kvm_svm->avic_vm_id = vm_id; + hash_add(svm_vm_data_hash, &kvm_svm->hnode, kvm_svm->avic_vm_id); + spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); + + return 0; + +free_avic: + avic_vm_destroy(kvm); + return err; +} + +void avic_init_vmcb(struct vcpu_svm *svm) +{ + struct vmcb *vmcb = svm->vmcb; + struct kvm_svm *kvm_svm = to_kvm_svm(svm->vcpu.kvm); + phys_addr_t bpa = __sme_set(page_to_phys(svm->avic_backing_page)); + phys_addr_t lpa = __sme_set(page_to_phys(kvm_svm->avic_logical_id_table_page)); + phys_addr_t ppa = __sme_set(page_to_phys(kvm_svm->avic_physical_id_table_page)); + + vmcb->control.avic_backing_page = bpa & AVIC_HPA_MASK; + vmcb->control.avic_logical_id = lpa & AVIC_HPA_MASK; + vmcb->control.avic_physical_id = ppa & AVIC_HPA_MASK; + vmcb->control.avic_physical_id |= AVIC_MAX_PHYSICAL_ID_COUNT; + if (kvm_apicv_activated(svm->vcpu.kvm)) + vmcb->control.int_ctl |= AVIC_ENABLE_MASK; + else + vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK; +} + +static u64 *avic_get_physical_id_entry(struct kvm_vcpu *vcpu, + unsigned int index) +{ + u64 *avic_physical_id_table; + struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm); + + if (index >= AVIC_MAX_PHYSICAL_ID_COUNT) + return NULL; + + avic_physical_id_table = page_address(kvm_svm->avic_physical_id_table_page); + + return &avic_physical_id_table[index]; +} + +/** + * Note: + * AVIC hardware walks the nested page table to check permissions, + * but does not use the SPA address specified in the leaf page + * table entry since it uses address in the AVIC_BACKING_PAGE pointer + * field of the VMCB. Therefore, we set up the + * APIC_ACCESS_PAGE_PRIVATE_MEMSLOT (4KB) here. + */ +static int avic_update_access_page(struct kvm *kvm, bool activate) +{ + int ret = 0; + + mutex_lock(&kvm->slots_lock); + /* + * During kvm_destroy_vm(), kvm_pit_set_reinject() could trigger + * APICv mode change, which update APIC_ACCESS_PAGE_PRIVATE_MEMSLOT + * memory region. So, we need to ensure that kvm->mm == current->mm. + */ + if ((kvm->arch.apic_access_page_done == activate) || + (kvm->mm != current->mm)) + goto out; + + ret = __x86_set_memory_region(kvm, + APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, + APIC_DEFAULT_PHYS_BASE, + activate ? PAGE_SIZE : 0); + if (ret) + goto out; + + kvm->arch.apic_access_page_done = activate; +out: + mutex_unlock(&kvm->slots_lock); + return ret; +} + +static int avic_init_backing_page(struct kvm_vcpu *vcpu) +{ + u64 *entry, new_entry; + int id = vcpu->vcpu_id; + struct vcpu_svm *svm = to_svm(vcpu); + + if (id >= AVIC_MAX_PHYSICAL_ID_COUNT) + return -EINVAL; + + if (!svm->vcpu.arch.apic->regs) + return -EINVAL; + + if (kvm_apicv_activated(vcpu->kvm)) { + int ret; + + ret = avic_update_access_page(vcpu->kvm, true); + if (ret) + return ret; + } + + svm->avic_backing_page = virt_to_page(svm->vcpu.arch.apic->regs); + + /* Setting AVIC backing page address in the phy APIC ID table */ + entry = avic_get_physical_id_entry(vcpu, id); + if (!entry) + return -EINVAL; + + new_entry = __sme_set((page_to_phys(svm->avic_backing_page) & + AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK) | + AVIC_PHYSICAL_ID_ENTRY_VALID_MASK); + WRITE_ONCE(*entry, new_entry); + + svm->avic_physical_id_cache = entry; + + return 0; +} + +int avic_incomplete_ipi_interception(struct vcpu_svm *svm) +{ + u32 icrh = svm->vmcb->control.exit_info_1 >> 32; + u32 icrl = svm->vmcb->control.exit_info_1; + u32 id = svm->vmcb->control.exit_info_2 >> 32; + u32 index = svm->vmcb->control.exit_info_2 & 0xFF; + struct kvm_lapic *apic = svm->vcpu.arch.apic; + + trace_kvm_avic_incomplete_ipi(svm->vcpu.vcpu_id, icrh, icrl, id, index); + + switch (id) { + case AVIC_IPI_FAILURE_INVALID_INT_TYPE: + /* + * AVIC hardware handles the generation of + * IPIs when the specified Message Type is Fixed + * (also known as fixed delivery mode) and + * the Trigger Mode is edge-triggered. The hardware + * also supports self and broadcast delivery modes + * specified via the Destination Shorthand(DSH) + * field of the ICRL. Logical and physical APIC ID + * formats are supported. All other IPI types cause + * a #VMEXIT, which needs to emulated. + */ + kvm_lapic_reg_write(apic, APIC_ICR2, icrh); + kvm_lapic_reg_write(apic, APIC_ICR, icrl); + break; + case AVIC_IPI_FAILURE_TARGET_NOT_RUNNING: { + int i; + struct kvm_vcpu *vcpu; + struct kvm *kvm = svm->vcpu.kvm; + struct kvm_lapic *apic = svm->vcpu.arch.apic; + + /* + * At this point, we expect that the AVIC HW has already + * set the appropriate IRR bits on the valid target + * vcpus. So, we just need to kick the appropriate vcpu. + */ + kvm_for_each_vcpu(i, vcpu, kvm) { + bool m = kvm_apic_match_dest(vcpu, apic, + icrl & APIC_SHORT_MASK, + GET_APIC_DEST_FIELD(icrh), + icrl & APIC_DEST_MASK); + + if (m && !avic_vcpu_is_running(vcpu)) + kvm_vcpu_wake_up(vcpu); + } + break; + } + case AVIC_IPI_FAILURE_INVALID_TARGET: + WARN_ONCE(1, "Invalid IPI target: index=%u, vcpu=%d, icr=%#0x:%#0x\n", + index, svm->vcpu.vcpu_id, icrh, icrl); + break; + case AVIC_IPI_FAILURE_INVALID_BACKING_PAGE: + WARN_ONCE(1, "Invalid backing page\n"); + break; + default: + pr_err("Unknown IPI interception\n"); + } + + return 1; +} + +static u32 *avic_get_logical_id_entry(struct kvm_vcpu *vcpu, u32 ldr, bool flat) +{ + struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm); + int index; + u32 *logical_apic_id_table; + int dlid = GET_APIC_LOGICAL_ID(ldr); + + if (!dlid) + return NULL; + + if (flat) { /* flat */ + index = ffs(dlid) - 1; + if (index > 7) + return NULL; + } else { /* cluster */ + int cluster = (dlid & 0xf0) >> 4; + int apic = ffs(dlid & 0x0f) - 1; + + if ((apic < 0) || (apic > 7) || + (cluster >= 0xf)) + return NULL; + index = (cluster << 2) + apic; + } + + logical_apic_id_table = (u32 *) page_address(kvm_svm->avic_logical_id_table_page); + + return &logical_apic_id_table[index]; +} + +static int avic_ldr_write(struct kvm_vcpu *vcpu, u8 g_physical_id, u32 ldr) +{ + bool flat; + u32 *entry, new_entry; + + flat = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR) == APIC_DFR_FLAT; + entry = avic_get_logical_id_entry(vcpu, ldr, flat); + if (!entry) + return -EINVAL; + + new_entry = READ_ONCE(*entry); + new_entry &= ~AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK; + new_entry |= (g_physical_id & AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK); + new_entry |= AVIC_LOGICAL_ID_ENTRY_VALID_MASK; + WRITE_ONCE(*entry, new_entry); + + return 0; +} + +static void avic_invalidate_logical_id_entry(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + bool flat = svm->dfr_reg == APIC_DFR_FLAT; + u32 *entry = avic_get_logical_id_entry(vcpu, svm->ldr_reg, flat); + + if (entry) + clear_bit(AVIC_LOGICAL_ID_ENTRY_VALID_BIT, (unsigned long *)entry); +} + +static int avic_handle_ldr_update(struct kvm_vcpu *vcpu) +{ + int ret = 0; + struct vcpu_svm *svm = to_svm(vcpu); + u32 ldr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LDR); + u32 id = kvm_xapic_id(vcpu->arch.apic); + + if (ldr == svm->ldr_reg) + return 0; + + avic_invalidate_logical_id_entry(vcpu); + + if (ldr) + ret = avic_ldr_write(vcpu, id, ldr); + + if (!ret) + svm->ldr_reg = ldr; + + return ret; +} + +static int avic_handle_apic_id_update(struct kvm_vcpu *vcpu) +{ + u64 *old, *new; + struct vcpu_svm *svm = to_svm(vcpu); + u32 id = kvm_xapic_id(vcpu->arch.apic); + + if (vcpu->vcpu_id == id) + return 0; + + old = avic_get_physical_id_entry(vcpu, vcpu->vcpu_id); + new = avic_get_physical_id_entry(vcpu, id); + if (!new || !old) + return 1; + + /* We need to move physical_id_entry to new offset */ + *new = *old; + *old = 0ULL; + to_svm(vcpu)->avic_physical_id_cache = new; + + /* + * Also update the guest physical APIC ID in the logical + * APIC ID table entry if already setup the LDR. + */ + if (svm->ldr_reg) + avic_handle_ldr_update(vcpu); + + return 0; +} + +static void avic_handle_dfr_update(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + u32 dfr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR); + + if (svm->dfr_reg == dfr) + return; + + avic_invalidate_logical_id_entry(vcpu); + svm->dfr_reg = dfr; +} + +static int avic_unaccel_trap_write(struct vcpu_svm *svm) +{ + struct kvm_lapic *apic = svm->vcpu.arch.apic; + u32 offset = svm->vmcb->control.exit_info_1 & + AVIC_UNACCEL_ACCESS_OFFSET_MASK; + + switch (offset) { + case APIC_ID: + if (avic_handle_apic_id_update(&svm->vcpu)) + return 0; + break; + case APIC_LDR: + if (avic_handle_ldr_update(&svm->vcpu)) + return 0; + break; + case APIC_DFR: + avic_handle_dfr_update(&svm->vcpu); + break; + default: + break; + } + + kvm_lapic_reg_write(apic, offset, kvm_lapic_get_reg(apic, offset)); + + return 1; +} + +static bool is_avic_unaccelerated_access_trap(u32 offset) +{ + bool ret = false; + + switch (offset) { + case APIC_ID: + case APIC_EOI: + case APIC_RRR: + case APIC_LDR: + case APIC_DFR: + case APIC_SPIV: + case APIC_ESR: + case APIC_ICR: + case APIC_LVTT: + case APIC_LVTTHMR: + case APIC_LVTPC: + case APIC_LVT0: + case APIC_LVT1: + case APIC_LVTERR: + case APIC_TMICT: + case APIC_TDCR: + ret = true; + break; + default: + break; + } + return ret; +} + +int avic_unaccelerated_access_interception(struct vcpu_svm *svm) +{ + int ret = 0; + u32 offset = svm->vmcb->control.exit_info_1 & + AVIC_UNACCEL_ACCESS_OFFSET_MASK; + u32 vector = svm->vmcb->control.exit_info_2 & + AVIC_UNACCEL_ACCESS_VECTOR_MASK; + bool write = (svm->vmcb->control.exit_info_1 >> 32) & + AVIC_UNACCEL_ACCESS_WRITE_MASK; + bool trap = is_avic_unaccelerated_access_trap(offset); + + trace_kvm_avic_unaccelerated_access(svm->vcpu.vcpu_id, offset, + trap, write, vector); + if (trap) { + /* Handling Trap */ + WARN_ONCE(!write, "svm: Handling trap read.\n"); + ret = avic_unaccel_trap_write(svm); + } else { + /* Handling Fault */ + ret = kvm_emulate_instruction(&svm->vcpu, 0); + } + + return ret; +} + +int avic_init_vcpu(struct vcpu_svm *svm) +{ + int ret; + struct kvm_vcpu *vcpu = &svm->vcpu; + + if (!avic || !irqchip_in_kernel(vcpu->kvm)) + return 0; + + ret = avic_init_backing_page(&svm->vcpu); + if (ret) + return ret; + + INIT_LIST_HEAD(&svm->ir_list); + spin_lock_init(&svm->ir_list_lock); + svm->dfr_reg = APIC_DFR_FLAT; + + return ret; +} + +void avic_post_state_restore(struct kvm_vcpu *vcpu) +{ + if (avic_handle_apic_id_update(vcpu) != 0) + return; + avic_handle_dfr_update(vcpu); + avic_handle_ldr_update(vcpu); +} + +void svm_toggle_avic_for_irq_window(struct kvm_vcpu *vcpu, bool activate) +{ + if (!avic || !lapic_in_kernel(vcpu)) + return; + + srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); + kvm_request_apicv_update(vcpu->kvm, activate, + APICV_INHIBIT_REASON_IRQWIN); + vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); +} + +void svm_set_virtual_apic_mode(struct kvm_vcpu *vcpu) +{ + return; +} + +void svm_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr) +{ +} + +void svm_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr) +{ +} + +static int svm_set_pi_irte_mode(struct kvm_vcpu *vcpu, bool activate) +{ + int ret = 0; + unsigned long flags; + struct amd_svm_iommu_ir *ir; + struct vcpu_svm *svm = to_svm(vcpu); + + if (!kvm_arch_has_assigned_device(vcpu->kvm)) + return 0; + + /* + * Here, we go through the per-vcpu ir_list to update all existing + * interrupt remapping table entry targeting this vcpu. + */ + spin_lock_irqsave(&svm->ir_list_lock, flags); + + if (list_empty(&svm->ir_list)) + goto out; + + list_for_each_entry(ir, &svm->ir_list, node) { + if (activate) + ret = amd_iommu_activate_guest_mode(ir->data); + else + ret = amd_iommu_deactivate_guest_mode(ir->data); + if (ret) + break; + } +out: + spin_unlock_irqrestore(&svm->ir_list_lock, flags); + return ret; +} + +void svm_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + struct vmcb *vmcb = svm->vmcb; + bool activated = kvm_vcpu_apicv_active(vcpu); + + if (!avic) + return; + + if (activated) { + /** + * During AVIC temporary deactivation, guest could update + * APIC ID, DFR and LDR registers, which would not be trapped + * by avic_unaccelerated_access_interception(). In this case, + * we need to check and update the AVIC logical APIC ID table + * accordingly before re-activating. + */ + avic_post_state_restore(vcpu); + vmcb->control.int_ctl |= AVIC_ENABLE_MASK; + } else { + vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK; + } + mark_dirty(vmcb, VMCB_AVIC); + + svm_set_pi_irte_mode(vcpu, activated); +} + +void svm_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap) +{ + return; +} + +int svm_deliver_avic_intr(struct kvm_vcpu *vcpu, int vec) +{ + if (!vcpu->arch.apicv_active) + return -1; + + kvm_lapic_set_irr(vec, vcpu->arch.apic); + smp_mb__after_atomic(); + + if (avic_vcpu_is_running(vcpu)) { + int cpuid = vcpu->cpu; + + if (cpuid != get_cpu()) + wrmsrl(SVM_AVIC_DOORBELL, kvm_cpu_get_apicid(cpuid)); + put_cpu(); + } else + kvm_vcpu_wake_up(vcpu); + + return 0; +} + +bool svm_dy_apicv_has_pending_interrupt(struct kvm_vcpu *vcpu) +{ + return false; +} + +static void svm_ir_list_del(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi) +{ + unsigned long flags; + struct amd_svm_iommu_ir *cur; + + spin_lock_irqsave(&svm->ir_list_lock, flags); + list_for_each_entry(cur, &svm->ir_list, node) { + if (cur->data != pi->ir_data) + continue; + list_del(&cur->node); + kfree(cur); + break; + } + spin_unlock_irqrestore(&svm->ir_list_lock, flags); +} + +static int svm_ir_list_add(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi) +{ + int ret = 0; + unsigned long flags; + struct amd_svm_iommu_ir *ir; + + /** + * In some cases, the existing irte is updaed and re-set, + * so we need to check here if it's already been * added + * to the ir_list. + */ + if (pi->ir_data && (pi->prev_ga_tag != 0)) { + struct kvm *kvm = svm->vcpu.kvm; + u32 vcpu_id = AVIC_GATAG_TO_VCPUID(pi->prev_ga_tag); + struct kvm_vcpu *prev_vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id); + struct vcpu_svm *prev_svm; + + if (!prev_vcpu) { + ret = -EINVAL; + goto out; + } + + prev_svm = to_svm(prev_vcpu); + svm_ir_list_del(prev_svm, pi); + } + + /** + * Allocating new amd_iommu_pi_data, which will get + * add to the per-vcpu ir_list. + */ + ir = kzalloc(sizeof(struct amd_svm_iommu_ir), GFP_KERNEL_ACCOUNT); + if (!ir) { + ret = -ENOMEM; + goto out; + } + ir->data = pi->ir_data; + + spin_lock_irqsave(&svm->ir_list_lock, flags); + list_add(&ir->node, &svm->ir_list); + spin_unlock_irqrestore(&svm->ir_list_lock, flags); +out: + return ret; +} + +/** + * Note: + * The HW cannot support posting multicast/broadcast + * interrupts to a vCPU. So, we still use legacy interrupt + * remapping for these kind of interrupts. + * + * For lowest-priority interrupts, we only support + * those with single CPU as the destination, e.g. user + * configures the interrupts via /proc/irq or uses + * irqbalance to make the interrupts single-CPU. + */ +static int +get_pi_vcpu_info(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, + struct vcpu_data *vcpu_info, struct vcpu_svm **svm) +{ + struct kvm_lapic_irq irq; + struct kvm_vcpu *vcpu = NULL; + + kvm_set_msi_irq(kvm, e, &irq); + + if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu) || + !kvm_irq_is_postable(&irq)) { + pr_debug("SVM: %s: use legacy intr remap mode for irq %u\n", + __func__, irq.vector); + return -1; + } + + pr_debug("SVM: %s: use GA mode for irq %u\n", __func__, + irq.vector); + *svm = to_svm(vcpu); + vcpu_info->pi_desc_addr = __sme_set(page_to_phys((*svm)->avic_backing_page)); + vcpu_info->vector = irq.vector; + + return 0; +} + +/* + * svm_update_pi_irte - set IRTE for Posted-Interrupts + * + * @kvm: kvm + * @host_irq: host irq of the interrupt + * @guest_irq: gsi of the interrupt + * @set: set or unset PI + * returns 0 on success, < 0 on failure + */ +int svm_update_pi_irte(struct kvm *kvm, unsigned int host_irq, + uint32_t guest_irq, bool set) +{ + struct kvm_kernel_irq_routing_entry *e; + struct kvm_irq_routing_table *irq_rt; + int idx, ret = -EINVAL; + + if (!kvm_arch_has_assigned_device(kvm) || + !irq_remapping_cap(IRQ_POSTING_CAP)) + return 0; + + pr_debug("SVM: %s: host_irq=%#x, guest_irq=%#x, set=%#x\n", + __func__, host_irq, guest_irq, set); + + idx = srcu_read_lock(&kvm->irq_srcu); + irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu); + WARN_ON(guest_irq >= irq_rt->nr_rt_entries); + + hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) { + struct vcpu_data vcpu_info; + struct vcpu_svm *svm = NULL; + + if (e->type != KVM_IRQ_ROUTING_MSI) + continue; + + /** + * Here, we setup with legacy mode in the following cases: + * 1. When cannot target interrupt to a specific vcpu. + * 2. Unsetting posted interrupt. + * 3. APIC virtialization is disabled for the vcpu. + * 4. IRQ has incompatible delivery mode (SMI, INIT, etc) + */ + if (!get_pi_vcpu_info(kvm, e, &vcpu_info, &svm) && set && + kvm_vcpu_apicv_active(&svm->vcpu)) { + struct amd_iommu_pi_data pi; + + /* Try to enable guest_mode in IRTE */ + pi.base = __sme_set(page_to_phys(svm->avic_backing_page) & + AVIC_HPA_MASK); + pi.ga_tag = AVIC_GATAG(to_kvm_svm(kvm)->avic_vm_id, + svm->vcpu.vcpu_id); + pi.is_guest_mode = true; + pi.vcpu_data = &vcpu_info; + ret = irq_set_vcpu_affinity(host_irq, &pi); + + /** + * Here, we successfully setting up vcpu affinity in + * IOMMU guest mode. Now, we need to store the posted + * interrupt information in a per-vcpu ir_list so that + * we can reference to them directly when we update vcpu + * scheduling information in IOMMU irte. + */ + if (!ret && pi.is_guest_mode) + svm_ir_list_add(svm, &pi); + } else { + /* Use legacy mode in IRTE */ + struct amd_iommu_pi_data pi; + + /** + * Here, pi is used to: + * - Tell IOMMU to use legacy mode for this interrupt. + * - Retrieve ga_tag of prior interrupt remapping data. + */ + pi.is_guest_mode = false; + ret = irq_set_vcpu_affinity(host_irq, &pi); + + /** + * Check if the posted interrupt was previously + * setup with the guest_mode by checking if the ga_tag + * was cached. If so, we need to clean up the per-vcpu + * ir_list. + */ + if (!ret && pi.prev_ga_tag) { + int id = AVIC_GATAG_TO_VCPUID(pi.prev_ga_tag); + struct kvm_vcpu *vcpu; + + vcpu = kvm_get_vcpu_by_id(kvm, id); + if (vcpu) + svm_ir_list_del(to_svm(vcpu), &pi); + } + } + + if (!ret && svm) { + trace_kvm_pi_irte_update(host_irq, svm->vcpu.vcpu_id, + e->gsi, vcpu_info.vector, + vcpu_info.pi_desc_addr, set); + } + + if (ret < 0) { + pr_err("%s: failed to update PI IRTE\n", __func__); + goto out; + } + } + + ret = 0; +out: + srcu_read_unlock(&kvm->irq_srcu, idx); + return ret; +} + +bool svm_check_apicv_inhibit_reasons(ulong bit) +{ + ulong supported = BIT(APICV_INHIBIT_REASON_DISABLE) | + BIT(APICV_INHIBIT_REASON_HYPERV) | + BIT(APICV_INHIBIT_REASON_NESTED) | + BIT(APICV_INHIBIT_REASON_IRQWIN) | + BIT(APICV_INHIBIT_REASON_PIT_REINJ) | + BIT(APICV_INHIBIT_REASON_X2APIC); + + return supported & BIT(bit); +} + +void svm_pre_update_apicv_exec_ctrl(struct kvm *kvm, bool activate) +{ + avic_update_access_page(kvm, activate); +} + +static inline int +avic_update_iommu_vcpu_affinity(struct kvm_vcpu *vcpu, int cpu, bool r) +{ + int ret = 0; + unsigned long flags; + struct amd_svm_iommu_ir *ir; + struct vcpu_svm *svm = to_svm(vcpu); + + if (!kvm_arch_has_assigned_device(vcpu->kvm)) + return 0; + + /* + * Here, we go through the per-vcpu ir_list to update all existing + * interrupt remapping table entry targeting this vcpu. + */ + spin_lock_irqsave(&svm->ir_list_lock, flags); + + if (list_empty(&svm->ir_list)) + goto out; + + list_for_each_entry(ir, &svm->ir_list, node) { + ret = amd_iommu_update_ga(cpu, r, ir->data); + if (ret) + break; + } +out: + spin_unlock_irqrestore(&svm->ir_list_lock, flags); + return ret; +} + +void avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu) +{ + u64 entry; + /* ID = 0xff (broadcast), ID > 0xff (reserved) */ + int h_physical_id = kvm_cpu_get_apicid(cpu); + struct vcpu_svm *svm = to_svm(vcpu); + + if (!kvm_vcpu_apicv_active(vcpu)) + return; + + /* + * Since the host physical APIC id is 8 bits, + * we can support host APIC ID upto 255. + */ + if (WARN_ON(h_physical_id > AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK)) + return; + + entry = READ_ONCE(*(svm->avic_physical_id_cache)); + WARN_ON(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK); + + entry &= ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK; + entry |= (h_physical_id & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK); + + entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; + if (svm->avic_is_running) + entry |= AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; + + WRITE_ONCE(*(svm->avic_physical_id_cache), entry); + avic_update_iommu_vcpu_affinity(vcpu, h_physical_id, + svm->avic_is_running); +} + +void avic_vcpu_put(struct kvm_vcpu *vcpu) +{ + u64 entry; + struct vcpu_svm *svm = to_svm(vcpu); + + if (!kvm_vcpu_apicv_active(vcpu)) + return; + + entry = READ_ONCE(*(svm->avic_physical_id_cache)); + if (entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK) + avic_update_iommu_vcpu_affinity(vcpu, -1, 0); + + entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; + WRITE_ONCE(*(svm->avic_physical_id_cache), entry); +} + +/** + * This function is called during VCPU halt/unhalt. + */ +static void avic_set_running(struct kvm_vcpu *vcpu, bool is_run) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + svm->avic_is_running = is_run; + if (is_run) + avic_vcpu_load(vcpu, vcpu->cpu); + else + avic_vcpu_put(vcpu); +} + +void svm_vcpu_blocking(struct kvm_vcpu *vcpu) +{ + avic_set_running(vcpu, false); +} + +void svm_vcpu_unblocking(struct kvm_vcpu *vcpu) +{ + if (kvm_check_request(KVM_REQ_APICV_UPDATE, vcpu)) + kvm_vcpu_update_apicv(vcpu); + avic_set_running(vcpu, true); +} diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index a24e2de4acda..e32b4956fa06 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -1,17 +1,3 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Kernel-based Virtual Machine driver for Linux - * - * AMD SVM support - * - * Copyright (C) 2006 Qumranet, Inc. - * Copyright 2010 Red Hat, Inc. and/or its affiliates. - * - * Authors: - * Yaniv Kamay - * Avi Kivity - */ - #define pr_fmt(fmt) "SVM: " fmt #include @@ -28,10 +14,10 @@ #include #include #include +#include #include #include #include -#include #include #include #include @@ -82,39 +68,12 @@ MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id); #define SVM_FEATURE_DECODE_ASSIST (1 << 7) #define SVM_FEATURE_PAUSE_FILTER (1 << 10) -#define SVM_AVIC_DOORBELL 0xc001011b - #define DEBUGCTL_RESERVED_BITS (~(0x3fULL)) #define TSC_RATIO_RSVD 0xffffff0000000000ULL #define TSC_RATIO_MIN 0x0000000000000001ULL #define TSC_RATIO_MAX 0x000000ffffffffffULL -#define AVIC_HPA_MASK ~((0xFFFULL << 52) | 0xFFF) - -/* - * 0xff is broadcast, so the max index allowed for physical APIC ID - * table is 0xfe. APIC IDs above 0xff are reserved. - */ -#define AVIC_MAX_PHYSICAL_ID_COUNT 255 - -#define AVIC_UNACCEL_ACCESS_WRITE_MASK 1 -#define AVIC_UNACCEL_ACCESS_OFFSET_MASK 0xFF0 -#define AVIC_UNACCEL_ACCESS_VECTOR_MASK 0xFFFFFFFF - -/* AVIC GATAG is encoded using VM and VCPU IDs */ -#define AVIC_VCPU_ID_BITS 8 -#define AVIC_VCPU_ID_MASK ((1 << AVIC_VCPU_ID_BITS) - 1) - -#define AVIC_VM_ID_BITS 24 -#define AVIC_VM_ID_NR (1 << AVIC_VM_ID_BITS) -#define AVIC_VM_ID_MASK ((1 << AVIC_VM_ID_BITS) - 1) - -#define AVIC_GATAG(x, y) (((x & AVIC_VM_ID_MASK) << AVIC_VCPU_ID_BITS) | \ - (y & AVIC_VCPU_ID_MASK)) -#define AVIC_GATAG_TO_VMID(x) ((x >> AVIC_VCPU_ID_BITS) & AVIC_VM_ID_MASK) -#define AVIC_GATAG_TO_VCPUID(x) (x & AVIC_VCPU_ID_MASK) - static bool erratum_383_found __read_mostly; u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly; @@ -125,23 +84,6 @@ u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly; */ static uint64_t osvw_len = 4, osvw_status; -/* - * This is a wrapper of struct amd_iommu_ir_data. - */ -struct amd_svm_iommu_ir { - struct list_head node; /* Used by SVM for per-vcpu ir_list */ - void *data; /* Storing pointer to struct amd_ir_data */ -}; - -#define AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK (0xFF) -#define AVIC_LOGICAL_ID_ENTRY_VALID_BIT 31 -#define AVIC_LOGICAL_ID_ENTRY_VALID_MASK (1 << 31) - -#define AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK (0xFFULL) -#define AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK (0xFFFFFFFFFFULL << 12) -#define AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK (1ULL << 62) -#define AVIC_PHYSICAL_ID_ENTRY_VALID_MASK (1ULL << 63) - static DEFINE_PER_CPU(u64, current_tsc_ratio); #define TSC_RATIO_DEFAULT 0x0100000000ULL @@ -231,12 +173,6 @@ module_param(npt, int, S_IRUGO); static int nested = true; module_param(nested, int, S_IRUGO); -/* enable / disable AVIC */ -static int avic; -#ifdef CONFIG_X86_LOCAL_APIC -module_param(avic, int, S_IRUGO); -#endif - /* enable/disable Next RIP Save */ static int nrips = true; module_param(nrips, int, 0444); @@ -259,10 +195,6 @@ module_param(dump_invalid_vmcb, bool, 0644); static u8 rsm_ins_bytes[] = "\x0f\xaa"; static void svm_complete_interrupts(struct vcpu_svm *svm); -static void svm_toggle_avic_for_irq_window(struct kvm_vcpu *vcpu, bool activate); -static inline void avic_post_state_restore(struct kvm_vcpu *vcpu); - -#define VMCB_AVIC_APIC_BAR_MASK 0xFFFFFFFFFF000ULL static int sev_flush_asids(void); static DECLARE_RWSEM(sev_deactivate_lock); @@ -282,11 +214,6 @@ struct enc_region { }; -static inline struct kvm_svm *to_kvm_svm(struct kvm *kvm) -{ - return container_of(kvm, struct kvm_svm, kvm); -} - static inline bool svm_sev_enabled(void) { return IS_ENABLED(CONFIG_KVM_AMD_SEV) ? max_sev_asid : 0; @@ -310,23 +237,6 @@ static inline int sev_get_asid(struct kvm *kvm) return sev->asid; } -static inline void avic_update_vapic_bar(struct vcpu_svm *svm, u64 data) -{ - svm->vmcb->control.avic_vapic_bar = data & VMCB_AVIC_APIC_BAR_MASK; - mark_dirty(svm->vmcb, VMCB_AVIC); -} - -static inline bool avic_vcpu_is_running(struct kvm_vcpu *vcpu) -{ - struct vcpu_svm *svm = to_svm(vcpu); - u64 *entry = svm->avic_physical_id_cache; - - if (!entry) - return false; - - return (READ_ONCE(*entry) & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK); -} - static unsigned long iopm_base; struct kvm_ldttss_desc { @@ -853,52 +763,6 @@ void disable_nmi_singlestep(struct vcpu_svm *svm) } } -/* Note: - * This hash table is used to map VM_ID to a struct kvm_svm, - * when handling AMD IOMMU GALOG notification to schedule in - * a particular vCPU. - */ -#define SVM_VM_DATA_HASH_BITS 8 -static DEFINE_HASHTABLE(svm_vm_data_hash, SVM_VM_DATA_HASH_BITS); -static u32 next_vm_id = 0; -static bool next_vm_id_wrapped = 0; -static DEFINE_SPINLOCK(svm_vm_data_hash_lock); - -/* Note: - * This function is called from IOMMU driver to notify - * SVM to schedule in a particular vCPU of a particular VM. - */ -static int avic_ga_log_notifier(u32 ga_tag) -{ - unsigned long flags; - struct kvm_svm *kvm_svm; - struct kvm_vcpu *vcpu = NULL; - u32 vm_id = AVIC_GATAG_TO_VMID(ga_tag); - u32 vcpu_id = AVIC_GATAG_TO_VCPUID(ga_tag); - - pr_debug("SVM: %s: vm_id=%#x, vcpu_id=%#x\n", __func__, vm_id, vcpu_id); - trace_kvm_avic_ga_log(vm_id, vcpu_id); - - spin_lock_irqsave(&svm_vm_data_hash_lock, flags); - hash_for_each_possible(svm_vm_data_hash, kvm_svm, hnode, vm_id) { - if (kvm_svm->avic_vm_id != vm_id) - continue; - vcpu = kvm_get_vcpu_by_id(&kvm_svm->kvm, vcpu_id); - break; - } - spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); - - /* Note: - * At this point, the IOMMU should have already set the pending - * bit in the vAPIC backing page. So, we just need to schedule - * in the vcpu. - */ - if (vcpu) - kvm_vcpu_wake_up(vcpu); - - return 0; -} - static __init int sev_hardware_setup(void) { struct sev_user_data_status *status; @@ -1227,24 +1091,6 @@ static u64 svm_write_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 offset) return svm->vmcb->control.tsc_offset; } -static void avic_init_vmcb(struct vcpu_svm *svm) -{ - struct vmcb *vmcb = svm->vmcb; - struct kvm_svm *kvm_svm = to_kvm_svm(svm->vcpu.kvm); - phys_addr_t bpa = __sme_set(page_to_phys(svm->avic_backing_page)); - phys_addr_t lpa = __sme_set(page_to_phys(kvm_svm->avic_logical_id_table_page)); - phys_addr_t ppa = __sme_set(page_to_phys(kvm_svm->avic_physical_id_table_page)); - - vmcb->control.avic_backing_page = bpa & AVIC_HPA_MASK; - vmcb->control.avic_logical_id = lpa & AVIC_HPA_MASK; - vmcb->control.avic_physical_id = ppa & AVIC_HPA_MASK; - vmcb->control.avic_physical_id |= AVIC_MAX_PHYSICAL_ID_COUNT; - if (kvm_apicv_activated(svm->vcpu.kvm)) - vmcb->control.int_ctl |= AVIC_ENABLE_MASK; - else - vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK; -} - static void init_vmcb(struct vcpu_svm *svm) { struct vmcb_control_area *control = &svm->vmcb->control; @@ -1404,92 +1250,6 @@ static void init_vmcb(struct vcpu_svm *svm) } -static u64 *avic_get_physical_id_entry(struct kvm_vcpu *vcpu, - unsigned int index) -{ - u64 *avic_physical_id_table; - struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm); - - if (index >= AVIC_MAX_PHYSICAL_ID_COUNT) - return NULL; - - avic_physical_id_table = page_address(kvm_svm->avic_physical_id_table_page); - - return &avic_physical_id_table[index]; -} - -/** - * Note: - * AVIC hardware walks the nested page table to check permissions, - * but does not use the SPA address specified in the leaf page - * table entry since it uses address in the AVIC_BACKING_PAGE pointer - * field of the VMCB. Therefore, we set up the - * APIC_ACCESS_PAGE_PRIVATE_MEMSLOT (4KB) here. - */ -static int avic_update_access_page(struct kvm *kvm, bool activate) -{ - int ret = 0; - - mutex_lock(&kvm->slots_lock); - /* - * During kvm_destroy_vm(), kvm_pit_set_reinject() could trigger - * APICv mode change, which update APIC_ACCESS_PAGE_PRIVATE_MEMSLOT - * memory region. So, we need to ensure that kvm->mm == current->mm. - */ - if ((kvm->arch.apic_access_page_done == activate) || - (kvm->mm != current->mm)) - goto out; - - ret = __x86_set_memory_region(kvm, - APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, - APIC_DEFAULT_PHYS_BASE, - activate ? PAGE_SIZE : 0); - if (ret) - goto out; - - kvm->arch.apic_access_page_done = activate; -out: - mutex_unlock(&kvm->slots_lock); - return ret; -} - -static int avic_init_backing_page(struct kvm_vcpu *vcpu) -{ - u64 *entry, new_entry; - int id = vcpu->vcpu_id; - struct vcpu_svm *svm = to_svm(vcpu); - - if (id >= AVIC_MAX_PHYSICAL_ID_COUNT) - return -EINVAL; - - if (!svm->vcpu.arch.apic->regs) - return -EINVAL; - - if (kvm_apicv_activated(vcpu->kvm)) { - int ret; - - ret = avic_update_access_page(vcpu->kvm, true); - if (ret) - return ret; - } - - svm->avic_backing_page = virt_to_page(svm->vcpu.arch.apic->regs); - - /* Setting AVIC backing page address in the phy APIC ID table */ - entry = avic_get_physical_id_entry(vcpu, id); - if (!entry) - return -EINVAL; - - new_entry = __sme_set((page_to_phys(svm->avic_backing_page) & - AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK) | - AVIC_PHYSICAL_ID_ENTRY_VALID_MASK); - WRITE_ONCE(*entry, new_entry); - - svm->avic_physical_id_cache = entry; - - return 0; -} - static void sev_asid_free(int asid) { struct svm_cpu_data *sd; @@ -1665,84 +1425,12 @@ static void sev_vm_destroy(struct kvm *kvm) sev_asid_free(sev->asid); } -static void avic_vm_destroy(struct kvm *kvm) -{ - unsigned long flags; - struct kvm_svm *kvm_svm = to_kvm_svm(kvm); - - if (!avic) - return; - - if (kvm_svm->avic_logical_id_table_page) - __free_page(kvm_svm->avic_logical_id_table_page); - if (kvm_svm->avic_physical_id_table_page) - __free_page(kvm_svm->avic_physical_id_table_page); - - spin_lock_irqsave(&svm_vm_data_hash_lock, flags); - hash_del(&kvm_svm->hnode); - spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); -} - static void svm_vm_destroy(struct kvm *kvm) { avic_vm_destroy(kvm); sev_vm_destroy(kvm); } -static int avic_vm_init(struct kvm *kvm) -{ - unsigned long flags; - int err = -ENOMEM; - struct kvm_svm *kvm_svm = to_kvm_svm(kvm); - struct kvm_svm *k2; - struct page *p_page; - struct page *l_page; - u32 vm_id; - - if (!avic) - return 0; - - /* Allocating physical APIC ID table (4KB) */ - p_page = alloc_page(GFP_KERNEL_ACCOUNT); - if (!p_page) - goto free_avic; - - kvm_svm->avic_physical_id_table_page = p_page; - clear_page(page_address(p_page)); - - /* Allocating logical APIC ID table (4KB) */ - l_page = alloc_page(GFP_KERNEL_ACCOUNT); - if (!l_page) - goto free_avic; - - kvm_svm->avic_logical_id_table_page = l_page; - clear_page(page_address(l_page)); - - spin_lock_irqsave(&svm_vm_data_hash_lock, flags); - again: - vm_id = next_vm_id = (next_vm_id + 1) & AVIC_VM_ID_MASK; - if (vm_id == 0) { /* id is 1-based, zero is not okay */ - next_vm_id_wrapped = 1; - goto again; - } - /* Is it still in use? Only possible if wrapped at least once */ - if (next_vm_id_wrapped) { - hash_for_each_possible(svm_vm_data_hash, k2, hnode, vm_id) { - if (k2->avic_vm_id == vm_id) - goto again; - } - } - kvm_svm->avic_vm_id = vm_id; - hash_add(svm_vm_data_hash, &kvm_svm->hnode, kvm_svm->avic_vm_id); - spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); - - return 0; - -free_avic: - avic_vm_destroy(kvm); - return err; -} - static int svm_vm_init(struct kvm *kvm) { if (avic) { @@ -1755,98 +1443,6 @@ static int svm_vm_init(struct kvm *kvm) return 0; } -static inline int -avic_update_iommu_vcpu_affinity(struct kvm_vcpu *vcpu, int cpu, bool r) -{ - int ret = 0; - unsigned long flags; - struct amd_svm_iommu_ir *ir; - struct vcpu_svm *svm = to_svm(vcpu); - - if (!kvm_arch_has_assigned_device(vcpu->kvm)) - return 0; - - /* - * Here, we go through the per-vcpu ir_list to update all existing - * interrupt remapping table entry targeting this vcpu. - */ - spin_lock_irqsave(&svm->ir_list_lock, flags); - - if (list_empty(&svm->ir_list)) - goto out; - - list_for_each_entry(ir, &svm->ir_list, node) { - ret = amd_iommu_update_ga(cpu, r, ir->data); - if (ret) - break; - } -out: - spin_unlock_irqrestore(&svm->ir_list_lock, flags); - return ret; -} - -static void avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu) -{ - u64 entry; - /* ID = 0xff (broadcast), ID > 0xff (reserved) */ - int h_physical_id = kvm_cpu_get_apicid(cpu); - struct vcpu_svm *svm = to_svm(vcpu); - - if (!kvm_vcpu_apicv_active(vcpu)) - return; - - /* - * Since the host physical APIC id is 8 bits, - * we can support host APIC ID upto 255. - */ - if (WARN_ON(h_physical_id > AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK)) - return; - - entry = READ_ONCE(*(svm->avic_physical_id_cache)); - WARN_ON(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK); - - entry &= ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK; - entry |= (h_physical_id & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK); - - entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; - if (svm->avic_is_running) - entry |= AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; - - WRITE_ONCE(*(svm->avic_physical_id_cache), entry); - avic_update_iommu_vcpu_affinity(vcpu, h_physical_id, - svm->avic_is_running); -} - -static void avic_vcpu_put(struct kvm_vcpu *vcpu) -{ - u64 entry; - struct vcpu_svm *svm = to_svm(vcpu); - - if (!kvm_vcpu_apicv_active(vcpu)) - return; - - entry = READ_ONCE(*(svm->avic_physical_id_cache)); - if (entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK) - avic_update_iommu_vcpu_affinity(vcpu, -1, 0); - - entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; - WRITE_ONCE(*(svm->avic_physical_id_cache), entry); -} - -/** - * This function is called during VCPU halt/unhalt. - */ -static void avic_set_running(struct kvm_vcpu *vcpu, bool is_run) -{ - struct vcpu_svm *svm = to_svm(vcpu); - - svm->avic_is_running = is_run; - if (is_run) - avic_vcpu_load(vcpu, vcpu->cpu); - else - avic_vcpu_put(vcpu); -} - static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) { struct vcpu_svm *svm = to_svm(vcpu); @@ -1871,25 +1467,6 @@ static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) avic_update_vapic_bar(svm, APIC_DEFAULT_PHYS_BASE); } -static int avic_init_vcpu(struct vcpu_svm *svm) -{ - int ret; - struct kvm_vcpu *vcpu = &svm->vcpu; - - if (!avic || !irqchip_in_kernel(vcpu->kvm)) - return 0; - - ret = avic_init_backing_page(&svm->vcpu); - if (ret) - return ret; - - INIT_LIST_HEAD(&svm->ir_list); - spin_lock_init(&svm->ir_list_lock); - svm->dfr_reg = APIC_DFR_FLAT; - - return ret; -} - static int svm_create_vcpu(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm; @@ -2046,18 +1623,6 @@ static void svm_vcpu_put(struct kvm_vcpu *vcpu) wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]); } -static void svm_vcpu_blocking(struct kvm_vcpu *vcpu) -{ - avic_set_running(vcpu, false); -} - -static void svm_vcpu_unblocking(struct kvm_vcpu *vcpu) -{ - if (kvm_check_request(KVM_REQ_APICV_UPDATE, vcpu)) - kvm_vcpu_update_apicv(vcpu); - avic_set_running(vcpu, true); -} - static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); @@ -3437,276 +3002,6 @@ static int mwait_interception(struct vcpu_svm *svm) return nop_interception(svm); } -enum avic_ipi_failure_cause { - AVIC_IPI_FAILURE_INVALID_INT_TYPE, - AVIC_IPI_FAILURE_TARGET_NOT_RUNNING, - AVIC_IPI_FAILURE_INVALID_TARGET, - AVIC_IPI_FAILURE_INVALID_BACKING_PAGE, -}; - -static int avic_incomplete_ipi_interception(struct vcpu_svm *svm) -{ - u32 icrh = svm->vmcb->control.exit_info_1 >> 32; - u32 icrl = svm->vmcb->control.exit_info_1; - u32 id = svm->vmcb->control.exit_info_2 >> 32; - u32 index = svm->vmcb->control.exit_info_2 & 0xFF; - struct kvm_lapic *apic = svm->vcpu.arch.apic; - - trace_kvm_avic_incomplete_ipi(svm->vcpu.vcpu_id, icrh, icrl, id, index); - - switch (id) { - case AVIC_IPI_FAILURE_INVALID_INT_TYPE: - /* - * AVIC hardware handles the generation of - * IPIs when the specified Message Type is Fixed - * (also known as fixed delivery mode) and - * the Trigger Mode is edge-triggered. The hardware - * also supports self and broadcast delivery modes - * specified via the Destination Shorthand(DSH) - * field of the ICRL. Logical and physical APIC ID - * formats are supported. All other IPI types cause - * a #VMEXIT, which needs to emulated. - */ - kvm_lapic_reg_write(apic, APIC_ICR2, icrh); - kvm_lapic_reg_write(apic, APIC_ICR, icrl); - break; - case AVIC_IPI_FAILURE_TARGET_NOT_RUNNING: { - int i; - struct kvm_vcpu *vcpu; - struct kvm *kvm = svm->vcpu.kvm; - struct kvm_lapic *apic = svm->vcpu.arch.apic; - - /* - * At this point, we expect that the AVIC HW has already - * set the appropriate IRR bits on the valid target - * vcpus. So, we just need to kick the appropriate vcpu. - */ - kvm_for_each_vcpu(i, vcpu, kvm) { - bool m = kvm_apic_match_dest(vcpu, apic, - icrl & APIC_SHORT_MASK, - GET_APIC_DEST_FIELD(icrh), - icrl & APIC_DEST_MASK); - - if (m && !avic_vcpu_is_running(vcpu)) - kvm_vcpu_wake_up(vcpu); - } - break; - } - case AVIC_IPI_FAILURE_INVALID_TARGET: - WARN_ONCE(1, "Invalid IPI target: index=%u, vcpu=%d, icr=%#0x:%#0x\n", - index, svm->vcpu.vcpu_id, icrh, icrl); - break; - case AVIC_IPI_FAILURE_INVALID_BACKING_PAGE: - WARN_ONCE(1, "Invalid backing page\n"); - break; - default: - pr_err("Unknown IPI interception\n"); - } - - return 1; -} - -static u32 *avic_get_logical_id_entry(struct kvm_vcpu *vcpu, u32 ldr, bool flat) -{ - struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm); - int index; - u32 *logical_apic_id_table; - int dlid = GET_APIC_LOGICAL_ID(ldr); - - if (!dlid) - return NULL; - - if (flat) { /* flat */ - index = ffs(dlid) - 1; - if (index > 7) - return NULL; - } else { /* cluster */ - int cluster = (dlid & 0xf0) >> 4; - int apic = ffs(dlid & 0x0f) - 1; - - if ((apic < 0) || (apic > 7) || - (cluster >= 0xf)) - return NULL; - index = (cluster << 2) + apic; - } - - logical_apic_id_table = (u32 *) page_address(kvm_svm->avic_logical_id_table_page); - - return &logical_apic_id_table[index]; -} - -static int avic_ldr_write(struct kvm_vcpu *vcpu, u8 g_physical_id, u32 ldr) -{ - bool flat; - u32 *entry, new_entry; - - flat = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR) == APIC_DFR_FLAT; - entry = avic_get_logical_id_entry(vcpu, ldr, flat); - if (!entry) - return -EINVAL; - - new_entry = READ_ONCE(*entry); - new_entry &= ~AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK; - new_entry |= (g_physical_id & AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK); - new_entry |= AVIC_LOGICAL_ID_ENTRY_VALID_MASK; - WRITE_ONCE(*entry, new_entry); - - return 0; -} - -static void avic_invalidate_logical_id_entry(struct kvm_vcpu *vcpu) -{ - struct vcpu_svm *svm = to_svm(vcpu); - bool flat = svm->dfr_reg == APIC_DFR_FLAT; - u32 *entry = avic_get_logical_id_entry(vcpu, svm->ldr_reg, flat); - - if (entry) - clear_bit(AVIC_LOGICAL_ID_ENTRY_VALID_BIT, (unsigned long *)entry); -} - -static int avic_handle_ldr_update(struct kvm_vcpu *vcpu) -{ - int ret = 0; - struct vcpu_svm *svm = to_svm(vcpu); - u32 ldr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LDR); - u32 id = kvm_xapic_id(vcpu->arch.apic); - - if (ldr == svm->ldr_reg) - return 0; - - avic_invalidate_logical_id_entry(vcpu); - - if (ldr) - ret = avic_ldr_write(vcpu, id, ldr); - - if (!ret) - svm->ldr_reg = ldr; - - return ret; -} - -static int avic_handle_apic_id_update(struct kvm_vcpu *vcpu) -{ - u64 *old, *new; - struct vcpu_svm *svm = to_svm(vcpu); - u32 id = kvm_xapic_id(vcpu->arch.apic); - - if (vcpu->vcpu_id == id) - return 0; - - old = avic_get_physical_id_entry(vcpu, vcpu->vcpu_id); - new = avic_get_physical_id_entry(vcpu, id); - if (!new || !old) - return 1; - - /* We need to move physical_id_entry to new offset */ - *new = *old; - *old = 0ULL; - to_svm(vcpu)->avic_physical_id_cache = new; - - /* - * Also update the guest physical APIC ID in the logical - * APIC ID table entry if already setup the LDR. - */ - if (svm->ldr_reg) - avic_handle_ldr_update(vcpu); - - return 0; -} - -static void avic_handle_dfr_update(struct kvm_vcpu *vcpu) -{ - struct vcpu_svm *svm = to_svm(vcpu); - u32 dfr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR); - - if (svm->dfr_reg == dfr) - return; - - avic_invalidate_logical_id_entry(vcpu); - svm->dfr_reg = dfr; -} - -static int avic_unaccel_trap_write(struct vcpu_svm *svm) -{ - struct kvm_lapic *apic = svm->vcpu.arch.apic; - u32 offset = svm->vmcb->control.exit_info_1 & - AVIC_UNACCEL_ACCESS_OFFSET_MASK; - - switch (offset) { - case APIC_ID: - if (avic_handle_apic_id_update(&svm->vcpu)) - return 0; - break; - case APIC_LDR: - if (avic_handle_ldr_update(&svm->vcpu)) - return 0; - break; - case APIC_DFR: - avic_handle_dfr_update(&svm->vcpu); - break; - default: - break; - } - - kvm_lapic_reg_write(apic, offset, kvm_lapic_get_reg(apic, offset)); - - return 1; -} - -static bool is_avic_unaccelerated_access_trap(u32 offset) -{ - bool ret = false; - - switch (offset) { - case APIC_ID: - case APIC_EOI: - case APIC_RRR: - case APIC_LDR: - case APIC_DFR: - case APIC_SPIV: - case APIC_ESR: - case APIC_ICR: - case APIC_LVTT: - case APIC_LVTTHMR: - case APIC_LVTPC: - case APIC_LVT0: - case APIC_LVT1: - case APIC_LVTERR: - case APIC_TMICT: - case APIC_TDCR: - ret = true; - break; - default: - break; - } - return ret; -} - -static int avic_unaccelerated_access_interception(struct vcpu_svm *svm) -{ - int ret = 0; - u32 offset = svm->vmcb->control.exit_info_1 & - AVIC_UNACCEL_ACCESS_OFFSET_MASK; - u32 vector = svm->vmcb->control.exit_info_2 & - AVIC_UNACCEL_ACCESS_VECTOR_MASK; - bool write = (svm->vmcb->control.exit_info_1 >> 32) & - AVIC_UNACCEL_ACCESS_WRITE_MASK; - bool trap = is_avic_unaccelerated_access_trap(offset); - - trace_kvm_avic_unaccelerated_access(svm->vcpu.vcpu_id, offset, - trap, write, vector); - if (trap) { - /* Handling Trap */ - WARN_ONCE(!write, "svm: Handling trap read.\n"); - ret = avic_unaccel_trap_write(svm); - } else { - /* Handling Fault */ - ret = kvm_emulate_instruction(&svm->vcpu, 0); - } - - return ret; -} - static int (*const svm_exit_handlers[])(struct vcpu_svm *svm) = { [SVM_EXIT_READ_CR0] = cr_interception, [SVM_EXIT_READ_CR3] = cr_interception, @@ -4074,324 +3369,6 @@ static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) set_cr_intercept(svm, INTERCEPT_CR8_WRITE); } -static void svm_set_virtual_apic_mode(struct kvm_vcpu *vcpu) -{ - return; -} - -static void svm_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr) -{ -} - -static void svm_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr) -{ -} - -static void svm_toggle_avic_for_irq_window(struct kvm_vcpu *vcpu, bool activate) -{ - if (!avic || !lapic_in_kernel(vcpu)) - return; - - srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); - kvm_request_apicv_update(vcpu->kvm, activate, - APICV_INHIBIT_REASON_IRQWIN); - vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); -} - -static int svm_set_pi_irte_mode(struct kvm_vcpu *vcpu, bool activate) -{ - int ret = 0; - unsigned long flags; - struct amd_svm_iommu_ir *ir; - struct vcpu_svm *svm = to_svm(vcpu); - - if (!kvm_arch_has_assigned_device(vcpu->kvm)) - return 0; - - /* - * Here, we go through the per-vcpu ir_list to update all existing - * interrupt remapping table entry targeting this vcpu. - */ - spin_lock_irqsave(&svm->ir_list_lock, flags); - - if (list_empty(&svm->ir_list)) - goto out; - - list_for_each_entry(ir, &svm->ir_list, node) { - if (activate) - ret = amd_iommu_activate_guest_mode(ir->data); - else - ret = amd_iommu_deactivate_guest_mode(ir->data); - if (ret) - break; - } -out: - spin_unlock_irqrestore(&svm->ir_list_lock, flags); - return ret; -} - -static void svm_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu) -{ - struct vcpu_svm *svm = to_svm(vcpu); - struct vmcb *vmcb = svm->vmcb; - bool activated = kvm_vcpu_apicv_active(vcpu); - - if (!avic) - return; - - if (activated) { - /** - * During AVIC temporary deactivation, guest could update - * APIC ID, DFR and LDR registers, which would not be trapped - * by avic_unaccelerated_access_interception(). In this case, - * we need to check and update the AVIC logical APIC ID table - * accordingly before re-activating. - */ - avic_post_state_restore(vcpu); - vmcb->control.int_ctl |= AVIC_ENABLE_MASK; - } else { - vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK; - } - mark_dirty(vmcb, VMCB_AVIC); - - svm_set_pi_irte_mode(vcpu, activated); -} - -static void svm_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap) -{ - return; -} - -static int svm_deliver_avic_intr(struct kvm_vcpu *vcpu, int vec) -{ - if (!vcpu->arch.apicv_active) - return -1; - - kvm_lapic_set_irr(vec, vcpu->arch.apic); - smp_mb__after_atomic(); - - if (avic_vcpu_is_running(vcpu)) { - int cpuid = vcpu->cpu; - - if (cpuid != get_cpu()) - wrmsrl(SVM_AVIC_DOORBELL, kvm_cpu_get_apicid(cpuid)); - put_cpu(); - } else - kvm_vcpu_wake_up(vcpu); - - return 0; -} - -static bool svm_dy_apicv_has_pending_interrupt(struct kvm_vcpu *vcpu) -{ - return false; -} - -static void svm_ir_list_del(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi) -{ - unsigned long flags; - struct amd_svm_iommu_ir *cur; - - spin_lock_irqsave(&svm->ir_list_lock, flags); - list_for_each_entry(cur, &svm->ir_list, node) { - if (cur->data != pi->ir_data) - continue; - list_del(&cur->node); - kfree(cur); - break; - } - spin_unlock_irqrestore(&svm->ir_list_lock, flags); -} - -static int svm_ir_list_add(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi) -{ - int ret = 0; - unsigned long flags; - struct amd_svm_iommu_ir *ir; - - /** - * In some cases, the existing irte is updaed and re-set, - * so we need to check here if it's already been * added - * to the ir_list. - */ - if (pi->ir_data && (pi->prev_ga_tag != 0)) { - struct kvm *kvm = svm->vcpu.kvm; - u32 vcpu_id = AVIC_GATAG_TO_VCPUID(pi->prev_ga_tag); - struct kvm_vcpu *prev_vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id); - struct vcpu_svm *prev_svm; - - if (!prev_vcpu) { - ret = -EINVAL; - goto out; - } - - prev_svm = to_svm(prev_vcpu); - svm_ir_list_del(prev_svm, pi); - } - - /** - * Allocating new amd_iommu_pi_data, which will get - * add to the per-vcpu ir_list. - */ - ir = kzalloc(sizeof(struct amd_svm_iommu_ir), GFP_KERNEL_ACCOUNT); - if (!ir) { - ret = -ENOMEM; - goto out; - } - ir->data = pi->ir_data; - - spin_lock_irqsave(&svm->ir_list_lock, flags); - list_add(&ir->node, &svm->ir_list); - spin_unlock_irqrestore(&svm->ir_list_lock, flags); -out: - return ret; -} - -/** - * Note: - * The HW cannot support posting multicast/broadcast - * interrupts to a vCPU. So, we still use legacy interrupt - * remapping for these kind of interrupts. - * - * For lowest-priority interrupts, we only support - * those with single CPU as the destination, e.g. user - * configures the interrupts via /proc/irq or uses - * irqbalance to make the interrupts single-CPU. - */ -static int -get_pi_vcpu_info(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, - struct vcpu_data *vcpu_info, struct vcpu_svm **svm) -{ - struct kvm_lapic_irq irq; - struct kvm_vcpu *vcpu = NULL; - - kvm_set_msi_irq(kvm, e, &irq); - - if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu) || - !kvm_irq_is_postable(&irq)) { - pr_debug("SVM: %s: use legacy intr remap mode for irq %u\n", - __func__, irq.vector); - return -1; - } - - pr_debug("SVM: %s: use GA mode for irq %u\n", __func__, - irq.vector); - *svm = to_svm(vcpu); - vcpu_info->pi_desc_addr = __sme_set(page_to_phys((*svm)->avic_backing_page)); - vcpu_info->vector = irq.vector; - - return 0; -} - -/* - * svm_update_pi_irte - set IRTE for Posted-Interrupts - * - * @kvm: kvm - * @host_irq: host irq of the interrupt - * @guest_irq: gsi of the interrupt - * @set: set or unset PI - * returns 0 on success, < 0 on failure - */ -static int svm_update_pi_irte(struct kvm *kvm, unsigned int host_irq, - uint32_t guest_irq, bool set) -{ - struct kvm_kernel_irq_routing_entry *e; - struct kvm_irq_routing_table *irq_rt; - int idx, ret = -EINVAL; - - if (!kvm_arch_has_assigned_device(kvm) || - !irq_remapping_cap(IRQ_POSTING_CAP)) - return 0; - - pr_debug("SVM: %s: host_irq=%#x, guest_irq=%#x, set=%#x\n", - __func__, host_irq, guest_irq, set); - - idx = srcu_read_lock(&kvm->irq_srcu); - irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu); - WARN_ON(guest_irq >= irq_rt->nr_rt_entries); - - hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) { - struct vcpu_data vcpu_info; - struct vcpu_svm *svm = NULL; - - if (e->type != KVM_IRQ_ROUTING_MSI) - continue; - - /** - * Here, we setup with legacy mode in the following cases: - * 1. When cannot target interrupt to a specific vcpu. - * 2. Unsetting posted interrupt. - * 3. APIC virtialization is disabled for the vcpu. - * 4. IRQ has incompatible delivery mode (SMI, INIT, etc) - */ - if (!get_pi_vcpu_info(kvm, e, &vcpu_info, &svm) && set && - kvm_vcpu_apicv_active(&svm->vcpu)) { - struct amd_iommu_pi_data pi; - - /* Try to enable guest_mode in IRTE */ - pi.base = __sme_set(page_to_phys(svm->avic_backing_page) & - AVIC_HPA_MASK); - pi.ga_tag = AVIC_GATAG(to_kvm_svm(kvm)->avic_vm_id, - svm->vcpu.vcpu_id); - pi.is_guest_mode = true; - pi.vcpu_data = &vcpu_info; - ret = irq_set_vcpu_affinity(host_irq, &pi); - - /** - * Here, we successfully setting up vcpu affinity in - * IOMMU guest mode. Now, we need to store the posted - * interrupt information in a per-vcpu ir_list so that - * we can reference to them directly when we update vcpu - * scheduling information in IOMMU irte. - */ - if (!ret && pi.is_guest_mode) - svm_ir_list_add(svm, &pi); - } else { - /* Use legacy mode in IRTE */ - struct amd_iommu_pi_data pi; - - /** - * Here, pi is used to: - * - Tell IOMMU to use legacy mode for this interrupt. - * - Retrieve ga_tag of prior interrupt remapping data. - */ - pi.is_guest_mode = false; - ret = irq_set_vcpu_affinity(host_irq, &pi); - - /** - * Check if the posted interrupt was previously - * setup with the guest_mode by checking if the ga_tag - * was cached. If so, we need to clean up the per-vcpu - * ir_list. - */ - if (!ret && pi.prev_ga_tag) { - int id = AVIC_GATAG_TO_VCPUID(pi.prev_ga_tag); - struct kvm_vcpu *vcpu; - - vcpu = kvm_get_vcpu_by_id(kvm, id); - if (vcpu) - svm_ir_list_del(to_svm(vcpu), &pi); - } - } - - if (!ret && svm) { - trace_kvm_pi_irte_update(host_irq, svm->vcpu.vcpu_id, - e->gsi, vcpu_info.vector, - vcpu_info.pi_desc_addr, set); - } - - if (ret < 0) { - pr_err("%s: failed to update PI IRTE\n", __func__); - goto out; - } - } - - ret = 0; -out: - srcu_read_unlock(&kvm->irq_srcu, idx); - return ret; -} - static int svm_nmi_allowed(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); @@ -5159,14 +4136,6 @@ static void svm_sched_in(struct kvm_vcpu *vcpu, int cpu) shrink_ple_window(vcpu); } -static inline void avic_post_state_restore(struct kvm_vcpu *vcpu) -{ - if (avic_handle_apic_id_update(vcpu) != 0) - return; - avic_handle_dfr_update(vcpu); - avic_handle_ldr_update(vcpu); -} - static void svm_setup_mce(struct kvm_vcpu *vcpu) { /* [63:9] are reserved. */ @@ -6214,23 +5183,6 @@ static bool svm_apic_init_signal_blocked(struct kvm_vcpu *vcpu) (svm->vmcb->control.intercept & (1ULL << INTERCEPT_INIT)); } -static bool svm_check_apicv_inhibit_reasons(ulong bit) -{ - ulong supported = BIT(APICV_INHIBIT_REASON_DISABLE) | - BIT(APICV_INHIBIT_REASON_HYPERV) | - BIT(APICV_INHIBIT_REASON_NESTED) | - BIT(APICV_INHIBIT_REASON_IRQWIN) | - BIT(APICV_INHIBIT_REASON_PIT_REINJ) | - BIT(APICV_INHIBIT_REASON_X2APIC); - - return supported & BIT(bit); -} - -static void svm_pre_update_apicv_exec_ctrl(struct kvm *kvm, bool activate) -{ - avic_update_access_page(kvm, activate); -} - static struct kvm_x86_ops svm_x86_ops __initdata = { .hardware_unsetup = svm_hardware_teardown, .hardware_enable = svm_hardware_enable, diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index f4c446d7a31e..c7abc1fede97 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -173,6 +173,11 @@ struct vcpu_svm { void recalc_intercepts(struct vcpu_svm *svm); +static inline struct kvm_svm *to_kvm_svm(struct kvm *kvm) +{ + return container_of(kvm, struct kvm_svm, kvm); +} + static inline void mark_all_dirty(struct vmcb *vmcb) { vmcb->control.clean = 0; @@ -378,4 +383,61 @@ int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr, int svm_check_nested_events(struct kvm_vcpu *vcpu); int nested_svm_exit_special(struct vcpu_svm *svm); +/* avic.c */ + +#define AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK (0xFF) +#define AVIC_LOGICAL_ID_ENTRY_VALID_BIT 31 +#define AVIC_LOGICAL_ID_ENTRY_VALID_MASK (1 << 31) + +#define AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK (0xFFULL) +#define AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK (0xFFFFFFFFFFULL << 12) +#define AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK (1ULL << 62) +#define AVIC_PHYSICAL_ID_ENTRY_VALID_MASK (1ULL << 63) + +#define VMCB_AVIC_APIC_BAR_MASK 0xFFFFFFFFFF000ULL + +extern int avic; + +static inline void avic_update_vapic_bar(struct vcpu_svm *svm, u64 data) +{ + svm->vmcb->control.avic_vapic_bar = data & VMCB_AVIC_APIC_BAR_MASK; + mark_dirty(svm->vmcb, VMCB_AVIC); +} + +static inline bool avic_vcpu_is_running(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + u64 *entry = svm->avic_physical_id_cache; + + if (!entry) + return false; + + return (READ_ONCE(*entry) & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK); +} + +int avic_ga_log_notifier(u32 ga_tag); +void avic_vm_destroy(struct kvm *kvm); +int avic_vm_init(struct kvm *kvm); +void avic_init_vmcb(struct vcpu_svm *svm); +void svm_toggle_avic_for_irq_window(struct kvm_vcpu *vcpu, bool activate); +int avic_incomplete_ipi_interception(struct vcpu_svm *svm); +int avic_unaccelerated_access_interception(struct vcpu_svm *svm); +int avic_init_vcpu(struct vcpu_svm *svm); +void avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu); +void avic_vcpu_put(struct kvm_vcpu *vcpu); +void avic_post_state_restore(struct kvm_vcpu *vcpu); +void svm_set_virtual_apic_mode(struct kvm_vcpu *vcpu); +void svm_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu); +bool svm_check_apicv_inhibit_reasons(ulong bit); +void svm_pre_update_apicv_exec_ctrl(struct kvm *kvm, bool activate); +void svm_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap); +void svm_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr); +void svm_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr); +int svm_deliver_avic_intr(struct kvm_vcpu *vcpu, int vec); +bool svm_dy_apicv_has_pending_interrupt(struct kvm_vcpu *vcpu); +int svm_update_pi_irte(struct kvm *kvm, unsigned int host_irq, + uint32_t guest_irq, bool set); +void svm_vcpu_blocking(struct kvm_vcpu *vcpu); +void svm_vcpu_unblocking(struct kvm_vcpu *vcpu); + #endif From eaf78265a4ab33935d3a0f1407ce4a91aac4d4d5 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 24 Mar 2020 10:41:54 +0100 Subject: [PATCH 440/635] KVM: SVM: Move SEV code to separate file Move the SEV specific parts of svm.c into the new sev.c file. Signed-off-by: Joerg Roedel Message-Id: <20200324094154.32352-5-joro@8bytes.org> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/Makefile | 2 +- arch/x86/kvm/svm/sev.c | 1187 ++++++++++++++++++++++++++++++++++++++ arch/x86/kvm/svm/svm.c | 1241 +--------------------------------------- arch/x86/kvm/svm/svm.h | 48 ++ 4 files changed, 1257 insertions(+), 1221 deletions(-) create mode 100644 arch/x86/kvm/svm/sev.c diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index 31aca07d8f12..e5a71aa0967b 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -14,7 +14,7 @@ kvm-y += x86.o emulate.o i8259.o irq.o lapic.o \ hyperv.o debugfs.o mmu/mmu.o mmu/page_track.o kvm-intel-y += vmx/vmx.o vmx/vmenter.o vmx/pmu_intel.o vmx/vmcs12.o vmx/evmcs.o vmx/nested.o -kvm-amd-y += svm/svm.o svm/pmu.o svm/nested.o svm/avic.o +kvm-amd-y += svm/svm.o svm/pmu.o svm/nested.o svm/avic.o svm/sev.o obj-$(CONFIG_KVM) += kvm.o obj-$(CONFIG_KVM_INTEL) += kvm-intel.o diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c new file mode 100644 index 000000000000..0e3fc311d7da --- /dev/null +++ b/arch/x86/kvm/svm/sev.c @@ -0,0 +1,1187 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Kernel-based Virtual Machine driver for Linux + * + * AMD SVM-SEV support + * + * Copyright 2010 Red Hat, Inc. and/or its affiliates. + */ + +#include +#include +#include +#include +#include +#include + +#include "x86.h" +#include "svm.h" + +static int sev_flush_asids(void); +static DECLARE_RWSEM(sev_deactivate_lock); +static DEFINE_MUTEX(sev_bitmap_lock); +unsigned int max_sev_asid; +static unsigned int min_sev_asid; +static unsigned long *sev_asid_bitmap; +static unsigned long *sev_reclaim_asid_bitmap; +#define __sme_page_pa(x) __sme_set(page_to_pfn(x) << PAGE_SHIFT) + +struct enc_region { + struct list_head list; + unsigned long npages; + struct page **pages; + unsigned long uaddr; + unsigned long size; +}; + +static int sev_flush_asids(void) +{ + int ret, error = 0; + + /* + * DEACTIVATE will clear the WBINVD indicator causing DF_FLUSH to fail, + * so it must be guarded. + */ + down_write(&sev_deactivate_lock); + + wbinvd_on_all_cpus(); + ret = sev_guest_df_flush(&error); + + up_write(&sev_deactivate_lock); + + if (ret) + pr_err("SEV: DF_FLUSH failed, ret=%d, error=%#x\n", ret, error); + + return ret; +} + +/* Must be called with the sev_bitmap_lock held */ +static bool __sev_recycle_asids(void) +{ + int pos; + + /* Check if there are any ASIDs to reclaim before performing a flush */ + pos = find_next_bit(sev_reclaim_asid_bitmap, + max_sev_asid, min_sev_asid - 1); + if (pos >= max_sev_asid) + return false; + + if (sev_flush_asids()) + return false; + + bitmap_xor(sev_asid_bitmap, sev_asid_bitmap, sev_reclaim_asid_bitmap, + max_sev_asid); + bitmap_zero(sev_reclaim_asid_bitmap, max_sev_asid); + + return true; +} + +static int sev_asid_new(void) +{ + bool retry = true; + int pos; + + mutex_lock(&sev_bitmap_lock); + + /* + * SEV-enabled guest must use asid from min_sev_asid to max_sev_asid. + */ +again: + pos = find_next_zero_bit(sev_asid_bitmap, max_sev_asid, min_sev_asid - 1); + if (pos >= max_sev_asid) { + if (retry && __sev_recycle_asids()) { + retry = false; + goto again; + } + mutex_unlock(&sev_bitmap_lock); + return -EBUSY; + } + + __set_bit(pos, sev_asid_bitmap); + + mutex_unlock(&sev_bitmap_lock); + + return pos + 1; +} + +static int sev_get_asid(struct kvm *kvm) +{ + struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; + + return sev->asid; +} + +static void sev_asid_free(int asid) +{ + struct svm_cpu_data *sd; + int cpu, pos; + + mutex_lock(&sev_bitmap_lock); + + pos = asid - 1; + __set_bit(pos, sev_reclaim_asid_bitmap); + + for_each_possible_cpu(cpu) { + sd = per_cpu(svm_data, cpu); + sd->sev_vmcbs[pos] = NULL; + } + + mutex_unlock(&sev_bitmap_lock); +} + +static void sev_unbind_asid(struct kvm *kvm, unsigned int handle) +{ + struct sev_data_decommission *decommission; + struct sev_data_deactivate *data; + + if (!handle) + return; + + data = kzalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return; + + /* deactivate handle */ + data->handle = handle; + + /* Guard DEACTIVATE against WBINVD/DF_FLUSH used in ASID recycling */ + down_read(&sev_deactivate_lock); + sev_guest_deactivate(data, NULL); + up_read(&sev_deactivate_lock); + + kfree(data); + + decommission = kzalloc(sizeof(*decommission), GFP_KERNEL); + if (!decommission) + return; + + /* decommission handle */ + decommission->handle = handle; + sev_guest_decommission(decommission, NULL); + + kfree(decommission); +} + +static int sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp) +{ + struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; + int asid, ret; + + ret = -EBUSY; + if (unlikely(sev->active)) + return ret; + + asid = sev_asid_new(); + if (asid < 0) + return ret; + + ret = sev_platform_init(&argp->error); + if (ret) + goto e_free; + + sev->active = true; + sev->asid = asid; + INIT_LIST_HEAD(&sev->regions_list); + + return 0; + +e_free: + sev_asid_free(asid); + return ret; +} + +static int sev_bind_asid(struct kvm *kvm, unsigned int handle, int *error) +{ + struct sev_data_activate *data; + int asid = sev_get_asid(kvm); + int ret; + + data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT); + if (!data) + return -ENOMEM; + + /* activate ASID on the given handle */ + data->handle = handle; + data->asid = asid; + ret = sev_guest_activate(data, error); + kfree(data); + + return ret; +} + +static int __sev_issue_cmd(int fd, int id, void *data, int *error) +{ + struct fd f; + int ret; + + f = fdget(fd); + if (!f.file) + return -EBADF; + + ret = sev_issue_cmd_external_user(f.file, id, data, error); + + fdput(f); + return ret; +} + +static int sev_issue_cmd(struct kvm *kvm, int id, void *data, int *error) +{ + struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; + + return __sev_issue_cmd(sev->fd, id, data, error); +} + +static int sev_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp) +{ + struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; + struct sev_data_launch_start *start; + struct kvm_sev_launch_start params; + void *dh_blob, *session_blob; + int *error = &argp->error; + int ret; + + if (!sev_guest(kvm)) + return -ENOTTY; + + if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, sizeof(params))) + return -EFAULT; + + start = kzalloc(sizeof(*start), GFP_KERNEL_ACCOUNT); + if (!start) + return -ENOMEM; + + dh_blob = NULL; + if (params.dh_uaddr) { + dh_blob = psp_copy_user_blob(params.dh_uaddr, params.dh_len); + if (IS_ERR(dh_blob)) { + ret = PTR_ERR(dh_blob); + goto e_free; + } + + start->dh_cert_address = __sme_set(__pa(dh_blob)); + start->dh_cert_len = params.dh_len; + } + + session_blob = NULL; + if (params.session_uaddr) { + session_blob = psp_copy_user_blob(params.session_uaddr, params.session_len); + if (IS_ERR(session_blob)) { + ret = PTR_ERR(session_blob); + goto e_free_dh; + } + + start->session_address = __sme_set(__pa(session_blob)); + start->session_len = params.session_len; + } + + start->handle = params.handle; + start->policy = params.policy; + + /* create memory encryption context */ + ret = __sev_issue_cmd(argp->sev_fd, SEV_CMD_LAUNCH_START, start, error); + if (ret) + goto e_free_session; + + /* Bind ASID to this guest */ + ret = sev_bind_asid(kvm, start->handle, error); + if (ret) + goto e_free_session; + + /* return handle to userspace */ + params.handle = start->handle; + if (copy_to_user((void __user *)(uintptr_t)argp->data, ¶ms, sizeof(params))) { + sev_unbind_asid(kvm, start->handle); + ret = -EFAULT; + goto e_free_session; + } + + sev->handle = start->handle; + sev->fd = argp->sev_fd; + +e_free_session: + kfree(session_blob); +e_free_dh: + kfree(dh_blob); +e_free: + kfree(start); + return ret; +} + +static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr, + unsigned long ulen, unsigned long *n, + int write) +{ + struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; + unsigned long npages, npinned, size; + unsigned long locked, lock_limit; + struct page **pages; + unsigned long first, last; + + if (ulen == 0 || uaddr + ulen < uaddr) + return NULL; + + /* Calculate number of pages. */ + first = (uaddr & PAGE_MASK) >> PAGE_SHIFT; + last = ((uaddr + ulen - 1) & PAGE_MASK) >> PAGE_SHIFT; + npages = (last - first + 1); + + locked = sev->pages_locked + npages; + lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; + if (locked > lock_limit && !capable(CAP_IPC_LOCK)) { + pr_err("SEV: %lu locked pages exceed the lock limit of %lu.\n", locked, lock_limit); + return NULL; + } + + /* Avoid using vmalloc for smaller buffers. */ + size = npages * sizeof(struct page *); + if (size > PAGE_SIZE) + pages = __vmalloc(size, GFP_KERNEL_ACCOUNT | __GFP_ZERO, + PAGE_KERNEL); + else + pages = kmalloc(size, GFP_KERNEL_ACCOUNT); + + if (!pages) + return NULL; + + /* Pin the user virtual address. */ + npinned = get_user_pages_fast(uaddr, npages, FOLL_WRITE, pages); + if (npinned != npages) { + pr_err("SEV: Failure locking %lu pages.\n", npages); + goto err; + } + + *n = npages; + sev->pages_locked = locked; + + return pages; + +err: + if (npinned > 0) + release_pages(pages, npinned); + + kvfree(pages); + return NULL; +} + +static void sev_unpin_memory(struct kvm *kvm, struct page **pages, + unsigned long npages) +{ + struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; + + release_pages(pages, npages); + kvfree(pages); + sev->pages_locked -= npages; +} + +static void sev_clflush_pages(struct page *pages[], unsigned long npages) +{ + uint8_t *page_virtual; + unsigned long i; + + if (npages == 0 || pages == NULL) + return; + + for (i = 0; i < npages; i++) { + page_virtual = kmap_atomic(pages[i]); + clflush_cache_range(page_virtual, PAGE_SIZE); + kunmap_atomic(page_virtual); + } +} + +static unsigned long get_num_contig_pages(unsigned long idx, + struct page **inpages, unsigned long npages) +{ + unsigned long paddr, next_paddr; + unsigned long i = idx + 1, pages = 1; + + /* find the number of contiguous pages starting from idx */ + paddr = __sme_page_pa(inpages[idx]); + while (i < npages) { + next_paddr = __sme_page_pa(inpages[i++]); + if ((paddr + PAGE_SIZE) == next_paddr) { + pages++; + paddr = next_paddr; + continue; + } + break; + } + + return pages; +} + +static int sev_launch_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp) +{ + unsigned long vaddr, vaddr_end, next_vaddr, npages, pages, size, i; + struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; + struct kvm_sev_launch_update_data params; + struct sev_data_launch_update_data *data; + struct page **inpages; + int ret; + + if (!sev_guest(kvm)) + return -ENOTTY; + + if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, sizeof(params))) + return -EFAULT; + + data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT); + if (!data) + return -ENOMEM; + + vaddr = params.uaddr; + size = params.len; + vaddr_end = vaddr + size; + + /* Lock the user memory. */ + inpages = sev_pin_memory(kvm, vaddr, size, &npages, 1); + if (!inpages) { + ret = -ENOMEM; + goto e_free; + } + + /* + * The LAUNCH_UPDATE command will perform in-place encryption of the + * memory content (i.e it will write the same memory region with C=1). + * It's possible that the cache may contain the data with C=0, i.e., + * unencrypted so invalidate it first. + */ + sev_clflush_pages(inpages, npages); + + for (i = 0; vaddr < vaddr_end; vaddr = next_vaddr, i += pages) { + int offset, len; + + /* + * If the user buffer is not page-aligned, calculate the offset + * within the page. + */ + offset = vaddr & (PAGE_SIZE - 1); + + /* Calculate the number of pages that can be encrypted in one go. */ + pages = get_num_contig_pages(i, inpages, npages); + + len = min_t(size_t, ((pages * PAGE_SIZE) - offset), size); + + data->handle = sev->handle; + data->len = len; + data->address = __sme_page_pa(inpages[i]) + offset; + ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_DATA, data, &argp->error); + if (ret) + goto e_unpin; + + size -= len; + next_vaddr = vaddr + len; + } + +e_unpin: + /* content of memory is updated, mark pages dirty */ + for (i = 0; i < npages; i++) { + set_page_dirty_lock(inpages[i]); + mark_page_accessed(inpages[i]); + } + /* unlock the user pages */ + sev_unpin_memory(kvm, inpages, npages); +e_free: + kfree(data); + return ret; +} + +static int sev_launch_measure(struct kvm *kvm, struct kvm_sev_cmd *argp) +{ + void __user *measure = (void __user *)(uintptr_t)argp->data; + struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; + struct sev_data_launch_measure *data; + struct kvm_sev_launch_measure params; + void __user *p = NULL; + void *blob = NULL; + int ret; + + if (!sev_guest(kvm)) + return -ENOTTY; + + if (copy_from_user(¶ms, measure, sizeof(params))) + return -EFAULT; + + data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT); + if (!data) + return -ENOMEM; + + /* User wants to query the blob length */ + if (!params.len) + goto cmd; + + p = (void __user *)(uintptr_t)params.uaddr; + if (p) { + if (params.len > SEV_FW_BLOB_MAX_SIZE) { + ret = -EINVAL; + goto e_free; + } + + ret = -ENOMEM; + blob = kmalloc(params.len, GFP_KERNEL); + if (!blob) + goto e_free; + + data->address = __psp_pa(blob); + data->len = params.len; + } + +cmd: + data->handle = sev->handle; + ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_MEASURE, data, &argp->error); + + /* + * If we query the session length, FW responded with expected data. + */ + if (!params.len) + goto done; + + if (ret) + goto e_free_blob; + + if (blob) { + if (copy_to_user(p, blob, params.len)) + ret = -EFAULT; + } + +done: + params.len = data->len; + if (copy_to_user(measure, ¶ms, sizeof(params))) + ret = -EFAULT; +e_free_blob: + kfree(blob); +e_free: + kfree(data); + return ret; +} + +static int sev_launch_finish(struct kvm *kvm, struct kvm_sev_cmd *argp) +{ + struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; + struct sev_data_launch_finish *data; + int ret; + + if (!sev_guest(kvm)) + return -ENOTTY; + + data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT); + if (!data) + return -ENOMEM; + + data->handle = sev->handle; + ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_FINISH, data, &argp->error); + + kfree(data); + return ret; +} + +static int sev_guest_status(struct kvm *kvm, struct kvm_sev_cmd *argp) +{ + struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; + struct kvm_sev_guest_status params; + struct sev_data_guest_status *data; + int ret; + + if (!sev_guest(kvm)) + return -ENOTTY; + + data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT); + if (!data) + return -ENOMEM; + + data->handle = sev->handle; + ret = sev_issue_cmd(kvm, SEV_CMD_GUEST_STATUS, data, &argp->error); + if (ret) + goto e_free; + + params.policy = data->policy; + params.state = data->state; + params.handle = data->handle; + + if (copy_to_user((void __user *)(uintptr_t)argp->data, ¶ms, sizeof(params))) + ret = -EFAULT; +e_free: + kfree(data); + return ret; +} + +static int __sev_issue_dbg_cmd(struct kvm *kvm, unsigned long src, + unsigned long dst, int size, + int *error, bool enc) +{ + struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; + struct sev_data_dbg *data; + int ret; + + data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT); + if (!data) + return -ENOMEM; + + data->handle = sev->handle; + data->dst_addr = dst; + data->src_addr = src; + data->len = size; + + ret = sev_issue_cmd(kvm, + enc ? SEV_CMD_DBG_ENCRYPT : SEV_CMD_DBG_DECRYPT, + data, error); + kfree(data); + return ret; +} + +static int __sev_dbg_decrypt(struct kvm *kvm, unsigned long src_paddr, + unsigned long dst_paddr, int sz, int *err) +{ + int offset; + + /* + * Its safe to read more than we are asked, caller should ensure that + * destination has enough space. + */ + src_paddr = round_down(src_paddr, 16); + offset = src_paddr & 15; + sz = round_up(sz + offset, 16); + + return __sev_issue_dbg_cmd(kvm, src_paddr, dst_paddr, sz, err, false); +} + +static int __sev_dbg_decrypt_user(struct kvm *kvm, unsigned long paddr, + unsigned long __user dst_uaddr, + unsigned long dst_paddr, + int size, int *err) +{ + struct page *tpage = NULL; + int ret, offset; + + /* if inputs are not 16-byte then use intermediate buffer */ + if (!IS_ALIGNED(dst_paddr, 16) || + !IS_ALIGNED(paddr, 16) || + !IS_ALIGNED(size, 16)) { + tpage = (void *)alloc_page(GFP_KERNEL); + if (!tpage) + return -ENOMEM; + + dst_paddr = __sme_page_pa(tpage); + } + + ret = __sev_dbg_decrypt(kvm, paddr, dst_paddr, size, err); + if (ret) + goto e_free; + + if (tpage) { + offset = paddr & 15; + if (copy_to_user((void __user *)(uintptr_t)dst_uaddr, + page_address(tpage) + offset, size)) + ret = -EFAULT; + } + +e_free: + if (tpage) + __free_page(tpage); + + return ret; +} + +static int __sev_dbg_encrypt_user(struct kvm *kvm, unsigned long paddr, + unsigned long __user vaddr, + unsigned long dst_paddr, + unsigned long __user dst_vaddr, + int size, int *error) +{ + struct page *src_tpage = NULL; + struct page *dst_tpage = NULL; + int ret, len = size; + + /* If source buffer is not aligned then use an intermediate buffer */ + if (!IS_ALIGNED(vaddr, 16)) { + src_tpage = alloc_page(GFP_KERNEL); + if (!src_tpage) + return -ENOMEM; + + if (copy_from_user(page_address(src_tpage), + (void __user *)(uintptr_t)vaddr, size)) { + __free_page(src_tpage); + return -EFAULT; + } + + paddr = __sme_page_pa(src_tpage); + } + + /* + * If destination buffer or length is not aligned then do read-modify-write: + * - decrypt destination in an intermediate buffer + * - copy the source buffer in an intermediate buffer + * - use the intermediate buffer as source buffer + */ + if (!IS_ALIGNED(dst_vaddr, 16) || !IS_ALIGNED(size, 16)) { + int dst_offset; + + dst_tpage = alloc_page(GFP_KERNEL); + if (!dst_tpage) { + ret = -ENOMEM; + goto e_free; + } + + ret = __sev_dbg_decrypt(kvm, dst_paddr, + __sme_page_pa(dst_tpage), size, error); + if (ret) + goto e_free; + + /* + * If source is kernel buffer then use memcpy() otherwise + * copy_from_user(). + */ + dst_offset = dst_paddr & 15; + + if (src_tpage) + memcpy(page_address(dst_tpage) + dst_offset, + page_address(src_tpage), size); + else { + if (copy_from_user(page_address(dst_tpage) + dst_offset, + (void __user *)(uintptr_t)vaddr, size)) { + ret = -EFAULT; + goto e_free; + } + } + + paddr = __sme_page_pa(dst_tpage); + dst_paddr = round_down(dst_paddr, 16); + len = round_up(size, 16); + } + + ret = __sev_issue_dbg_cmd(kvm, paddr, dst_paddr, len, error, true); + +e_free: + if (src_tpage) + __free_page(src_tpage); + if (dst_tpage) + __free_page(dst_tpage); + return ret; +} + +static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec) +{ + unsigned long vaddr, vaddr_end, next_vaddr; + unsigned long dst_vaddr; + struct page **src_p, **dst_p; + struct kvm_sev_dbg debug; + unsigned long n; + unsigned int size; + int ret; + + if (!sev_guest(kvm)) + return -ENOTTY; + + if (copy_from_user(&debug, (void __user *)(uintptr_t)argp->data, sizeof(debug))) + return -EFAULT; + + if (!debug.len || debug.src_uaddr + debug.len < debug.src_uaddr) + return -EINVAL; + if (!debug.dst_uaddr) + return -EINVAL; + + vaddr = debug.src_uaddr; + size = debug.len; + vaddr_end = vaddr + size; + dst_vaddr = debug.dst_uaddr; + + for (; vaddr < vaddr_end; vaddr = next_vaddr) { + int len, s_off, d_off; + + /* lock userspace source and destination page */ + src_p = sev_pin_memory(kvm, vaddr & PAGE_MASK, PAGE_SIZE, &n, 0); + if (!src_p) + return -EFAULT; + + dst_p = sev_pin_memory(kvm, dst_vaddr & PAGE_MASK, PAGE_SIZE, &n, 1); + if (!dst_p) { + sev_unpin_memory(kvm, src_p, n); + return -EFAULT; + } + + /* + * The DBG_{DE,EN}CRYPT commands will perform {dec,en}cryption of the + * memory content (i.e it will write the same memory region with C=1). + * It's possible that the cache may contain the data with C=0, i.e., + * unencrypted so invalidate it first. + */ + sev_clflush_pages(src_p, 1); + sev_clflush_pages(dst_p, 1); + + /* + * Since user buffer may not be page aligned, calculate the + * offset within the page. + */ + s_off = vaddr & ~PAGE_MASK; + d_off = dst_vaddr & ~PAGE_MASK; + len = min_t(size_t, (PAGE_SIZE - s_off), size); + + if (dec) + ret = __sev_dbg_decrypt_user(kvm, + __sme_page_pa(src_p[0]) + s_off, + dst_vaddr, + __sme_page_pa(dst_p[0]) + d_off, + len, &argp->error); + else + ret = __sev_dbg_encrypt_user(kvm, + __sme_page_pa(src_p[0]) + s_off, + vaddr, + __sme_page_pa(dst_p[0]) + d_off, + dst_vaddr, + len, &argp->error); + + sev_unpin_memory(kvm, src_p, n); + sev_unpin_memory(kvm, dst_p, n); + + if (ret) + goto err; + + next_vaddr = vaddr + len; + dst_vaddr = dst_vaddr + len; + size -= len; + } +err: + return ret; +} + +static int sev_launch_secret(struct kvm *kvm, struct kvm_sev_cmd *argp) +{ + struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; + struct sev_data_launch_secret *data; + struct kvm_sev_launch_secret params; + struct page **pages; + void *blob, *hdr; + unsigned long n; + int ret, offset; + + if (!sev_guest(kvm)) + return -ENOTTY; + + if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, sizeof(params))) + return -EFAULT; + + pages = sev_pin_memory(kvm, params.guest_uaddr, params.guest_len, &n, 1); + if (!pages) + return -ENOMEM; + + /* + * The secret must be copied into contiguous memory region, lets verify + * that userspace memory pages are contiguous before we issue command. + */ + if (get_num_contig_pages(0, pages, n) != n) { + ret = -EINVAL; + goto e_unpin_memory; + } + + ret = -ENOMEM; + data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT); + if (!data) + goto e_unpin_memory; + + offset = params.guest_uaddr & (PAGE_SIZE - 1); + data->guest_address = __sme_page_pa(pages[0]) + offset; + data->guest_len = params.guest_len; + + blob = psp_copy_user_blob(params.trans_uaddr, params.trans_len); + if (IS_ERR(blob)) { + ret = PTR_ERR(blob); + goto e_free; + } + + data->trans_address = __psp_pa(blob); + data->trans_len = params.trans_len; + + hdr = psp_copy_user_blob(params.hdr_uaddr, params.hdr_len); + if (IS_ERR(hdr)) { + ret = PTR_ERR(hdr); + goto e_free_blob; + } + data->hdr_address = __psp_pa(hdr); + data->hdr_len = params.hdr_len; + + data->handle = sev->handle; + ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_SECRET, data, &argp->error); + + kfree(hdr); + +e_free_blob: + kfree(blob); +e_free: + kfree(data); +e_unpin_memory: + sev_unpin_memory(kvm, pages, n); + return ret; +} + +int svm_mem_enc_op(struct kvm *kvm, void __user *argp) +{ + struct kvm_sev_cmd sev_cmd; + int r; + + if (!svm_sev_enabled()) + return -ENOTTY; + + if (!argp) + return 0; + + if (copy_from_user(&sev_cmd, argp, sizeof(struct kvm_sev_cmd))) + return -EFAULT; + + mutex_lock(&kvm->lock); + + switch (sev_cmd.id) { + case KVM_SEV_INIT: + r = sev_guest_init(kvm, &sev_cmd); + break; + case KVM_SEV_LAUNCH_START: + r = sev_launch_start(kvm, &sev_cmd); + break; + case KVM_SEV_LAUNCH_UPDATE_DATA: + r = sev_launch_update_data(kvm, &sev_cmd); + break; + case KVM_SEV_LAUNCH_MEASURE: + r = sev_launch_measure(kvm, &sev_cmd); + break; + case KVM_SEV_LAUNCH_FINISH: + r = sev_launch_finish(kvm, &sev_cmd); + break; + case KVM_SEV_GUEST_STATUS: + r = sev_guest_status(kvm, &sev_cmd); + break; + case KVM_SEV_DBG_DECRYPT: + r = sev_dbg_crypt(kvm, &sev_cmd, true); + break; + case KVM_SEV_DBG_ENCRYPT: + r = sev_dbg_crypt(kvm, &sev_cmd, false); + break; + case KVM_SEV_LAUNCH_SECRET: + r = sev_launch_secret(kvm, &sev_cmd); + break; + default: + r = -EINVAL; + goto out; + } + + if (copy_to_user(argp, &sev_cmd, sizeof(struct kvm_sev_cmd))) + r = -EFAULT; + +out: + mutex_unlock(&kvm->lock); + return r; +} + +int svm_register_enc_region(struct kvm *kvm, + struct kvm_enc_region *range) +{ + struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; + struct enc_region *region; + int ret = 0; + + if (!sev_guest(kvm)) + return -ENOTTY; + + if (range->addr > ULONG_MAX || range->size > ULONG_MAX) + return -EINVAL; + + region = kzalloc(sizeof(*region), GFP_KERNEL_ACCOUNT); + if (!region) + return -ENOMEM; + + region->pages = sev_pin_memory(kvm, range->addr, range->size, ®ion->npages, 1); + if (!region->pages) { + ret = -ENOMEM; + goto e_free; + } + + /* + * The guest may change the memory encryption attribute from C=0 -> C=1 + * or vice versa for this memory range. Lets make sure caches are + * flushed to ensure that guest data gets written into memory with + * correct C-bit. + */ + sev_clflush_pages(region->pages, region->npages); + + region->uaddr = range->addr; + region->size = range->size; + + mutex_lock(&kvm->lock); + list_add_tail(®ion->list, &sev->regions_list); + mutex_unlock(&kvm->lock); + + return ret; + +e_free: + kfree(region); + return ret; +} + +static struct enc_region * +find_enc_region(struct kvm *kvm, struct kvm_enc_region *range) +{ + struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; + struct list_head *head = &sev->regions_list; + struct enc_region *i; + + list_for_each_entry(i, head, list) { + if (i->uaddr == range->addr && + i->size == range->size) + return i; + } + + return NULL; +} + +static void __unregister_enc_region_locked(struct kvm *kvm, + struct enc_region *region) +{ + sev_unpin_memory(kvm, region->pages, region->npages); + list_del(®ion->list); + kfree(region); +} + +int svm_unregister_enc_region(struct kvm *kvm, + struct kvm_enc_region *range) +{ + struct enc_region *region; + int ret; + + mutex_lock(&kvm->lock); + + if (!sev_guest(kvm)) { + ret = -ENOTTY; + goto failed; + } + + region = find_enc_region(kvm, range); + if (!region) { + ret = -EINVAL; + goto failed; + } + + /* + * Ensure that all guest tagged cache entries are flushed before + * releasing the pages back to the system for use. CLFLUSH will + * not do this, so issue a WBINVD. + */ + wbinvd_on_all_cpus(); + + __unregister_enc_region_locked(kvm, region); + + mutex_unlock(&kvm->lock); + return 0; + +failed: + mutex_unlock(&kvm->lock); + return ret; +} + +void sev_vm_destroy(struct kvm *kvm) +{ + struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; + struct list_head *head = &sev->regions_list; + struct list_head *pos, *q; + + if (!sev_guest(kvm)) + return; + + mutex_lock(&kvm->lock); + + /* + * Ensure that all guest tagged cache entries are flushed before + * releasing the pages back to the system for use. CLFLUSH will + * not do this, so issue a WBINVD. + */ + wbinvd_on_all_cpus(); + + /* + * if userspace was terminated before unregistering the memory regions + * then lets unpin all the registered memory. + */ + if (!list_empty(head)) { + list_for_each_safe(pos, q, head) { + __unregister_enc_region_locked(kvm, + list_entry(pos, struct enc_region, list)); + } + } + + mutex_unlock(&kvm->lock); + + sev_unbind_asid(kvm, sev->handle); + sev_asid_free(sev->asid); +} + +int __init sev_hardware_setup(void) +{ + struct sev_user_data_status *status; + int rc; + + /* Maximum number of encrypted guests supported simultaneously */ + max_sev_asid = cpuid_ecx(0x8000001F); + + if (!max_sev_asid) + return 1; + + /* Minimum ASID value that should be used for SEV guest */ + min_sev_asid = cpuid_edx(0x8000001F); + + /* Initialize SEV ASID bitmaps */ + sev_asid_bitmap = bitmap_zalloc(max_sev_asid, GFP_KERNEL); + if (!sev_asid_bitmap) + return 1; + + sev_reclaim_asid_bitmap = bitmap_zalloc(max_sev_asid, GFP_KERNEL); + if (!sev_reclaim_asid_bitmap) + return 1; + + status = kmalloc(sizeof(*status), GFP_KERNEL); + if (!status) + return 1; + + /* + * Check SEV platform status. + * + * PLATFORM_STATUS can be called in any state, if we failed to query + * the PLATFORM status then either PSP firmware does not support SEV + * feature or SEV firmware is dead. + */ + rc = sev_platform_status(status, NULL); + if (rc) + goto err; + + pr_info("SEV supported\n"); + +err: + kfree(status); + return rc; +} + +void sev_hardware_teardown(void) +{ + bitmap_free(sev_asid_bitmap); + bitmap_free(sev_reclaim_asid_bitmap); + + sev_flush_asids(); +} + +void pre_sev_run(struct vcpu_svm *svm, int cpu) +{ + struct svm_cpu_data *sd = per_cpu(svm_data, cpu); + int asid = sev_get_asid(svm->vcpu.kvm); + + /* Assign the asid allocated with this SEV guest */ + svm->vmcb->control.asid = asid; + + /* + * Flush guest TLB: + * + * 1) when different VMCB for the same ASID is to be run on the same host CPU. + * 2) or this VMCB was executed on different host CPU in previous VMRUNs. + */ + if (sd->sev_vmcbs[asid] == svm->vmcb && + svm->last_cpu == cpu) + return; + + svm->last_cpu = cpu; + sd->sev_vmcbs[asid] = svm->vmcb; + svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID; + mark_dirty(svm->vmcb, VMCB_ASID); +} diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index e32b4956fa06..05d77b395fc1 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -196,47 +196,6 @@ static u8 rsm_ins_bytes[] = "\x0f\xaa"; static void svm_complete_interrupts(struct vcpu_svm *svm); -static int sev_flush_asids(void); -static DECLARE_RWSEM(sev_deactivate_lock); -static DEFINE_MUTEX(sev_bitmap_lock); -static unsigned int max_sev_asid; -static unsigned int min_sev_asid; -static unsigned long *sev_asid_bitmap; -static unsigned long *sev_reclaim_asid_bitmap; -#define __sme_page_pa(x) __sme_set(page_to_pfn(x) << PAGE_SHIFT) - -struct enc_region { - struct list_head list; - unsigned long npages; - struct page **pages; - unsigned long uaddr; - unsigned long size; -}; - - -static inline bool svm_sev_enabled(void) -{ - return IS_ENABLED(CONFIG_KVM_AMD_SEV) ? max_sev_asid : 0; -} - -static inline bool sev_guest(struct kvm *kvm) -{ -#ifdef CONFIG_KVM_AMD_SEV - struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; - - return sev->active; -#else - return false; -#endif -} - -static inline int sev_get_asid(struct kvm *kvm) -{ - struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; - - return sev->asid; -} - static unsigned long iopm_base; struct kvm_ldttss_desc { @@ -248,23 +207,7 @@ struct kvm_ldttss_desc { u32 zero1; } __attribute__((packed)); -struct svm_cpu_data { - int cpu; - - u64 asid_generation; - u32 max_asid; - u32 next_asid; - u32 min_asid; - struct kvm_ldttss_desc *tss_desc; - - struct page *save_area; - struct vmcb *current_vmcb; - - /* index = sev_asid, value = vmcb pointer */ - struct vmcb **sev_vmcbs; -}; - -static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data); +DEFINE_PER_CPU(struct svm_cpu_data *, svm_data); static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000}; @@ -763,51 +706,6 @@ void disable_nmi_singlestep(struct vcpu_svm *svm) } } -static __init int sev_hardware_setup(void) -{ - struct sev_user_data_status *status; - int rc; - - /* Maximum number of encrypted guests supported simultaneously */ - max_sev_asid = cpuid_ecx(0x8000001F); - - if (!max_sev_asid) - return 1; - - /* Minimum ASID value that should be used for SEV guest */ - min_sev_asid = cpuid_edx(0x8000001F); - - /* Initialize SEV ASID bitmaps */ - sev_asid_bitmap = bitmap_zalloc(max_sev_asid, GFP_KERNEL); - if (!sev_asid_bitmap) - return 1; - - sev_reclaim_asid_bitmap = bitmap_zalloc(max_sev_asid, GFP_KERNEL); - if (!sev_reclaim_asid_bitmap) - return 1; - - status = kmalloc(sizeof(*status), GFP_KERNEL); - if (!status) - return 1; - - /* - * Check SEV platform status. - * - * PLATFORM_STATUS can be called in any state, if we failed to query - * the PLATFORM status then either PSP firmware does not support SEV - * feature or SEV firmware is dead. - */ - rc = sev_platform_status(status, NULL); - if (rc) - goto err; - - pr_info("SEV supported\n"); - -err: - kfree(status); - return rc; -} - static void grow_ple_window(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); @@ -889,12 +787,8 @@ static void svm_hardware_teardown(void) { int cpu; - if (svm_sev_enabled()) { - bitmap_free(sev_asid_bitmap); - bitmap_free(sev_reclaim_asid_bitmap); - - sev_flush_asids(); - } + if (svm_sev_enabled()) + sev_hardware_teardown(); for_each_possible_cpu(cpu) svm_cpu_uninit(cpu); @@ -1250,199 +1144,6 @@ static void init_vmcb(struct vcpu_svm *svm) } -static void sev_asid_free(int asid) -{ - struct svm_cpu_data *sd; - int cpu, pos; - - mutex_lock(&sev_bitmap_lock); - - pos = asid - 1; - __set_bit(pos, sev_reclaim_asid_bitmap); - - for_each_possible_cpu(cpu) { - sd = per_cpu(svm_data, cpu); - sd->sev_vmcbs[pos] = NULL; - } - - mutex_unlock(&sev_bitmap_lock); -} - -static void sev_unbind_asid(struct kvm *kvm, unsigned int handle) -{ - struct sev_data_decommission *decommission; - struct sev_data_deactivate *data; - - if (!handle) - return; - - data = kzalloc(sizeof(*data), GFP_KERNEL); - if (!data) - return; - - /* deactivate handle */ - data->handle = handle; - - /* Guard DEACTIVATE against WBINVD/DF_FLUSH used in ASID recycling */ - down_read(&sev_deactivate_lock); - sev_guest_deactivate(data, NULL); - up_read(&sev_deactivate_lock); - - kfree(data); - - decommission = kzalloc(sizeof(*decommission), GFP_KERNEL); - if (!decommission) - return; - - /* decommission handle */ - decommission->handle = handle; - sev_guest_decommission(decommission, NULL); - - kfree(decommission); -} - -static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr, - unsigned long ulen, unsigned long *n, - int write) -{ - struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; - unsigned long npages, npinned, size; - unsigned long locked, lock_limit; - struct page **pages; - unsigned long first, last; - - if (ulen == 0 || uaddr + ulen < uaddr) - return NULL; - - /* Calculate number of pages. */ - first = (uaddr & PAGE_MASK) >> PAGE_SHIFT; - last = ((uaddr + ulen - 1) & PAGE_MASK) >> PAGE_SHIFT; - npages = (last - first + 1); - - locked = sev->pages_locked + npages; - lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; - if (locked > lock_limit && !capable(CAP_IPC_LOCK)) { - pr_err("SEV: %lu locked pages exceed the lock limit of %lu.\n", locked, lock_limit); - return NULL; - } - - /* Avoid using vmalloc for smaller buffers. */ - size = npages * sizeof(struct page *); - if (size > PAGE_SIZE) - pages = __vmalloc(size, GFP_KERNEL_ACCOUNT | __GFP_ZERO, - PAGE_KERNEL); - else - pages = kmalloc(size, GFP_KERNEL_ACCOUNT); - - if (!pages) - return NULL; - - /* Pin the user virtual address. */ - npinned = get_user_pages_fast(uaddr, npages, FOLL_WRITE, pages); - if (npinned != npages) { - pr_err("SEV: Failure locking %lu pages.\n", npages); - goto err; - } - - *n = npages; - sev->pages_locked = locked; - - return pages; - -err: - if (npinned > 0) - release_pages(pages, npinned); - - kvfree(pages); - return NULL; -} - -static void sev_unpin_memory(struct kvm *kvm, struct page **pages, - unsigned long npages) -{ - struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; - - release_pages(pages, npages); - kvfree(pages); - sev->pages_locked -= npages; -} - -static void sev_clflush_pages(struct page *pages[], unsigned long npages) -{ - uint8_t *page_virtual; - unsigned long i; - - if (npages == 0 || pages == NULL) - return; - - for (i = 0; i < npages; i++) { - page_virtual = kmap_atomic(pages[i]); - clflush_cache_range(page_virtual, PAGE_SIZE); - kunmap_atomic(page_virtual); - } -} - -static void __unregister_enc_region_locked(struct kvm *kvm, - struct enc_region *region) -{ - sev_unpin_memory(kvm, region->pages, region->npages); - list_del(®ion->list); - kfree(region); -} - -static void sev_vm_destroy(struct kvm *kvm) -{ - struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; - struct list_head *head = &sev->regions_list; - struct list_head *pos, *q; - - if (!sev_guest(kvm)) - return; - - mutex_lock(&kvm->lock); - - /* - * Ensure that all guest tagged cache entries are flushed before - * releasing the pages back to the system for use. CLFLUSH will - * not do this, so issue a WBINVD. - */ - wbinvd_on_all_cpus(); - - /* - * if userspace was terminated before unregistering the memory regions - * then lets unpin all the registered memory. - */ - if (!list_empty(head)) { - list_for_each_safe(pos, q, head) { - __unregister_enc_region_locked(kvm, - list_entry(pos, struct enc_region, list)); - } - } - - mutex_unlock(&kvm->lock); - - sev_unbind_asid(kvm, sev->handle); - sev_asid_free(sev->asid); -} - -static void svm_vm_destroy(struct kvm *kvm) -{ - avic_vm_destroy(kvm); - sev_vm_destroy(kvm); -} - -static int svm_vm_init(struct kvm *kvm) -{ - if (avic) { - int ret = avic_vm_init(kvm); - if (ret) - return ret; - } - - kvm_apicv_init(kvm, avic); - return 0; -} - static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) { struct vcpu_svm *svm = to_svm(vcpu); @@ -3292,30 +2993,6 @@ static void reload_tss(struct kvm_vcpu *vcpu) load_TR_desc(); } -static void pre_sev_run(struct vcpu_svm *svm, int cpu) -{ - struct svm_cpu_data *sd = per_cpu(svm_data, cpu); - int asid = sev_get_asid(svm->vcpu.kvm); - - /* Assign the asid allocated with this SEV guest */ - svm->vmcb->control.asid = asid; - - /* - * Flush guest TLB: - * - * 1) when different VMCB for the same ASID is to be run on the same host CPU. - * 2) or this VMCB was executed on different host CPU in previous VMRUNs. - */ - if (sd->sev_vmcbs[asid] == svm->vmcb && - svm->last_cpu == cpu) - return; - - svm->last_cpu = cpu; - sd->sev_vmcbs[asid] = svm->vmcb; - svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID; - mark_dirty(svm->vmcb, VMCB_ASID); -} - static void pre_svm_run(struct vcpu_svm *svm) { int cpu = raw_smp_processor_id(); @@ -4216,900 +3893,6 @@ static int enable_smi_window(struct kvm_vcpu *vcpu) return 0; } -static int sev_flush_asids(void) -{ - int ret, error; - - /* - * DEACTIVATE will clear the WBINVD indicator causing DF_FLUSH to fail, - * so it must be guarded. - */ - down_write(&sev_deactivate_lock); - - wbinvd_on_all_cpus(); - ret = sev_guest_df_flush(&error); - - up_write(&sev_deactivate_lock); - - if (ret) - pr_err("SEV: DF_FLUSH failed, ret=%d, error=%#x\n", ret, error); - - return ret; -} - -/* Must be called with the sev_bitmap_lock held */ -static bool __sev_recycle_asids(void) -{ - int pos; - - /* Check if there are any ASIDs to reclaim before performing a flush */ - pos = find_next_bit(sev_reclaim_asid_bitmap, - max_sev_asid, min_sev_asid - 1); - if (pos >= max_sev_asid) - return false; - - if (sev_flush_asids()) - return false; - - bitmap_xor(sev_asid_bitmap, sev_asid_bitmap, sev_reclaim_asid_bitmap, - max_sev_asid); - bitmap_zero(sev_reclaim_asid_bitmap, max_sev_asid); - - return true; -} - -static int sev_asid_new(void) -{ - bool retry = true; - int pos; - - mutex_lock(&sev_bitmap_lock); - - /* - * SEV-enabled guest must use asid from min_sev_asid to max_sev_asid. - */ -again: - pos = find_next_zero_bit(sev_asid_bitmap, max_sev_asid, min_sev_asid - 1); - if (pos >= max_sev_asid) { - if (retry && __sev_recycle_asids()) { - retry = false; - goto again; - } - mutex_unlock(&sev_bitmap_lock); - return -EBUSY; - } - - __set_bit(pos, sev_asid_bitmap); - - mutex_unlock(&sev_bitmap_lock); - - return pos + 1; -} - -static int sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp) -{ - struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; - int asid, ret; - - ret = -EBUSY; - if (unlikely(sev->active)) - return ret; - - asid = sev_asid_new(); - if (asid < 0) - return ret; - - ret = sev_platform_init(&argp->error); - if (ret) - goto e_free; - - sev->active = true; - sev->asid = asid; - INIT_LIST_HEAD(&sev->regions_list); - - return 0; - -e_free: - sev_asid_free(asid); - return ret; -} - -static int sev_bind_asid(struct kvm *kvm, unsigned int handle, int *error) -{ - struct sev_data_activate *data; - int asid = sev_get_asid(kvm); - int ret; - - data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT); - if (!data) - return -ENOMEM; - - /* activate ASID on the given handle */ - data->handle = handle; - data->asid = asid; - ret = sev_guest_activate(data, error); - kfree(data); - - return ret; -} - -static int __sev_issue_cmd(int fd, int id, void *data, int *error) -{ - struct fd f; - int ret; - - f = fdget(fd); - if (!f.file) - return -EBADF; - - ret = sev_issue_cmd_external_user(f.file, id, data, error); - - fdput(f); - return ret; -} - -static int sev_issue_cmd(struct kvm *kvm, int id, void *data, int *error) -{ - struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; - - return __sev_issue_cmd(sev->fd, id, data, error); -} - -static int sev_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp) -{ - struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; - struct sev_data_launch_start *start; - struct kvm_sev_launch_start params; - void *dh_blob, *session_blob; - int *error = &argp->error; - int ret; - - if (!sev_guest(kvm)) - return -ENOTTY; - - if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, sizeof(params))) - return -EFAULT; - - start = kzalloc(sizeof(*start), GFP_KERNEL_ACCOUNT); - if (!start) - return -ENOMEM; - - dh_blob = NULL; - if (params.dh_uaddr) { - dh_blob = psp_copy_user_blob(params.dh_uaddr, params.dh_len); - if (IS_ERR(dh_blob)) { - ret = PTR_ERR(dh_blob); - goto e_free; - } - - start->dh_cert_address = __sme_set(__pa(dh_blob)); - start->dh_cert_len = params.dh_len; - } - - session_blob = NULL; - if (params.session_uaddr) { - session_blob = psp_copy_user_blob(params.session_uaddr, params.session_len); - if (IS_ERR(session_blob)) { - ret = PTR_ERR(session_blob); - goto e_free_dh; - } - - start->session_address = __sme_set(__pa(session_blob)); - start->session_len = params.session_len; - } - - start->handle = params.handle; - start->policy = params.policy; - - /* create memory encryption context */ - ret = __sev_issue_cmd(argp->sev_fd, SEV_CMD_LAUNCH_START, start, error); - if (ret) - goto e_free_session; - - /* Bind ASID to this guest */ - ret = sev_bind_asid(kvm, start->handle, error); - if (ret) - goto e_free_session; - - /* return handle to userspace */ - params.handle = start->handle; - if (copy_to_user((void __user *)(uintptr_t)argp->data, ¶ms, sizeof(params))) { - sev_unbind_asid(kvm, start->handle); - ret = -EFAULT; - goto e_free_session; - } - - sev->handle = start->handle; - sev->fd = argp->sev_fd; - -e_free_session: - kfree(session_blob); -e_free_dh: - kfree(dh_blob); -e_free: - kfree(start); - return ret; -} - -static unsigned long get_num_contig_pages(unsigned long idx, - struct page **inpages, unsigned long npages) -{ - unsigned long paddr, next_paddr; - unsigned long i = idx + 1, pages = 1; - - /* find the number of contiguous pages starting from idx */ - paddr = __sme_page_pa(inpages[idx]); - while (i < npages) { - next_paddr = __sme_page_pa(inpages[i++]); - if ((paddr + PAGE_SIZE) == next_paddr) { - pages++; - paddr = next_paddr; - continue; - } - break; - } - - return pages; -} - -static int sev_launch_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp) -{ - unsigned long vaddr, vaddr_end, next_vaddr, npages, pages, size, i; - struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; - struct kvm_sev_launch_update_data params; - struct sev_data_launch_update_data *data; - struct page **inpages; - int ret; - - if (!sev_guest(kvm)) - return -ENOTTY; - - if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, sizeof(params))) - return -EFAULT; - - data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT); - if (!data) - return -ENOMEM; - - vaddr = params.uaddr; - size = params.len; - vaddr_end = vaddr + size; - - /* Lock the user memory. */ - inpages = sev_pin_memory(kvm, vaddr, size, &npages, 1); - if (!inpages) { - ret = -ENOMEM; - goto e_free; - } - - /* - * The LAUNCH_UPDATE command will perform in-place encryption of the - * memory content (i.e it will write the same memory region with C=1). - * It's possible that the cache may contain the data with C=0, i.e., - * unencrypted so invalidate it first. - */ - sev_clflush_pages(inpages, npages); - - for (i = 0; vaddr < vaddr_end; vaddr = next_vaddr, i += pages) { - int offset, len; - - /* - * If the user buffer is not page-aligned, calculate the offset - * within the page. - */ - offset = vaddr & (PAGE_SIZE - 1); - - /* Calculate the number of pages that can be encrypted in one go. */ - pages = get_num_contig_pages(i, inpages, npages); - - len = min_t(size_t, ((pages * PAGE_SIZE) - offset), size); - - data->handle = sev->handle; - data->len = len; - data->address = __sme_page_pa(inpages[i]) + offset; - ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_DATA, data, &argp->error); - if (ret) - goto e_unpin; - - size -= len; - next_vaddr = vaddr + len; - } - -e_unpin: - /* content of memory is updated, mark pages dirty */ - for (i = 0; i < npages; i++) { - set_page_dirty_lock(inpages[i]); - mark_page_accessed(inpages[i]); - } - /* unlock the user pages */ - sev_unpin_memory(kvm, inpages, npages); -e_free: - kfree(data); - return ret; -} - -static int sev_launch_measure(struct kvm *kvm, struct kvm_sev_cmd *argp) -{ - void __user *measure = (void __user *)(uintptr_t)argp->data; - struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; - struct sev_data_launch_measure *data; - struct kvm_sev_launch_measure params; - void __user *p = NULL; - void *blob = NULL; - int ret; - - if (!sev_guest(kvm)) - return -ENOTTY; - - if (copy_from_user(¶ms, measure, sizeof(params))) - return -EFAULT; - - data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT); - if (!data) - return -ENOMEM; - - /* User wants to query the blob length */ - if (!params.len) - goto cmd; - - p = (void __user *)(uintptr_t)params.uaddr; - if (p) { - if (params.len > SEV_FW_BLOB_MAX_SIZE) { - ret = -EINVAL; - goto e_free; - } - - ret = -ENOMEM; - blob = kmalloc(params.len, GFP_KERNEL); - if (!blob) - goto e_free; - - data->address = __psp_pa(blob); - data->len = params.len; - } - -cmd: - data->handle = sev->handle; - ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_MEASURE, data, &argp->error); - - /* - * If we query the session length, FW responded with expected data. - */ - if (!params.len) - goto done; - - if (ret) - goto e_free_blob; - - if (blob) { - if (copy_to_user(p, blob, params.len)) - ret = -EFAULT; - } - -done: - params.len = data->len; - if (copy_to_user(measure, ¶ms, sizeof(params))) - ret = -EFAULT; -e_free_blob: - kfree(blob); -e_free: - kfree(data); - return ret; -} - -static int sev_launch_finish(struct kvm *kvm, struct kvm_sev_cmd *argp) -{ - struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; - struct sev_data_launch_finish *data; - int ret; - - if (!sev_guest(kvm)) - return -ENOTTY; - - data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT); - if (!data) - return -ENOMEM; - - data->handle = sev->handle; - ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_FINISH, data, &argp->error); - - kfree(data); - return ret; -} - -static int sev_guest_status(struct kvm *kvm, struct kvm_sev_cmd *argp) -{ - struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; - struct kvm_sev_guest_status params; - struct sev_data_guest_status *data; - int ret; - - if (!sev_guest(kvm)) - return -ENOTTY; - - data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT); - if (!data) - return -ENOMEM; - - data->handle = sev->handle; - ret = sev_issue_cmd(kvm, SEV_CMD_GUEST_STATUS, data, &argp->error); - if (ret) - goto e_free; - - params.policy = data->policy; - params.state = data->state; - params.handle = data->handle; - - if (copy_to_user((void __user *)(uintptr_t)argp->data, ¶ms, sizeof(params))) - ret = -EFAULT; -e_free: - kfree(data); - return ret; -} - -static int __sev_issue_dbg_cmd(struct kvm *kvm, unsigned long src, - unsigned long dst, int size, - int *error, bool enc) -{ - struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; - struct sev_data_dbg *data; - int ret; - - data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT); - if (!data) - return -ENOMEM; - - data->handle = sev->handle; - data->dst_addr = dst; - data->src_addr = src; - data->len = size; - - ret = sev_issue_cmd(kvm, - enc ? SEV_CMD_DBG_ENCRYPT : SEV_CMD_DBG_DECRYPT, - data, error); - kfree(data); - return ret; -} - -static int __sev_dbg_decrypt(struct kvm *kvm, unsigned long src_paddr, - unsigned long dst_paddr, int sz, int *err) -{ - int offset; - - /* - * Its safe to read more than we are asked, caller should ensure that - * destination has enough space. - */ - src_paddr = round_down(src_paddr, 16); - offset = src_paddr & 15; - sz = round_up(sz + offset, 16); - - return __sev_issue_dbg_cmd(kvm, src_paddr, dst_paddr, sz, err, false); -} - -static int __sev_dbg_decrypt_user(struct kvm *kvm, unsigned long paddr, - unsigned long __user dst_uaddr, - unsigned long dst_paddr, - int size, int *err) -{ - struct page *tpage = NULL; - int ret, offset; - - /* if inputs are not 16-byte then use intermediate buffer */ - if (!IS_ALIGNED(dst_paddr, 16) || - !IS_ALIGNED(paddr, 16) || - !IS_ALIGNED(size, 16)) { - tpage = (void *)alloc_page(GFP_KERNEL); - if (!tpage) - return -ENOMEM; - - dst_paddr = __sme_page_pa(tpage); - } - - ret = __sev_dbg_decrypt(kvm, paddr, dst_paddr, size, err); - if (ret) - goto e_free; - - if (tpage) { - offset = paddr & 15; - if (copy_to_user((void __user *)(uintptr_t)dst_uaddr, - page_address(tpage) + offset, size)) - ret = -EFAULT; - } - -e_free: - if (tpage) - __free_page(tpage); - - return ret; -} - -static int __sev_dbg_encrypt_user(struct kvm *kvm, unsigned long paddr, - unsigned long __user vaddr, - unsigned long dst_paddr, - unsigned long __user dst_vaddr, - int size, int *error) -{ - struct page *src_tpage = NULL; - struct page *dst_tpage = NULL; - int ret, len = size; - - /* If source buffer is not aligned then use an intermediate buffer */ - if (!IS_ALIGNED(vaddr, 16)) { - src_tpage = alloc_page(GFP_KERNEL); - if (!src_tpage) - return -ENOMEM; - - if (copy_from_user(page_address(src_tpage), - (void __user *)(uintptr_t)vaddr, size)) { - __free_page(src_tpage); - return -EFAULT; - } - - paddr = __sme_page_pa(src_tpage); - } - - /* - * If destination buffer or length is not aligned then do read-modify-write: - * - decrypt destination in an intermediate buffer - * - copy the source buffer in an intermediate buffer - * - use the intermediate buffer as source buffer - */ - if (!IS_ALIGNED(dst_vaddr, 16) || !IS_ALIGNED(size, 16)) { - int dst_offset; - - dst_tpage = alloc_page(GFP_KERNEL); - if (!dst_tpage) { - ret = -ENOMEM; - goto e_free; - } - - ret = __sev_dbg_decrypt(kvm, dst_paddr, - __sme_page_pa(dst_tpage), size, error); - if (ret) - goto e_free; - - /* - * If source is kernel buffer then use memcpy() otherwise - * copy_from_user(). - */ - dst_offset = dst_paddr & 15; - - if (src_tpage) - memcpy(page_address(dst_tpage) + dst_offset, - page_address(src_tpage), size); - else { - if (copy_from_user(page_address(dst_tpage) + dst_offset, - (void __user *)(uintptr_t)vaddr, size)) { - ret = -EFAULT; - goto e_free; - } - } - - paddr = __sme_page_pa(dst_tpage); - dst_paddr = round_down(dst_paddr, 16); - len = round_up(size, 16); - } - - ret = __sev_issue_dbg_cmd(kvm, paddr, dst_paddr, len, error, true); - -e_free: - if (src_tpage) - __free_page(src_tpage); - if (dst_tpage) - __free_page(dst_tpage); - return ret; -} - -static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec) -{ - unsigned long vaddr, vaddr_end, next_vaddr; - unsigned long dst_vaddr; - struct page **src_p, **dst_p; - struct kvm_sev_dbg debug; - unsigned long n; - unsigned int size; - int ret; - - if (!sev_guest(kvm)) - return -ENOTTY; - - if (copy_from_user(&debug, (void __user *)(uintptr_t)argp->data, sizeof(debug))) - return -EFAULT; - - if (!debug.len || debug.src_uaddr + debug.len < debug.src_uaddr) - return -EINVAL; - if (!debug.dst_uaddr) - return -EINVAL; - - vaddr = debug.src_uaddr; - size = debug.len; - vaddr_end = vaddr + size; - dst_vaddr = debug.dst_uaddr; - - for (; vaddr < vaddr_end; vaddr = next_vaddr) { - int len, s_off, d_off; - - /* lock userspace source and destination page */ - src_p = sev_pin_memory(kvm, vaddr & PAGE_MASK, PAGE_SIZE, &n, 0); - if (!src_p) - return -EFAULT; - - dst_p = sev_pin_memory(kvm, dst_vaddr & PAGE_MASK, PAGE_SIZE, &n, 1); - if (!dst_p) { - sev_unpin_memory(kvm, src_p, n); - return -EFAULT; - } - - /* - * The DBG_{DE,EN}CRYPT commands will perform {dec,en}cryption of the - * memory content (i.e it will write the same memory region with C=1). - * It's possible that the cache may contain the data with C=0, i.e., - * unencrypted so invalidate it first. - */ - sev_clflush_pages(src_p, 1); - sev_clflush_pages(dst_p, 1); - - /* - * Since user buffer may not be page aligned, calculate the - * offset within the page. - */ - s_off = vaddr & ~PAGE_MASK; - d_off = dst_vaddr & ~PAGE_MASK; - len = min_t(size_t, (PAGE_SIZE - s_off), size); - - if (dec) - ret = __sev_dbg_decrypt_user(kvm, - __sme_page_pa(src_p[0]) + s_off, - dst_vaddr, - __sme_page_pa(dst_p[0]) + d_off, - len, &argp->error); - else - ret = __sev_dbg_encrypt_user(kvm, - __sme_page_pa(src_p[0]) + s_off, - vaddr, - __sme_page_pa(dst_p[0]) + d_off, - dst_vaddr, - len, &argp->error); - - sev_unpin_memory(kvm, src_p, n); - sev_unpin_memory(kvm, dst_p, n); - - if (ret) - goto err; - - next_vaddr = vaddr + len; - dst_vaddr = dst_vaddr + len; - size -= len; - } -err: - return ret; -} - -static int sev_launch_secret(struct kvm *kvm, struct kvm_sev_cmd *argp) -{ - struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; - struct sev_data_launch_secret *data; - struct kvm_sev_launch_secret params; - struct page **pages; - void *blob, *hdr; - unsigned long n; - int ret, offset; - - if (!sev_guest(kvm)) - return -ENOTTY; - - if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, sizeof(params))) - return -EFAULT; - - pages = sev_pin_memory(kvm, params.guest_uaddr, params.guest_len, &n, 1); - if (!pages) - return -ENOMEM; - - /* - * The secret must be copied into contiguous memory region, lets verify - * that userspace memory pages are contiguous before we issue command. - */ - if (get_num_contig_pages(0, pages, n) != n) { - ret = -EINVAL; - goto e_unpin_memory; - } - - ret = -ENOMEM; - data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT); - if (!data) - goto e_unpin_memory; - - offset = params.guest_uaddr & (PAGE_SIZE - 1); - data->guest_address = __sme_page_pa(pages[0]) + offset; - data->guest_len = params.guest_len; - - blob = psp_copy_user_blob(params.trans_uaddr, params.trans_len); - if (IS_ERR(blob)) { - ret = PTR_ERR(blob); - goto e_free; - } - - data->trans_address = __psp_pa(blob); - data->trans_len = params.trans_len; - - hdr = psp_copy_user_blob(params.hdr_uaddr, params.hdr_len); - if (IS_ERR(hdr)) { - ret = PTR_ERR(hdr); - goto e_free_blob; - } - data->hdr_address = __psp_pa(hdr); - data->hdr_len = params.hdr_len; - - data->handle = sev->handle; - ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_SECRET, data, &argp->error); - - kfree(hdr); - -e_free_blob: - kfree(blob); -e_free: - kfree(data); -e_unpin_memory: - sev_unpin_memory(kvm, pages, n); - return ret; -} - -static int svm_mem_enc_op(struct kvm *kvm, void __user *argp) -{ - struct kvm_sev_cmd sev_cmd; - int r; - - if (!svm_sev_enabled()) - return -ENOTTY; - - if (!argp) - return 0; - - if (copy_from_user(&sev_cmd, argp, sizeof(struct kvm_sev_cmd))) - return -EFAULT; - - mutex_lock(&kvm->lock); - - switch (sev_cmd.id) { - case KVM_SEV_INIT: - r = sev_guest_init(kvm, &sev_cmd); - break; - case KVM_SEV_LAUNCH_START: - r = sev_launch_start(kvm, &sev_cmd); - break; - case KVM_SEV_LAUNCH_UPDATE_DATA: - r = sev_launch_update_data(kvm, &sev_cmd); - break; - case KVM_SEV_LAUNCH_MEASURE: - r = sev_launch_measure(kvm, &sev_cmd); - break; - case KVM_SEV_LAUNCH_FINISH: - r = sev_launch_finish(kvm, &sev_cmd); - break; - case KVM_SEV_GUEST_STATUS: - r = sev_guest_status(kvm, &sev_cmd); - break; - case KVM_SEV_DBG_DECRYPT: - r = sev_dbg_crypt(kvm, &sev_cmd, true); - break; - case KVM_SEV_DBG_ENCRYPT: - r = sev_dbg_crypt(kvm, &sev_cmd, false); - break; - case KVM_SEV_LAUNCH_SECRET: - r = sev_launch_secret(kvm, &sev_cmd); - break; - default: - r = -EINVAL; - goto out; - } - - if (copy_to_user(argp, &sev_cmd, sizeof(struct kvm_sev_cmd))) - r = -EFAULT; - -out: - mutex_unlock(&kvm->lock); - return r; -} - -static int svm_register_enc_region(struct kvm *kvm, - struct kvm_enc_region *range) -{ - struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; - struct enc_region *region; - int ret = 0; - - if (!sev_guest(kvm)) - return -ENOTTY; - - if (range->addr > ULONG_MAX || range->size > ULONG_MAX) - return -EINVAL; - - region = kzalloc(sizeof(*region), GFP_KERNEL_ACCOUNT); - if (!region) - return -ENOMEM; - - region->pages = sev_pin_memory(kvm, range->addr, range->size, ®ion->npages, 1); - if (!region->pages) { - ret = -ENOMEM; - goto e_free; - } - - /* - * The guest may change the memory encryption attribute from C=0 -> C=1 - * or vice versa for this memory range. Lets make sure caches are - * flushed to ensure that guest data gets written into memory with - * correct C-bit. - */ - sev_clflush_pages(region->pages, region->npages); - - region->uaddr = range->addr; - region->size = range->size; - - mutex_lock(&kvm->lock); - list_add_tail(®ion->list, &sev->regions_list); - mutex_unlock(&kvm->lock); - - return ret; - -e_free: - kfree(region); - return ret; -} - -static struct enc_region * -find_enc_region(struct kvm *kvm, struct kvm_enc_region *range) -{ - struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; - struct list_head *head = &sev->regions_list; - struct enc_region *i; - - list_for_each_entry(i, head, list) { - if (i->uaddr == range->addr && - i->size == range->size) - return i; - } - - return NULL; -} - - -static int svm_unregister_enc_region(struct kvm *kvm, - struct kvm_enc_region *range) -{ - struct enc_region *region; - int ret; - - mutex_lock(&kvm->lock); - - if (!sev_guest(kvm)) { - ret = -ENOTTY; - goto failed; - } - - region = find_enc_region(kvm, range); - if (!region) { - ret = -EINVAL; - goto failed; - } - - /* - * Ensure that all guest tagged cache entries are flushed before - * releasing the pages back to the system for use. CLFLUSH will - * not do this, so issue a WBINVD. - */ - wbinvd_on_all_cpus(); - - __unregister_enc_region_locked(kvm, region); - - mutex_unlock(&kvm->lock); - return 0; - -failed: - mutex_unlock(&kvm->lock); - return ret; -} - static bool svm_need_emulation_on_page_fault(struct kvm_vcpu *vcpu) { unsigned long cr4 = kvm_read_cr4(vcpu); @@ -5183,6 +3966,24 @@ static bool svm_apic_init_signal_blocked(struct kvm_vcpu *vcpu) (svm->vmcb->control.intercept & (1ULL << INTERCEPT_INIT)); } +static void svm_vm_destroy(struct kvm *kvm) +{ + avic_vm_destroy(kvm); + sev_vm_destroy(kvm); +} + +static int svm_vm_init(struct kvm *kvm) +{ + if (avic) { + int ret = avic_vm_init(kvm); + if (ret) + return ret; + } + + kvm_apicv_init(kvm, avic); + return 0; +} + static struct kvm_x86_ops svm_x86_ops __initdata = { .hardware_unsetup = svm_hardware_teardown, .hardware_enable = svm_hardware_enable, diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index c7abc1fede97..df3474f4fb02 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -171,6 +171,24 @@ struct vcpu_svm { unsigned int last_cpu; }; +struct svm_cpu_data { + int cpu; + + u64 asid_generation; + u32 max_asid; + u32 next_asid; + u32 min_asid; + struct kvm_ldttss_desc *tss_desc; + + struct page *save_area; + struct vmcb *current_vmcb; + + /* index = sev_asid, value = vmcb pointer */ + struct vmcb **sev_vmcbs; +}; + +DECLARE_PER_CPU(struct svm_cpu_data *, svm_data); + void recalc_intercepts(struct vcpu_svm *svm); static inline struct kvm_svm *to_kvm_svm(struct kvm *kvm) @@ -440,4 +458,34 @@ int svm_update_pi_irte(struct kvm *kvm, unsigned int host_irq, void svm_vcpu_blocking(struct kvm_vcpu *vcpu); void svm_vcpu_unblocking(struct kvm_vcpu *vcpu); +/* sev.c */ + +extern unsigned int max_sev_asid; + +static inline bool sev_guest(struct kvm *kvm) +{ +#ifdef CONFIG_KVM_AMD_SEV + struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; + + return sev->active; +#else + return false; +#endif +} + +static inline bool svm_sev_enabled(void) +{ + return IS_ENABLED(CONFIG_KVM_AMD_SEV) ? max_sev_asid : 0; +} + +void sev_vm_destroy(struct kvm *kvm); +int svm_mem_enc_op(struct kvm *kvm, void __user *argp); +int svm_register_enc_region(struct kvm *kvm, + struct kvm_enc_region *range); +int svm_unregister_enc_region(struct kvm *kvm, + struct kvm_enc_region *range); +void pre_sev_run(struct vcpu_svm *svm, int cpu); +int __init sev_hardware_setup(void); +void sev_hardware_teardown(void); + #endif From 199cd1d7b5348de4b58208420687676c658efed3 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Mon, 30 Mar 2020 15:02:13 +0200 Subject: [PATCH 441/635] KVM: SVM: Split svm_vcpu_run inline assembly to separate file The compiler (GCC) does not like the situation, where there is inline assembly block that clobbers all available machine registers in the middle of the function. This situation can be found in function svm_vcpu_run in file kvm/svm.c and results in many register spills and fills to/from stack frame. This patch fixes the issue with the same approach as was done for VMX some time ago. The big inline assembly is moved to a separate assembly .S file, taking into account all ABI requirements. There are two main benefits of the above approach: * elimination of several register spills and fills to/from stack frame, and consequently smaller function .text size. The binary size of svm_vcpu_run is lowered from 2019 to 1626 bytes. * more efficient access to a register save array. Currently, register save array is accessed as: 7b00: 48 8b 98 28 02 00 00 mov 0x228(%rax),%rbx 7b07: 48 8b 88 18 02 00 00 mov 0x218(%rax),%rcx 7b0e: 48 8b 90 20 02 00 00 mov 0x220(%rax),%rdx and passing ia pointer to a register array as an argument to a function one gets: 12: 48 8b 48 08 mov 0x8(%rax),%rcx 16: 48 8b 50 10 mov 0x10(%rax),%rdx 1a: 48 8b 58 18 mov 0x18(%rax),%rbx As a result, the total size, considering that the new function size is 229 bytes, gets lowered by 164 bytes. Signed-off-by: Uros Bizjak Signed-off-by: Paolo Bonzini --- arch/x86/kvm/Makefile | 2 +- arch/x86/kvm/svm/svm.c | 92 +-------------------- arch/x86/kvm/svm/vmenter.S | 162 +++++++++++++++++++++++++++++++++++++ 3 files changed, 166 insertions(+), 90 deletions(-) create mode 100644 arch/x86/kvm/svm/vmenter.S diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index e5a71aa0967b..a789759b7261 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -14,7 +14,7 @@ kvm-y += x86.o emulate.o i8259.o irq.o lapic.o \ hyperv.o debugfs.o mmu/mmu.o mmu/page_track.o kvm-intel-y += vmx/vmx.o vmx/vmenter.o vmx/pmu_intel.o vmx/vmcs12.o vmx/evmcs.o vmx/nested.o -kvm-amd-y += svm/svm.o svm/pmu.o svm/nested.o svm/avic.o svm/sev.o +kvm-amd-y += svm/svm.o svm/vmenter.o svm/pmu.o svm/nested.o svm/avic.o svm/sev.o obj-$(CONFIG_KVM) += kvm.o obj-$(CONFIG_KVM_INTEL) += kvm-intel.o diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 05d77b395fc1..2be5bbae3a40 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3276,6 +3276,8 @@ static void svm_cancel_injection(struct kvm_vcpu *vcpu) svm_complete_interrupts(svm); } +bool __svm_vcpu_run(unsigned long vmcb_pa, unsigned long *regs); + static void svm_vcpu_run(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); @@ -3330,95 +3332,7 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu) local_irq_enable(); - asm volatile ( - "push %%" _ASM_BP "; \n\t" - "mov %c[rbx](%[svm]), %%" _ASM_BX " \n\t" - "mov %c[rcx](%[svm]), %%" _ASM_CX " \n\t" - "mov %c[rdx](%[svm]), %%" _ASM_DX " \n\t" - "mov %c[rsi](%[svm]), %%" _ASM_SI " \n\t" - "mov %c[rdi](%[svm]), %%" _ASM_DI " \n\t" - "mov %c[rbp](%[svm]), %%" _ASM_BP " \n\t" -#ifdef CONFIG_X86_64 - "mov %c[r8](%[svm]), %%r8 \n\t" - "mov %c[r9](%[svm]), %%r9 \n\t" - "mov %c[r10](%[svm]), %%r10 \n\t" - "mov %c[r11](%[svm]), %%r11 \n\t" - "mov %c[r12](%[svm]), %%r12 \n\t" - "mov %c[r13](%[svm]), %%r13 \n\t" - "mov %c[r14](%[svm]), %%r14 \n\t" - "mov %c[r15](%[svm]), %%r15 \n\t" -#endif - - /* Enter guest mode */ - "push %%" _ASM_AX " \n\t" - "mov %c[vmcb](%[svm]), %%" _ASM_AX " \n\t" - __ex("vmload %%" _ASM_AX) "\n\t" - __ex("vmrun %%" _ASM_AX) "\n\t" - __ex("vmsave %%" _ASM_AX) "\n\t" - "pop %%" _ASM_AX " \n\t" - - /* Save guest registers, load host registers */ - "mov %%" _ASM_BX ", %c[rbx](%[svm]) \n\t" - "mov %%" _ASM_CX ", %c[rcx](%[svm]) \n\t" - "mov %%" _ASM_DX ", %c[rdx](%[svm]) \n\t" - "mov %%" _ASM_SI ", %c[rsi](%[svm]) \n\t" - "mov %%" _ASM_DI ", %c[rdi](%[svm]) \n\t" - "mov %%" _ASM_BP ", %c[rbp](%[svm]) \n\t" -#ifdef CONFIG_X86_64 - "mov %%r8, %c[r8](%[svm]) \n\t" - "mov %%r9, %c[r9](%[svm]) \n\t" - "mov %%r10, %c[r10](%[svm]) \n\t" - "mov %%r11, %c[r11](%[svm]) \n\t" - "mov %%r12, %c[r12](%[svm]) \n\t" - "mov %%r13, %c[r13](%[svm]) \n\t" - "mov %%r14, %c[r14](%[svm]) \n\t" - "mov %%r15, %c[r15](%[svm]) \n\t" - /* - * Clear host registers marked as clobbered to prevent - * speculative use. - */ - "xor %%r8d, %%r8d \n\t" - "xor %%r9d, %%r9d \n\t" - "xor %%r10d, %%r10d \n\t" - "xor %%r11d, %%r11d \n\t" - "xor %%r12d, %%r12d \n\t" - "xor %%r13d, %%r13d \n\t" - "xor %%r14d, %%r14d \n\t" - "xor %%r15d, %%r15d \n\t" -#endif - "xor %%ebx, %%ebx \n\t" - "xor %%ecx, %%ecx \n\t" - "xor %%edx, %%edx \n\t" - "xor %%esi, %%esi \n\t" - "xor %%edi, %%edi \n\t" - "pop %%" _ASM_BP - : - : [svm]"a"(svm), - [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)), - [rbx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBX])), - [rcx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RCX])), - [rdx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDX])), - [rsi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RSI])), - [rdi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDI])), - [rbp]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBP])) -#ifdef CONFIG_X86_64 - , [r8]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R8])), - [r9]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R9])), - [r10]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R10])), - [r11]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R11])), - [r12]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R12])), - [r13]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R13])), - [r14]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R14])), - [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15])) -#endif - : "cc", "memory" -#ifdef CONFIG_X86_64 - , "rbx", "rcx", "rdx", "rsi", "rdi" - , "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15" -#else - , "ebx", "ecx", "edx", "esi", "edi" -#endif - ); + __svm_vcpu_run(svm->vmcb_pa, (unsigned long *)&svm->vcpu.arch.regs); /* Eliminate branch target predictions from guest mode */ vmexit_fill_RSB(); diff --git a/arch/x86/kvm/svm/vmenter.S b/arch/x86/kvm/svm/vmenter.S new file mode 100644 index 000000000000..fa1af90067e9 --- /dev/null +++ b/arch/x86/kvm/svm/vmenter.S @@ -0,0 +1,162 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include +#include +#include + +#define WORD_SIZE (BITS_PER_LONG / 8) + +/* Intentionally omit RAX as it's context switched by hardware */ +#define VCPU_RCX __VCPU_REGS_RCX * WORD_SIZE +#define VCPU_RDX __VCPU_REGS_RDX * WORD_SIZE +#define VCPU_RBX __VCPU_REGS_RBX * WORD_SIZE +/* Intentionally omit RSP as it's context switched by hardware */ +#define VCPU_RBP __VCPU_REGS_RBP * WORD_SIZE +#define VCPU_RSI __VCPU_REGS_RSI * WORD_SIZE +#define VCPU_RDI __VCPU_REGS_RDI * WORD_SIZE + +#ifdef CONFIG_X86_64 +#define VCPU_R8 __VCPU_REGS_R8 * WORD_SIZE +#define VCPU_R9 __VCPU_REGS_R9 * WORD_SIZE +#define VCPU_R10 __VCPU_REGS_R10 * WORD_SIZE +#define VCPU_R11 __VCPU_REGS_R11 * WORD_SIZE +#define VCPU_R12 __VCPU_REGS_R12 * WORD_SIZE +#define VCPU_R13 __VCPU_REGS_R13 * WORD_SIZE +#define VCPU_R14 __VCPU_REGS_R14 * WORD_SIZE +#define VCPU_R15 __VCPU_REGS_R15 * WORD_SIZE +#endif + + .text + +/** + * __svm_vcpu_run - Run a vCPU via a transition to SVM guest mode + * @vmcb_pa: unsigned long + * @regs: unsigned long * (to guest registers) + */ +SYM_FUNC_START(__svm_vcpu_run) + push %_ASM_BP + mov %_ASM_SP, %_ASM_BP +#ifdef CONFIG_X86_64 + push %r15 + push %r14 + push %r13 + push %r12 +#else + push %edi + push %esi +#endif + push %_ASM_BX + + /* Save @regs. */ + push %_ASM_ARG2 + + /* Save @vmcb. */ + push %_ASM_ARG1 + + /* Move @regs to RAX. */ + mov %_ASM_ARG2, %_ASM_AX + + /* Load guest registers. */ + mov VCPU_RCX(%_ASM_AX), %_ASM_CX + mov VCPU_RDX(%_ASM_AX), %_ASM_DX + mov VCPU_RBX(%_ASM_AX), %_ASM_BX + mov VCPU_RBP(%_ASM_AX), %_ASM_BP + mov VCPU_RSI(%_ASM_AX), %_ASM_SI + mov VCPU_RDI(%_ASM_AX), %_ASM_DI +#ifdef CONFIG_X86_64 + mov VCPU_R8 (%_ASM_AX), %r8 + mov VCPU_R9 (%_ASM_AX), %r9 + mov VCPU_R10(%_ASM_AX), %r10 + mov VCPU_R11(%_ASM_AX), %r11 + mov VCPU_R12(%_ASM_AX), %r12 + mov VCPU_R13(%_ASM_AX), %r13 + mov VCPU_R14(%_ASM_AX), %r14 + mov VCPU_R15(%_ASM_AX), %r15 +#endif + + /* "POP" @vmcb to RAX. */ + pop %_ASM_AX + + /* Enter guest mode */ +1: vmload %_ASM_AX + jmp 3f +2: cmpb $0, kvm_rebooting + jne 3f + ud2 + _ASM_EXTABLE(1b, 2b) + +3: vmrun %_ASM_AX + jmp 5f +4: cmpb $0, kvm_rebooting + jne 5f + ud2 + _ASM_EXTABLE(3b, 4b) + +5: vmsave %_ASM_AX + jmp 7f +6: cmpb $0, kvm_rebooting + jne 7f + ud2 + _ASM_EXTABLE(5b, 6b) +7: + /* "POP" @regs to RAX. */ + pop %_ASM_AX + + /* Save all guest registers. */ + mov %_ASM_CX, VCPU_RCX(%_ASM_AX) + mov %_ASM_DX, VCPU_RDX(%_ASM_AX) + mov %_ASM_BX, VCPU_RBX(%_ASM_AX) + mov %_ASM_BP, VCPU_RBP(%_ASM_AX) + mov %_ASM_SI, VCPU_RSI(%_ASM_AX) + mov %_ASM_DI, VCPU_RDI(%_ASM_AX) +#ifdef CONFIG_X86_64 + mov %r8, VCPU_R8 (%_ASM_AX) + mov %r9, VCPU_R9 (%_ASM_AX) + mov %r10, VCPU_R10(%_ASM_AX) + mov %r11, VCPU_R11(%_ASM_AX) + mov %r12, VCPU_R12(%_ASM_AX) + mov %r13, VCPU_R13(%_ASM_AX) + mov %r14, VCPU_R14(%_ASM_AX) + mov %r15, VCPU_R15(%_ASM_AX) +#endif + + /* + * Clear all general purpose registers except RSP and RAX to prevent + * speculative use of the guest's values, even those that are reloaded + * via the stack. In theory, an L1 cache miss when restoring registers + * could lead to speculative execution with the guest's values. + * Zeroing XORs are dirt cheap, i.e. the extra paranoia is essentially + * free. RSP and RAX are exempt as they are restored by hardware + * during VM-Exit. + */ + xor %ecx, %ecx + xor %edx, %edx + xor %ebx, %ebx + xor %ebp, %ebp + xor %esi, %esi + xor %edi, %edi +#ifdef CONFIG_X86_64 + xor %r8d, %r8d + xor %r9d, %r9d + xor %r10d, %r10d + xor %r11d, %r11d + xor %r12d, %r12d + xor %r13d, %r13d + xor %r14d, %r14d + xor %r15d, %r15d +#endif + + pop %_ASM_BX + +#ifdef CONFIG_X86_64 + pop %r12 + pop %r13 + pop %r14 + pop %r15 +#else + pop %esi + pop %edi +#endif + pop %_ASM_BP + ret +SYM_FUNC_END(__svm_vcpu_run) From fc51b1cf391dd91ff9362861ed226ef51048e863 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Thu, 2 Apr 2020 15:27:09 -0400 Subject: [PATCH 442/635] NFS: Beware when dereferencing the delegation cred When we look up the delegation cred, we are usually doing so in conjunction with a read of the stateid, and we want to ensure that the look up is atomic with that read. Fixes: 57f188e04773 ("NFSv4: nfs_update_inplace_delegation() should update delegation cred") [sfr@canb.auug.org.au: Fixed up borken Fixes: line from Trond :-)] Signed-off-by: Trond Myklebust --- fs/nfs/delegation.c | 9 ++++++++- fs/nfs/nfs4proc.c | 3 +++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/fs/nfs/delegation.c b/fs/nfs/delegation.c index 01974f17afc9..816e1427f17e 100644 --- a/fs/nfs/delegation.c +++ b/fs/nfs/delegation.c @@ -1243,8 +1243,10 @@ restart_locked: inode = nfs_delegation_grab_inode(delegation); if (inode == NULL) goto restart_locked; + spin_lock(&delegation->lock); cred = get_cred_rcu(delegation->cred); nfs4_stateid_copy(&stateid, &delegation->stateid); + spin_unlock(&delegation->lock); clear_bit(NFS_DELEGATION_TEST_EXPIRED, &delegation->flags); rcu_read_unlock(); nfs_delegation_test_free_expired(inode, &stateid, cred); @@ -1363,11 +1365,14 @@ bool nfs4_copy_delegation_stateid(struct inode *inode, fmode_t flags, { struct nfs_inode *nfsi = NFS_I(inode); struct nfs_delegation *delegation; - bool ret; + bool ret = false; flags &= FMODE_READ|FMODE_WRITE; rcu_read_lock(); delegation = rcu_dereference(nfsi->delegation); + if (!delegation) + goto out; + spin_lock(&delegation->lock); ret = nfs4_is_valid_delegation(delegation, flags); if (ret) { nfs4_stateid_copy(dst, &delegation->stateid); @@ -1375,6 +1380,8 @@ bool nfs4_copy_delegation_stateid(struct inode *inode, fmode_t flags, if (cred) *cred = get_cred(delegation->cred); } + spin_unlock(&delegation->lock); +out: rcu_read_unlock(); return ret; } diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 905c7d1bc277..e4f8311e506c 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -2790,16 +2790,19 @@ static int nfs41_check_delegation_stateid(struct nfs4_state *state) return NFS_OK; } + spin_lock(&delegation->lock); nfs4_stateid_copy(&stateid, &delegation->stateid); if (!test_and_clear_bit(NFS_DELEGATION_TEST_EXPIRED, &delegation->flags)) { + spin_unlock(&delegation->lock); rcu_read_unlock(); return NFS_OK; } if (delegation->cred) cred = get_cred(delegation->cred); + spin_unlock(&delegation->lock); rcu_read_unlock(); status = nfs41_test_and_free_expired_stateid(server, &stateid, cred); trace_nfs4_test_delegation_stateid(state, NULL, status); From 97a728f5e2ad62a069dd395d3e516bb0acdbb3ec Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Thu, 2 Apr 2020 15:47:08 -0400 Subject: [PATCH 443/635] NFS/pnfs: Fix dereference of layout cred in pnfs_layoutcommit_inode() Ensure that the dereference of the layout cred is atomic with the stateid. Signed-off-by: Trond Myklebust --- fs/nfs/pnfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index 6fcf26b16816..84029c9b2b1b 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -3137,10 +3137,10 @@ pnfs_layoutcommit_inode(struct inode *inode, bool sync) end_pos = nfsi->layout->plh_lwb; nfs4_stateid_copy(&data->args.stateid, &nfsi->layout->plh_stateid); + data->cred = get_cred(nfsi->layout->plh_lc_cred); spin_unlock(&inode->i_lock); data->args.inode = inode; - data->cred = get_cred(nfsi->layout->plh_lc_cred); nfs_fattr_init(&data->fattr); data->args.bitmask = NFS_SERVER(inode)->cache_consistency_bitmask; data->res.fattr = &data->fattr; From 44ea8dfce021db1db6a31970b1f13a80366a1f21 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Thu, 2 Apr 2020 15:37:02 -0400 Subject: [PATCH 444/635] NFS/pnfs: Reference the layout cred in pnfs_prepare_layoutreturn() When we're sending a layoutreturn, ensure that we reference the layout cred atomically with the copy of the stateid. Signed-off-by: Trond Myklebust --- fs/nfs/nfs4proc.c | 1 + fs/nfs/pnfs.c | 52 ++++++++++++++++++++++++++++++----------------- 2 files changed, 34 insertions(+), 19 deletions(-) diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index e4f8311e506c..99e9f2ee7e7a 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -9297,6 +9297,7 @@ static void nfs4_layoutreturn_release(void *calldata) lrp->ld_private.ops->free(&lrp->ld_private); pnfs_put_layout_hdr(lrp->args.layout); nfs_iput_and_deactive(lrp->inode); + put_cred(lrp->cred); kfree(calldata); dprintk("<-- %s\n", __func__); } diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index 84029c9b2b1b..f2dc35c22964 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -1144,6 +1144,7 @@ out_unlock: static bool pnfs_prepare_layoutreturn(struct pnfs_layout_hdr *lo, nfs4_stateid *stateid, + const struct cred **cred, enum pnfs_iomode *iomode) { /* Serialise LAYOUTGET/LAYOUTRETURN */ @@ -1154,18 +1155,17 @@ pnfs_prepare_layoutreturn(struct pnfs_layout_hdr *lo, set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags); pnfs_get_layout_hdr(lo); if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) { - if (stateid != NULL) { - nfs4_stateid_copy(stateid, &lo->plh_stateid); - if (lo->plh_return_seq != 0) - stateid->seqid = cpu_to_be32(lo->plh_return_seq); - } + nfs4_stateid_copy(stateid, &lo->plh_stateid); + *cred = get_cred(lo->plh_lc_cred); + if (lo->plh_return_seq != 0) + stateid->seqid = cpu_to_be32(lo->plh_return_seq); if (iomode != NULL) *iomode = lo->plh_return_iomode; pnfs_clear_layoutreturn_info(lo); return true; } - if (stateid != NULL) - nfs4_stateid_copy(stateid, &lo->plh_stateid); + nfs4_stateid_copy(stateid, &lo->plh_stateid); + *cred = get_cred(lo->plh_lc_cred); if (iomode != NULL) *iomode = IOMODE_ANY; return true; @@ -1189,20 +1189,26 @@ pnfs_init_layoutreturn_args(struct nfs4_layoutreturn_args *args, } static int -pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, const nfs4_stateid *stateid, - enum pnfs_iomode iomode, bool sync) +pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, + const nfs4_stateid *stateid, + const struct cred **pcred, + enum pnfs_iomode iomode, + bool sync) { struct inode *ino = lo->plh_inode; struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld; struct nfs4_layoutreturn *lrp; + const struct cred *cred = *pcred; int status = 0; + *pcred = NULL; lrp = kzalloc(sizeof(*lrp), GFP_NOFS); if (unlikely(lrp == NULL)) { status = -ENOMEM; spin_lock(&ino->i_lock); pnfs_clear_layoutreturn_waitbit(lo); spin_unlock(&ino->i_lock); + put_cred(cred); pnfs_put_layout_hdr(lo); goto out; } @@ -1210,7 +1216,7 @@ pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, const nfs4_stateid *stateid, pnfs_init_layoutreturn_args(&lrp->args, lo, stateid, iomode); lrp->args.ld_private = &lrp->ld_private; lrp->clp = NFS_SERVER(ino)->nfs_client; - lrp->cred = lo->plh_lc_cred; + lrp->cred = cred; if (ld->prepare_layoutreturn) ld->prepare_layoutreturn(&lrp->args); @@ -1255,15 +1261,16 @@ static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo) return; spin_lock(&inode->i_lock); if (pnfs_layout_need_return(lo)) { + const struct cred *cred; nfs4_stateid stateid; enum pnfs_iomode iomode; bool send; - send = pnfs_prepare_layoutreturn(lo, &stateid, &iomode); + send = pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode); spin_unlock(&inode->i_lock); if (send) { /* Send an async layoutreturn so we dont deadlock */ - pnfs_send_layoutreturn(lo, &stateid, iomode, false); + pnfs_send_layoutreturn(lo, &stateid, &cred, iomode, false); } } else spin_unlock(&inode->i_lock); @@ -1283,6 +1290,7 @@ _pnfs_return_layout(struct inode *ino) struct pnfs_layout_hdr *lo = NULL; struct nfs_inode *nfsi = NFS_I(ino); LIST_HEAD(tmp_list); + const struct cred *cred; nfs4_stateid stateid; int status = 0; bool send, valid_layout; @@ -1327,10 +1335,10 @@ _pnfs_return_layout(struct inode *ino) goto out_put_layout_hdr; } - send = pnfs_prepare_layoutreturn(lo, &stateid, NULL); + send = pnfs_prepare_layoutreturn(lo, &stateid, &cred, NULL); spin_unlock(&ino->i_lock); if (send) - status = pnfs_send_layoutreturn(lo, &stateid, IOMODE_ANY, true); + status = pnfs_send_layoutreturn(lo, &stateid, &cred, IOMODE_ANY, true); out_put_layout_hdr: pnfs_free_lseg_list(&tmp_list); pnfs_put_layout_hdr(lo); @@ -1376,6 +1384,7 @@ bool pnfs_roc(struct inode *ino, struct nfs4_state *state; struct pnfs_layout_hdr *lo; struct pnfs_layout_segment *lseg, *next; + const struct cred *lc_cred; nfs4_stateid stateid; enum pnfs_iomode iomode = 0; bool layoutreturn = false, roc = false; @@ -1445,16 +1454,20 @@ retry: * 2. we don't send layoutreturn */ /* lo ref dropped in pnfs_roc_release() */ - layoutreturn = pnfs_prepare_layoutreturn(lo, &stateid, &iomode); + layoutreturn = pnfs_prepare_layoutreturn(lo, &stateid, &lc_cred, &iomode); /* If the creds don't match, we can't compound the layoutreturn */ - if (!layoutreturn || cred_fscmp(cred, lo->plh_lc_cred) != 0) + if (!layoutreturn) goto out_noroc; + if (cred_fscmp(cred, lc_cred) != 0) + goto out_noroc_put_cred; roc = layoutreturn; pnfs_init_layoutreturn_args(args, lo, &stateid, iomode); res->lrs_present = 0; layoutreturn = false; +out_noroc_put_cred: + put_cred(lc_cred); out_noroc: spin_unlock(&ino->i_lock); rcu_read_unlock(); @@ -1467,7 +1480,7 @@ out_noroc: return true; } if (layoutreturn) - pnfs_send_layoutreturn(lo, &stateid, iomode, true); + pnfs_send_layoutreturn(lo, &stateid, &lc_cred, iomode, true); pnfs_put_layout_hdr(lo); return false; } @@ -2464,13 +2477,14 @@ pnfs_mark_layout_for_return(struct inode *inode, * for how it works. */ if (pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, range, 0) != -EBUSY) { + const struct cred *cred; nfs4_stateid stateid; enum pnfs_iomode iomode; - return_now = pnfs_prepare_layoutreturn(lo, &stateid, &iomode); + return_now = pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode); spin_unlock(&inode->i_lock); if (return_now) - pnfs_send_layoutreturn(lo, &stateid, iomode, false); + pnfs_send_layoutreturn(lo, &stateid, &cred, iomode, false); } else { spin_unlock(&inode->i_lock); nfs_commit_inode(inode, 0); From 1fab7dc477241c12f977955aa6baea7938b6f08d Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Sat, 4 Apr 2020 19:52:21 -0400 Subject: [PATCH 445/635] SUNRPC: Don't start a timer on an already queued rpc task Move the test for whether a task is already queued to prevent corruption of the timer list in __rpc_sleep_on_priority_timeout(). Signed-off-by: Trond Myklebust --- net/sunrpc/sched.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/net/sunrpc/sched.c b/net/sunrpc/sched.c index 6eff14119a88..7eba20a88438 100644 --- a/net/sunrpc/sched.c +++ b/net/sunrpc/sched.c @@ -204,10 +204,6 @@ static void __rpc_add_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task, unsigned char queue_priority) { - WARN_ON_ONCE(RPC_IS_QUEUED(task)); - if (RPC_IS_QUEUED(task)) - return; - INIT_LIST_HEAD(&task->u.tk_wait.timer_list); if (RPC_IS_PRIORITY(queue)) __rpc_add_wait_queue_priority(queue, task, queue_priority); @@ -382,7 +378,7 @@ static void rpc_make_runnable(struct workqueue_struct *wq, * NB: An RPC task will only receive interrupt-driven events as long * as it's on a wait queue. */ -static void __rpc_sleep_on_priority(struct rpc_wait_queue *q, +static void __rpc_do_sleep_on_priority(struct rpc_wait_queue *q, struct rpc_task *task, unsigned char queue_priority) { @@ -395,12 +391,23 @@ static void __rpc_sleep_on_priority(struct rpc_wait_queue *q, } +static void __rpc_sleep_on_priority(struct rpc_wait_queue *q, + struct rpc_task *task, + unsigned char queue_priority) +{ + if (WARN_ON_ONCE(RPC_IS_QUEUED(task))) + return; + __rpc_do_sleep_on_priority(q, task, queue_priority); +} + static void __rpc_sleep_on_priority_timeout(struct rpc_wait_queue *q, struct rpc_task *task, unsigned long timeout, unsigned char queue_priority) { + if (WARN_ON_ONCE(RPC_IS_QUEUED(task))) + return; if (time_is_after_jiffies(timeout)) { - __rpc_sleep_on_priority(q, task, queue_priority); + __rpc_do_sleep_on_priority(q, task, queue_priority); __rpc_add_timer(q, task, timeout); } else task->tk_status = -ETIMEDOUT; From 93ce4af774bc3d8a72ce2271d03241c96383629d Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 6 Apr 2020 13:39:29 -0400 Subject: [PATCH 446/635] NFS: Clean up process of marking inode stale. Instead of the various open coded calls to set the NFS_INO_STALE bit and call nfs_zap_caches(), consolidate them into a single function nfs_set_inode_stale(). Signed-off-by: Trond Myklebust --- fs/nfs/dir.c | 5 +++-- fs/nfs/inode.c | 18 +++++++++++++----- fs/nfs/nfstrace.h | 1 + fs/nfs/read.c | 2 +- include/linux/nfs_fs.h | 1 + 5 files changed, 19 insertions(+), 8 deletions(-) diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index f14184d0ba82..d729d8311c7e 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -2669,9 +2669,10 @@ static int nfs_do_access(struct inode *inode, const struct cred *cred, int mask) status = NFS_PROTO(inode)->access(inode, &cache); if (status != 0) { if (status == -ESTALE) { - nfs_zap_caches(inode); if (!S_ISDIR(inode->i_mode)) - set_bit(NFS_INO_STALE, &NFS_I(inode)->flags); + nfs_set_inode_stale(inode); + else + nfs_zap_caches(inode); } goto out; } diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index a10fb87c6ac3..b9d0921cb4fe 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -62,7 +62,6 @@ /* Default is to see 64-bit inode numbers */ static bool enable_ino64 = NFS_64_BIT_INODE_NUMBERS_ENABLED; -static void nfs_invalidate_inode(struct inode *); static int nfs_update_inode(struct inode *, struct nfs_fattr *); static struct kmem_cache * nfs_inode_cachep; @@ -284,10 +283,18 @@ EXPORT_SYMBOL_GPL(nfs_invalidate_atime); * Invalidate, but do not unhash, the inode. * NB: must be called with inode->i_lock held! */ -static void nfs_invalidate_inode(struct inode *inode) +static void nfs_set_inode_stale_locked(struct inode *inode) { set_bit(NFS_INO_STALE, &NFS_I(inode)->flags); nfs_zap_caches_locked(inode); + trace_nfs_set_inode_stale(inode); +} + +void nfs_set_inode_stale(struct inode *inode) +{ + spin_lock(&inode->i_lock); + nfs_set_inode_stale_locked(inode); + spin_unlock(&inode->i_lock); } struct nfs_find_desc { @@ -1163,9 +1170,10 @@ __nfs_revalidate_inode(struct nfs_server *server, struct inode *inode) status = 0; break; case -ESTALE: - nfs_zap_caches(inode); if (!S_ISDIR(inode->i_mode)) - set_bit(NFS_INO_STALE, &NFS_I(inode)->flags); + nfs_set_inode_stale(inode); + else + nfs_zap_caches(inode); } goto err_out; } @@ -2064,7 +2072,7 @@ static int nfs_update_inode(struct inode *inode, struct nfs_fattr *fattr) * lookup validation will know that the inode is bad. * (But we fall through to invalidate the caches.) */ - nfs_invalidate_inode(inode); + nfs_set_inode_stale_locked(inode); return -ESTALE; } diff --git a/fs/nfs/nfstrace.h b/fs/nfs/nfstrace.h index a9588d19a5ae..7e7a97ae21ed 100644 --- a/fs/nfs/nfstrace.h +++ b/fs/nfs/nfstrace.h @@ -181,6 +181,7 @@ DECLARE_EVENT_CLASS(nfs_inode_event_done, int error \ ), \ TP_ARGS(inode, error)) +DEFINE_NFS_INODE_EVENT(nfs_set_inode_stale); DEFINE_NFS_INODE_EVENT(nfs_refresh_inode_enter); DEFINE_NFS_INODE_EVENT_DONE(nfs_refresh_inode_exit); DEFINE_NFS_INODE_EVENT(nfs_revalidate_inode_enter); diff --git a/fs/nfs/read.c b/fs/nfs/read.c index 34bb9add2302..13b22e898116 100644 --- a/fs/nfs/read.c +++ b/fs/nfs/read.c @@ -250,7 +250,7 @@ static int nfs_readpage_done(struct rpc_task *task, trace_nfs_readpage_done(task, hdr); if (task->tk_status == -ESTALE) { - set_bit(NFS_INO_STALE, &NFS_I(inode)->flags); + nfs_set_inode_stale(inode); nfs_mark_for_revalidate(inode); } return 0; diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h index 5d5b91e54f73..73eda45f1cfd 100644 --- a/include/linux/nfs_fs.h +++ b/include/linux/nfs_fs.h @@ -354,6 +354,7 @@ static inline unsigned long nfs_save_change_attribute(struct inode *dir) extern int nfs_sync_mapping(struct address_space *mapping); extern void nfs_zap_mapping(struct inode *inode, struct address_space *mapping); extern void nfs_zap_caches(struct inode *); +extern void nfs_set_inode_stale(struct inode *inode); extern void nfs_invalidate_atime(struct inode *); extern struct inode *nfs_fhget(struct super_block *, struct nfs_fh *, struct nfs_fattr *, struct nfs4_label *); From 28799272acbf385d820b1fa86bd43a80aedd51fa Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Fri, 13 Mar 2020 17:19:37 +0000 Subject: [PATCH 447/635] leds: lm3532: make bitfield 'enabled' unsigned The bitfield 'enabled' should bit unsigned, so make it unsigned. Signed-off-by: Colin Ian King Signed-off-by: Pavel Machek --- drivers/leds/leds-lm3532.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/leds/leds-lm3532.c b/drivers/leds/leds-lm3532.c index 188a57da981a..aa9bf8cda673 100644 --- a/drivers/leds/leds-lm3532.c +++ b/drivers/leds/leds-lm3532.c @@ -140,7 +140,7 @@ struct lm3532_led { int ctrl_brt_pointer; int num_leds; int full_scale_current; - int enabled:1; + unsigned int enabled:1; u32 led_strings[LM3532_MAX_CONTROL_BANKS]; char label[LED_MAX_NAME_SIZE]; }; From 19aa98409066b6e0f1f7ecfb404a983724f946f0 Mon Sep 17 00:00:00 2001 From: Pavel Machek Date: Sat, 21 Mar 2020 17:00:27 +0100 Subject: [PATCH 448/635] leds: ip30: label power LED as such Make label "white:power" to be consistent with dt-bindings/leds/common.h . Signed-off-by: Pavel Machek --- drivers/leds/leds-ip30.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/leds/leds-ip30.c b/drivers/leds/leds-ip30.c index e95ea786a43e..d4ec7361c616 100644 --- a/drivers/leds/leds-ip30.c +++ b/drivers/leds/leds-ip30.c @@ -44,7 +44,7 @@ static int ip30led_create(struct platform_device *pdev, int num) switch (num) { case IP30_LED_SYSTEM: - data->cdev.name = "white:system"; + data->cdev.name = "white:power"; break; case IP30_LED_FAULT: data->cdev.name = "red:fault"; From c7e4ea68c1626cceb966323a4b572e2f8d805138 Mon Sep 17 00:00:00 2001 From: Pavel Machek Date: Sat, 21 Mar 2020 17:01:53 +0100 Subject: [PATCH 449/635] leds: old enums are not really applicable to new code Warn about old defines that probably should not be used. Signed-off-by: Pavel Machek --- include/linux/leds.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/linux/leds.h b/include/linux/leds.h index 75353e5f9d13..2451962d1ec5 100644 --- a/include/linux/leds.h +++ b/include/linux/leds.h @@ -25,6 +25,7 @@ struct device_node; * LED Core */ +/* This is obsolete/useless. We now support variable maximum brightness. */ enum led_brightness { LED_OFF = 0, LED_ON = 1, From 457386350e6a49e0b26ad7e2f303bb81e094d4a2 Mon Sep 17 00:00:00 2001 From: Pavel Machek Date: Sat, 21 Mar 2020 17:09:00 +0100 Subject: [PATCH 450/635] leds: sort Makefile entries Sort Makefile entries to reduce risk of rejects. Signed-off-by: Pavel Machek --- drivers/leds/Makefile | 100 +++++++++++++++++++++--------------------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/drivers/leds/Makefile b/drivers/leds/Makefile index 46bd611a03a9..45235d5fb218 100644 --- a/drivers/leds/Makefile +++ b/drivers/leds/Makefile @@ -6,92 +6,92 @@ obj-$(CONFIG_LEDS_CLASS) += led-class.o obj-$(CONFIG_LEDS_CLASS_FLASH) += led-class-flash.o obj-$(CONFIG_LEDS_TRIGGERS) += led-triggers.o -# LED Platform Drivers +# LED Platform Drivers (keep this sorted, M-| sort) obj-$(CONFIG_LEDS_88PM860X) += leds-88pm860x.o obj-$(CONFIG_LEDS_AAT1290) += leds-aat1290.o +obj-$(CONFIG_LEDS_ADP5520) += leds-adp5520.o +obj-$(CONFIG_LEDS_AN30259A) += leds-an30259a.o obj-$(CONFIG_LEDS_APU) += leds-apu.o obj-$(CONFIG_LEDS_AS3645A) += leds-as3645a.o -obj-$(CONFIG_LEDS_AN30259A) += leds-an30259a.o +obj-$(CONFIG_LEDS_ASIC3) += leds-asic3.o obj-$(CONFIG_LEDS_BCM6328) += leds-bcm6328.o obj-$(CONFIG_LEDS_BCM6358) += leds-bcm6358.o obj-$(CONFIG_LEDS_BD2802) += leds-bd2802.o +obj-$(CONFIG_LEDS_BLINKM) += leds-blinkm.o +obj-$(CONFIG_LEDS_CLEVO_MAIL) += leds-clevo-mail.o +obj-$(CONFIG_LEDS_COBALT_QUBE) += leds-cobalt-qube.o +obj-$(CONFIG_LEDS_COBALT_RAQ) += leds-cobalt-raq.o obj-$(CONFIG_LEDS_CPCAP) += leds-cpcap.o -obj-$(CONFIG_LEDS_LOCOMO) += leds-locomo.o +obj-$(CONFIG_LEDS_DA903X) += leds-da903x.o +obj-$(CONFIG_LEDS_DA9052) += leds-da9052.o +obj-$(CONFIG_LEDS_FSG) += leds-fsg.o +obj-$(CONFIG_LEDS_GPIO) += leds-gpio.o +obj-$(CONFIG_LEDS_GPIO_REGISTER) += leds-gpio-register.o +obj-$(CONFIG_LEDS_HP6XX) += leds-hp6xx.o +obj-$(CONFIG_LEDS_INTEL_SS4200) += leds-ss4200.o +obj-$(CONFIG_LEDS_IP30) += leds-ip30.o +obj-$(CONFIG_LEDS_IPAQ_MICRO) += leds-ipaq-micro.o +obj-$(CONFIG_LEDS_IS31FL319X) += leds-is31fl319x.o +obj-$(CONFIG_LEDS_IS31FL32XX) += leds-is31fl32xx.o +obj-$(CONFIG_LEDS_KTD2692) += leds-ktd2692.o obj-$(CONFIG_LEDS_LM3530) += leds-lm3530.o obj-$(CONFIG_LEDS_LM3532) += leds-lm3532.o obj-$(CONFIG_LEDS_LM3533) += leds-lm3533.o +obj-$(CONFIG_LEDS_LM355x) += leds-lm355x.o +obj-$(CONFIG_LEDS_LM3601X) += leds-lm3601x.o +obj-$(CONFIG_LEDS_LM36274) += leds-lm36274.o obj-$(CONFIG_LEDS_LM3642) += leds-lm3642.o -obj-$(CONFIG_LEDS_MIKROTIK_RB532) += leds-rb532.o -obj-$(CONFIG_LEDS_S3C24XX) += leds-s3c24xx.o -obj-$(CONFIG_LEDS_NET48XX) += leds-net48xx.o -obj-$(CONFIG_LEDS_WRAP) += leds-wrap.o -obj-$(CONFIG_LEDS_COBALT_QUBE) += leds-cobalt-qube.o -obj-$(CONFIG_LEDS_COBALT_RAQ) += leds-cobalt-raq.o -obj-$(CONFIG_LEDS_SUNFIRE) += leds-sunfire.o -obj-$(CONFIG_LEDS_PCA9532) += leds-pca9532.o -obj-$(CONFIG_LEDS_GPIO_REGISTER) += leds-gpio-register.o -obj-$(CONFIG_LEDS_GPIO) += leds-gpio.o +obj-$(CONFIG_LEDS_LM3692X) += leds-lm3692x.o +obj-$(CONFIG_LEDS_LM3697) += leds-lm3697.o +obj-$(CONFIG_LEDS_LOCOMO) += leds-locomo.o obj-$(CONFIG_LEDS_LP3944) += leds-lp3944.o obj-$(CONFIG_LEDS_LP3952) += leds-lp3952.o -obj-$(CONFIG_LEDS_LP55XX_COMMON) += leds-lp55xx-common.o obj-$(CONFIG_LEDS_LP5521) += leds-lp5521.o obj-$(CONFIG_LEDS_LP5523) += leds-lp5523.o obj-$(CONFIG_LEDS_LP5562) += leds-lp5562.o +obj-$(CONFIG_LEDS_LP55XX_COMMON) += leds-lp55xx-common.o obj-$(CONFIG_LEDS_LP8501) += leds-lp8501.o obj-$(CONFIG_LEDS_LP8788) += leds-lp8788.o obj-$(CONFIG_LEDS_LP8860) += leds-lp8860.o -obj-$(CONFIG_LEDS_TCA6507) += leds-tca6507.o -obj-$(CONFIG_LEDS_TLC591XX) += leds-tlc591xx.o -obj-$(CONFIG_LEDS_CLEVO_MAIL) += leds-clevo-mail.o -obj-$(CONFIG_LEDS_IPAQ_MICRO) += leds-ipaq-micro.o -obj-$(CONFIG_LEDS_HP6XX) += leds-hp6xx.o -obj-$(CONFIG_LEDS_OT200) += leds-ot200.o -obj-$(CONFIG_LEDS_FSG) += leds-fsg.o -obj-$(CONFIG_LEDS_PCA955X) += leds-pca955x.o -obj-$(CONFIG_LEDS_PCA963X) += leds-pca963x.o -obj-$(CONFIG_LEDS_DA903X) += leds-da903x.o -obj-$(CONFIG_LEDS_DA9052) += leds-da9052.o -obj-$(CONFIG_LEDS_WM831X_STATUS) += leds-wm831x-status.o -obj-$(CONFIG_LEDS_WM8350) += leds-wm8350.o -obj-$(CONFIG_LEDS_PWM) += leds-pwm.o -obj-$(CONFIG_LEDS_REGULATOR) += leds-regulator.o -obj-$(CONFIG_LEDS_INTEL_SS4200) += leds-ss4200.o obj-$(CONFIG_LEDS_LT3593) += leds-lt3593.o -obj-$(CONFIG_LEDS_ADP5520) += leds-adp5520.o -obj-$(CONFIG_LEDS_MC13783) += leds-mc13783.o -obj-$(CONFIG_LEDS_NS2) += leds-ns2.o -obj-$(CONFIG_LEDS_NETXBIG) += leds-netxbig.o -obj-$(CONFIG_LEDS_ASIC3) += leds-asic3.o obj-$(CONFIG_LEDS_MAX77650) += leds-max77650.o obj-$(CONFIG_LEDS_MAX77693) += leds-max77693.o obj-$(CONFIG_LEDS_MAX8997) += leds-max8997.o -obj-$(CONFIG_LEDS_LM355x) += leds-lm355x.o -obj-$(CONFIG_LEDS_BLINKM) += leds-blinkm.o -obj-$(CONFIG_LEDS_SYSCON) += leds-syscon.o +obj-$(CONFIG_LEDS_MC13783) += leds-mc13783.o obj-$(CONFIG_LEDS_MENF21BMC) += leds-menf21bmc.o -obj-$(CONFIG_LEDS_KTD2692) += leds-ktd2692.o -obj-$(CONFIG_LEDS_POWERNV) += leds-powernv.o -obj-$(CONFIG_LEDS_IS31FL319X) += leds-is31fl319x.o -obj-$(CONFIG_LEDS_IS31FL32XX) += leds-is31fl32xx.o -obj-$(CONFIG_LEDS_PM8058) += leds-pm8058.o +obj-$(CONFIG_LEDS_MIKROTIK_RB532) += leds-rb532.o obj-$(CONFIG_LEDS_MLXCPLD) += leds-mlxcpld.o obj-$(CONFIG_LEDS_MLXREG) += leds-mlxreg.o -obj-$(CONFIG_LEDS_NIC78BX) += leds-nic78bx.o -obj-$(CONFIG_LEDS_SPI_BYTE) += leds-spi-byte.o obj-$(CONFIG_LEDS_MT6323) += leds-mt6323.o -obj-$(CONFIG_LEDS_LM3692X) += leds-lm3692x.o +obj-$(CONFIG_LEDS_NET48XX) += leds-net48xx.o +obj-$(CONFIG_LEDS_NETXBIG) += leds-netxbig.o +obj-$(CONFIG_LEDS_NIC78BX) += leds-nic78bx.o +obj-$(CONFIG_LEDS_NS2) += leds-ns2.o +obj-$(CONFIG_LEDS_OT200) += leds-ot200.o +obj-$(CONFIG_LEDS_PCA9532) += leds-pca9532.o +obj-$(CONFIG_LEDS_PCA955X) += leds-pca955x.o +obj-$(CONFIG_LEDS_PCA963X) += leds-pca963x.o +obj-$(CONFIG_LEDS_PM8058) += leds-pm8058.o +obj-$(CONFIG_LEDS_POWERNV) += leds-powernv.o +obj-$(CONFIG_LEDS_PWM) += leds-pwm.o +obj-$(CONFIG_LEDS_REGULATOR) += leds-regulator.o +obj-$(CONFIG_LEDS_S3C24XX) += leds-s3c24xx.o obj-$(CONFIG_LEDS_SC27XX_BLTC) += leds-sc27xx-bltc.o -obj-$(CONFIG_LEDS_LM3601X) += leds-lm3601x.o +obj-$(CONFIG_LEDS_SUNFIRE) += leds-sunfire.o +obj-$(CONFIG_LEDS_SYSCON) += leds-syscon.o +obj-$(CONFIG_LEDS_TCA6507) += leds-tca6507.o obj-$(CONFIG_LEDS_TI_LMU_COMMON) += leds-ti-lmu-common.o -obj-$(CONFIG_LEDS_LM3697) += leds-lm3697.o -obj-$(CONFIG_LEDS_LM36274) += leds-lm36274.o +obj-$(CONFIG_LEDS_TLC591XX) += leds-tlc591xx.o obj-$(CONFIG_LEDS_TPS6105X) += leds-tps6105x.o -obj-$(CONFIG_LEDS_IP30) += leds-ip30.o +obj-$(CONFIG_LEDS_WM831X_STATUS) += leds-wm831x-status.o +obj-$(CONFIG_LEDS_WM8350) += leds-wm8350.o +obj-$(CONFIG_LEDS_WRAP) += leds-wrap.o # LED SPI Drivers obj-$(CONFIG_LEDS_CR0014114) += leds-cr0014114.o obj-$(CONFIG_LEDS_DAC124S085) += leds-dac124s085.o obj-$(CONFIG_LEDS_EL15203000) += leds-el15203000.o +obj-$(CONFIG_LEDS_SPI_BYTE) += leds-spi-byte.o # LED Userspace Drivers obj-$(CONFIG_LEDS_USER) += uleds.o From 4b37883a8c07c7c7ec166ad5b66d8d0f39e7994d Mon Sep 17 00:00:00 2001 From: Pavel Machek Date: Sat, 21 Mar 2020 22:20:43 +0100 Subject: [PATCH 451/635] leds: make functions easier to understand Group LED functions according to functionality, and add some explaining comments. Signed-off-by: Pavel Machek --- include/dt-bindings/leds/common.h | 36 ++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/include/dt-bindings/leds/common.h b/include/dt-bindings/leds/common.h index 9e1256a7c1bf..0ce7dfc00dcb 100644 --- a/include/dt-bindings/leds/common.h +++ b/include/dt-bindings/leds/common.h @@ -6,6 +6,7 @@ * Author: Jacek Anaszewski * * Copyright (C) 2019 Jacek Anaszewski + * Copyright (C) 2020 Pavel Machek */ #ifndef __DT_BINDINGS_LEDS_H @@ -32,16 +33,38 @@ #define LED_COLOR_ID_MAX 8 /* Standard LED functions */ +/* Keyboard LEDs, usually it would be input4::capslock etc. */ +/* Obsolete equivalent: "shift-key-light" */ +#define LED_FUNCTION_CAPSLOCK "capslock" +#define LED_FUNCTION_SCROLLLOCK "scrolllock" +#define LED_FUNCTION_NUMLOCK "numlock" +/* Obsolete equivalents: "tpacpi::thinklight" (IBM/Lenovo Thinkpads), + "lp5523:kb{1,2,3,4,5,6}" (Nokia N900) */ +#define LED_FUNCTION_KBD_BACKLIGHT "kbd_backlight" + +/* System LEDs, usually found on system body. + platform::mute (etc) is sometimes seen, :mute would be better */ +#define LED_FUNCTION_POWER "power" +#define LED_FUNCTION_DISK "disk" + +/* Obsolete: "platform:*:charging" (allwinner sun50i) */ +#define LED_FUNCTION_CHARGING "charging" +/* Used RGB notification LEDs common on phones. + Obsolete equivalents: "status-led:{red,green,blue}" (Motorola Droid 4), + "lp5523:{r,g,b}" (Nokia N900) */ +#define LED_FUNCTION_STATUS "status" + +#define LED_FUNCTION_MICMUTE "micmute" +#define LED_FUNCTION_MUTE "mute" + +/* Miscelleaus functions. Use functions above if you can. */ #define LED_FUNCTION_ACTIVITY "activity" #define LED_FUNCTION_ALARM "alarm" #define LED_FUNCTION_BACKLIGHT "backlight" #define LED_FUNCTION_BLUETOOTH "bluetooth" #define LED_FUNCTION_BOOT "boot" #define LED_FUNCTION_CPU "cpu" -#define LED_FUNCTION_CAPSLOCK "capslock" -#define LED_FUNCTION_CHARGING "charging" #define LED_FUNCTION_DEBUG "debug" -#define LED_FUNCTION_DISK "disk" #define LED_FUNCTION_DISK_ACTIVITY "disk-activity" #define LED_FUNCTION_DISK_ERR "disk-err" #define LED_FUNCTION_DISK_READ "disk-read" @@ -50,21 +73,14 @@ #define LED_FUNCTION_FLASH "flash" #define LED_FUNCTION_HEARTBEAT "heartbeat" #define LED_FUNCTION_INDICATOR "indicator" -#define LED_FUNCTION_KBD_BACKLIGHT "kbd_backlight" #define LED_FUNCTION_LAN "lan" #define LED_FUNCTION_MAIL "mail" #define LED_FUNCTION_MTD "mtd" -#define LED_FUNCTION_MICMUTE "micmute" -#define LED_FUNCTION_MUTE "mute" -#define LED_FUNCTION_NUMLOCK "numlock" #define LED_FUNCTION_PANIC "panic" #define LED_FUNCTION_PROGRAMMING "programming" -#define LED_FUNCTION_POWER "power" #define LED_FUNCTION_RX "rx" #define LED_FUNCTION_SD "sd" -#define LED_FUNCTION_SCROLLLOCK "scrolllock" #define LED_FUNCTION_STANDBY "standby" -#define LED_FUNCTION_STATUS "status" #define LED_FUNCTION_TORCH "torch" #define LED_FUNCTION_TX "tx" #define LED_FUNCTION_USB "usb" From 64ed6588c2ea618d3f9ca9d8b365ae4c19f76225 Mon Sep 17 00:00:00 2001 From: Ricardo Ribalda Delgado Date: Wed, 1 Apr 2020 11:51:47 +0200 Subject: [PATCH 452/635] leds: core: Fix warning message when init_data The warning message when a led is renamed due to name collition can fail to show proper original name if init_data is used. Eg: [ 9.073996] leds-gpio a0040000.leds_0: Led (null) renamed to red_led_1 due to name collision Fixes: bb4e9af0348d ("leds: core: Add support for composing LED class device names") Signed-off-by: Ricardo Ribalda Delgado Acked-by: Jacek Anaszewski Signed-off-by: Pavel Machek --- drivers/leds/led-class.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/leds/led-class.c b/drivers/leds/led-class.c index 1fc40e8af75e..3363a6551a70 100644 --- a/drivers/leds/led-class.c +++ b/drivers/leds/led-class.c @@ -376,7 +376,7 @@ int led_classdev_register_ext(struct device *parent, if (ret) dev_warn(parent, "Led %s renamed to %s due to name collision", - led_cdev->name, dev_name(led_cdev->dev)); + proposed_name, dev_name(led_cdev->dev)); if (led_cdev->flags & LED_BRIGHT_HW_CHANGED) { ret = led_add_brightness_hw_changed(led_cdev); From 8a39db76e79ef502f5d2c1865358e9025b5fa38a Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Mon, 6 Jan 2020 12:16:02 +1000 Subject: [PATCH 453/635] drm/nouveau/ttm: evict other IO mappings when running out of BAR1 space Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_bo.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index 2b4b21b02e40..c40f127de3d0 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -1494,8 +1494,13 @@ nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *reg) ret = nvif_object_map_handle(&mem->mem.object, &args, argc, &handle, &length); - if (ret != 1) - return ret ? ret : -EINVAL; + if (ret != 1) { + if (WARN_ON(ret == 0)) + return -EINVAL; + if (ret == -ENOSPC) + return -EAGAIN; + return ret; + } reg->bus.base = 0; reg->bus.offset = handle; From e18c6e8b6f5b567c54cbdf422c89c877d49e9c4e Mon Sep 17 00:00:00 2001 From: Wambui Karuga Date: Tue, 18 Feb 2020 20:28:17 +0300 Subject: [PATCH 454/635] drm/nouveau: remove checks for return value of debugfs functions As there is no need to check for the return value of debugfs_create_file and drm_debugfs_create_files, remove unnecessary checks and error handling in nouveau_drm_debugfs_init. Signed-off-by: Wambui Karuga Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_debugfs.c | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_debugfs.c b/drivers/gpu/drm/nouveau/nouveau_debugfs.c index 7dfbbbc1beea..15a3d40edf02 100644 --- a/drivers/gpu/drm/nouveau/nouveau_debugfs.c +++ b/drivers/gpu/drm/nouveau/nouveau_debugfs.c @@ -222,22 +222,18 @@ nouveau_drm_debugfs_init(struct drm_minor *minor) { struct nouveau_drm *drm = nouveau_drm(minor->dev); struct dentry *dentry; - int i, ret; + int i; for (i = 0; i < ARRAY_SIZE(nouveau_debugfs_files); i++) { - dentry = debugfs_create_file(nouveau_debugfs_files[i].name, - S_IRUGO | S_IWUSR, - minor->debugfs_root, minor->dev, - nouveau_debugfs_files[i].fops); - if (!dentry) - return -ENOMEM; + debugfs_create_file(nouveau_debugfs_files[i].name, + S_IRUGO | S_IWUSR, + minor->debugfs_root, minor->dev, + nouveau_debugfs_files[i].fops); } - ret = drm_debugfs_create_files(nouveau_debugfs_list, - NOUVEAU_DEBUGFS_ENTRIES, - minor->debugfs_root, minor); - if (ret) - return ret; + drm_debugfs_create_files(nouveau_debugfs_list, + NOUVEAU_DEBUGFS_ENTRIES, + minor->debugfs_root, minor); /* Set the size of the vbios since we know it, and it's confusing to * userspace if it wants to seek() but the file has a length of 0 From b92103b559c77abc5f8b7bec269230a219c880b7 Mon Sep 17 00:00:00 2001 From: Ralph Campbell Date: Tue, 3 Mar 2020 16:13:36 -0800 Subject: [PATCH 455/635] drm/nouveau/svm: fix vma range check for migration find_vma_intersection(mm, start, end) only guarantees that end is greater than or equal to vma->vm_start but doesn't guarantee that start is greater than or equal to vma->vm_start. The calculation for the intersecting range in nouveau_svmm_bind() isn't accounting for this and can call migrate_vma_setup() with a starting address less than vma->vm_start. This results in migrate_vma_setup() returning -EINVAL for the range instead of nouveau skipping that part of the range and migrating the rest. Signed-off-by: Ralph Campbell Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_svm.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/nouveau/nouveau_svm.c b/drivers/gpu/drm/nouveau/nouveau_svm.c index df9bf1fd1bc0..169320409286 100644 --- a/drivers/gpu/drm/nouveau/nouveau_svm.c +++ b/drivers/gpu/drm/nouveau/nouveau_svm.c @@ -179,6 +179,7 @@ nouveau_svmm_bind(struct drm_device *dev, void *data, if (!vma) break; + addr = max(addr, vma->vm_start); next = min(vma->vm_end, end); /* This is a best effort so we ignore errors */ nouveau_dmem_migrate_vma(cli->drm, vma, addr, next); From 822cab6150d3002952407a8297ff5a0d32bb7b54 Mon Sep 17 00:00:00 2001 From: Ralph Campbell Date: Tue, 3 Mar 2020 16:13:37 -0800 Subject: [PATCH 456/635] drm/nouveau/svm: check for SVM initialized before migrating When migrating system memory to GPU memory, check that SVM has been enabled. Even though most errors can be ignored since migration is a performance optimization, return an error because this is a violation of the API. Signed-off-by: Ralph Campbell Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_svm.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/gpu/drm/nouveau/nouveau_svm.c b/drivers/gpu/drm/nouveau/nouveau_svm.c index 169320409286..c567526b75b8 100644 --- a/drivers/gpu/drm/nouveau/nouveau_svm.c +++ b/drivers/gpu/drm/nouveau/nouveau_svm.c @@ -171,6 +171,11 @@ nouveau_svmm_bind(struct drm_device *dev, void *data, mm = get_task_mm(current); down_read(&mm->mmap_sem); + if (!cli->svm.svmm) { + up_read(&mm->mmap_sem); + return -EINVAL; + } + for (addr = args->va_start, end = args->va_start + size; addr < end;) { struct vm_area_struct *vma; unsigned long next; From bc7b188023e77d69d8dd584c51e62f0e99d9e2a2 Mon Sep 17 00:00:00 2001 From: Ralph Campbell Date: Tue, 3 Mar 2020 16:13:38 -0800 Subject: [PATCH 457/635] drm/nouveau/svm: remove useless SVM range check When nouveau processes GPU faults, it checks to see if the fault address falls within the "unmanaged" range which is reserved for fixed allocations instead of addresses chosen by the core mm code. If start is greater than or equal to svmm->unmanaged.limit, then limit will also be greater than svmm->unmanaged.limit which is greater than svmm->unmanaged.start and the start = max_t(u64, start, svmm->unmanaged.limit) will change nothing. Just remove the useless lines of code. Signed-off-by: Ralph Campbell Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_svm.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_svm.c b/drivers/gpu/drm/nouveau/nouveau_svm.c index c567526b75b8..8dfa5cb74826 100644 --- a/drivers/gpu/drm/nouveau/nouveau_svm.c +++ b/drivers/gpu/drm/nouveau/nouveau_svm.c @@ -663,9 +663,6 @@ nouveau_svm_fault(struct nvif_notify *notify) limit = start + (ARRAY_SIZE(args.phys) << PAGE_SHIFT); if (start < svmm->unmanaged.limit) limit = min_t(u64, limit, svmm->unmanaged.start); - else - if (limit > svmm->unmanaged.start) - start = max_t(u64, start, svmm->unmanaged.limit); SVMM_DBG(svmm, "wndw %016llx-%016llx", start, limit); mm = svmm->notifier.mm; From 434fdb51513bf3057ac144d152e6f2f2b509e857 Mon Sep 17 00:00:00 2001 From: Karol Herbst Date: Tue, 24 Mar 2020 21:29:23 +0100 Subject: [PATCH 458/635] drm/nouveau: workaround runpm fail by disabling PCI power management on certain intel bridges Fixes the infamous 'runtime PM' bug many users are facing on Laptops with Nvidia Pascal GPUs by skipping said PCI power state changes on the GPU. Depending on the used kernel there might be messages like those in demsg: "nouveau 0000:01:00.0: Refused to change power state, currently in D3" "nouveau 0000:01:00.0: can't change power state from D3cold to D0 (config space inaccessible)" followed by backtraces of kernel crashes or timeouts within nouveau. It's still unkown why this issue exists, but this is a reliable workaround and solves a very annoying issue for user having to choose between a crashing kernel or higher power consumption of their Laptops. Signed-off-by: Karol Herbst Cc: Bjorn Helgaas Cc: Lyude Paul Cc: Rafael J. Wysocki Cc: Mika Westerberg Cc: linux-pci@vger.kernel.org Cc: linux-pm@vger.kernel.org Cc: dri-devel@lists.freedesktop.org Cc: nouveau@lists.freedesktop.org Bugzilla: https://bugzilla.kernel.org/show_bug.cgi?id=205623 Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_drm.c | 63 +++++++++++++++++++++++++++ drivers/gpu/drm/nouveau/nouveau_drv.h | 2 + 2 files changed, 65 insertions(+) diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c index 6b1629c14dd7..ca4087f5a15b 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drm.c +++ b/drivers/gpu/drm/nouveau/nouveau_drm.c @@ -618,6 +618,64 @@ nouveau_drm_device_fini(struct drm_device *dev) kfree(drm); } +/* + * On some Intel PCIe bridge controllers doing a + * D0 -> D3hot -> D3cold -> D0 sequence causes Nvidia GPUs to not reappear. + * Skipping the intermediate D3hot step seems to make it work again. This is + * probably caused by not meeting the expectation the involved AML code has + * when the GPU is put into D3hot state before invoking it. + * + * This leads to various manifestations of this issue: + * - AML code execution to power on the GPU hits an infinite loop (as the + * code waits on device memory to change). + * - kernel crashes, as all PCI reads return -1, which most code isn't able + * to handle well enough. + * + * In all cases dmesg will contain at least one line like this: + * 'nouveau 0000:01:00.0: Refused to change power state, currently in D3' + * followed by a lot of nouveau timeouts. + * + * In the \_SB.PCI0.PEG0.PG00._OFF code deeper down writes bit 0x80 to the not + * documented PCI config space register 0x248 of the Intel PCIe bridge + * controller (0x1901) in order to change the state of the PCIe link between + * the PCIe port and the GPU. There are alternative code paths using other + * registers, which seem to work fine (executed pre Windows 8): + * - 0xbc bit 0x20 (publicly available documentation claims 'reserved') + * - 0xb0 bit 0x10 (link disable) + * Changing the conditions inside the firmware by poking into the relevant + * addresses does resolve the issue, but it seemed to be ACPI private memory + * and not any device accessible memory at all, so there is no portable way of + * changing the conditions. + * On a XPS 9560 that means bits [0,3] on \CPEX need to be cleared. + * + * The only systems where this behavior can be seen are hybrid graphics laptops + * with a secondary Nvidia Maxwell, Pascal or Turing GPU. It's unclear whether + * this issue only occurs in combination with listed Intel PCIe bridge + * controllers and the mentioned GPUs or other devices as well. + * + * documentation on the PCIe bridge controller can be found in the + * "7th Generation Intel® Processor Families for H Platforms Datasheet Volume 2" + * Section "12 PCI Express* Controller (x16) Registers" + */ + +static void quirk_broken_nv_runpm(struct pci_dev *pdev) +{ + struct drm_device *dev = pci_get_drvdata(pdev); + struct nouveau_drm *drm = nouveau_drm(dev); + struct pci_dev *bridge = pci_upstream_bridge(pdev); + + if (!bridge || bridge->vendor != PCI_VENDOR_ID_INTEL) + return; + + switch (bridge->device) { + case 0x1901: + drm->old_pm_cap = pdev->pm_cap; + pdev->pm_cap = 0; + NV_INFO(drm, "Disabling PCI power management to avoid bug\n"); + break; + } +} + static int nouveau_drm_probe(struct pci_dev *pdev, const struct pci_device_id *pent) { @@ -699,6 +757,7 @@ static int nouveau_drm_probe(struct pci_dev *pdev, if (ret) goto fail_drm_dev_init; + quirk_broken_nv_runpm(pdev); return 0; fail_drm_dev_init: @@ -734,7 +793,11 @@ static void nouveau_drm_remove(struct pci_dev *pdev) { struct drm_device *dev = pci_get_drvdata(pdev); + struct nouveau_drm *drm = nouveau_drm(dev); + /* revert our workaround */ + if (drm->old_pm_cap) + pdev->pm_cap = drm->old_pm_cap; nouveau_drm_device_remove(dev); pci_disable_device(pdev); } diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h index c2c332fbde97..2a6519737800 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drv.h +++ b/drivers/gpu/drm/nouveau/nouveau_drv.h @@ -140,6 +140,8 @@ struct nouveau_drm { struct list_head clients; + u8 old_pm_cap; + struct { struct agp_bridge_data *bridge; u32 base; From 028a12f5aa829b4ba6ac011530b815eda4960e89 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Tue, 31 Mar 2020 16:08:44 +1000 Subject: [PATCH 459/635] drm/nouveau/gr/gp107,gp108: implement workaround for HW hanging during init Certain boards with GP107/GP108 chipsets hang (often, but randomly) for unknown reasons during GR initialisation. The first tell-tale symptom of this issue is: nouveau 0000:01:00.0: bus: MMIO read of 00000000 FAULT at 409800 [ TIMEOUT ] appearing in dmesg, likely followed by many other failures being logged. Karol found this WAR for the issue a while back, but efforts to isolate the root cause and proper fix have not yielded success so far. I've modified the original patch to include a few more details, limit it to GP107/GP108 by default, and added a config option to override this choice. Signed-off-by: Ben Skeggs Reviewed-by: Karol Herbst --- .../gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index dd8f85b8b3a7..f2f5636efac4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -1981,8 +1981,34 @@ gf100_gr_init_(struct nvkm_gr *base) { struct gf100_gr *gr = gf100_gr(base); struct nvkm_subdev *subdev = &base->engine.subdev; + struct nvkm_device *device = subdev->device; + bool reset = device->chipset == 0x137 || device->chipset == 0x138; u32 ret; + /* On certain GP107/GP108 boards, we trigger a weird issue where + * GR will stop responding to PRI accesses after we've asked the + * SEC2 RTOS to boot the GR falcons. This happens with far more + * frequency when cold-booting a board (ie. returning from D3). + * + * The root cause for this is not known and has proven difficult + * to isolate, with many avenues being dead-ends. + * + * A workaround was discovered by Karol, whereby putting GR into + * reset for an extended period right before initialisation + * prevents the problem from occuring. + * + * XXX: As RM does not require any such workaround, this is more + * of a hack than a true fix. + */ + reset = nvkm_boolopt(device->cfgopt, "NvGrResetWar", reset); + if (reset) { + nvkm_mask(device, 0x000200, 0x00001000, 0x00000000); + nvkm_rd32(device, 0x000200); + msleep(50); + nvkm_mask(device, 0x000200, 0x00001000, 0x00001000); + nvkm_rd32(device, 0x000200); + } + nvkm_pmu_pgob(gr->base.engine.subdev.device->pmu, false); ret = nvkm_falcon_get(&gr->fecs.falcon, subdev); From 58e92b5ee8743ebb77b824cce3330521f049bbac Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Mon, 17 Feb 2020 14:55:49 +1000 Subject: [PATCH 460/635] drm/nouveau/nvif: access PTIMER through usermode class, if available Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvif/user.h | 1 + drivers/gpu/drm/nouveau/nvif/device.c | 14 +++++++++----- drivers/gpu/drm/nouveau/nvif/userc361.c | 14 ++++++++++++++ 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvif/user.h b/drivers/gpu/drm/nouveau/include/nvif/user.h index 03c11826b693..6825574d93c2 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/user.h +++ b/drivers/gpu/drm/nouveau/include/nvif/user.h @@ -10,6 +10,7 @@ struct nvif_user { struct nvif_user_func { void (*doorbell)(struct nvif_user *, u32 token); + u64 (*time)(struct nvif_user *); }; int nvif_user_init(struct nvif_device *); diff --git a/drivers/gpu/drm/nouveau/nvif/device.c b/drivers/gpu/drm/nouveau/nvif/device.c index 1ec101ba3b42..0e92db44bbc8 100644 --- a/drivers/gpu/drm/nouveau/nvif/device.c +++ b/drivers/gpu/drm/nouveau/nvif/device.c @@ -27,11 +27,15 @@ u64 nvif_device_time(struct nvif_device *device) { - struct nv_device_time_v0 args = {}; - int ret = nvif_object_mthd(&device->object, NV_DEVICE_V0_TIME, - &args, sizeof(args)); - WARN_ON_ONCE(ret != 0); - return args.time; + if (!device->user.func) { + struct nv_device_time_v0 args = {}; + int ret = nvif_object_mthd(&device->object, NV_DEVICE_V0_TIME, + &args, sizeof(args)); + WARN_ON_ONCE(ret != 0); + return args.time; + } + + return device->user.func->time(&device->user); } void diff --git a/drivers/gpu/drm/nouveau/nvif/userc361.c b/drivers/gpu/drm/nouveau/nvif/userc361.c index 19f9958e7e01..1116f871b272 100644 --- a/drivers/gpu/drm/nouveau/nvif/userc361.c +++ b/drivers/gpu/drm/nouveau/nvif/userc361.c @@ -21,6 +21,19 @@ */ #include +static u64 +nvif_userc361_time(struct nvif_user *user) +{ + u32 hi, lo; + + do { + hi = nvif_rd32(&user->object, 0x084); + lo = nvif_rd32(&user->object, 0x080); + } while (hi != nvif_rd32(&user->object, 0x084)); + + return ((u64)hi << 32 | lo); +} + static void nvif_userc361_doorbell(struct nvif_user *user, u32 token) { @@ -30,4 +43,5 @@ nvif_userc361_doorbell(struct nvif_user *user, u32 token) const struct nvif_user_func nvif_userc361 = { .doorbell = nvif_userc361_doorbell, + .time = nvif_userc361_time, }; From ed3d1489d2717a13f3b47b296bb2faa7ce4168da Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Mon, 17 Feb 2020 14:58:02 +1000 Subject: [PATCH 461/635] drm/nouveau/nvif: protect waits against GPU falling off the bus Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/dispnv04/dac.c | 3 +- drivers/gpu/drm/nouveau/dispnv04/hw.c | 1 + drivers/gpu/drm/nouveau/dispnv50/base507c.c | 1 + drivers/gpu/drm/nouveau/dispnv50/core507d.c | 1 + drivers/gpu/drm/nouveau/dispnv50/corec37d.c | 2 + drivers/gpu/drm/nouveau/dispnv50/disp.c | 1 + drivers/gpu/drm/nouveau/dispnv50/ovly827e.c | 2 + drivers/gpu/drm/nouveau/include/nvif/device.h | 21 ------- drivers/gpu/drm/nouveau/include/nvif/timer.h | 35 ++++++++++++ drivers/gpu/drm/nouveau/nvif/Kbuild | 1 + drivers/gpu/drm/nouveau/nvif/timer.c | 56 +++++++++++++++++++ 11 files changed, 102 insertions(+), 22 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/include/nvif/timer.h create mode 100644 drivers/gpu/drm/nouveau/nvif/timer.c diff --git a/drivers/gpu/drm/nouveau/dispnv04/dac.c b/drivers/gpu/drm/nouveau/dispnv04/dac.c index e8eef88a8382..ffdd447d8706 100644 --- a/drivers/gpu/drm/nouveau/dispnv04/dac.c +++ b/drivers/gpu/drm/nouveau/dispnv04/dac.c @@ -35,7 +35,8 @@ #include #include -#include + +#include int nv04_dac_output_offset(struct drm_encoder *encoder) { diff --git a/drivers/gpu/drm/nouveau/dispnv04/hw.c b/drivers/gpu/drm/nouveau/dispnv04/hw.c index 3fdfafa8b0ad..b674d68ef28a 100644 --- a/drivers/gpu/drm/nouveau/dispnv04/hw.c +++ b/drivers/gpu/drm/nouveau/dispnv04/hw.c @@ -26,6 +26,7 @@ #include "hw.h" #include +#include #define CHIPSET_NFORCE 0x01a0 #define CHIPSET_NFORCE2 0x01f0 diff --git a/drivers/gpu/drm/nouveau/dispnv50/base507c.c b/drivers/gpu/drm/nouveau/dispnv50/base507c.c index 00a85f1e1a4a..ee782151d332 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/base507c.c +++ b/drivers/gpu/drm/nouveau/dispnv50/base507c.c @@ -23,6 +23,7 @@ #include #include +#include #include #include diff --git a/drivers/gpu/drm/nouveau/dispnv50/core507d.c b/drivers/gpu/drm/nouveau/dispnv50/core507d.c index e7fcfa6e6467..c5152c39c684 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/core507d.c +++ b/drivers/gpu/drm/nouveau/dispnv50/core507d.c @@ -23,6 +23,7 @@ #include "head.h" #include +#include #include "nouveau_bo.h" diff --git a/drivers/gpu/drm/nouveau/dispnv50/corec37d.c b/drivers/gpu/drm/nouveau/dispnv50/corec37d.c index 3b36dc8d36b2..c03cb987856b 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/corec37d.c +++ b/drivers/gpu/drm/nouveau/dispnv50/corec37d.c @@ -24,6 +24,8 @@ #include +#include + void corec37d_wndw_owner(struct nv50_core *core) { diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.c b/drivers/gpu/drm/nouveau/dispnv50/disp.c index 4d1c58468dbc..6be9df1820c5 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/disp.c +++ b/drivers/gpu/drm/nouveau/dispnv50/disp.c @@ -45,6 +45,7 @@ #include #include #include +#include #include "nouveau_drv.h" #include "nouveau_dma.h" diff --git a/drivers/gpu/drm/nouveau/dispnv50/ovly827e.c b/drivers/gpu/drm/nouveau/dispnv50/ovly827e.c index 2e68fc736fe1..4f7ce57f2036 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/ovly827e.c +++ b/drivers/gpu/drm/nouveau/dispnv50/ovly827e.c @@ -24,6 +24,8 @@ #include +#include + static void ovly827e_image_set(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw) { diff --git a/drivers/gpu/drm/nouveau/include/nvif/device.h b/drivers/gpu/drm/nouveau/include/nvif/device.h index 25d969dcf67d..c2a572c67a76 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/device.h +++ b/drivers/gpu/drm/nouveau/include/nvif/device.h @@ -23,27 +23,6 @@ int nvif_device_init(struct nvif_object *, u32 handle, s32 oclass, void *, u32, void nvif_device_fini(struct nvif_device *); u64 nvif_device_time(struct nvif_device *); -/* Delay based on GPU time (ie. PTIMER). - * - * Will return -ETIMEDOUT unless the loop was terminated with 'break', - * where it will return the number of nanoseconds taken instead. - */ -#define nvif_nsec(d,n,cond...) ({ \ - struct nvif_device *_device = (d); \ - u64 _nsecs = (n), _time0 = nvif_device_time(_device); \ - s64 _taken = 0; \ - \ - do { \ - cond \ - } while (_taken = nvif_device_time(_device) - _time0, _taken < _nsecs);\ - \ - if (_taken >= _nsecs) \ - _taken = -ETIMEDOUT; \ - _taken; \ -}) -#define nvif_usec(d,u,cond...) nvif_nsec((d), (u) * 1000, ##cond) -#define nvif_msec(d,m,cond...) nvif_usec((d), (m) * 1000, ##cond) - /*XXX*/ #include #include diff --git a/drivers/gpu/drm/nouveau/include/nvif/timer.h b/drivers/gpu/drm/nouveau/include/nvif/timer.h new file mode 100644 index 000000000000..57587a985c4b --- /dev/null +++ b/drivers/gpu/drm/nouveau/include/nvif/timer.h @@ -0,0 +1,35 @@ +#ifndef __NVIF_TIMER_H__ +#define __NVIF_TIMER_H__ +#include + +struct nvif_timer_wait { + struct nvif_device *device; + u64 limit; + u64 time0; + u64 time1; + int reads; +}; + +void nvif_timer_wait_init(struct nvif_device *, u64 nsec, + struct nvif_timer_wait *); +s64 nvif_timer_wait_test(struct nvif_timer_wait *); + +/* Delay based on GPU time (ie. PTIMER). + * + * Will return -ETIMEDOUT unless the loop was terminated with 'break', + * where it will return the number of nanoseconds taken instead. + */ +#define nvif_nsec(d,n,cond...) ({ \ + struct nvif_timer_wait _wait; \ + s64 _taken = 0; \ + \ + nvif_timer_wait_init((d), (n), &_wait); \ + do { \ + cond \ + } while ((_taken = nvif_timer_wait_test(&_wait)) >= 0); \ + \ + _taken; \ +}) +#define nvif_usec(d,u,cond...) nvif_nsec((d), (u) * 1000, ##cond) +#define nvif_msec(d,m,cond...) nvif_usec((d), (m) * 1000, ##cond) +#endif diff --git a/drivers/gpu/drm/nouveau/nvif/Kbuild b/drivers/gpu/drm/nouveau/nvif/Kbuild index 50d583d63807..f194d354c1f5 100644 --- a/drivers/gpu/drm/nouveau/nvif/Kbuild +++ b/drivers/gpu/drm/nouveau/nvif/Kbuild @@ -8,6 +8,7 @@ nvif-y += nvif/fifo.o nvif-y += nvif/mem.o nvif-y += nvif/mmu.o nvif-y += nvif/notify.o +nvif-y += nvif/timer.o nvif-y += nvif/vmm.o # Usermode classes diff --git a/drivers/gpu/drm/nouveau/nvif/timer.c b/drivers/gpu/drm/nouveau/nvif/timer.c new file mode 100644 index 000000000000..602c1a258d10 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvif/timer.c @@ -0,0 +1,56 @@ +/* + * Copyright 2020 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include +#include + +s64 +nvif_timer_wait_test(struct nvif_timer_wait *wait) +{ + u64 time = nvif_device_time(wait->device); + + if (wait->reads == 0) { + wait->time0 = time; + wait->time1 = time; + } + + if (wait->time1 == time) { + if (WARN_ON(wait->reads++ == 16)) + return -ETIMEDOUT; + } else { + wait->time1 = time; + wait->reads = 1; + } + + if (wait->time1 - wait->time0 > wait->limit) + return -ETIMEDOUT; + + return wait->time1 - wait->time0; +} + +void +nvif_timer_wait_init(struct nvif_device *device, u64 nsec, + struct nvif_timer_wait *wait) +{ + wait->device = device; + wait->limit = nsec; + wait->reads = 0; +} From 374b55802cd567e9f340b7f27d2c5e20b069ac3c Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Mon, 17 Feb 2020 14:33:57 +1000 Subject: [PATCH 462/635] drm/nouveau/kms/nv50-: wait for FIFO space on PIO channels Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/dispnv50/curs507a.c | 21 ++++++++++++++++++--- drivers/gpu/drm/nouveau/dispnv50/cursc37a.c | 9 ++++++--- drivers/gpu/drm/nouveau/dispnv50/wndw.h | 1 + 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/nouveau/dispnv50/curs507a.c b/drivers/gpu/drm/nouveau/dispnv50/curs507a.c index 397143b639c6..8c5cf096f69b 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/curs507a.c +++ b/drivers/gpu/drm/nouveau/dispnv50/curs507a.c @@ -24,21 +24,36 @@ #include "head.h" #include +#include #include #include +bool +curs507a_space(struct nv50_wndw *wndw) +{ + nvif_msec(&nouveau_drm(wndw->plane.dev)->client.device, 2, + if (nvif_rd32(&wndw->wimm.base.user, 0x0008) >= 4) + return true; + ); + WARN_ON(1); + return false; +} + static void curs507a_update(struct nv50_wndw *wndw, u32 *interlock) { - nvif_wr32(&wndw->wimm.base.user, 0x0080, 0x00000000); + if (curs507a_space(wndw)) + nvif_wr32(&wndw->wimm.base.user, 0x0080, 0x00000000); } static void curs507a_point(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw) { - nvif_wr32(&wndw->wimm.base.user, 0x0084, asyw->point.y << 16 | - asyw->point.x); + if (curs507a_space(wndw)) { + nvif_wr32(&wndw->wimm.base.user, 0x0084, asyw->point.y << 16 | + asyw->point.x); + } } const struct nv50_wimm_func diff --git a/drivers/gpu/drm/nouveau/dispnv50/cursc37a.c b/drivers/gpu/drm/nouveau/dispnv50/cursc37a.c index 23fb29d41efe..96dff4f09f57 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/cursc37a.c +++ b/drivers/gpu/drm/nouveau/dispnv50/cursc37a.c @@ -25,14 +25,17 @@ static void cursc37a_update(struct nv50_wndw *wndw, u32 *interlock) { - nvif_wr32(&wndw->wimm.base.user, 0x0200, 0x00000001); + if (curs507a_space(wndw)) + nvif_wr32(&wndw->wimm.base.user, 0x0200, 0x00000001); } static void cursc37a_point(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw) { - nvif_wr32(&wndw->wimm.base.user, 0x0208, asyw->point.y << 16 | - asyw->point.x); + if (curs507a_space(wndw)) { + nvif_wr32(&wndw->wimm.base.user, 0x0208, asyw->point.y << 16 | + asyw->point.x); + } } static const struct nv50_wimm_func diff --git a/drivers/gpu/drm/nouveau/dispnv50/wndw.h b/drivers/gpu/drm/nouveau/dispnv50/wndw.h index caf397475918..a7412b9d3a98 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/wndw.h +++ b/drivers/gpu/drm/nouveau/dispnv50/wndw.h @@ -97,6 +97,7 @@ struct nv50_wimm_func { }; extern const struct nv50_wimm_func curs507a; +bool curs507a_space(struct nv50_wndw *); int wndwc37e_new(struct nouveau_drm *, enum drm_plane_type, int, s32, struct nv50_wndw **); From da7e42320940059d571c233375d873c3a9de88c9 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Mon, 6 Apr 2020 22:21:08 +0200 Subject: [PATCH 463/635] KVM: VMX: Remove unnecessary exception trampoline in vmx_vmenter The exception trampoline in .fixup section is not needed, the exception handling code can jump directly to the label in the .text section. Changes since v1: - Fix commit message. Cc: Sean Christopherson Cc: Paolo Bonzini Reviewed-by: Sean Christopherson Signed-off-by: Uros Bizjak Message-Id: <20200406202108.74300-1-ubizjak@gmail.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/vmenter.S | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/vmx/vmenter.S b/arch/x86/kvm/vmx/vmenter.S index 9651ba388ba9..87f3f24fef37 100644 --- a/arch/x86/kvm/vmx/vmenter.S +++ b/arch/x86/kvm/vmx/vmenter.S @@ -58,12 +58,8 @@ SYM_FUNC_START(vmx_vmenter) ret 4: ud2 - .pushsection .fixup, "ax" -5: jmp 3b - .popsection - - _ASM_EXTABLE(1b, 5b) - _ASM_EXTABLE(2b, 5b) + _ASM_EXTABLE(1b, 3b) + _ASM_EXTABLE(2b, 3b) SYM_FUNC_END(vmx_vmenter) From 5c8beb474665220374abcafa79e40d78778606d1 Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Mon, 6 Apr 2020 20:12:37 +0000 Subject: [PATCH 464/635] KVM: nVMX: don't clear mtf_pending when nested events are blocked If nested events are blocked, don't clear the mtf_pending flag to avoid missing later delivery of the MTF VM-exit. Fixes: 5ef8acbdd687c ("KVM: nVMX: Emulate MTF when performing instruction emulation") Signed-off-by: Oliver Upton Message-Id: <20200406201237.178725-1-oupton@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index de232306561a..cbc9ea2de28f 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -3645,7 +3645,8 @@ static int vmx_check_nested_events(struct kvm_vcpu *vcpu) * Clear the MTF state. If a higher priority VM-exit is delivered first, * this state is discarded. */ - vmx->nested.mtf_pending = false; + if (!block_nested_events) + vmx->nested.mtf_pending = false; if (lapic_in_kernel(vcpu) && test_bit(KVM_APIC_INIT, &apic->pending_events)) { From 3f5b9959041e0db6dacbea80bb833bff5900999f Mon Sep 17 00:00:00 2001 From: Martin Blumenstingl Date: Fri, 3 Apr 2020 22:51:33 +0200 Subject: [PATCH 465/635] thermal: devfreq_cooling: inline all stubs for CONFIG_DEVFREQ_THERMAL=n When CONFIG_DEVFREQ_THERMAL is disabled all functions except of_devfreq_cooling_register_power() were already inlined. Also inline the last function to avoid compile errors when multiple drivers call of_devfreq_cooling_register_power() when CONFIG_DEVFREQ_THERMAL is not set. Compilation failed with the following message: multiple definition of `of_devfreq_cooling_register_power' (which then lists all usages of of_devfreq_cooling_register_power()) Thomas Zimmermann reported this problem [0] on a kernel config with CONFIG_DRM_LIMA={m,y}, CONFIG_DRM_PANFROST={m,y} and CONFIG_DEVFREQ_THERMAL=n after both, the lima and panfrost drivers gained devfreq cooling support. [0] https://www.spinics.net/lists/dri-devel/msg252825.html Fixes: a76caf55e5b356 ("thermal: Add devfreq cooling") Cc: stable@vger.kernel.org Reported-by: Thomas Zimmermann Signed-off-by: Martin Blumenstingl Tested-by: Thomas Zimmermann Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200403205133.1101808-1-martin.blumenstingl@googlemail.com --- include/linux/devfreq_cooling.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/devfreq_cooling.h b/include/linux/devfreq_cooling.h index 4635f95000a4..79a6e37a1d6f 100644 --- a/include/linux/devfreq_cooling.h +++ b/include/linux/devfreq_cooling.h @@ -75,7 +75,7 @@ void devfreq_cooling_unregister(struct thermal_cooling_device *dfc); #else /* !CONFIG_DEVFREQ_THERMAL */ -struct thermal_cooling_device * +static inline struct thermal_cooling_device * of_devfreq_cooling_register_power(struct device_node *np, struct devfreq *df, struct devfreq_cooling_power *dfc_power) { From 11700fcb90b40a38fbfba55b2bedfc9909be8ec8 Mon Sep 17 00:00:00 2001 From: Sergey Vidishev Date: Fri, 3 Apr 2020 22:12:42 +0300 Subject: [PATCH 466/635] Documentation: cpu-idle-cooling: Fix diagram for 33% duty cycle Signed-off-by: Sergey Vidishev Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/2374188.AZIXMmL6Zy@sergeyv-box --- Documentation/driver-api/thermal/cpu-idle-cooling.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/driver-api/thermal/cpu-idle-cooling.rst b/Documentation/driver-api/thermal/cpu-idle-cooling.rst index 9f0016ee4cfb..a1c3edecae00 100644 --- a/Documentation/driver-api/thermal/cpu-idle-cooling.rst +++ b/Documentation/driver-api/thermal/cpu-idle-cooling.rst @@ -105,8 +105,8 @@ and this variation will modulate the cooling effect. idle <--------------> running - <-----------------------------> - duty cycle 33% + <---------------------> + duty cycle 33% ^ From 835a6a649d0dd1b1f46759eb60fff2f63ed253a7 Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Tue, 7 Apr 2020 05:43:30 -0400 Subject: [PATCH 467/635] virtio-balloon: Revert "virtio-balloon: Switch back to OOM handler for VIRTIO_BALLOON_F_DEFLATE_ON_OOM" This reverts commit 5a6b4cc5b7a1892a8d7f63d6cbac6e0ae2a9d031. It has been queued properly in the akpm tree, this version is just creating conflicts. Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_balloon.c | 107 +++++++++++++++++++------------- 1 file changed, 63 insertions(+), 44 deletions(-) diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c index 44375a22307b..341458fd95ca 100644 --- a/drivers/virtio/virtio_balloon.c +++ b/drivers/virtio/virtio_balloon.c @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -28,9 +27,7 @@ */ #define VIRTIO_BALLOON_PAGES_PER_PAGE (unsigned)(PAGE_SIZE >> VIRTIO_BALLOON_PFN_SHIFT) #define VIRTIO_BALLOON_ARRAY_PFNS_MAX 256 -/* Maximum number of (4k) pages to deflate on OOM notifications. */ -#define VIRTIO_BALLOON_OOM_NR_PAGES 256 -#define VIRTIO_BALLOON_OOM_NOTIFY_PRIORITY 80 +#define VIRTBALLOON_OOM_NOTIFY_PRIORITY 80 #define VIRTIO_BALLOON_FREE_PAGE_ALLOC_FLAG (__GFP_NORETRY | __GFP_NOWARN | \ __GFP_NOMEMALLOC) @@ -115,11 +112,8 @@ struct virtio_balloon { /* Memory statistics */ struct virtio_balloon_stat stats[VIRTIO_BALLOON_S_NR]; - /* Shrinker to return free pages - VIRTIO_BALLOON_F_FREE_PAGE_HINT */ + /* To register a shrinker to shrink memory upon memory pressure */ struct shrinker shrinker; - - /* OOM notifier to deflate on OOM - VIRTIO_BALLOON_F_DEFLATE_ON_OOM */ - struct notifier_block oom_nb; }; static struct virtio_device_id id_table[] = { @@ -794,13 +788,50 @@ static unsigned long shrink_free_pages(struct virtio_balloon *vb, return blocks_freed * VIRTIO_BALLOON_HINT_BLOCK_PAGES; } +static unsigned long leak_balloon_pages(struct virtio_balloon *vb, + unsigned long pages_to_free) +{ + return leak_balloon(vb, pages_to_free * VIRTIO_BALLOON_PAGES_PER_PAGE) / + VIRTIO_BALLOON_PAGES_PER_PAGE; +} + +static unsigned long shrink_balloon_pages(struct virtio_balloon *vb, + unsigned long pages_to_free) +{ + unsigned long pages_freed = 0; + + /* + * One invocation of leak_balloon can deflate at most + * VIRTIO_BALLOON_ARRAY_PFNS_MAX balloon pages, so we call it + * multiple times to deflate pages till reaching pages_to_free. + */ + while (vb->num_pages && pages_freed < pages_to_free) + pages_freed += leak_balloon_pages(vb, + pages_to_free - pages_freed); + + update_balloon_size(vb); + + return pages_freed; +} + static unsigned long virtio_balloon_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) { + unsigned long pages_to_free, pages_freed = 0; struct virtio_balloon *vb = container_of(shrinker, struct virtio_balloon, shrinker); - return shrink_free_pages(vb, sc->nr_to_scan); + pages_to_free = sc->nr_to_scan; + + if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) + pages_freed = shrink_free_pages(vb, pages_to_free); + + if (pages_freed >= pages_to_free) + return pages_freed; + + pages_freed += shrink_balloon_pages(vb, pages_to_free - pages_freed); + + return pages_freed; } static unsigned long virtio_balloon_shrinker_count(struct shrinker *shrinker, @@ -808,22 +839,26 @@ static unsigned long virtio_balloon_shrinker_count(struct shrinker *shrinker, { struct virtio_balloon *vb = container_of(shrinker, struct virtio_balloon, shrinker); + unsigned long count; - return vb->num_free_page_blocks * VIRTIO_BALLOON_HINT_BLOCK_PAGES; + count = vb->num_pages / VIRTIO_BALLOON_PAGES_PER_PAGE; + count += vb->num_free_page_blocks * VIRTIO_BALLOON_HINT_BLOCK_PAGES; + + return count; } -static int virtio_balloon_oom_notify(struct notifier_block *nb, - unsigned long dummy, void *parm) +static void virtio_balloon_unregister_shrinker(struct virtio_balloon *vb) { - struct virtio_balloon *vb = container_of(nb, - struct virtio_balloon, oom_nb); - unsigned long *freed = parm; + unregister_shrinker(&vb->shrinker); +} - *freed += leak_balloon(vb, VIRTIO_BALLOON_OOM_NR_PAGES) / - VIRTIO_BALLOON_PAGES_PER_PAGE; - update_balloon_size(vb); +static int virtio_balloon_register_shrinker(struct virtio_balloon *vb) +{ + vb->shrinker.scan_objects = virtio_balloon_shrinker_scan; + vb->shrinker.count_objects = virtio_balloon_shrinker_count; + vb->shrinker.seeks = DEFAULT_SEEKS; - return NOTIFY_OK; + return register_shrinker(&vb->shrinker); } static int virtballoon_probe(struct virtio_device *vdev) @@ -900,35 +935,22 @@ static int virtballoon_probe(struct virtio_device *vdev) virtio_cwrite(vb->vdev, struct virtio_balloon_config, poison_val, &poison_val); } - - /* - * We're allowed to reuse any free pages, even if they are - * still to be processed by the host. - */ - vb->shrinker.scan_objects = virtio_balloon_shrinker_scan; - vb->shrinker.count_objects = virtio_balloon_shrinker_count; - vb->shrinker.seeks = DEFAULT_SEEKS; - err = register_shrinker(&vb->shrinker); + } + /* + * We continue to use VIRTIO_BALLOON_F_DEFLATE_ON_OOM to decide if a + * shrinker needs to be registered to relieve memory pressure. + */ + if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) { + err = virtio_balloon_register_shrinker(vb); if (err) goto out_del_balloon_wq; } - if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) { - vb->oom_nb.notifier_call = virtio_balloon_oom_notify; - vb->oom_nb.priority = VIRTIO_BALLOON_OOM_NOTIFY_PRIORITY; - err = register_oom_notifier(&vb->oom_nb); - if (err < 0) - goto out_unregister_shrinker; - } - virtio_device_ready(vdev); if (towards_target(vb)) virtballoon_changed(vdev); return 0; -out_unregister_shrinker: - if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) - unregister_shrinker(&vb->shrinker); out_del_balloon_wq: if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) destroy_workqueue(vb->balloon_wq); @@ -967,11 +989,8 @@ static void virtballoon_remove(struct virtio_device *vdev) { struct virtio_balloon *vb = vdev->priv; - if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) - unregister_oom_notifier(&vb->oom_nb); - if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) - unregister_shrinker(&vb->shrinker); - + if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) + virtio_balloon_unregister_shrinker(vb); spin_lock_irq(&vb->stop_update_lock); vb->stop_update = true; spin_unlock_irq(&vb->stop_update_lock); From a1d032a49522cb5368e5dfb945a85899b4c74f65 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Fri, 3 Apr 2020 17:30:46 +0200 Subject: [PATCH 468/635] KVM: s390: vsie: Fix region 1 ASCE sanity shadow address checks In case we have a region 1 the following calculation (31 + ((gmap->asce & _ASCE_TYPE_MASK) >> 2)*11) results in 64. As shifts beyond the size are undefined the compiler is free to use instructions like sllg. sllg will only use 6 bits of the shift value (here 64) resulting in no shift at all. That means that ALL addresses will be rejected. The can result in endless loops, e.g. when prefix cannot get mapped. Fixes: 4be130a08420 ("s390/mm: add shadow gmap support") Tested-by: Janosch Frank Reported-by: Janosch Frank Cc: # v4.8+ Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20200403153050.20569-2-david@redhat.com Reviewed-by: Claudio Imbrenda Reviewed-by: Christian Borntraeger [borntraeger@de.ibm.com: fix patch description, remove WARN_ON_ONCE] Signed-off-by: Christian Borntraeger --- arch/s390/mm/gmap.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c index 2fbece47ef6f..7e9a6761f254 100644 --- a/arch/s390/mm/gmap.c +++ b/arch/s390/mm/gmap.c @@ -787,14 +787,18 @@ static void gmap_call_notifier(struct gmap *gmap, unsigned long start, static inline unsigned long *gmap_table_walk(struct gmap *gmap, unsigned long gaddr, int level) { + const int asce_type = gmap->asce & _ASCE_TYPE_MASK; unsigned long *table; if ((gmap->asce & _ASCE_TYPE_MASK) + 4 < (level * 4)) return NULL; if (gmap_is_shadow(gmap) && gmap->removed) return NULL; - if (gaddr & (-1UL << (31 + ((gmap->asce & _ASCE_TYPE_MASK) >> 2)*11))) + + if (asce_type != _ASCE_TYPE_REGION1 && + gaddr & (-1UL << (31 + (asce_type >> 2) * 11))) return NULL; + table = gmap->table; switch (gmap->asce & _ASCE_TYPE_MASK) { case _ASCE_TYPE_REGION1: From 4d4cee96fb7a3cc53702a9be8299bf525be4ee98 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Fri, 3 Apr 2020 17:30:47 +0200 Subject: [PATCH 469/635] KVM: s390: vsie: Fix delivery of addressing exceptions Whenever we get an -EFAULT, we failed to read in guest 2 physical address space. Such addressing exceptions are reported via a program intercept to the nested hypervisor. We faked the intercept, we have to return to guest 2. Instead, right now we would be returning -EFAULT from the intercept handler, eventually crashing the VM. the correct thing to do is to return 1 as rc == 1 is the internal representation of "we have to go back into g2". Addressing exceptions can only happen if the g2->g3 page tables reference invalid g2 addresses (say, either a table or the final page is not accessible - so something that basically never happens in sane environments. Identified by manual code inspection. Fixes: a3508fbe9dc6 ("KVM: s390: vsie: initial support for nested virtualization") Cc: # v4.8+ Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20200403153050.20569-3-david@redhat.com Reviewed-by: Claudio Imbrenda Reviewed-by: Christian Borntraeger [borntraeger@de.ibm.com: fix patch description] Signed-off-by: Christian Borntraeger --- arch/s390/kvm/vsie.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c index 076090f9e666..4f6c22d72072 100644 --- a/arch/s390/kvm/vsie.c +++ b/arch/s390/kvm/vsie.c @@ -1202,6 +1202,7 @@ static int vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) scb_s->iprcc = PGM_ADDRESSING; scb_s->pgmilc = 4; scb_s->gpsw.addr = __rewind_psw(scb_s->gpsw, 4); + rc = 1; } return rc; } From 1493e0f944f3c319d11e067c185c904d01c17ae5 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Fri, 3 Apr 2020 17:30:48 +0200 Subject: [PATCH 470/635] KVM: s390: vsie: Fix possible race when shadowing region 3 tables We have to properly retry again by returning -EINVAL immediately in case somebody else instantiated the table concurrently. We missed to add the goto in this function only. The code now matches the other, similar shadowing functions. We are overwriting an existing region 2 table entry. All allocated pages are added to the crst_list to be freed later, so they are not lost forever. However, when unshadowing the region 2 table, we wouldn't trigger unshadowing of the original shadowed region 3 table that we replaced. It would get unshadowed when the original region 3 table is modified. As it's not connected to the page table hierarchy anymore, it's not going to get used anymore. However, for a limited time, this page table will stick around, so it's in some sense a temporary memory leak. Identified by manual code inspection. I don't think this classifies as stable material. Fixes: 998f637cc4b9 ("s390/mm: avoid races on region/segment/page table shadowing") Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20200403153050.20569-4-david@redhat.com Reviewed-by: Claudio Imbrenda Reviewed-by: Christian Borntraeger Signed-off-by: Christian Borntraeger --- arch/s390/mm/gmap.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c index 7e9a6761f254..1a95d8809cc3 100644 --- a/arch/s390/mm/gmap.c +++ b/arch/s390/mm/gmap.c @@ -1844,6 +1844,7 @@ int gmap_shadow_r3t(struct gmap *sg, unsigned long saddr, unsigned long r3t, goto out_free; } else if (*table & _REGION_ENTRY_ORIGIN) { rc = -EAGAIN; /* Race with shadow */ + goto out_free; } crst_table_init(s_r3t, _REGION3_ENTRY_EMPTY); /* mark as invalid as long as the parent table is not protected */ From 4064a4c6a1f90d169f36259647be3a8ddb91fa96 Mon Sep 17 00:00:00 2001 From: Wanpeng Li Date: Thu, 2 Apr 2020 16:20:26 +0800 Subject: [PATCH 471/635] KVM: X86: Filter out the broadcast dest for IPI fastpath Except destination shorthand, a destination value 0xffffffff is used to broadcast interrupts, let's also filter out this for single target IPI fastpath. Reviewed-by: Vitaly Kuznetsov Signed-off-by: Wanpeng Li Message-Id: <1585815626-28370-1-git-send-email-wanpengli@tencent.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/lapic.c | 3 --- arch/x86/kvm/lapic.h | 3 +++ arch/x86/kvm/x86.c | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index ca80daf8f878..9af25c97612a 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -59,9 +59,6 @@ #define MAX_APIC_VECTOR 256 #define APIC_VECTORS_PER_REG 32 -#define APIC_BROADCAST 0xFF -#define X2APIC_BROADCAST 0xFFFFFFFFul - static bool lapic_timer_advance_dynamic __read_mostly; #define LAPIC_TIMER_ADVANCE_ADJUST_MIN 100 /* clock cycles */ #define LAPIC_TIMER_ADVANCE_ADJUST_MAX 10000 /* clock cycles */ diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index 40ed6ed22751..a0ffb4331418 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -17,6 +17,9 @@ #define APIC_BUS_CYCLE_NS 1 #define APIC_BUS_FREQUENCY (1000000000ULL / APIC_BUS_CYCLE_NS) +#define APIC_BROADCAST 0xFF +#define X2APIC_BROADCAST 0xFFFFFFFFul + enum lapic_mode { LAPIC_MODE_DISABLED = 0, LAPIC_MODE_INVALID = X2APIC_ENABLE, diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index b8124b562dea..027dfd278a97 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1586,7 +1586,8 @@ static int handle_fastpath_set_x2apic_icr_irqoff(struct kvm_vcpu *vcpu, u64 data if (((data & APIC_SHORT_MASK) == APIC_DEST_NOSHORT) && ((data & APIC_DEST_MASK) == APIC_DEST_PHYSICAL) && - ((data & APIC_MODE_MASK) == APIC_DM_FIXED)) { + ((data & APIC_MODE_MASK) == APIC_DM_FIXED) && + ((u32)(data >> 32) != X2APIC_BROADCAST)) { data &= ~(1 << 12); kvm_apic_send_ipi(vcpu->arch.apic, (u32)data, (u32)(data >> 32)); From dbef2808af6c594922fe32833b30f55f35e9da6d Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Wed, 1 Apr 2020 10:13:48 +0200 Subject: [PATCH 472/635] KVM: VMX: fix crash cleanup when KVM wasn't used If KVM wasn't used at all before we crash the cleanup procedure fails with BUG: unable to handle page fault for address: ffffffffffffffc8 #PF: supervisor read access in kernel mode #PF: error_code(0x0000) - not-present page PGD 23215067 P4D 23215067 PUD 23217067 PMD 0 Oops: 0000 [#8] SMP PTI CPU: 0 PID: 3542 Comm: bash Kdump: loaded Tainted: G D 5.6.0-rc2+ #823 RIP: 0010:crash_vmclear_local_loaded_vmcss.cold+0x19/0x51 [kvm_intel] The root cause is that loaded_vmcss_on_cpu list is not yet initialized, we initialize it in hardware_enable() but this only happens when we start a VM. Previously, we used to have a bitmap with enabled CPUs and that was preventing [masking] the issue. Initialized loaded_vmcss_on_cpu list earlier, right before we assign crash_vmclear_loaded_vmcss pointer. blocked_vcpu_on_cpu list and blocked_vcpu_on_cpu_lock are moved altogether for consistency. Fixes: 31603d4fc2bb ("KVM: VMX: Always VMCLEAR in-use VMCSes during crash with kexec support") Signed-off-by: Vitaly Kuznetsov Message-Id: <20200401081348.1345307-1-vkuznets@redhat.com> Reviewed-by: Sean Christopherson Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/vmx.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 91749f1254e8..8959514eaf0f 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -2261,10 +2261,6 @@ static int hardware_enable(void) !hv_get_vp_assist_page(cpu)) return -EFAULT; - INIT_LIST_HEAD(&per_cpu(loaded_vmcss_on_cpu, cpu)); - INIT_LIST_HEAD(&per_cpu(blocked_vcpu_on_cpu, cpu)); - spin_lock_init(&per_cpu(blocked_vcpu_on_cpu_lock, cpu)); - r = kvm_cpu_vmxon(phys_addr); if (r) return r; @@ -8044,7 +8040,7 @@ module_exit(vmx_exit); static int __init vmx_init(void) { - int r; + int r, cpu; #if IS_ENABLED(CONFIG_HYPERV) /* @@ -8098,6 +8094,12 @@ static int __init vmx_init(void) return r; } + for_each_possible_cpu(cpu) { + INIT_LIST_HEAD(&per_cpu(loaded_vmcss_on_cpu, cpu)); + INIT_LIST_HEAD(&per_cpu(blocked_vcpu_on_cpu, cpu)); + spin_lock_init(&per_cpu(blocked_vcpu_on_cpu_lock, cpu)); + } + #ifdef CONFIG_KEXEC_CORE rcu_assign_pointer(crash_vmclear_loaded_vmcss, crash_vmclear_local_loaded_vmcss); From 4bf173072cd69cb2e059d710429dc6c46c896548 Mon Sep 17 00:00:00 2001 From: Chris Down Date: Mon, 6 Apr 2020 20:03:30 -0700 Subject: [PATCH 473/635] mm, memcg: bypass high reclaim iteration for cgroup hierarchy root The root of the hierarchy cannot have high set, so we will never reclaim based on it. This makes that clearer and avoids another entry. Signed-off-by: Chris Down Signed-off-by: Andrew Morton Acked-by: Johannes Weiner Cc: Tejun Heo Cc: Roman Gushchin Cc: Michal Hocko Link: http://lkml.kernel.org/r/20200312164137.GA1753625@chrisdown.name Signed-off-by: Linus Torvalds --- mm/memcontrol.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index ca194864d802..9965a77d53e5 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2254,7 +2254,8 @@ static void reclaim_high(struct mem_cgroup *memcg, continue; memcg_memory_event(memcg, MEMCG_HIGH); try_to_free_mem_cgroup_pages(memcg, nr_pages, gfp_mask, true); - } while ((memcg = parent_mem_cgroup(memcg))); + } while ((memcg = parent_mem_cgroup(memcg)) && + !mem_cgroup_is_root(memcg)); } static void high_work_func(struct work_struct *work) From 93949bb21b52aae924fd26f9775c8655dd9e885e Mon Sep 17 00:00:00 2001 From: Li Xinhai Date: Mon, 6 Apr 2020 20:03:33 -0700 Subject: [PATCH 474/635] mm: don't prepare anon_vma if vma has VM_WIPEONFORK Patch series "mm: Fix misuse of parent anon_vma in dup_mmap path". This patchset fixes the misuse of parenet anon_vma, which mainly caused by child vma's vm_next and vm_prev are left same as its parent after duplicate vma. Finally, code reached parent vma's neighbor by referring pointer of child vma and executed wrong logic. The first two patches fix relevant issues, and the third patch sets vm_next and vm_prev to NULL when duplicate vma to prevent potential misuse in future. Effects of the first bug is that causes rmap code to check both parent and child's page table, although a page couldn't be mapped by both parent and child, because child vma has WIPEONFORK so all pages mapped by child are 'new' and not relevant to parent. Effects of the second bug is that the relationship of anon_vma of parent and child are totallyconvoluted. It would cause 'son', 'grandson', ..., etc, to share 'parent' anon_vma, which disobey the design rule of reusing anon_vma (the rule to be followed is that reusing should among vma of same process, and vma should not gone through fork). So, both issues should cause unnecessary rmap walking and have unexpected complexity. These two issues would not be directly visible, I used debugging code to check the anon_vma pointers of parent and child when inspecting the suspicious implementation of issue #2, then find the problem. This patch (of 3): In dup_mmap(), anon_vma_prepare() is called for vma has VM_WIPEONFORK, and parameter 'tmp' (i.e., the new vma of child) has same ->vm_next and ->vm_prev as its parent vma. That allows anon_vma used by parent been mistakenly shared by child (find_mergeable_anon_vma() will do this reuse work). Besides this issue, call anon_vma_prepare() should be avoided because we don't copy page for this vma. Preparing anon_vma will be handled during fault. Fixes: d2cd9ede6e19 ("mm,fork: introduce MADV_WIPEONFORK") Signed-off-by: Li Xinhai Signed-off-by: Andrew Morton Acked-by: Kirill A. Shutemov Cc: Rik van Riel Cc: Kirill A. Shutemov Cc: Johannes Weiner Link: http://lkml.kernel.org/r/1581150928-3214-2-git-send-email-lixinhai.lxh@gmail.com Signed-off-by: Linus Torvalds --- kernel/fork.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/kernel/fork.c b/kernel/fork.c index d2a967bf85d5..d3a5915d3cfe 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -553,10 +553,12 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, if (retval) goto fail_nomem_anon_vma_fork; if (tmp->vm_flags & VM_WIPEONFORK) { - /* VM_WIPEONFORK gets a clean slate in the child. */ + /* + * VM_WIPEONFORK gets a clean slate in the child. + * Don't prepare anon_vma until fault since we don't + * copy page for current vma. + */ tmp->anon_vma = NULL; - if (anon_vma_prepare(tmp)) - goto fail_nomem_anon_vma_fork; } else if (anon_vma_fork(tmp, mpnt)) goto fail_nomem_anon_vma_fork; tmp->vm_flags &= ~(VM_LOCKED | VM_LOCKONFAULT); From 23ab76bf90a66d45e557f238f8806d147fc3be5d Mon Sep 17 00:00:00 2001 From: Li Xinhai Date: Mon, 6 Apr 2020 20:03:36 -0700 Subject: [PATCH 475/635] Revert "mm/rmap.c: reuse mergeable anon_vma as parent when fork" This reverts commit 4e4a9eb921332b9d1 ("mm/rmap.c: reuse mergeable anon_vma as parent when fork"). In dup_mmap(), anon_vma_fork() is called for attaching anon_vma and parameter 'tmp' (i.e., the new vma of child) has same ->vm_next and ->vm_prev as its parent vma. That causes the anon_vma used by parent been mistakenly shared by child (In anon_vma_clone(), the code added by that commit will do this reuse work). Besides this issue, the design of reusing anon_vma from vma which has gone through fork should be avoided ([1]). So, this patch reverts that commit and maintains the consistent logic of reusing anon_vma for fork/split/merge vma. Reusing anon_vma within the process is fine. But if a vma has gone through fork(), then that vma's anon_vma should not be shared with its neighbor vma. As explained in [1], when vma gone through fork(), the check for list_is_singular(vma->anon_vma_chain) will be false, and don't share anon_vma. With current issue, one example can clarify more. Parent process do below two steps: 1. p_vma_1 is created and p_anon_vma_1 is prepared; 2. p_vma_2 is created and share p_anon_vma_1; (this is allowed, becaues p_vma_1 didn't gothrough fork()); parent process do fork(): 3. c_vma_1 is dup from p_vma_1, and has its own c_anon_vma_1 prepared; at this point, c_vma_1->anon_vma_chain has two items, one for p_anon_vma_1 and one for c_anon_vma_1; 4. c_vma_2 is dup from p_vma_2, it is not allowed to share c_anon_vma_1, because c_vma_1->anon_vma_chain has two items. [1] commit d0e9fe1758f2 ("Simplify and comment on anon_vma re-use for anon_vma_prepare()") explains the test of "list_is_singular()". Fixes: 4e4a9eb92133 ("mm/rmap.c: reuse mergeable anon_vma as parent when fork") Signed-off-by: Li Xinhai Signed-off-by: Andrew Morton Cc: Kirill A. Shutemov Cc: Matthew Wilcox Cc: Johannes Weiner Cc: Rik van Riel Link: http://lkml.kernel.org/r/1581150928-3214-3-git-send-email-lixinhai.lxh@gmail.com Signed-off-by: Linus Torvalds --- mm/rmap.c | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/mm/rmap.c b/mm/rmap.c index 2df75a119c83..68fe0472c803 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -275,19 +275,6 @@ int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src) { struct anon_vma_chain *avc, *pavc; struct anon_vma *root = NULL; - struct vm_area_struct *prev = dst->vm_prev, *pprev = src->vm_prev; - - /* - * If parent share anon_vma with its vm_prev, keep this sharing in in - * child. - * - * 1. Parent has vm_prev, which implies we have vm_prev. - * 2. Parent and its vm_prev have the same anon_vma. - */ - if (!dst->anon_vma && src->anon_vma && - pprev && pprev->anon_vma == src->anon_vma) - dst->anon_vma = prev->anon_vma; - list_for_each_entry_reverse(pavc, &src->anon_vma_chain, same_vma) { struct anon_vma *anon_vma; From e39a4b332df69f6850efda5ce6d932ef14a39042 Mon Sep 17 00:00:00 2001 From: Li Xinhai Date: Mon, 6 Apr 2020 20:03:39 -0700 Subject: [PATCH 476/635] mm: set vm_next and vm_prev to NULL in vm_area_dup() Set ->vm_next and ->vm_prev to NULL to prevent potential misuse from the new duplicated vma. Currently, only in fork path there are misuse for handling anon_vma. No other bugs been revealed with this patch applied. Signed-off-by: Li Xinhai Signed-off-by: Andrew Morton Acked-by: Kirill A. Shutemov Cc: Matthew Wilcox Cc: Johannes Weiner Cc: Rik van Riel Link: http://lkml.kernel.org/r/1581150928-3214-4-git-send-email-lixinhai.lxh@gmail.com Signed-off-by: Linus Torvalds --- kernel/fork.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/fork.c b/kernel/fork.c index d3a5915d3cfe..4385f3d639f2 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -361,6 +361,7 @@ struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig) if (new) { *new = *orig; INIT_LIST_HEAD(&new->anon_vma_chain); + new->vm_next = new->vm_prev = NULL; } return new; } @@ -562,7 +563,6 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, } else if (anon_vma_fork(tmp, mpnt)) goto fail_nomem_anon_vma_fork; tmp->vm_flags &= ~(VM_LOCKED | VM_LOCKONFAULT); - tmp->vm_next = tmp->vm_prev = NULL; file = tmp->vm_file; if (file) { struct inode *inode = file_inode(file); From 7e96fb5710a806fd41d2eb6b41bac14a26a094e2 Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Mon, 6 Apr 2020 20:03:43 -0700 Subject: [PATCH 477/635] mm/vma: add missing VMA flag readable name for VM_SYNC Patch series "mm/vma: Use all available wrappers when possible", v2. Apart from adding a VMA flag readable name for trace purpose, this series does some open encoding replacements with availabe VMA specific wrappers. This skips VM_HUGETLB check in vma_migratable() as its already being done with another patch (https://patchwork.kernel.org/patch/11347831/) which is yet to be merged. This patch (of 4): This just adds the missing readable name for VM_SYNC. Signed-off-by: Anshuman Khandual Signed-off-by: Andrew Morton Cc: Vlastimil Babka Cc: Steven Rostedt Cc: Ingo Molnar Cc: Alexander Viro Cc: Andy Lutomirski Cc: "Aneesh Kumar K.V" Cc: Arnaldo Carvalho de Melo Cc: Arnd Bergmann Cc: Benjamin Herrenschmidt Cc: Dave Hansen Cc: Geert Uytterhoeven Cc: Guo Ren Cc: Mel Gorman Cc: Michael Ellerman Cc: Nick Piggin Cc: Paul Burton Cc: Paul Mackerras Cc: Paul Mackerras Cc: Peter Zijlstra Cc: Ralf Baechle Cc: Rich Felker Cc: Thomas Gleixner Cc: Will Deacon Cc: Yoshinori Sato Link: http://lkml.kernel.org/r/1582520593-30704-2-git-send-email-anshuman.khandual@arm.com Signed-off-by: Linus Torvalds --- include/trace/events/mmflags.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/trace/events/mmflags.h b/include/trace/events/mmflags.h index a1675d43777e..5fb752034386 100644 --- a/include/trace/events/mmflags.h +++ b/include/trace/events/mmflags.h @@ -154,6 +154,7 @@ IF_HAVE_PG_IDLE(PG_idle, "idle" ) {VM_ACCOUNT, "account" }, \ {VM_NORESERVE, "noreserve" }, \ {VM_HUGETLB, "hugetlb" }, \ + {VM_SYNC, "sync" }, \ __VM_ARCH_SPECIFIC_1 , \ {VM_WIPEONFORK, "wipeonfork" }, \ {VM_DONTDUMP, "dontdump" }, \ From 3122e80efc0faf4a2accba7a46c7ed795edbfded Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Mon, 6 Apr 2020 20:03:47 -0700 Subject: [PATCH 478/635] mm/vma: make vma_is_accessible() available for general use Lets move vma_is_accessible() helper to include/linux/mm.h which makes it available for general use. While here, this replaces all remaining open encodings for VMA access check with vma_is_accessible(). Signed-off-by: Anshuman Khandual Signed-off-by: Andrew Morton Acked-by: Geert Uytterhoeven Acked-by: Guo Ren Acked-by: Vlastimil Babka Cc: Guo Ren Cc: Geert Uytterhoeven Cc: Ralf Baechle Cc: Paul Burton Cc: Benjamin Herrenschmidt Cc: Paul Mackerras Cc: Michael Ellerman Cc: Yoshinori Sato Cc: Rich Felker Cc: Dave Hansen Cc: Andy Lutomirski Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Steven Rostedt Cc: Mel Gorman Cc: Alexander Viro Cc: "Aneesh Kumar K.V" Cc: Arnaldo Carvalho de Melo Cc: Arnd Bergmann Cc: Nick Piggin Cc: Paul Mackerras Cc: Will Deacon Link: http://lkml.kernel.org/r/1582520593-30704-3-git-send-email-anshuman.khandual@arm.com Signed-off-by: Linus Torvalds --- arch/csky/mm/fault.c | 2 +- arch/m68k/mm/fault.c | 2 +- arch/mips/mm/fault.c | 2 +- arch/powerpc/mm/fault.c | 2 +- arch/sh/mm/fault.c | 2 +- arch/x86/mm/fault.c | 2 +- include/linux/mm.h | 6 ++++++ kernel/sched/fair.c | 2 +- mm/gup.c | 2 +- mm/memory.c | 5 ----- mm/mempolicy.c | 3 +-- mm/mmap.c | 5 ++--- 12 files changed, 17 insertions(+), 18 deletions(-) diff --git a/arch/csky/mm/fault.c b/arch/csky/mm/fault.c index d3c61b83e195..a6e8230b6fbf 100644 --- a/arch/csky/mm/fault.c +++ b/arch/csky/mm/fault.c @@ -141,7 +141,7 @@ good_area: if (!(vma->vm_flags & VM_WRITE)) goto bad_area; } else { - if (!(vma->vm_flags & (VM_READ | VM_WRITE | VM_EXEC))) + if (!vma_is_accessible(vma)) goto bad_area; } diff --git a/arch/m68k/mm/fault.c b/arch/m68k/mm/fault.c index f7afb9897966..0c4a21a685d5 100644 --- a/arch/m68k/mm/fault.c +++ b/arch/m68k/mm/fault.c @@ -125,7 +125,7 @@ good_area: case 1: /* read, present */ goto acc_err; case 0: /* read, not present */ - if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))) + if (!vma_is_accessible(vma)) goto acc_err; } diff --git a/arch/mips/mm/fault.c b/arch/mips/mm/fault.c index 4a0eafe3d932..fb048ba2b91d 100644 --- a/arch/mips/mm/fault.c +++ b/arch/mips/mm/fault.c @@ -142,7 +142,7 @@ good_area: goto bad_area; } } else { - if (!(vma->vm_flags & (VM_READ | VM_WRITE | VM_EXEC))) + if (!vma_is_accessible(vma)) goto bad_area; } } diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c index d15f0f0ee806..84af6c8eecf7 100644 --- a/arch/powerpc/mm/fault.c +++ b/arch/powerpc/mm/fault.c @@ -314,7 +314,7 @@ static bool access_error(bool is_write, bool is_exec, return false; } - if (unlikely(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)))) + if (unlikely(!vma_is_accessible(vma))) return true; /* * We should ideally do the vma pkey access check here. But in the diff --git a/arch/sh/mm/fault.c b/arch/sh/mm/fault.c index 13ee4d20e622..5f23d7907597 100644 --- a/arch/sh/mm/fault.c +++ b/arch/sh/mm/fault.c @@ -355,7 +355,7 @@ static inline int access_error(int error_code, struct vm_area_struct *vma) return 1; /* read, not present: */ - if (unlikely(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)))) + if (unlikely(!vma_is_accessible(vma))) return 1; return 0; diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 859519f5b342..a51df516b87b 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -1222,7 +1222,7 @@ access_error(unsigned long error_code, struct vm_area_struct *vma) return 1; /* read, not present: */ - if (unlikely(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)))) + if (unlikely(!vma_is_accessible(vma))) return 1; return 0; diff --git a/include/linux/mm.h b/include/linux/mm.h index 7dd5c4ccbf85..be49e371e4b5 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -629,6 +629,12 @@ static inline bool vma_is_foreign(struct vm_area_struct *vma) return false; } + +static inline bool vma_is_accessible(struct vm_area_struct *vma) +{ + return vma->vm_flags & (VM_READ | VM_WRITE | VM_EXEC); +} + #ifdef CONFIG_SHMEM /* * The vma_is_shmem is not inline because it is used only by slow diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index d7fb20adabeb..1ea3dddafe69 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -2799,7 +2799,7 @@ static void task_numa_work(struct callback_head *work) * Skip inaccessible VMAs to avoid any confusion between * PROT_NONE and NUMA hinting ptes */ - if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))) + if (!vma_is_accessible(vma)) continue; do { diff --git a/mm/gup.c b/mm/gup.c index da3e03185144..4d505c994623 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -1416,7 +1416,7 @@ long populate_vma_page_range(struct vm_area_struct *vma, * We want mlock to succeed for regions that have any permissions * other than PROT_NONE. */ - if (vma->vm_flags & (VM_READ | VM_WRITE | VM_EXEC)) + if (vma_is_accessible(vma)) gup_flags |= FOLL_FORCE; /* diff --git a/mm/memory.c b/mm/memory.c index 586271f3efc6..d2a353c345ad 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3964,11 +3964,6 @@ static inline vm_fault_t wp_huge_pmd(struct vm_fault *vmf, pmd_t orig_pmd) return VM_FAULT_FALLBACK; } -static inline bool vma_is_accessible(struct vm_area_struct *vma) -{ - return vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE); -} - static vm_fault_t create_huge_pud(struct vm_fault *vmf) { #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 5fb427aed612..b36926ba02e2 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -678,8 +678,7 @@ static int queue_pages_test_walk(unsigned long start, unsigned long end, if (flags & MPOL_MF_LAZY) { /* Similar to task_numa_work, skip inaccessible VMAs */ - if (!is_vm_hugetlb_page(vma) && - (vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)) && + if (!is_vm_hugetlb_page(vma) && vma_is_accessible(vma) && !(vma->vm_flags & VM_MIXEDMAP)) change_prot_numa(vma, start, endvma); return 1; diff --git a/mm/mmap.c b/mm/mmap.c index 94ae18398c59..aa09429d5888 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -2358,8 +2358,7 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) gap_addr = TASK_SIZE; next = vma->vm_next; - if (next && next->vm_start < gap_addr && - (next->vm_flags & (VM_WRITE|VM_READ|VM_EXEC))) { + if (next && next->vm_start < gap_addr && vma_is_accessible(next)) { if (!(next->vm_flags & VM_GROWSUP)) return -ENOMEM; /* Check that both stack segments have the same anon_vma? */ @@ -2440,7 +2439,7 @@ int expand_downwards(struct vm_area_struct *vma, prev = vma->vm_prev; /* Check that both stack segments have the same anon_vma? */ if (prev && !(prev->vm_flags & VM_GROWSDOWN) && - (prev->vm_flags & (VM_WRITE|VM_READ|VM_EXEC))) { + vma_is_accessible(prev)) { if (address - prev->vm_end < stack_guard_gap) return -ENOMEM; } From 03911132aafd6727e59408e497c049402a5a11fa Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Mon, 6 Apr 2020 20:03:51 -0700 Subject: [PATCH 479/635] mm/vma: replace all remaining open encodings with is_vm_hugetlb_page() This replaces all remaining open encodings with is_vm_hugetlb_page(). Signed-off-by: Anshuman Khandual Signed-off-by: Andrew Morton Acked-by: Vlastimil Babka Cc: Paul Mackerras Cc: Benjamin Herrenschmidt Cc: Michael Ellerman Cc: Alexander Viro Cc: Will Deacon Cc: "Aneesh Kumar K.V" Cc: Nick Piggin Cc: Peter Zijlstra Cc: Arnd Bergmann Cc: Ingo Molnar Cc: Arnaldo Carvalho de Melo Cc: Andy Lutomirski Cc: Dave Hansen Cc: Geert Uytterhoeven Cc: Guo Ren Cc: Mel Gorman Cc: Paul Burton Cc: Paul Mackerras Cc: Ralf Baechle Cc: Rich Felker Cc: Steven Rostedt Cc: Thomas Gleixner Cc: Yoshinori Sato Link: http://lkml.kernel.org/r/1582520593-30704-4-git-send-email-anshuman.khandual@arm.com Signed-off-by: Linus Torvalds --- arch/powerpc/kvm/e500_mmu_host.c | 2 +- fs/binfmt_elf.c | 3 ++- include/asm-generic/tlb.h | 3 ++- kernel/events/core.c | 3 ++- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/arch/powerpc/kvm/e500_mmu_host.c b/arch/powerpc/kvm/e500_mmu_host.c index 425d13806645..df9989cf7ba3 100644 --- a/arch/powerpc/kvm/e500_mmu_host.c +++ b/arch/powerpc/kvm/e500_mmu_host.c @@ -422,7 +422,7 @@ static inline int kvmppc_e500_shadow_map(struct kvmppc_vcpu_e500 *vcpu_e500, break; } } else if (vma && hva >= vma->vm_start && - (vma->vm_flags & VM_HUGETLB)) { + is_vm_hugetlb_page(vma)) { unsigned long psize = vma_kernel_pagesize(vma); tsize = (gtlbe->mas1 & MAS1_TSIZE_MASK) >> diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index f4713ea76e82..1eb63867e266 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -1317,7 +1318,7 @@ static unsigned long vma_dump_size(struct vm_area_struct *vma, } /* Hugetlb memory check */ - if (vma->vm_flags & VM_HUGETLB) { + if (is_vm_hugetlb_page(vma)) { if ((vma->vm_flags & VM_SHARED) && FILTER(HUGETLB_SHARED)) goto whole; if (!(vma->vm_flags & VM_SHARED) && FILTER(HUGETLB_PRIVATE)) diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index f391f6b500b4..3f1649a8cf55 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -398,7 +399,7 @@ tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma) * We rely on tlb_end_vma() to issue a flush, such that when we reset * these values the batch is empty. */ - tlb->vma_huge = !!(vma->vm_flags & VM_HUGETLB); + tlb->vma_huge = is_vm_hugetlb_page(vma); tlb->vma_exec = !!(vma->vm_flags & VM_EXEC); } diff --git a/kernel/events/core.c b/kernel/events/core.c index 81e6d80cb219..55e44417f66d 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -7973,7 +7974,7 @@ static void perf_event_mmap_event(struct perf_mmap_event *mmap_event) flags |= MAP_EXECUTABLE; if (vma->vm_flags & VM_LOCKED) flags |= MAP_LOCKED; - if (vma->vm_flags & VM_HUGETLB) + if (is_vm_hugetlb_page(vma)) flags |= MAP_HUGETLB; if (file) { From a0137f16dfe870d63bfb9acdc34b67745270ab4b Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Mon, 6 Apr 2020 20:03:55 -0700 Subject: [PATCH 480/635] mm/vma: replace all remaining open encodings with vma_is_anonymous() This replaces all remaining open encodings with vma_is_anonymous(). Signed-off-by: Anshuman Khandual Signed-off-by: Andrew Morton Acked-by: Vlastimil Babka Cc: Andy Lutomirski Cc: "Aneesh Kumar K.V" Cc: Arnaldo Carvalho de Melo Cc: Arnd Bergmann Cc: Benjamin Herrenschmidt Cc: Dave Hansen Cc: Geert Uytterhoeven Cc: Guo Ren Cc: Ingo Molnar Cc: Mel Gorman Cc: Michael Ellerman Cc: Nick Piggin Cc: Paul Burton Cc: Paul Mackerras Cc: Paul Mackerras Cc: Peter Zijlstra Cc: Ralf Baechle Cc: Rich Felker Cc: Steven Rostedt Cc: Thomas Gleixner Cc: Will Deacon Cc: Yoshinori Sato Link: http://lkml.kernel.org/r/1582520593-30704-5-git-send-email-anshuman.khandual@arm.com Signed-off-by: Linus Torvalds --- mm/gup.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/gup.c b/mm/gup.c index 4d505c994623..96af7e08db4b 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -351,7 +351,8 @@ static struct page *no_page_table(struct vm_area_struct *vma, * But we can only make this optimization where a hole would surely * be zero-filled if handle_mm_fault() actually did handle it. */ - if ((flags & FOLL_DUMP) && (!vma->vm_ops || !vma->vm_ops->fault)) + if ((flags & FOLL_DUMP) && + (vma_is_anonymous(vma) || !vma->vm_ops->fault)) return ERR_PTR(-EFAULT); return NULL; } From 5093c5872be3a041917828b2b53e9f981d754768 Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Mon, 6 Apr 2020 20:03:59 -0700 Subject: [PATCH 481/635] mm/vma: append unlikely() while testing VMA access permissions It is unlikely that an inaccessible VMA without required permission flags will get a page fault. Hence lets just append unlikely() directive to such checks in order to improve performance while also standardizing it across various platforms. Signed-off-by: Anshuman Khandual Signed-off-by: Andrew Morton Reviewed-by: Andrew Morton Cc: Guo Ren Cc: Geert Uytterhoeven Cc: Ralf Baechle Cc: Paul Burton Cc: Mike Rapoport Link: http://lkml.kernel.org/r/1582525304-32113-1-git-send-email-anshuman.khandual@arm.com Signed-off-by: Linus Torvalds --- arch/csky/mm/fault.c | 2 +- arch/m68k/mm/fault.c | 2 +- arch/mips/mm/fault.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/csky/mm/fault.c b/arch/csky/mm/fault.c index a6e8230b6fbf..4e6dc68f3258 100644 --- a/arch/csky/mm/fault.c +++ b/arch/csky/mm/fault.c @@ -141,7 +141,7 @@ good_area: if (!(vma->vm_flags & VM_WRITE)) goto bad_area; } else { - if (!vma_is_accessible(vma)) + if (unlikely(!vma_is_accessible(vma))) goto bad_area; } diff --git a/arch/m68k/mm/fault.c b/arch/m68k/mm/fault.c index 0c4a21a685d5..3bfb5c8ac3c7 100644 --- a/arch/m68k/mm/fault.c +++ b/arch/m68k/mm/fault.c @@ -125,7 +125,7 @@ good_area: case 1: /* read, present */ goto acc_err; case 0: /* read, not present */ - if (!vma_is_accessible(vma)) + if (unlikely(!vma_is_accessible(vma))) goto acc_err; } diff --git a/arch/mips/mm/fault.c b/arch/mips/mm/fault.c index fb048ba2b91d..f8d62cd83b36 100644 --- a/arch/mips/mm/fault.c +++ b/arch/mips/mm/fault.c @@ -142,7 +142,7 @@ good_area: goto bad_area; } } else { - if (!vma_is_accessible(vma)) + if (unlikely(!vma_is_accessible(vma))) goto bad_area; } } From d8cc323d951e7903b5efffad292116148815dc4e Mon Sep 17 00:00:00 2001 From: Qiujun Huang Date: Mon, 6 Apr 2020 20:04:02 -0700 Subject: [PATCH 482/635] mm/vmalloc: fix a typo in comment There is a typo in comment, fix it. "exeeds" -> "exceeds" Signed-off-by: Qiujun Huang Signed-off-by: Andrew Morton Reviewed-by: Andrew Morton Link: http://lkml.kernel.org/r/20200404060136.10838-1-hqjagain@gmail.com Signed-off-by: Linus Torvalds --- mm/vmalloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 6b8eeb0ecee5..399f219544f7 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -3368,7 +3368,7 @@ retry: goto overflow; /* - * If required width exeeds current VA block, move + * If required width exceeds current VA block, move * base downwards and then recheck. */ if (base + end > va->va_end) { From 29fd1897070125ab49634524a20f146fb4240a51 Mon Sep 17 00:00:00 2001 From: Michal Hocko Date: Mon, 6 Apr 2020 20:04:06 -0700 Subject: [PATCH 483/635] mm: make it clear that gfp reclaim modifiers are valid only for sleepable allocations While it might be really clear to MM developers that gfp reclaim modifiers are applicable only to sleepable allocations (those with __GFP_DIRECT_RECLAIM) it seems that actual users of the API are not always sure. Make it explicit that they are not applicable for GFP_NOWAIT or GFP_ATOMIC allocations which are the most commonly used non-sleepable allocation masks. Signed-off-by: Michal Hocko Signed-off-by: Andrew Morton Reviewed-by: Joel Fernandes (Google) Acked-by: Paul E. McKenney Acked-by: David Rientjes Cc: Neil Brown Link: http://lkml.kernel.org/r/20200403083543.11552-3-mhocko@kernel.org Signed-off-by: Linus Torvalds --- include/linux/gfp.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/linux/gfp.h b/include/linux/gfp.h index be2754841369..4aba4c86c626 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -124,6 +124,8 @@ struct vm_area_struct; * * Reclaim modifiers * ~~~~~~~~~~~~~~~~~ + * Please note that all the following flags are only applicable to sleepable + * allocations (e.g. %GFP_NOWAIT and %GFP_ATOMIC will ignore them). * * %__GFP_IO can start physical IO. * From 4afdacec2e313509bf9c25f3370be299a0d60b9a Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Mon, 6 Apr 2020 20:04:09 -0700 Subject: [PATCH 484/635] mm/migrate.c: no need to check for i > start in do_pages_move() Patch series "cleanup on do_pages_move()", v5. The logic in do_pages_move() is a little mess for audience to read and has some potential error on handling the return value. Especially there are three calls on do_move_pages_to_node() and store_status() with almost the same form. This patch set tries to make the code a little friendly for audience by consolidate the calls. This patch (of 4): At this point, we always have i >= start. If i == start, store_status() will return 0. So we can drop the check for i > start. [david@redhat.com rephrase changelog] Signed-off-by: Wei Yang Signed-off-by: Andrew Morton Reviewed-by: David Hildenbrand Acked-by: Michal Hocko Link: http://lkml.kernel.org/r/20200214003017.25558-2-richardw.yang@linux.intel.com Signed-off-by: Linus Torvalds --- mm/migrate.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/mm/migrate.c b/mm/migrate.c index 7ded07081be9..20bc562f499e 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -1694,11 +1694,9 @@ static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes, err += nr_pages - i - 1; goto out; } - if (i > start) { - err = store_status(status, start, current_node, i - start); - if (err) - goto out; - } + err = store_status(status, start, current_node, i - start); + if (err) + goto out; current_node = NUMA_NO_NODE; } out_flush: From 7ca8783ad816972f0c64a189b8ba6615c6b78521 Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Mon, 6 Apr 2020 20:04:12 -0700 Subject: [PATCH 485/635] mm/migrate.c: wrap do_move_pages_to_node() and store_status() Usually, do_move_pages_to_node() and store_status() are used in combination. We have three similar call sites. Let's provide a wrapper for both function calls - move_pages_and_store_status - to make the calling code easier to maintain and fix (as noted by Yang Shi, the return value handling of do_move_pages_to_node() has a flaw). [david@redhat.com rephrase changelog] Signed-off-by: Wei Yang Signed-off-by: Andrew Morton Reviewed-by: David Hildenbrand Acked-by: Michal Hocko Link: http://lkml.kernel.org/r/20200214003017.25558-3-richardw.yang@linux.intel.com Signed-off-by: Linus Torvalds --- mm/migrate.c | 61 +++++++++++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 32 deletions(-) diff --git a/mm/migrate.c b/mm/migrate.c index 20bc562f499e..6f6317903007 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -1602,6 +1602,29 @@ out: return err; } +static int move_pages_and_store_status(struct mm_struct *mm, int node, + struct list_head *pagelist, int __user *status, + int start, int i, unsigned long nr_pages) +{ + int err; + + err = do_move_pages_to_node(mm, pagelist, node); + if (err) { + /* + * Positive err means the number of failed + * pages to migrate. Since we are going to + * abort and return the number of non-migrated + * pages, so need to incude the rest of the + * nr_pages that have not been attempted as + * well. + */ + if (err > 0) + err += nr_pages - i - 1; + return err; + } + return store_status(status, start, node, i - start); +} + /* * Migrate an array of page address onto an array of nodes and fill * the corresponding array of status. @@ -1645,21 +1668,8 @@ static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes, current_node = node; start = i; } else if (node != current_node) { - err = do_move_pages_to_node(mm, &pagelist, current_node); - if (err) { - /* - * Positive err means the number of failed - * pages to migrate. Since we are going to - * abort and return the number of non-migrated - * pages, so need to incude the rest of the - * nr_pages that have not been attempted as - * well. - */ - if (err > 0) - err += nr_pages - i - 1; - goto out; - } - err = store_status(status, start, current_node, i - start); + err = move_pages_and_store_status(mm, current_node, + &pagelist, status, start, i, nr_pages); if (err) goto out; start = i; @@ -1688,13 +1698,8 @@ static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes, if (err) goto out_flush; - err = do_move_pages_to_node(mm, &pagelist, current_node); - if (err) { - if (err > 0) - err += nr_pages - i - 1; - goto out; - } - err = store_status(status, start, current_node, i - start); + err = move_pages_and_store_status(mm, current_node, &pagelist, + status, start, i, nr_pages); if (err) goto out; current_node = NUMA_NO_NODE; @@ -1704,16 +1709,8 @@ out_flush: return err; /* Make sure we do not overwrite the existing error */ - err1 = do_move_pages_to_node(mm, &pagelist, current_node); - /* - * Don't have to report non-attempted pages here since: - * - If the above loop is done gracefully all pages have been - * attempted. - * - If the above loop is aborted it means a fatal error - * happened, should return ret. - */ - if (!err1) - err1 = store_status(status, start, current_node, i - start); + err1 = move_pages_and_store_status(mm, current_node, &pagelist, + status, start, i, nr_pages); if (err >= 0) err = err1; out: From 5d7ae891cdc68f30e8b6db35bbd7f0b1aeee189c Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Mon, 6 Apr 2020 20:04:15 -0700 Subject: [PATCH 486/635] mm/migrate.c: check pagelist in move_pages_and_store_status() When pagelist is empty, it is not necessary to do the move and store. Also it consolidate the empty list check in one place. Signed-off-by: Wei Yang Signed-off-by: Andrew Morton Reviewed-by: David Hildenbrand Acked-by: Michal Hocko Link: http://lkml.kernel.org/r/20200214003017.25558-4-richardw.yang@linux.intel.com Signed-off-by: Linus Torvalds --- mm/migrate.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/mm/migrate.c b/mm/migrate.c index 6f6317903007..80eaa99bed39 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -1518,9 +1518,6 @@ static int do_move_pages_to_node(struct mm_struct *mm, { int err; - if (list_empty(pagelist)) - return 0; - err = migrate_pages(pagelist, alloc_new_node_page, NULL, node, MIGRATE_SYNC, MR_SYSCALL); if (err) @@ -1608,6 +1605,9 @@ static int move_pages_and_store_status(struct mm_struct *mm, int node, { int err; + if (list_empty(pagelist)) + return 0; + err = do_move_pages_to_node(mm, pagelist, node); if (err) { /* @@ -1705,9 +1705,6 @@ static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes, current_node = NUMA_NO_NODE; } out_flush: - if (list_empty(&pagelist)) - return err; - /* Make sure we do not overwrite the existing error */ err1 = move_pages_and_store_status(mm, current_node, &pagelist, status, start, i, nr_pages); From d08221a0807b0489f0081476bcd36da88722560b Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Mon, 6 Apr 2020 20:04:18 -0700 Subject: [PATCH 487/635] mm/migrate.c: unify "not queued for migration" handling in do_pages_move() It can currently happen that we store the status of a page twice: * Once we detect that it is already on the target node * Once we moved a bunch of pages, and a page that's already on the target node is contained in the current interval. Let's simplify the code and always call do_move_pages_to_node() in case we did not queue a page for migration. Note that pages that are already on the target node are not added to the pagelist and are, therefore, ignored by do_move_pages_to_node() - there is no functional change. The status of such a page is now only stored once. [david@redhat.com rephrase changelog] Signed-off-by: Wei Yang Signed-off-by: Andrew Morton Reviewed-by: David Hildenbrand Acked-by: Michal Hocko Link: http://lkml.kernel.org/r/20200214003017.25558-5-richardw.yang@linux.intel.com Signed-off-by: Linus Torvalds --- mm/migrate.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/mm/migrate.c b/mm/migrate.c index 80eaa99bed39..c550230664b1 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -1683,18 +1683,16 @@ static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes, err = add_page_for_migration(mm, addr, current_node, &pagelist, flags & MPOL_MF_MOVE_ALL); - if (!err) { - /* The page is already on the target node */ - err = store_status(status, i, current_node, 1); - if (err) - goto out_flush; - continue; - } else if (err > 0) { + if (err > 0) { /* The page is successfully queued for migration */ continue; } - err = store_status(status, i, err, 1); + /* + * If the page is already on the target node (!err), store the + * node, otherwise, store the err. + */ + err = store_status(status, i, err ? : current_node, 1); if (err) goto out_flush; From 6aeff241fe6c4561a842b344c1ca14a700ec8441 Mon Sep 17 00:00:00 2001 From: Yang Shi Date: Mon, 6 Apr 2020 20:04:21 -0700 Subject: [PATCH 488/635] mm/migrate.c: migrate PG_readahead flag Currently the migration code doesn't migrate PG_readahead flag. Theoretically this would incur slight performance loss as the application might have to ramp its readahead back up again. Even though such problem happens, it might be hidden by something else since migration is typically triggered by compaction and NUMA balancing, any of which should be more noticeable. Migrate the flag after end_page_writeback() since it may clear PG_reclaim flag, which is the same bit as PG_readahead, for the new page. [akpm@linux-foundation.org: tweak comment] Signed-off-by: Yang Shi Signed-off-by: Andrew Morton Cc: Matthew Wilcox Cc: Michal Hocko Cc: Mel Gorman Link: http://lkml.kernel.org/r/1581640185-95731-1-git-send-email-yang.shi@linux.alibaba.com Signed-off-by: Linus Torvalds --- mm/migrate.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mm/migrate.c b/mm/migrate.c index c550230664b1..1a205503be3f 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -647,6 +647,14 @@ void migrate_page_states(struct page *newpage, struct page *page) if (PageWriteback(newpage)) end_page_writeback(newpage); + /* + * PG_readahead shares the same bit with PG_reclaim. The above + * end_page_writeback() may clear PG_readahead mistakenly, so set the + * bit after that. + */ + if (PageReadahead(page)) + SetPageReadahead(newpage); + copy_page_owner(page, newpage); mem_cgroup_migrate(page, newpage); From dcdf11ee144133328664d90836e712d840d047d9 Mon Sep 17 00:00:00 2001 From: David Rientjes Date: Mon, 6 Apr 2020 20:04:25 -0700 Subject: [PATCH 489/635] mm, shmem: add vmstat for hugepage fallback The existing thp_fault_fallback indicates when thp attempts to allocate a hugepage but fails, or if the hugepage cannot be charged to the mem cgroup hierarchy. Extend this to shmem as well. Adds a new thp_file_fallback to complement thp_file_alloc that gets incremented when a hugepage is attempted to be allocated but fails, or if it cannot be charged to the mem cgroup hierarchy. Additionally, remove the check for CONFIG_TRANSPARENT_HUGE_PAGECACHE from shmem_alloc_hugepage() since it is only called with this configuration option. Signed-off-by: David Rientjes Signed-off-by: Andrew Morton Reviewed-by: Yang Shi Acked-by: Kirill A. Shutemov Cc: Mike Rapoport Cc: Jeremy Cline Cc: Andrea Arcangeli Cc: Mike Kravetz Cc: Michal Hocko Cc: Vlastimil Babka Link: http://lkml.kernel.org/r/alpine.DEB.2.21.2003061421240.7412@chino.kir.corp.google.com Signed-off-by: Linus Torvalds --- Documentation/admin-guide/mm/transhuge.rst | 4 ++++ include/linux/vm_event_item.h | 2 ++ mm/shmem.c | 10 ++++++---- mm/vmstat.c | 1 + 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/Documentation/admin-guide/mm/transhuge.rst b/Documentation/admin-guide/mm/transhuge.rst index bd5714547cee..f79ebbcd6725 100644 --- a/Documentation/admin-guide/mm/transhuge.rst +++ b/Documentation/admin-guide/mm/transhuge.rst @@ -319,6 +319,10 @@ thp_file_alloc is incremented every time a file huge page is successfully allocated. +thp_file_fallback + is incremented if a file huge page is attempted to be allocated + but fails and instead falls back to using small pages. + thp_file_mapped is incremented every time a file huge page is mapped into user address space. diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h index 47a3441cf4c4..41a4c7568748 100644 --- a/include/linux/vm_event_item.h +++ b/include/linux/vm_event_item.h @@ -76,6 +76,7 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, THP_COLLAPSE_ALLOC, THP_COLLAPSE_ALLOC_FAILED, THP_FILE_ALLOC, + THP_FILE_FALLBACK, THP_FILE_MAPPED, THP_SPLIT_PAGE, THP_SPLIT_PAGE_FAILED, @@ -115,6 +116,7 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, #ifndef CONFIG_TRANSPARENT_HUGEPAGE #define THP_FILE_ALLOC ({ BUILD_BUG(); 0; }) +#define THP_FILE_FALLBACK ({ BUILD_BUG(); 0; }) #define THP_FILE_MAPPED ({ BUILD_BUG(); 0; }) #endif diff --git a/mm/shmem.c b/mm/shmem.c index f47347cb30f6..8160d0762bf5 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1472,9 +1472,6 @@ static struct page *shmem_alloc_hugepage(gfp_t gfp, pgoff_t hindex; struct page *page; - if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) - return NULL; - hindex = round_down(index, HPAGE_PMD_NR); if (xa_find(&mapping->i_pages, &hindex, hindex + HPAGE_PMD_NR - 1, XA_PRESENT)) @@ -1486,6 +1483,8 @@ static struct page *shmem_alloc_hugepage(gfp_t gfp, shmem_pseudo_vma_destroy(&pvma); if (page) prep_transhuge_page(page); + else + count_vm_event(THP_FILE_FALLBACK); return page; } @@ -1871,8 +1870,11 @@ alloc_nohuge: error = mem_cgroup_try_charge_delay(page, charge_mm, gfp, &memcg, PageTransHuge(page)); - if (error) + if (error) { + if (PageTransHuge(page)) + count_vm_event(THP_FILE_FALLBACK); goto unacct; + } error = shmem_add_to_page_cache(page, mapping, hindex, NULL, gfp & GFP_RECLAIM_MASK); if (error) { diff --git a/mm/vmstat.c b/mm/vmstat.c index c9c0d71f917f..ebc1dcaa0539 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1259,6 +1259,7 @@ const char * const vmstat_text[] = { "thp_collapse_alloc", "thp_collapse_alloc_failed", "thp_file_alloc", + "thp_file_fallback", "thp_file_mapped", "thp_split_page", "thp_split_page_failed", From 85b9f46e8ea451633ccd60a7d8cacbfff9f34047 Mon Sep 17 00:00:00 2001 From: David Rientjes Date: Mon, 6 Apr 2020 20:04:28 -0700 Subject: [PATCH 490/635] mm, thp: track fallbacks due to failed memcg charges separately The thp_fault_fallback and thp_file_fallback vmstats are incremented if either the hugepage allocation fails through the page allocator or the hugepage charge fails through mem cgroup. This patch leaves this field untouched but adds two new fields, thp_{fault,file}_fallback_charge, which is incremented only when the mem cgroup charge fails. This distinguishes between attempted hugepage allocations that fail due to fragmentation (or low memory conditions) and those that fail due to mem cgroup limits. That can be used to determine the impact of fragmentation on the system by excluding faults that failed due to memcg usage. Signed-off-by: David Rientjes Signed-off-by: Andrew Morton Reviewed-by: Yang Shi Acked-by: Kirill A. Shutemov Cc: Mike Rapoport Cc: Jeremy Cline Cc: Andrea Arcangeli Cc: Mike Kravetz Cc: Michal Hocko Cc: Vlastimil Babka Link: http://lkml.kernel.org/r/alpine.DEB.2.21.2003061422070.7412@chino.kir.corp.google.com Signed-off-by: Linus Torvalds --- Documentation/admin-guide/mm/transhuge.rst | 10 ++++++++++ include/linux/vm_event_item.h | 3 +++ mm/huge_memory.c | 2 ++ mm/shmem.c | 4 +++- mm/vmstat.c | 2 ++ 5 files changed, 20 insertions(+), 1 deletion(-) diff --git a/Documentation/admin-guide/mm/transhuge.rst b/Documentation/admin-guide/mm/transhuge.rst index f79ebbcd6725..2f31de8f7c74 100644 --- a/Documentation/admin-guide/mm/transhuge.rst +++ b/Documentation/admin-guide/mm/transhuge.rst @@ -310,6 +310,11 @@ thp_fault_fallback is incremented if a page fault fails to allocate a huge page and instead falls back to using small pages. +thp_fault_fallback_charge + is incremented if a page fault fails to charge a huge page and + instead falls back to using small pages even though the + allocation was successful. + thp_collapse_alloc_failed is incremented if khugepaged found a range of pages that should be collapsed into one huge page but failed @@ -323,6 +328,11 @@ thp_file_fallback is incremented if a file huge page is attempted to be allocated but fails and instead falls back to using small pages. +thp_file_fallback_charge + is incremented if a file huge page cannot be charged and instead + falls back to using small pages even though the allocation was + successful. + thp_file_mapped is incremented every time a file huge page is mapped into user address space. diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h index 41a4c7568748..ffef0f279747 100644 --- a/include/linux/vm_event_item.h +++ b/include/linux/vm_event_item.h @@ -73,10 +73,12 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, #ifdef CONFIG_TRANSPARENT_HUGEPAGE THP_FAULT_ALLOC, THP_FAULT_FALLBACK, + THP_FAULT_FALLBACK_CHARGE, THP_COLLAPSE_ALLOC, THP_COLLAPSE_ALLOC_FAILED, THP_FILE_ALLOC, THP_FILE_FALLBACK, + THP_FILE_FALLBACK_CHARGE, THP_FILE_MAPPED, THP_SPLIT_PAGE, THP_SPLIT_PAGE_FAILED, @@ -117,6 +119,7 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, #ifndef CONFIG_TRANSPARENT_HUGEPAGE #define THP_FILE_ALLOC ({ BUILD_BUG(); 0; }) #define THP_FILE_FALLBACK ({ BUILD_BUG(); 0; }) +#define THP_FILE_FALLBACK_CHARGE ({ BUILD_BUG(); 0; }) #define THP_FILE_MAPPED ({ BUILD_BUG(); 0; }) #endif diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 0f9389f9d1f8..0080e8df18ef 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -597,6 +597,7 @@ static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf, if (mem_cgroup_try_charge_delay(page, vma->vm_mm, gfp, &memcg, true)) { put_page(page); count_vm_event(THP_FAULT_FALLBACK); + count_vm_event(THP_FAULT_FALLBACK_CHARGE); return VM_FAULT_FALLBACK; } @@ -1446,6 +1447,7 @@ alloc: put_page(page); ret |= VM_FAULT_FALLBACK; count_vm_event(THP_FAULT_FALLBACK); + count_vm_event(THP_FAULT_FALLBACK_CHARGE); goto out; } diff --git a/mm/shmem.c b/mm/shmem.c index 8160d0762bf5..b48ac3806f8f 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1871,8 +1871,10 @@ alloc_nohuge: error = mem_cgroup_try_charge_delay(page, charge_mm, gfp, &memcg, PageTransHuge(page)); if (error) { - if (PageTransHuge(page)) + if (PageTransHuge(page)) { count_vm_event(THP_FILE_FALLBACK); + count_vm_event(THP_FILE_FALLBACK_CHARGE); + } goto unacct; } error = shmem_add_to_page_cache(page, mapping, hindex, diff --git a/mm/vmstat.c b/mm/vmstat.c index ebc1dcaa0539..96d21a792b57 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1256,10 +1256,12 @@ const char * const vmstat_text[] = { #ifdef CONFIG_TRANSPARENT_HUGEPAGE "thp_fault_alloc", "thp_fault_fallback", + "thp_fault_fallback_charge", "thp_collapse_alloc", "thp_collapse_alloc_failed", "thp_file_alloc", "thp_file_fallback", + "thp_file_fallback_charge", "thp_file_mapped", "thp_split_page", "thp_split_page_failed", From a0650604a707b1926cbc32feb2f006ebb74ef47b Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Mon, 6 Apr 2020 20:04:31 -0700 Subject: [PATCH 491/635] include/linux/pagemap.h: optimise find_subpage for !THP If THP is disabled, find_subpage() can become a no-op by using hpage_nr_pages() instead of compound_nr(). hpage_nr_pages() embeds a check for PageTail, so we can drop the check here. Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton Reviewed-by: Christoph Hellwig Acked-by: Kirill A. Shutemov Cc: Aneesh Kumar K.V Cc: Pankaj Gupta Link: http://lkml.kernel.org/r/20200318140253.6141-5-willy@infradead.org Signed-off-by: Linus Torvalds --- include/linux/pagemap.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index f56282491a48..a8f7bd8ea1c6 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -341,9 +341,7 @@ static inline struct page *find_subpage(struct page *head, pgoff_t index) if (PageHuge(head)) return head; - VM_BUG_ON_PAGE(PageTail(head), head); - - return head + (index & (compound_nr(head) - 1)); + return head + (index & (hpage_nr_pages(head) - 1)); } struct page *find_get_entry(struct address_space *mapping, pgoff_t offset); From 396bcc5299c281e9cf1737ad0efcd97be9f83845 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Mon, 6 Apr 2020 20:04:35 -0700 Subject: [PATCH 492/635] mm: remove CONFIG_TRANSPARENT_HUGE_PAGECACHE Commit e496cf3d7821 ("thp: introduce CONFIG_TRANSPARENT_HUGE_PAGECACHE") notes that it should be reverted when the PowerPC problem was fixed. The commit fixing the PowerPC problem (953c66c2b22a) did not revert the commit; instead setting CONFIG_TRANSPARENT_HUGE_PAGECACHE to the same as CONFIG_TRANSPARENT_HUGEPAGE. Checking with Kirill and Aneesh, this was an oversight, so remove the Kconfig symbol and undo the work of commit e496cf3d7821. Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton Acked-by: Kirill A. Shutemov Cc: Aneesh Kumar K.V Cc: Christoph Hellwig Cc: Pankaj Gupta Link: http://lkml.kernel.org/r/20200318140253.6141-6-willy@infradead.org Signed-off-by: Linus Torvalds --- include/linux/shmem_fs.h | 10 +--------- mm/Kconfig | 6 +----- mm/huge_memory.c | 2 +- mm/khugepaged.c | 12 ++++-------- mm/memory.c | 5 ++--- mm/rmap.c | 2 +- mm/shmem.c | 34 +++++++++++++++++----------------- 7 files changed, 27 insertions(+), 44 deletions(-) diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h index d56fefef8905..7a35a6901221 100644 --- a/include/linux/shmem_fs.h +++ b/include/linux/shmem_fs.h @@ -78,6 +78,7 @@ extern void shmem_truncate_range(struct inode *inode, loff_t start, loff_t end); extern int shmem_unuse(unsigned int type, bool frontswap, unsigned long *fs_pages_to_unuse); +extern bool shmem_huge_enabled(struct vm_area_struct *vma); extern unsigned long shmem_swap_usage(struct vm_area_struct *vma); extern unsigned long shmem_partial_swap_usage(struct address_space *mapping, pgoff_t start, pgoff_t end); @@ -114,15 +115,6 @@ static inline bool shmem_file(struct file *file) extern bool shmem_charge(struct inode *inode, long pages); extern void shmem_uncharge(struct inode *inode, long pages); -#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE -extern bool shmem_huge_enabled(struct vm_area_struct *vma); -#else -static inline bool shmem_huge_enabled(struct vm_area_struct *vma) -{ - return false; -} -#endif - #ifdef CONFIG_SHMEM extern int shmem_mcopy_atomic_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd, struct vm_area_struct *dst_vma, diff --git a/mm/Kconfig b/mm/Kconfig index ab80933be65f..211a70e8d5cf 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -420,10 +420,6 @@ config THP_SWAP For selection by architectures with reasonable THP sizes. -config TRANSPARENT_HUGE_PAGECACHE - def_bool y - depends on TRANSPARENT_HUGEPAGE - # # UP and nommu archs use km based percpu allocator # @@ -714,7 +710,7 @@ config GUP_GET_PTE_LOW_HIGH config READ_ONLY_THP_FOR_FS bool "Read-only THP for filesystems (EXPERIMENTAL)" - depends on TRANSPARENT_HUGE_PAGECACHE && SHMEM + depends on TRANSPARENT_HUGEPAGE && SHMEM help Allow khugepaged to put read-only file-backed pages in THP. diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 0080e8df18ef..c1e7c71db1e6 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -326,7 +326,7 @@ static struct attribute *hugepage_attr[] = { &defrag_attr.attr, &use_zero_page_attr.attr, &hpage_pmd_size_attr.attr, -#if defined(CONFIG_SHMEM) && defined(CONFIG_TRANSPARENT_HUGE_PAGECACHE) +#ifdef CONFIG_SHMEM &shmem_enabled_attr.attr, #endif #ifdef CONFIG_DEBUG_VM diff --git a/mm/khugepaged.c b/mm/khugepaged.c index c659c68728bc..b1d9a8e189b8 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -414,8 +414,6 @@ static bool hugepage_vma_check(struct vm_area_struct *vma, (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && vma->vm_file && (vm_flags & VM_DENYWRITE))) { - if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) - return false; return IS_ALIGNED((vma->vm_start >> PAGE_SHIFT) - vma->vm_pgoff, HPAGE_PMD_NR); } @@ -1258,7 +1256,7 @@ static void collect_mm_slot(struct mm_slot *mm_slot) } } -#if defined(CONFIG_SHMEM) && defined(CONFIG_TRANSPARENT_HUGE_PAGECACHE) +#ifdef CONFIG_SHMEM /* * Notify khugepaged that given addr of the mm is pte-mapped THP. Then * khugepaged should try to collapse the page table. @@ -1973,6 +1971,8 @@ skip: if (khugepaged_scan.address < hstart) khugepaged_scan.address = hstart; VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK); + if (shmem_file(vma->vm_file) && !shmem_huge_enabled(vma)) + goto skip; while (khugepaged_scan.address < hend) { int ret; @@ -1984,14 +1984,10 @@ skip: khugepaged_scan.address + HPAGE_PMD_SIZE > hend); if (IS_ENABLED(CONFIG_SHMEM) && vma->vm_file) { - struct file *file; + struct file *file = get_file(vma->vm_file); pgoff_t pgoff = linear_page_index(vma, khugepaged_scan.address); - if (shmem_file(vma->vm_file) - && !shmem_huge_enabled(vma)) - goto skip; - file = get_file(vma->vm_file); up_read(&mm->mmap_sem); ret = 1; khugepaged_scan_file(mm, file, pgoff, hpage); diff --git a/mm/memory.c b/mm/memory.c index d2a353c345ad..d527e0ec29c7 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3373,7 +3373,7 @@ map_pte: return 0; } -#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE +#ifdef CONFIG_TRANSPARENT_HUGEPAGE static void deposit_prealloc_pte(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; @@ -3475,8 +3475,7 @@ vm_fault_t alloc_set_pte(struct vm_fault *vmf, struct mem_cgroup *memcg, pte_t entry; vm_fault_t ret; - if (pmd_none(*vmf->pmd) && PageTransCompound(page) && - IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) { + if (pmd_none(*vmf->pmd) && PageTransCompound(page)) { /* THP on COW? */ VM_BUG_ON_PAGE(memcg, page); diff --git a/mm/rmap.c b/mm/rmap.c index 68fe0472c803..374a9bfdbffa 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -933,7 +933,7 @@ static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma, set_pte_at(vma->vm_mm, address, pte, entry); ret = 1; } else { -#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE +#ifdef CONFIG_TRANSPARENT_HUGEPAGE pmd_t *pmd = pvmw.pmd; pmd_t entry; diff --git a/mm/shmem.c b/mm/shmem.c index b48ac3806f8f..2c255f383608 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -410,7 +410,7 @@ static bool shmem_confirm_swap(struct address_space *mapping, #define SHMEM_HUGE_DENY (-1) #define SHMEM_HUGE_FORCE (-2) -#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE +#ifdef CONFIG_TRANSPARENT_HUGEPAGE /* ifdef here to avoid bloating shmem.o when not necessary */ static int shmem_huge __read_mostly; @@ -580,7 +580,7 @@ static long shmem_unused_huge_count(struct super_block *sb, struct shmem_sb_info *sbinfo = SHMEM_SB(sb); return READ_ONCE(sbinfo->shrinklist_len); } -#else /* !CONFIG_TRANSPARENT_HUGE_PAGECACHE */ +#else /* !CONFIG_TRANSPARENT_HUGEPAGE */ #define shmem_huge SHMEM_HUGE_DENY @@ -589,11 +589,11 @@ static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, { return 0; } -#endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE */ +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ static inline bool is_huge_enabled(struct shmem_sb_info *sbinfo) { - if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && + if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && (shmem_huge == SHMEM_HUGE_FORCE || sbinfo->huge) && shmem_huge != SHMEM_HUGE_DENY) return true; @@ -1059,7 +1059,7 @@ static int shmem_setattr(struct dentry *dentry, struct iattr *attr) * Part of the huge page can be beyond i_size: subject * to shrink under memory pressure. */ - if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) { + if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { spin_lock(&sbinfo->shrinklist_lock); /* * _careful to defend against unlocked access to @@ -1510,7 +1510,7 @@ static struct page *shmem_alloc_and_acct_page(gfp_t gfp, int nr; int err = -ENOSPC; - if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) huge = false; nr = huge ? HPAGE_PMD_NR : 1; @@ -2093,7 +2093,7 @@ unsigned long shmem_get_unmapped_area(struct file *file, get_area = current->mm->get_unmapped_area; addr = get_area(file, uaddr, len, pgoff, flags); - if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) return addr; if (IS_ERR_VALUE(addr)) return addr; @@ -2232,7 +2232,7 @@ static int shmem_mmap(struct file *file, struct vm_area_struct *vma) file_accessed(file); vma->vm_ops = &shmem_vm_ops; - if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && + if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) < (vma->vm_end & HPAGE_PMD_MASK)) { khugepaged_enter(vma, vma->vm_flags); @@ -3459,7 +3459,7 @@ static int shmem_parse_one(struct fs_context *fc, struct fs_parameter *param) case Opt_huge: ctx->huge = result.uint_32; if (ctx->huge != SHMEM_HUGE_NEVER && - !(IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && + !(IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && has_transparent_hugepage())) goto unsupported_parameter; ctx->seen |= SHMEM_SEEN_HUGE; @@ -3605,7 +3605,7 @@ static int shmem_show_options(struct seq_file *seq, struct dentry *root) if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID)) seq_printf(seq, ",gid=%u", from_kgid_munged(&init_user_ns, sbinfo->gid)); -#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE +#ifdef CONFIG_TRANSPARENT_HUGEPAGE /* Rightly or wrongly, show huge mount option unmasked by shmem_huge */ if (sbinfo->huge) seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge)); @@ -3850,7 +3850,7 @@ static const struct super_operations shmem_ops = { .evict_inode = shmem_evict_inode, .drop_inode = generic_delete_inode, .put_super = shmem_put_super, -#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE +#ifdef CONFIG_TRANSPARENT_HUGEPAGE .nr_cached_objects = shmem_unused_huge_count, .free_cached_objects = shmem_unused_huge_scan, #endif @@ -3912,7 +3912,7 @@ int __init shmem_init(void) goto out1; } -#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE +#ifdef CONFIG_TRANSPARENT_HUGEPAGE if (has_transparent_hugepage() && shmem_huge > SHMEM_HUGE_DENY) SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge; else @@ -3928,7 +3928,7 @@ out2: return error; } -#if defined(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && defined(CONFIG_SYSFS) +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SYSFS) static ssize_t shmem_enabled_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { @@ -3980,9 +3980,9 @@ static ssize_t shmem_enabled_store(struct kobject *kobj, struct kobj_attribute shmem_enabled_attr = __ATTR(shmem_enabled, 0644, shmem_enabled_show, shmem_enabled_store); -#endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE && CONFIG_SYSFS */ +#endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_SYSFS */ -#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE +#ifdef CONFIG_TRANSPARENT_HUGEPAGE bool shmem_huge_enabled(struct vm_area_struct *vma) { struct inode *inode = file_inode(vma->vm_file); @@ -4017,7 +4017,7 @@ bool shmem_huge_enabled(struct vm_area_struct *vma) return false; } } -#endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE */ +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ #else /* !CONFIG_SHMEM */ @@ -4186,7 +4186,7 @@ int shmem_zero_setup(struct vm_area_struct *vma) vma->vm_file = file; vma->vm_ops = &shmem_vm_ops; - if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && + if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) < (vma->vm_end & HPAGE_PMD_MASK)) { khugepaged_enter(vma, vma->vm_flags); From 7a9547fd4ebc5df52245e1df8a2e0bb481f20a92 Mon Sep 17 00:00:00 2001 From: Li Chen Date: Mon, 6 Apr 2020 20:04:38 -0700 Subject: [PATCH 493/635] mm/ksm.c: update get_user_pages() argument in comment This updates get_user_pages()'s argument in ksm_test_exit()'s comment Signed-off-by: Li Chen Signed-off-by: Andrew Morton Reviewed-by: Andrew Morton Link: http://lkml.kernel.org/r/30ac2417-f1c7-f337-0beb-df561295298c@uniontech.com Signed-off-by: Linus Torvalds --- mm/ksm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/ksm.c b/mm/ksm.c index d17c7d57d0d8..363ec8189561 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -455,7 +455,7 @@ static inline bool ksm_test_exit(struct mm_struct *mm) /* * We use break_ksm to break COW on a ksm page: it's a stripped down * - * if (get_user_pages(addr, 1, 1, 1, &page, NULL) == 1) + * if (get_user_pages(addr, 1, FOLL_WRITE, &page, NULL) == 1) * put_page(page); * * but taking great care only to touch a ksm page, in a VM_MERGEABLE vma, From 9de4f22a60f731943f050f4448bf2933ed3fa70b Mon Sep 17 00:00:00 2001 From: Huang Ying Date: Mon, 6 Apr 2020 20:04:41 -0700 Subject: [PATCH 494/635] mm: code cleanup for MADV_FREE Some comments for MADV_FREE is revised and added to help people understand the MADV_FREE code, especially the page flag, PG_swapbacked. This makes page_is_file_cache() isn't consistent with its comments. So the function is renamed to page_is_file_lru() to make them consistent again. All these are put in one patch as one logical change. Suggested-by: David Hildenbrand Suggested-by: Johannes Weiner Suggested-by: David Rientjes Signed-off-by: "Huang, Ying" Signed-off-by: Andrew Morton Acked-by: Johannes Weiner Acked-by: David Rientjes Acked-by: Michal Hocko Acked-by: Pankaj Gupta Acked-by: Vlastimil Babka Cc: Dave Hansen Cc: Mel Gorman Cc: Minchan Kim Cc: Hugh Dickins Cc: Rik van Riel Link: http://lkml.kernel.org/r/20200317100342.2730705-1-ying.huang@intel.com Signed-off-by: Linus Torvalds --- include/linux/mm_inline.h | 15 ++++++++------- include/linux/page-flags.h | 5 +++++ include/trace/events/vmscan.h | 2 +- mm/compaction.c | 2 +- mm/gup.c | 2 +- mm/khugepaged.c | 4 ++-- mm/memory-failure.c | 2 +- mm/memory_hotplug.c | 2 +- mm/mempolicy.c | 2 +- mm/migrate.c | 16 ++++++++-------- mm/mprotect.c | 2 +- mm/swap.c | 16 ++++++++-------- mm/vmscan.c | 12 ++++++------ 13 files changed, 44 insertions(+), 38 deletions(-) diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 6f2fef7b0784..219bef41d87c 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -6,19 +6,20 @@ #include /** - * page_is_file_cache - should the page be on a file LRU or anon LRU? + * page_is_file_lru - should the page be on a file LRU or anon LRU? * @page: the page to test * - * Returns 1 if @page is page cache page backed by a regular filesystem, - * or 0 if @page is anonymous, tmpfs or otherwise ram or swap backed. - * Used by functions that manipulate the LRU lists, to sort a page - * onto the right LRU list. + * Returns 1 if @page is a regular filesystem backed page cache page or a lazily + * freed anonymous page (e.g. via MADV_FREE). Returns 0 if @page is a normal + * anonymous page, a tmpfs page or otherwise ram or swap backed page. Used by + * functions that manipulate the LRU lists, to sort a page onto the right LRU + * list. * * We would like to get this info without a page flag, but the state * needs to survive until the page is last deleted from the LRU, which * could be as far down as __page_cache_release. */ -static inline int page_is_file_cache(struct page *page) +static inline int page_is_file_lru(struct page *page) { return !PageSwapBacked(page); } @@ -75,7 +76,7 @@ static __always_inline void del_page_from_lru_list(struct page *page, */ static inline enum lru_list page_lru_base_type(struct page *page) { - if (page_is_file_cache(page)) + if (page_is_file_lru(page)) return LRU_INACTIVE_FILE; return LRU_INACTIVE_ANON; } diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 77de28bfefb0..acf7988fd640 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -63,6 +63,11 @@ * page_waitqueue(page) is a wait queue of all tasks waiting for the page * to become unlocked. * + * PG_swapbacked is set when a page uses swap as a backing storage. This are + * usually PageAnon or shmem pages but please note that even anonymous pages + * might lose their PG_swapbacked flag when they simply can be dropped (e.g. as + * a result of MADV_FREE). + * * PG_uptodate tells whether the page's contents is valid. When a read * completes, the page becomes uptodate, unless a disk I/O error happened. * diff --git a/include/trace/events/vmscan.h b/include/trace/events/vmscan.h index a5ab2973e8dc..74bb594ccb25 100644 --- a/include/trace/events/vmscan.h +++ b/include/trace/events/vmscan.h @@ -323,7 +323,7 @@ TRACE_EVENT(mm_vmscan_writepage, TP_fast_assign( __entry->pfn = page_to_pfn(page); __entry->reclaim_flags = trace_reclaim_flags( - page_is_file_cache(page)); + page_is_file_lru(page)); ), TP_printk("page=%p pfn=%lu flags=%s", diff --git a/mm/compaction.c b/mm/compaction.c index df3da2f76fdc..46af63eb8212 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -989,7 +989,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, /* Successfully isolated */ del_page_from_lru_list(page, lruvec, page_lru(page)); mod_node_page_state(page_pgdat(page), - NR_ISOLATED_ANON + page_is_file_cache(page), + NR_ISOLATED_ANON + page_is_file_lru(page), hpage_nr_pages(page)); isolate_success: diff --git a/mm/gup.c b/mm/gup.c index 96af7e08db4b..b185377c38b7 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -1677,7 +1677,7 @@ check_again: list_add_tail(&head->lru, &cma_page_list); mod_node_page_state(page_pgdat(head), NR_ISOLATED_ANON + - page_is_file_cache(head), + page_is_file_lru(head), hpage_nr_pages(head)); } } diff --git a/mm/khugepaged.c b/mm/khugepaged.c index b1d9a8e189b8..3afc1e2d7a55 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -511,7 +511,7 @@ void __khugepaged_exit(struct mm_struct *mm) static void release_pte_page(struct page *page) { - dec_node_page_state(page, NR_ISOLATED_ANON + page_is_file_cache(page)); + dec_node_page_state(page, NR_ISOLATED_ANON + page_is_file_lru(page)); unlock_page(page); putback_lru_page(page); } @@ -611,7 +611,7 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma, goto out; } inc_node_page_state(page, - NR_ISOLATED_ANON + page_is_file_cache(page)); + NR_ISOLATED_ANON + page_is_file_lru(page)); VM_BUG_ON_PAGE(!PageLocked(page), page); VM_BUG_ON_PAGE(PageLRU(page), page); diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 1c961cd26c0b..a96364be8ab4 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -1810,7 +1810,7 @@ static int __soft_offline_page(struct page *page, int flags) */ if (!__PageMovable(page)) inc_node_page_state(page, NR_ISOLATED_ANON + - page_is_file_cache(page)); + page_is_file_lru(page)); list_add(&page->lru, &pagelist); ret = migrate_pages(&pagelist, new_page, NULL, MPOL_MF_MOVE_ALL, MIGRATE_SYNC, MR_MEMORY_FAILURE); diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 19389cdc16a5..005eab3411e5 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -1317,7 +1317,7 @@ do_migrate_range(unsigned long start_pfn, unsigned long end_pfn) list_add_tail(&page->lru, &source); if (!__PageMovable(page)) inc_node_page_state(page, NR_ISOLATED_ANON + - page_is_file_cache(page)); + page_is_file_lru(page)); } else { pr_warn("failed to isolate pfn %lx\n", pfn); diff --git a/mm/mempolicy.c b/mm/mempolicy.c index b36926ba02e2..037e5f548118 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -1022,7 +1022,7 @@ static int migrate_page_add(struct page *page, struct list_head *pagelist, if (!isolate_lru_page(head)) { list_add_tail(&head->lru, pagelist); mod_node_page_state(page_pgdat(head), - NR_ISOLATED_ANON + page_is_file_cache(head), + NR_ISOLATED_ANON + page_is_file_lru(head), hpage_nr_pages(head)); } else if (flags & MPOL_MF_STRICT) { /* diff --git a/mm/migrate.c b/mm/migrate.c index 1a205503be3f..c1412e04975e 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -193,7 +193,7 @@ void putback_movable_pages(struct list_head *l) put_page(page); } else { mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + - page_is_file_cache(page), -hpage_nr_pages(page)); + page_is_file_lru(page), -hpage_nr_pages(page)); putback_lru_page(page); } } @@ -1219,7 +1219,7 @@ out: */ if (likely(!__PageMovable(page))) mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + - page_is_file_cache(page), -hpage_nr_pages(page)); + page_is_file_lru(page), -hpage_nr_pages(page)); } /* @@ -1592,7 +1592,7 @@ static int add_page_for_migration(struct mm_struct *mm, unsigned long addr, err = 1; list_add_tail(&head->lru, pagelist); mod_node_page_state(page_pgdat(head), - NR_ISOLATED_ANON + page_is_file_cache(head), + NR_ISOLATED_ANON + page_is_file_lru(head), hpage_nr_pages(head)); } out_putpage: @@ -1955,7 +1955,7 @@ static int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page) return 0; } - page_lru = page_is_file_cache(page); + page_lru = page_is_file_lru(page); mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + page_lru, hpage_nr_pages(page)); @@ -1991,7 +1991,7 @@ int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma, * Don't migrate file pages that are mapped in multiple processes * with execute permissions as they are probably shared libraries. */ - if (page_mapcount(page) != 1 && page_is_file_cache(page) && + if (page_mapcount(page) != 1 && page_is_file_lru(page) && (vma->vm_flags & VM_EXEC)) goto out; @@ -1999,7 +1999,7 @@ int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma, * Also do not migrate dirty pages as not all filesystems can move * dirty pages in MIGRATE_ASYNC mode which is a waste of cycles. */ - if (page_is_file_cache(page) && PageDirty(page)) + if (page_is_file_lru(page) && PageDirty(page)) goto out; isolated = numamigrate_isolate_page(pgdat, page); @@ -2014,7 +2014,7 @@ int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma, if (!list_empty(&migratepages)) { list_del(&page->lru); dec_node_page_state(page, NR_ISOLATED_ANON + - page_is_file_cache(page)); + page_is_file_lru(page)); putback_lru_page(page); } isolated = 0; @@ -2044,7 +2044,7 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm, pg_data_t *pgdat = NODE_DATA(node); int isolated = 0; struct page *new_page = NULL; - int page_lru = page_is_file_cache(page); + int page_lru = page_is_file_lru(page); unsigned long start = address & HPAGE_PMD_MASK; new_page = alloc_pages_node(node, diff --git a/mm/mprotect.c b/mm/mprotect.c index 311c0dadf71c..0fee14b39416 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -98,7 +98,7 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd, * it cannot move them all from MIGRATE_ASYNC * context. */ - if (page_is_file_cache(page) && PageDirty(page)) + if (page_is_file_lru(page) && PageDirty(page)) continue; /* diff --git a/mm/swap.c b/mm/swap.c index a4af8c999963..18505990c3b1 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -276,7 +276,7 @@ static void __activate_page(struct page *page, struct lruvec *lruvec, void *arg) { if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { - int file = page_is_file_cache(page); + int file = page_is_file_lru(page); int lru = page_lru_base_type(page); del_page_from_lru_list(page, lruvec, lru); @@ -394,7 +394,7 @@ void mark_page_accessed(struct page *page) else __lru_cache_activate_page(page); ClearPageReferenced(page); - if (page_is_file_cache(page)) + if (page_is_file_lru(page)) workingset_activation(page); } if (page_is_idle(page)) @@ -515,7 +515,7 @@ static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec, return; active = PageActive(page); - file = page_is_file_cache(page); + file = page_is_file_lru(page); lru = page_lru_base_type(page); del_page_from_lru_list(page, lruvec, lru + active); @@ -548,7 +548,7 @@ static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec, void *arg) { if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) { - int file = page_is_file_cache(page); + int file = page_is_file_lru(page); int lru = page_lru_base_type(page); del_page_from_lru_list(page, lruvec, lru + LRU_ACTIVE); @@ -573,9 +573,9 @@ static void lru_lazyfree_fn(struct page *page, struct lruvec *lruvec, ClearPageActive(page); ClearPageReferenced(page); /* - * lazyfree pages are clean anonymous pages. They have - * SwapBacked flag cleared to distinguish normal anonymous - * pages + * Lazyfree pages are clean anonymous pages. They have + * PG_swapbacked flag cleared, to distinguish them from normal + * anonymous pages */ ClearPageSwapBacked(page); add_page_to_lru_list(page, lruvec, LRU_INACTIVE_FILE); @@ -962,7 +962,7 @@ static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec, if (page_evictable(page)) { lru = page_lru(page); - update_page_reclaim_stat(lruvec, page_is_file_cache(page), + update_page_reclaim_stat(lruvec, page_is_file_lru(page), PageActive(page)); if (was_unevictable) count_vm_event(UNEVICTABLE_PGRESCUED); diff --git a/mm/vmscan.c b/mm/vmscan.c index 2e8e690d2813..b06868fc4926 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -919,7 +919,7 @@ static int __remove_mapping(struct address_space *mapping, struct page *page, * exceptional entries and shadow exceptional entries in the * same address_space. */ - if (reclaimed && page_is_file_cache(page) && + if (reclaimed && page_is_file_lru(page) && !mapping_exiting(mapping) && !dax_mapping(mapping)) shadow = workingset_eviction(page, target_memcg); __delete_from_page_cache(page, shadow); @@ -1043,7 +1043,7 @@ static void page_check_dirty_writeback(struct page *page, * Anonymous pages are not handled by flushers and must be written * from reclaim context. Do not stall reclaim based on them */ - if (!page_is_file_cache(page) || + if (!page_is_file_lru(page) || (PageAnon(page) && !PageSwapBacked(page))) { *dirty = false; *writeback = false; @@ -1315,7 +1315,7 @@ static unsigned long shrink_page_list(struct list_head *page_list, * the rest of the LRU for clean pages and see * the same dirty pages again (PageReclaim). */ - if (page_is_file_cache(page) && + if (page_is_file_lru(page) && (!current_is_kswapd() || !PageReclaim(page) || !test_bit(PGDAT_DIRTY, &pgdat->flags))) { /* @@ -1459,7 +1459,7 @@ activate_locked: try_to_free_swap(page); VM_BUG_ON_PAGE(PageActive(page), page); if (!PageMlocked(page)) { - int type = page_is_file_cache(page); + int type = page_is_file_lru(page); SetPageActive(page); stat->nr_activate[type] += nr_pages; count_memcg_page_event(page, PGACTIVATE); @@ -1497,7 +1497,7 @@ unsigned long reclaim_clean_pages_from_list(struct zone *zone, LIST_HEAD(clean_pages); list_for_each_entry_safe(page, next, page_list, lru) { - if (page_is_file_cache(page) && !PageDirty(page) && + if (page_is_file_lru(page) && !PageDirty(page) && !__PageMovable(page) && !PageUnevictable(page)) { ClearPageActive(page); list_move(&page->lru, &clean_pages); @@ -2053,7 +2053,7 @@ static void shrink_active_list(unsigned long nr_to_scan, * IO, plus JVM can create lots of anon VM_EXEC pages, * so we ignore them here. */ - if ((vm_flags & VM_EXEC) && page_is_file_cache(page)) { + if ((vm_flags & VM_EXEC) && page_is_file_lru(page)) { list_add(&page->lru, &l_active); continue; } From a2129f24798a993abde9b4bf8b3713b52d56c121 Mon Sep 17 00:00:00 2001 From: Alexander Duyck Date: Mon, 6 Apr 2020 20:04:45 -0700 Subject: [PATCH 495/635] mm: adjust shuffle code to allow for future coalescing Patch series "mm / virtio: Provide support for free page reporting", v17. This series provides an asynchronous means of reporting free guest pages to a hypervisor so that the memory associated with those pages can be dropped and reused by other processes and/or guests on the host. Using this it is possible to avoid unnecessary I/O to disk and greatly improve performance in the case of memory overcommit on the host. When enabled we will be performing a scan of free memory every 2 seconds while pages of sufficiently high order are being freed. In each pass at least one sixteenth of each free list will be reported. By doing this we avoid racing against other threads that may be causing a high amount of memory churn. The lowest page order currently scanned when reporting pages is pageblock_order so that this feature will not interfere with the use of Transparent Huge Pages in the case of virtualization. Currently this is only in use by virtio-balloon however there is the hope that at some point in the future other hypervisors might be able to make use of it. In the virtio-balloon/QEMU implementation the hypervisor is currently using MADV_DONTNEED to indicate to the host kernel that the page is currently free. It will be zeroed and faulted back into the guest the next time the page is accessed. To track if a page is reported or not the Uptodate flag was repurposed and used as a Reported flag for Buddy pages. We walk though the free list isolating pages and adding them to the scatterlist until we either encounter the end of the list or have processed at least one sixteenth of the pages that were listed in nr_free prior to us starting. If we fill the scatterlist before we reach the end of the list we rotate the list so that the first unreported page we encounter is moved to the head of the list as that is where we will resume after we have freed the reported pages back into the tail of the list. Below are the results from various benchmarks. I primarily focused on two tests. The first is the will-it-scale/page_fault2 test, and the other is a modified version of will-it-scale/page_fault1 that was enabled to use THP. I did this as it allows for better visibility into different parts of the memory subsystem. The guest is running with 32G for RAM on one node of a E5-2630 v3. The host has had some features such as CPU turbo disabled in the BIOS. Test page_fault1 (THP) page_fault2 Name tasks Process Iter STDEV Process Iter STDEV Baseline 1 1012402.50 0.14% 361855.25 0.81% 16 8827457.25 0.09% 3282347.00 0.34% Patches Applied 1 1007897.00 0.23% 361887.00 0.26% 16 8784741.75 0.39% 3240669.25 0.48% Patches Enabled 1 1010227.50 0.39% 359749.25 0.56% 16 8756219.00 0.24% 3226608.75 0.97% Patches Enabled 1 1050982.00 4.26% 357966.25 0.14% page shuffle 16 8672601.25 0.49% 3223177.75 0.40% Patches enabled 1 1003238.00 0.22% 360211.00 0.22% shuffle w/ RFC 16 8767010.50 0.32% 3199874.00 0.71% The results above are for a baseline with a linux-next-20191219 kernel, that kernel with this patch set applied but page reporting disabled in virtio-balloon, the patches applied and page reporting fully enabled, the patches enabled with page shuffling enabled, and the patches applied with page shuffling enabled and an RFC patch that makes used of MADV_FREE in QEMU. These results include the deviation seen between the average value reported here versus the high and/or low value. I observed that during the test memory usage for the first three tests never dropped whereas with the patches fully enabled the VM would drop to using only a few GB of the host's memory when switching from memhog to page fault tests. Any of the overhead visible with this patch set enabled seems due to page faults caused by accessing the reported pages and the host zeroing the page before giving it back to the guest. This overhead is much more visible when using THP than with standard 4K pages. In addition page shuffling seemed to increase the amount of faults generated due to an increase in memory churn. The overehad is reduced when using MADV_FREE as we can avoid the extra zeroing of the pages when they are reintroduced to the host, as can be seen when the RFC is applied with shuffling enabled. The overall guest size is kept fairly small to only a few GB while the test is running. If the host memory were oversubscribed this patch set should result in a performance improvement as swapping memory in the host can be avoided. A brief history on the background of free page reporting can be found at: https://lore.kernel.org/lkml/29f43d5796feed0dec8e8bb98b187d9dac03b900.camel@linux.intel.com/ This patch (of 9): Move the head/tail adding logic out of the shuffle code and into the __free_one_page function since ultimately that is where it is really needed anyway. By doing this we should be able to reduce the overhead and can consolidate all of the list addition bits in one spot. Signed-off-by: Alexander Duyck Signed-off-by: Andrew Morton Reviewed-by: Dan Williams Acked-by: Mel Gorman Acked-by: David Hildenbrand Cc: Yang Zhang Cc: Pankaj Gupta Cc: Konrad Rzeszutek Wilk Cc: Nitesh Narayan Lal Cc: Rik van Riel Cc: Matthew Wilcox Cc: Luiz Capitulino Cc: Dave Hansen Cc: Wei Wang Cc: Andrea Arcangeli Cc: Paolo Bonzini Cc: Michal Hocko Cc: Vlastimil Babka Cc: Oscar Salvador Cc: Michael S. Tsirkin Cc: wei qi Link: http://lkml.kernel.org/r/20200211224602.29318.84523.stgit@localhost.localdomain Signed-off-by: Linus Torvalds --- include/linux/mmzone.h | 12 ------- mm/page_alloc.c | 73 +++++++++++++++++++++++++----------------- mm/shuffle.c | 12 +++---- mm/shuffle.h | 6 ++++ 4 files changed, 55 insertions(+), 48 deletions(-) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index e84d448988b6..c023d7968b14 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -116,18 +116,6 @@ static inline void add_to_free_area_tail(struct page *page, struct free_area *ar area->nr_free++; } -#ifdef CONFIG_SHUFFLE_PAGE_ALLOCATOR -/* Used to preserve page allocation order entropy */ -void add_to_free_area_random(struct page *page, struct free_area *area, - int migratetype); -#else -static inline void add_to_free_area_random(struct page *page, - struct free_area *area, int migratetype) -{ - add_to_free_area(page, area, migratetype); -} -#endif - /* Used for pages which are on another list */ static inline void move_to_free_area(struct page *page, struct free_area *area, int migratetype) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index e5f76da8cd4e..f2b8cb8f995f 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -864,6 +864,36 @@ compaction_capture(struct capture_control *capc, struct page *page, } #endif /* CONFIG_COMPACTION */ +/* + * If this is not the largest possible page, check if the buddy + * of the next-highest order is free. If it is, it's possible + * that pages are being freed that will coalesce soon. In case, + * that is happening, add the free page to the tail of the list + * so it's less likely to be used soon and more likely to be merged + * as a higher order page + */ +static inline bool +buddy_merge_likely(unsigned long pfn, unsigned long buddy_pfn, + struct page *page, unsigned int order) +{ + struct page *higher_page, *higher_buddy; + unsigned long combined_pfn; + + if (order >= MAX_ORDER - 2) + return false; + + if (!pfn_valid_within(buddy_pfn)) + return false; + + combined_pfn = buddy_pfn & pfn; + higher_page = page + (combined_pfn - pfn); + buddy_pfn = __find_buddy_pfn(combined_pfn, order + 1); + higher_buddy = higher_page + (buddy_pfn - combined_pfn); + + return pfn_valid_within(buddy_pfn) && + page_is_buddy(higher_page, higher_buddy, order + 1); +} + /* * Freeing function for a buddy system allocator. * @@ -893,11 +923,13 @@ static inline void __free_one_page(struct page *page, struct zone *zone, unsigned int order, int migratetype) { - unsigned long combined_pfn; - unsigned long uninitialized_var(buddy_pfn); - struct page *buddy; - unsigned int max_order; struct capture_control *capc = task_capc(zone); + unsigned long uninitialized_var(buddy_pfn); + unsigned long combined_pfn; + struct free_area *area; + unsigned int max_order; + struct page *buddy; + bool to_tail; max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1); @@ -966,35 +998,16 @@ continue_merging: done_merging: set_page_order(page, order); - /* - * If this is not the largest possible page, check if the buddy - * of the next-highest order is free. If it is, it's possible - * that pages are being freed that will coalesce soon. In case, - * that is happening, add the free page to the tail of the list - * so it's less likely to be used soon and more likely to be merged - * as a higher order page - */ - if ((order < MAX_ORDER-2) && pfn_valid_within(buddy_pfn) - && !is_shuffle_order(order)) { - struct page *higher_page, *higher_buddy; - combined_pfn = buddy_pfn & pfn; - higher_page = page + (combined_pfn - pfn); - buddy_pfn = __find_buddy_pfn(combined_pfn, order + 1); - higher_buddy = higher_page + (buddy_pfn - combined_pfn); - if (pfn_valid_within(buddy_pfn) && - page_is_buddy(higher_page, higher_buddy, order + 1)) { - add_to_free_area_tail(page, &zone->free_area[order], - migratetype); - return; - } - } - + area = &zone->free_area[order]; if (is_shuffle_order(order)) - add_to_free_area_random(page, &zone->free_area[order], - migratetype); + to_tail = shuffle_pick_tail(); else - add_to_free_area(page, &zone->free_area[order], migratetype); + to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order); + if (to_tail) + add_to_free_area_tail(page, area, migratetype); + else + add_to_free_area(page, area, migratetype); } /* diff --git a/mm/shuffle.c b/mm/shuffle.c index c716059cbd3c..44406d9977c7 100644 --- a/mm/shuffle.c +++ b/mm/shuffle.c @@ -183,11 +183,11 @@ void __meminit __shuffle_free_memory(pg_data_t *pgdat) shuffle_zone(z); } -void add_to_free_area_random(struct page *page, struct free_area *area, - int migratetype) +bool shuffle_pick_tail(void) { static u64 rand; static u8 rand_bits; + bool ret; /* * The lack of locking is deliberate. If 2 threads race to @@ -198,10 +198,10 @@ void add_to_free_area_random(struct page *page, struct free_area *area, rand = get_random_u64(); } - if (rand & 1) - add_to_free_area(page, area, migratetype); - else - add_to_free_area_tail(page, area, migratetype); + ret = rand & 1; + rand_bits--; rand >>= 1; + + return ret; } diff --git a/mm/shuffle.h b/mm/shuffle.h index 777a257a0d2f..4d79f03b6658 100644 --- a/mm/shuffle.h +++ b/mm/shuffle.h @@ -22,6 +22,7 @@ enum mm_shuffle_ctl { DECLARE_STATIC_KEY_FALSE(page_alloc_shuffle_key); extern void page_alloc_shuffle(enum mm_shuffle_ctl ctl); extern void __shuffle_free_memory(pg_data_t *pgdat); +extern bool shuffle_pick_tail(void); static inline void shuffle_free_memory(pg_data_t *pgdat) { if (!static_branch_unlikely(&page_alloc_shuffle_key)) @@ -44,6 +45,11 @@ static inline bool is_shuffle_order(int order) return order >= SHUFFLE_ORDER; } #else +static inline bool shuffle_pick_tail(void) +{ + return false; +} + static inline void shuffle_free_memory(pg_data_t *pgdat) { } From 6ab0136310961ebf4b5ecb565f0bf52c233dc093 Mon Sep 17 00:00:00 2001 From: Alexander Duyck Date: Mon, 6 Apr 2020 20:04:49 -0700 Subject: [PATCH 496/635] mm: use zone and order instead of free area in free_list manipulators In order to enable the use of the zone from the list manipulator functions I will need access to the zone pointer. As it turns out most of the accessors were always just being directly passed &zone->free_area[order] anyway so it would make sense to just fold that into the function itself and pass the zone and order as arguments instead of the free area. In order to be able to reference the zone we need to move the declaration of the functions down so that we have the zone defined before we define the list manipulation functions. Since the functions are only used in the file mm/page_alloc.c we can just move them there to reduce noise in the header. Signed-off-by: Alexander Duyck Signed-off-by: Andrew Morton Reviewed-by: Dan Williams Reviewed-by: David Hildenbrand Reviewed-by: Pankaj Gupta Acked-by: Mel Gorman Cc: Andrea Arcangeli Cc: Dave Hansen Cc: Konrad Rzeszutek Wilk Cc: Luiz Capitulino Cc: Matthew Wilcox Cc: Michael S. Tsirkin Cc: Michal Hocko Cc: Nitesh Narayan Lal Cc: Oscar Salvador Cc: Paolo Bonzini Cc: Rik van Riel Cc: Vlastimil Babka Cc: Wei Wang Cc: Yang Zhang Cc: wei qi Link: http://lkml.kernel.org/r/20200211224613.29318.43080.stgit@localhost.localdomain Signed-off-by: Linus Torvalds --- include/linux/mmzone.h | 32 -------------------- mm/page_alloc.c | 67 ++++++++++++++++++++++++++++++------------ 2 files changed, 49 insertions(+), 50 deletions(-) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index c023d7968b14..42b77d3b68e8 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -100,29 +100,6 @@ struct free_area { unsigned long nr_free; }; -/* Used for pages not on another list */ -static inline void add_to_free_area(struct page *page, struct free_area *area, - int migratetype) -{ - list_add(&page->lru, &area->free_list[migratetype]); - area->nr_free++; -} - -/* Used for pages not on another list */ -static inline void add_to_free_area_tail(struct page *page, struct free_area *area, - int migratetype) -{ - list_add_tail(&page->lru, &area->free_list[migratetype]); - area->nr_free++; -} - -/* Used for pages which are on another list */ -static inline void move_to_free_area(struct page *page, struct free_area *area, - int migratetype) -{ - list_move(&page->lru, &area->free_list[migratetype]); -} - static inline struct page *get_page_from_free_area(struct free_area *area, int migratetype) { @@ -130,15 +107,6 @@ static inline struct page *get_page_from_free_area(struct free_area *area, struct page, lru); } -static inline void del_page_from_free_area(struct page *page, - struct free_area *area) -{ - list_del(&page->lru); - __ClearPageBuddy(page); - set_page_private(page, 0); - area->nr_free--; -} - static inline bool free_area_empty(struct free_area *area, int migratetype) { return list_empty(&area->free_list[migratetype]); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index f2b8cb8f995f..14bdf3608a6b 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -864,6 +864,44 @@ compaction_capture(struct capture_control *capc, struct page *page, } #endif /* CONFIG_COMPACTION */ +/* Used for pages not on another list */ +static inline void add_to_free_list(struct page *page, struct zone *zone, + unsigned int order, int migratetype) +{ + struct free_area *area = &zone->free_area[order]; + + list_add(&page->lru, &area->free_list[migratetype]); + area->nr_free++; +} + +/* Used for pages not on another list */ +static inline void add_to_free_list_tail(struct page *page, struct zone *zone, + unsigned int order, int migratetype) +{ + struct free_area *area = &zone->free_area[order]; + + list_add_tail(&page->lru, &area->free_list[migratetype]); + area->nr_free++; +} + +/* Used for pages which are on another list */ +static inline void move_to_free_list(struct page *page, struct zone *zone, + unsigned int order, int migratetype) +{ + struct free_area *area = &zone->free_area[order]; + + list_move(&page->lru, &area->free_list[migratetype]); +} + +static inline void del_page_from_free_list(struct page *page, struct zone *zone, + unsigned int order) +{ + list_del(&page->lru); + __ClearPageBuddy(page); + set_page_private(page, 0); + zone->free_area[order].nr_free--; +} + /* * If this is not the largest possible page, check if the buddy * of the next-highest order is free. If it is, it's possible @@ -926,7 +964,6 @@ static inline void __free_one_page(struct page *page, struct capture_control *capc = task_capc(zone); unsigned long uninitialized_var(buddy_pfn); unsigned long combined_pfn; - struct free_area *area; unsigned int max_order; struct page *buddy; bool to_tail; @@ -964,7 +1001,7 @@ continue_merging: if (page_is_guard(buddy)) clear_page_guard(zone, buddy, order, migratetype); else - del_page_from_free_area(buddy, &zone->free_area[order]); + del_page_from_free_list(buddy, zone, order); combined_pfn = buddy_pfn & pfn; page = page + (combined_pfn - pfn); pfn = combined_pfn; @@ -998,16 +1035,15 @@ continue_merging: done_merging: set_page_order(page, order); - area = &zone->free_area[order]; if (is_shuffle_order(order)) to_tail = shuffle_pick_tail(); else to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order); if (to_tail) - add_to_free_area_tail(page, area, migratetype); + add_to_free_list_tail(page, zone, order, migratetype); else - add_to_free_area(page, area, migratetype); + add_to_free_list(page, zone, order, migratetype); } /* @@ -2021,13 +2057,11 @@ void __init init_cma_reserved_pageblock(struct page *page) * -- nyc */ static inline void expand(struct zone *zone, struct page *page, - int low, int high, struct free_area *area, - int migratetype) + int low, int high, int migratetype) { unsigned long size = 1 << high; while (high > low) { - area--; high--; size >>= 1; VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); @@ -2041,7 +2075,7 @@ static inline void expand(struct zone *zone, struct page *page, if (set_page_guard(zone, &page[size], high, migratetype)) continue; - add_to_free_area(&page[size], area, migratetype); + add_to_free_list(&page[size], zone, high, migratetype); set_page_order(&page[size], high); } } @@ -2199,8 +2233,8 @@ struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, page = get_page_from_free_area(area, migratetype); if (!page) continue; - del_page_from_free_area(page, area); - expand(zone, page, order, current_order, area, migratetype); + del_page_from_free_list(page, zone, current_order); + expand(zone, page, order, current_order, migratetype); set_pcppage_migratetype(page, migratetype); return page; } @@ -2274,7 +2308,7 @@ static int move_freepages(struct zone *zone, VM_BUG_ON_PAGE(page_zone(page) != zone, page); order = page_order(page); - move_to_free_area(page, &zone->free_area[order], migratetype); + move_to_free_list(page, zone, order, migratetype); page += 1 << order; pages_moved += 1 << order; } @@ -2390,7 +2424,6 @@ static void steal_suitable_fallback(struct zone *zone, struct page *page, unsigned int alloc_flags, int start_type, bool whole_block) { unsigned int current_order = page_order(page); - struct free_area *area; int free_pages, movable_pages, alike_pages; int old_block_type; @@ -2461,8 +2494,7 @@ static void steal_suitable_fallback(struct zone *zone, struct page *page, return; single_page: - area = &zone->free_area[current_order]; - move_to_free_area(page, area, start_type); + move_to_free_list(page, zone, current_order, start_type); } /* @@ -3133,7 +3165,6 @@ EXPORT_SYMBOL_GPL(split_page); int __isolate_free_page(struct page *page, unsigned int order) { - struct free_area *area = &page_zone(page)->free_area[order]; unsigned long watermark; struct zone *zone; int mt; @@ -3159,7 +3190,7 @@ int __isolate_free_page(struct page *page, unsigned int order) /* Remove page from free list */ - del_page_from_free_area(page, area); + del_page_from_free_list(page, zone, order); /* * Set the pageblock if the isolated page is at least half of a @@ -8726,7 +8757,7 @@ __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) BUG_ON(!PageBuddy(page)); order = page_order(page); offlined_pages += 1 << order; - del_page_from_free_area(page, &zone->free_area[order]); + del_page_from_free_list(page, zone, order); pfn += (1 << order); } spin_unlock_irqrestore(&zone->lock, flags); From 624f58d8f4639676d2fa1238425ab0148d501c4a Mon Sep 17 00:00:00 2001 From: Alexander Duyck Date: Mon, 6 Apr 2020 20:04:53 -0700 Subject: [PATCH 497/635] mm: add function __putback_isolated_page There are cases where we would benefit from avoiding having to go through the allocation and free cycle to return an isolated page. Examples for this might include page poisoning in which we isolate a page and then put it back in the free list without ever having actually allocated it. This will enable us to also avoid notifiers for the future free page reporting which will need to avoid retriggering page reporting when returning pages that have been reported on. Signed-off-by: Alexander Duyck Signed-off-by: Andrew Morton Acked-by: David Hildenbrand Acked-by: Mel Gorman Cc: Andrea Arcangeli Cc: Dan Williams Cc: Dave Hansen Cc: Konrad Rzeszutek Wilk Cc: Luiz Capitulino Cc: Matthew Wilcox Cc: Michael S. Tsirkin Cc: Michal Hocko Cc: Nitesh Narayan Lal Cc: Oscar Salvador Cc: Pankaj Gupta Cc: Paolo Bonzini Cc: Rik van Riel Cc: Vlastimil Babka Cc: Wei Wang Cc: Yang Zhang Cc: wei qi Link: http://lkml.kernel.org/r/20200211224624.29318.89287.stgit@localhost.localdomain Signed-off-by: Linus Torvalds --- mm/internal.h | 2 ++ mm/page_alloc.c | 19 +++++++++++++++++++ mm/page_isolation.c | 6 ++---- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/mm/internal.h b/mm/internal.h index 2d58ae15a958..b5634e78f01d 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -180,6 +180,8 @@ static inline struct page *pageblock_pfn_to_page(unsigned long start_pfn, } extern int __isolate_free_page(struct page *page, unsigned int order); +extern void __putback_isolated_page(struct page *page, unsigned int order, + int mt); extern void memblock_free_pages(struct page *page, unsigned long pfn, unsigned int order); extern void __free_pages_core(struct page *page, unsigned int order); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 14bdf3608a6b..448e439b75f2 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -3211,6 +3211,25 @@ int __isolate_free_page(struct page *page, unsigned int order) return 1UL << order; } +/** + * __putback_isolated_page - Return a now-isolated page back where we got it + * @page: Page that was isolated + * @order: Order of the isolated page + * + * This function is meant to return a page pulled from the free lists via + * __isolate_free_page back to the free lists they were pulled from. + */ +void __putback_isolated_page(struct page *page, unsigned int order, int mt) +{ + struct zone *zone = page_zone(page); + + /* zone lock should be held when this function is called */ + lockdep_assert_held(&zone->lock); + + /* Return isolated page to tail of freelist. */ + __free_one_page(page, page_to_pfn(page), zone, order, mt); +} + /* * Update NUMA hit/miss statistics * diff --git a/mm/page_isolation.c b/mm/page_isolation.c index a9fd7c740c23..2c11a38d6e87 100644 --- a/mm/page_isolation.c +++ b/mm/page_isolation.c @@ -117,13 +117,11 @@ static void unset_migratetype_isolate(struct page *page, unsigned migratetype) __mod_zone_freepage_state(zone, nr_pages, migratetype); } set_pageblock_migratetype(page, migratetype); + if (isolated_page) + __putback_isolated_page(page, order, migratetype); zone->nr_isolate_pageblock--; out: spin_unlock_irqrestore(&zone->lock, flags); - if (isolated_page) { - post_alloc_hook(page, order, __GFP_MOVABLE); - __free_pages(page, order); - } } static inline struct page * From 36e66c554b5c6a9d17a229faca7a61693527b0bd Mon Sep 17 00:00:00 2001 From: Alexander Duyck Date: Mon, 6 Apr 2020 20:04:56 -0700 Subject: [PATCH 498/635] mm: introduce Reported pages In order to pave the way for free page reporting in virtualized environments we will need a way to get pages out of the free lists and identify those pages after they have been returned. To accomplish this, this patch adds the concept of a Reported Buddy, which is essentially meant to just be the Uptodate flag used in conjunction with the Buddy page type. To prevent the reported pages from leaking outside of the buddy lists I added a check to clear the PageReported bit in the del_page_from_free_list function. As a result any reported page that is split, merged, or allocated will have the flag cleared prior to the PageBuddy value being cleared. The process for reporting pages is fairly simple. Once we free a page that meets the minimum order for page reporting we will schedule a worker thread to start 2s or more in the future. That worker thread will begin working from the lowest supported page reporting order up to MAX_ORDER - 1 pulling unreported pages from the free list and storing them in the scatterlist. When processing each individual free list it is necessary for the worker thread to release the zone lock when it needs to stop and report the full scatterlist of pages. To reduce the work of the next iteration the worker thread will rotate the free list so that the first unreported page in the free list becomes the first entry in the list. It will then call a reporting function providing information on how many entries are in the scatterlist. Once the function completes it will return the pages to the free area from which they were allocated and start over pulling more pages from the free areas until there are no longer enough pages to report on to keep the worker busy, or we have processed as many pages as were contained in the free area when we started processing the list. The worker thread will work in a round-robin fashion making its way though each zone requesting reporting, and through each reportable free list within that zone. Once all free areas within the zone have been processed it will check to see if there have been any requests for reporting while it was processing. If so it will reschedule the worker thread to start up again in roughly 2s and exit. Signed-off-by: Alexander Duyck Signed-off-by: Andrew Morton Acked-by: Mel Gorman Cc: Andrea Arcangeli Cc: Dan Williams Cc: Dave Hansen Cc: David Hildenbrand Cc: Konrad Rzeszutek Wilk Cc: Luiz Capitulino Cc: Matthew Wilcox Cc: Michael S. Tsirkin Cc: Michal Hocko Cc: Nitesh Narayan Lal Cc: Oscar Salvador Cc: Pankaj Gupta Cc: Paolo Bonzini Cc: Rik van Riel Cc: Vlastimil Babka Cc: Wei Wang Cc: Yang Zhang Cc: wei qi Link: http://lkml.kernel.org/r/20200211224635.29318.19750.stgit@localhost.localdomain Signed-off-by: Linus Torvalds --- include/linux/page-flags.h | 11 ++ include/linux/page_reporting.h | 25 +++ mm/Kconfig | 11 ++ mm/Makefile | 1 + mm/page_alloc.c | 17 +- mm/page_reporting.c | 319 +++++++++++++++++++++++++++++++++ mm/page_reporting.h | 54 ++++++ 7 files changed, 434 insertions(+), 4 deletions(-) create mode 100644 include/linux/page_reporting.h create mode 100644 mm/page_reporting.c create mode 100644 mm/page_reporting.h diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index acf7988fd640..222f6f7b2bb3 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -168,6 +168,9 @@ enum pageflags { /* non-lru isolated movable page */ PG_isolated = PG_reclaim, + + /* Only valid for buddy pages. Used to track pages that are reported */ + PG_reported = PG_uptodate, }; #ifndef __GENERATING_BOUNDS_H @@ -436,6 +439,14 @@ TESTCLEARFLAG(Young, young, PF_ANY) PAGEFLAG(Idle, idle, PF_ANY) #endif +/* + * PageReported() is used to track reported free pages within the Buddy + * allocator. We can use the non-atomic version of the test and set + * operations as both should be shielded with the zone lock to prevent + * any possible races on the setting or clearing of the bit. + */ +__PAGEFLAG(Reported, reported, PF_NO_COMPOUND) + /* * On an anonymous page mapped into a user virtual memory area, * page->mapping points to its anon_vma, not to a struct address_space; diff --git a/include/linux/page_reporting.h b/include/linux/page_reporting.h new file mode 100644 index 000000000000..32355486f572 --- /dev/null +++ b/include/linux/page_reporting.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_PAGE_REPORTING_H +#define _LINUX_PAGE_REPORTING_H + +#include +#include + +#define PAGE_REPORTING_CAPACITY 32 + +struct page_reporting_dev_info { + /* function that alters pages to make them "reported" */ + int (*report)(struct page_reporting_dev_info *prdev, + struct scatterlist *sg, unsigned int nents); + + /* work struct for processing reports */ + struct delayed_work work; + + /* Current state of page reporting */ + atomic_t state; +}; + +/* Tear-down and bring-up for page reporting devices */ +void page_reporting_unregister(struct page_reporting_dev_info *prdev); +int page_reporting_register(struct page_reporting_dev_info *prdev); +#endif /*_LINUX_PAGE_REPORTING_H */ diff --git a/mm/Kconfig b/mm/Kconfig index 211a70e8d5cf..d286dc54458c 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -236,6 +236,17 @@ config COMPACTION it and then we would be really interested to hear about that at linux-mm@kvack.org. +# +# support for free page reporting +config PAGE_REPORTING + bool "Free page reporting" + def_bool n + help + Free page reporting allows for the incremental acquisition of + free pages from the buddy allocator for the purpose of reporting + those pages to another entity, such as a hypervisor, so that the + memory can be freed within the host for other uses. + # # support for page migration # diff --git a/mm/Makefile b/mm/Makefile index dbc8346d16ca..fccd3756b25f 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -111,3 +111,4 @@ obj-$(CONFIG_HMM_MIRROR) += hmm.o obj-$(CONFIG_MEMFD_CREATE) += memfd.o obj-$(CONFIG_MAPPING_DIRTY_HELPERS) += mapping_dirty_helpers.o obj-$(CONFIG_PTDUMP_CORE) += ptdump.o +obj-$(CONFIG_PAGE_REPORTING) += page_reporting.o diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 448e439b75f2..114c56c3685d 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -74,6 +74,7 @@ #include #include "internal.h" #include "shuffle.h" +#include "page_reporting.h" /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */ static DEFINE_MUTEX(pcp_batch_high_lock); @@ -896,6 +897,10 @@ static inline void move_to_free_list(struct page *page, struct zone *zone, static inline void del_page_from_free_list(struct page *page, struct zone *zone, unsigned int order) { + /* clear reported state and update reported page count */ + if (page_reported(page)) + __ClearPageReported(page); + list_del(&page->lru); __ClearPageBuddy(page); set_page_private(page, 0); @@ -959,7 +964,7 @@ buddy_merge_likely(unsigned long pfn, unsigned long buddy_pfn, static inline void __free_one_page(struct page *page, unsigned long pfn, struct zone *zone, unsigned int order, - int migratetype) + int migratetype, bool report) { struct capture_control *capc = task_capc(zone); unsigned long uninitialized_var(buddy_pfn); @@ -1044,6 +1049,10 @@ done_merging: add_to_free_list_tail(page, zone, order, migratetype); else add_to_free_list(page, zone, order, migratetype); + + /* Notify page reporting subsystem of freed page */ + if (report) + page_reporting_notify_free(order); } /* @@ -1360,7 +1369,7 @@ static void free_pcppages_bulk(struct zone *zone, int count, if (unlikely(isolated_pageblocks)) mt = get_pageblock_migratetype(page); - __free_one_page(page, page_to_pfn(page), zone, 0, mt); + __free_one_page(page, page_to_pfn(page), zone, 0, mt, true); trace_mm_page_pcpu_drain(page, 0, mt); } spin_unlock(&zone->lock); @@ -1376,7 +1385,7 @@ static void free_one_page(struct zone *zone, is_migrate_isolate(migratetype))) { migratetype = get_pfnblock_migratetype(page, pfn); } - __free_one_page(page, pfn, zone, order, migratetype); + __free_one_page(page, pfn, zone, order, migratetype, true); spin_unlock(&zone->lock); } @@ -3227,7 +3236,7 @@ void __putback_isolated_page(struct page *page, unsigned int order, int mt) lockdep_assert_held(&zone->lock); /* Return isolated page to tail of freelist. */ - __free_one_page(page, page_to_pfn(page), zone, order, mt); + __free_one_page(page, page_to_pfn(page), zone, order, mt, false); } /* diff --git a/mm/page_reporting.c b/mm/page_reporting.c new file mode 100644 index 000000000000..1047c6872d4f --- /dev/null +++ b/mm/page_reporting.c @@ -0,0 +1,319 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include + +#include "page_reporting.h" +#include "internal.h" + +#define PAGE_REPORTING_DELAY (2 * HZ) +static struct page_reporting_dev_info __rcu *pr_dev_info __read_mostly; + +enum { + PAGE_REPORTING_IDLE = 0, + PAGE_REPORTING_REQUESTED, + PAGE_REPORTING_ACTIVE +}; + +/* request page reporting */ +static void +__page_reporting_request(struct page_reporting_dev_info *prdev) +{ + unsigned int state; + + /* Check to see if we are in desired state */ + state = atomic_read(&prdev->state); + if (state == PAGE_REPORTING_REQUESTED) + return; + + /* + * If reporting is already active there is nothing we need to do. + * Test against 0 as that represents PAGE_REPORTING_IDLE. + */ + state = atomic_xchg(&prdev->state, PAGE_REPORTING_REQUESTED); + if (state != PAGE_REPORTING_IDLE) + return; + + /* + * Delay the start of work to allow a sizable queue to build. For + * now we are limiting this to running no more than once every + * couple of seconds. + */ + schedule_delayed_work(&prdev->work, PAGE_REPORTING_DELAY); +} + +/* notify prdev of free page reporting request */ +void __page_reporting_notify(void) +{ + struct page_reporting_dev_info *prdev; + + /* + * We use RCU to protect the pr_dev_info pointer. In almost all + * cases this should be present, however in the unlikely case of + * a shutdown this will be NULL and we should exit. + */ + rcu_read_lock(); + prdev = rcu_dereference(pr_dev_info); + if (likely(prdev)) + __page_reporting_request(prdev); + + rcu_read_unlock(); +} + +static void +page_reporting_drain(struct page_reporting_dev_info *prdev, + struct scatterlist *sgl, unsigned int nents, bool reported) +{ + struct scatterlist *sg = sgl; + + /* + * Drain the now reported pages back into their respective + * free lists/areas. We assume at least one page is populated. + */ + do { + struct page *page = sg_page(sg); + int mt = get_pageblock_migratetype(page); + unsigned int order = get_order(sg->length); + + __putback_isolated_page(page, order, mt); + + /* If the pages were not reported due to error skip flagging */ + if (!reported) + continue; + + /* + * If page was not comingled with another page we can + * consider the result to be "reported" since the page + * hasn't been modified, otherwise we will need to + * report on the new larger page when we make our way + * up to that higher order. + */ + if (PageBuddy(page) && page_order(page) == order) + __SetPageReported(page); + } while ((sg = sg_next(sg))); + + /* reinitialize scatterlist now that it is empty */ + sg_init_table(sgl, nents); +} + +/* + * The page reporting cycle consists of 4 stages, fill, report, drain, and + * idle. We will cycle through the first 3 stages until we cannot obtain a + * full scatterlist of pages, in that case we will switch to idle. + */ +static int +page_reporting_cycle(struct page_reporting_dev_info *prdev, struct zone *zone, + unsigned int order, unsigned int mt, + struct scatterlist *sgl, unsigned int *offset) +{ + struct free_area *area = &zone->free_area[order]; + struct list_head *list = &area->free_list[mt]; + unsigned int page_len = PAGE_SIZE << order; + struct page *page, *next; + int err = 0; + + /* + * Perform early check, if free area is empty there is + * nothing to process so we can skip this free_list. + */ + if (list_empty(list)) + return err; + + spin_lock_irq(&zone->lock); + + /* loop through free list adding unreported pages to sg list */ + list_for_each_entry_safe(page, next, list, lru) { + /* We are going to skip over the reported pages. */ + if (PageReported(page)) + continue; + + /* Attempt to pull page from list */ + if (!__isolate_free_page(page, order)) + break; + + /* Add page to scatter list */ + --(*offset); + sg_set_page(&sgl[*offset], page, page_len, 0); + + /* If scatterlist isn't full grab more pages */ + if (*offset) + continue; + + /* release lock before waiting on report processing */ + spin_unlock_irq(&zone->lock); + + /* begin processing pages in local list */ + err = prdev->report(prdev, sgl, PAGE_REPORTING_CAPACITY); + + /* reset offset since the full list was reported */ + *offset = PAGE_REPORTING_CAPACITY; + + /* reacquire zone lock and resume processing */ + spin_lock_irq(&zone->lock); + + /* flush reported pages from the sg list */ + page_reporting_drain(prdev, sgl, PAGE_REPORTING_CAPACITY, !err); + + /* + * Reset next to first entry, the old next isn't valid + * since we dropped the lock to report the pages + */ + next = list_first_entry(list, struct page, lru); + + /* exit on error */ + if (err) + break; + } + + spin_unlock_irq(&zone->lock); + + return err; +} + +static int +page_reporting_process_zone(struct page_reporting_dev_info *prdev, + struct scatterlist *sgl, struct zone *zone) +{ + unsigned int order, mt, leftover, offset = PAGE_REPORTING_CAPACITY; + unsigned long watermark; + int err = 0; + + /* Generate minimum watermark to be able to guarantee progress */ + watermark = low_wmark_pages(zone) + + (PAGE_REPORTING_CAPACITY << PAGE_REPORTING_MIN_ORDER); + + /* + * Cancel request if insufficient free memory or if we failed + * to allocate page reporting statistics for the zone. + */ + if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA)) + return err; + + /* Process each free list starting from lowest order/mt */ + for (order = PAGE_REPORTING_MIN_ORDER; order < MAX_ORDER; order++) { + for (mt = 0; mt < MIGRATE_TYPES; mt++) { + /* We do not pull pages from the isolate free list */ + if (is_migrate_isolate(mt)) + continue; + + err = page_reporting_cycle(prdev, zone, order, mt, + sgl, &offset); + if (err) + return err; + } + } + + /* report the leftover pages before going idle */ + leftover = PAGE_REPORTING_CAPACITY - offset; + if (leftover) { + sgl = &sgl[offset]; + err = prdev->report(prdev, sgl, leftover); + + /* flush any remaining pages out from the last report */ + spin_lock_irq(&zone->lock); + page_reporting_drain(prdev, sgl, leftover, !err); + spin_unlock_irq(&zone->lock); + } + + return err; +} + +static void page_reporting_process(struct work_struct *work) +{ + struct delayed_work *d_work = to_delayed_work(work); + struct page_reporting_dev_info *prdev = + container_of(d_work, struct page_reporting_dev_info, work); + int err = 0, state = PAGE_REPORTING_ACTIVE; + struct scatterlist *sgl; + struct zone *zone; + + /* + * Change the state to "Active" so that we can track if there is + * anyone requests page reporting after we complete our pass. If + * the state is not altered by the end of the pass we will switch + * to idle and quit scheduling reporting runs. + */ + atomic_set(&prdev->state, state); + + /* allocate scatterlist to store pages being reported on */ + sgl = kmalloc_array(PAGE_REPORTING_CAPACITY, sizeof(*sgl), GFP_KERNEL); + if (!sgl) + goto err_out; + + sg_init_table(sgl, PAGE_REPORTING_CAPACITY); + + for_each_zone(zone) { + err = page_reporting_process_zone(prdev, sgl, zone); + if (err) + break; + } + + kfree(sgl); +err_out: + /* + * If the state has reverted back to requested then there may be + * additional pages to be processed. We will defer for 2s to allow + * more pages to accumulate. + */ + state = atomic_cmpxchg(&prdev->state, state, PAGE_REPORTING_IDLE); + if (state == PAGE_REPORTING_REQUESTED) + schedule_delayed_work(&prdev->work, PAGE_REPORTING_DELAY); +} + +static DEFINE_MUTEX(page_reporting_mutex); +DEFINE_STATIC_KEY_FALSE(page_reporting_enabled); + +int page_reporting_register(struct page_reporting_dev_info *prdev) +{ + int err = 0; + + mutex_lock(&page_reporting_mutex); + + /* nothing to do if already in use */ + if (rcu_access_pointer(pr_dev_info)) { + err = -EBUSY; + goto err_out; + } + + /* initialize state and work structures */ + atomic_set(&prdev->state, PAGE_REPORTING_IDLE); + INIT_DELAYED_WORK(&prdev->work, &page_reporting_process); + + /* Begin initial flush of zones */ + __page_reporting_request(prdev); + + /* Assign device to allow notifications */ + rcu_assign_pointer(pr_dev_info, prdev); + + /* enable page reporting notification */ + if (!static_key_enabled(&page_reporting_enabled)) { + static_branch_enable(&page_reporting_enabled); + pr_info("Free page reporting enabled\n"); + } +err_out: + mutex_unlock(&page_reporting_mutex); + + return err; +} +EXPORT_SYMBOL_GPL(page_reporting_register); + +void page_reporting_unregister(struct page_reporting_dev_info *prdev) +{ + mutex_lock(&page_reporting_mutex); + + if (rcu_access_pointer(pr_dev_info) == prdev) { + /* Disable page reporting notification */ + RCU_INIT_POINTER(pr_dev_info, NULL); + synchronize_rcu(); + + /* Flush any existing work, and lock it out */ + cancel_delayed_work_sync(&prdev->work); + } + + mutex_unlock(&page_reporting_mutex); +} +EXPORT_SYMBOL_GPL(page_reporting_unregister); diff --git a/mm/page_reporting.h b/mm/page_reporting.h new file mode 100644 index 000000000000..aa6d37f4dc22 --- /dev/null +++ b/mm/page_reporting.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _MM_PAGE_REPORTING_H +#define _MM_PAGE_REPORTING_H + +#include +#include +#include +#include +#include +#include +#include + +#define PAGE_REPORTING_MIN_ORDER pageblock_order + +#ifdef CONFIG_PAGE_REPORTING +DECLARE_STATIC_KEY_FALSE(page_reporting_enabled); +void __page_reporting_notify(void); + +static inline bool page_reported(struct page *page) +{ + return static_branch_unlikely(&page_reporting_enabled) && + PageReported(page); +} + +/** + * page_reporting_notify_free - Free page notification to start page processing + * + * This function is meant to act as a screener for __page_reporting_notify + * which will determine if a give zone has crossed over the high-water mark + * that will justify us beginning page treatment. If we have crossed that + * threshold then it will start the process of pulling some pages and + * placing them in the batch list for treatment. + */ +static inline void page_reporting_notify_free(unsigned int order) +{ + /* Called from hot path in __free_one_page() */ + if (!static_branch_unlikely(&page_reporting_enabled)) + return; + + /* Determine if we have crossed reporting threshold */ + if (order < PAGE_REPORTING_MIN_ORDER) + return; + + /* This will add a few cycles, but should be called infrequently */ + __page_reporting_notify(); +} +#else /* CONFIG_PAGE_REPORTING */ +#define page_reported(_page) false + +static inline void page_reporting_notify_free(unsigned int order) +{ +} +#endif /* CONFIG_PAGE_REPORTING */ +#endif /*_MM_PAGE_REPORTING_H */ From d74b78fabe043158e26196291d2e439b487395bd Mon Sep 17 00:00:00 2001 From: Alexander Duyck Date: Mon, 6 Apr 2020 20:05:01 -0700 Subject: [PATCH 499/635] virtio-balloon: pull page poisoning config out of free page hinting Currently the page poisoning setting wasn't being enabled unless free page hinting was enabled. However we will need the page poisoning tracking logic as well for free page reporting. As such pull it out and make it a separate bit of config in the probe function. In addition we need to add support for the more recent init_on_free feature which expects a behavior similar to page poisoning in that we expect the page to be pre-zeroed. Signed-off-by: Alexander Duyck Signed-off-by: Andrew Morton Reviewed-by: David Hildenbrand Acked-by: Michael S. Tsirkin Cc: Andrea Arcangeli Cc: Dan Williams Cc: Dave Hansen Cc: Konrad Rzeszutek Wilk Cc: Luiz Capitulino Cc: Matthew Wilcox Cc: Mel Gorman Cc: Michal Hocko Cc: Nitesh Narayan Lal Cc: Oscar Salvador Cc: Pankaj Gupta Cc: Paolo Bonzini Cc: Rik van Riel Cc: Vlastimil Babka Cc: Wei Wang Cc: Yang Zhang Cc: wei qi Link: http://lkml.kernel.org/r/20200211224646.29318.695.stgit@localhost.localdomain Signed-off-by: Linus Torvalds --- drivers/virtio/virtio_balloon.c | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c index 341458fd95ca..e41a62d84260 100644 --- a/drivers/virtio/virtio_balloon.c +++ b/drivers/virtio/virtio_balloon.c @@ -864,7 +864,6 @@ static int virtio_balloon_register_shrinker(struct virtio_balloon *vb) static int virtballoon_probe(struct virtio_device *vdev) { struct virtio_balloon *vb; - __u32 poison_val; int err; if (!vdev->config->get) { @@ -930,11 +929,20 @@ static int virtballoon_probe(struct virtio_device *vdev) VIRTIO_BALLOON_CMD_ID_STOP); spin_lock_init(&vb->free_page_list_lock); INIT_LIST_HEAD(&vb->free_page_list); - if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_PAGE_POISON)) { + } + if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_PAGE_POISON)) { + /* Start with poison val of 0 representing general init */ + __u32 poison_val = 0; + + /* + * Let the hypervisor know that we are expecting a + * specific value to be written back in balloon pages. + */ + if (!want_init_on_free()) memset(&poison_val, PAGE_POISON, sizeof(poison_val)); - virtio_cwrite(vb->vdev, struct virtio_balloon_config, - poison_val, &poison_val); - } + + virtio_cwrite(vb->vdev, struct virtio_balloon_config, + poison_val, &poison_val); } /* * We continue to use VIRTIO_BALLOON_F_DEFLATE_ON_OOM to decide if a @@ -1045,7 +1053,10 @@ static int virtballoon_restore(struct virtio_device *vdev) static int virtballoon_validate(struct virtio_device *vdev) { - if (!page_poisoning_enabled()) + /* Tell the host whether we care about poisoned pages. */ + if (!want_init_on_free() && + (IS_ENABLED(CONFIG_PAGE_POISONING_NO_SANITY) || + !page_poisoning_enabled())) __virtio_clear_bit(vdev, VIRTIO_BALLOON_F_PAGE_POISON); __virtio_clear_bit(vdev, VIRTIO_F_IOMMU_PLATFORM); From b0c504f154718904ae49349147e3b7e6ae91ffdc Mon Sep 17 00:00:00 2001 From: Alexander Duyck Date: Mon, 6 Apr 2020 20:05:05 -0700 Subject: [PATCH 500/635] virtio-balloon: add support for providing free page reports to host Add support for the page reporting feature provided by virtio-balloon. Reporting differs from the regular balloon functionality in that is is much less durable than a standard memory balloon. Instead of creating a list of pages that cannot be accessed the pages are only inaccessible while they are being indicated to the virtio interface. Once the interface has acknowledged them they are placed back into their respective free lists and are once again accessible by the guest system. Unlike a standard balloon we don't inflate and deflate the pages. Instead we perform the reporting, and once the reporting is completed it is assumed that the page has been dropped from the guest and will be faulted back in the next time the page is accessed. Signed-off-by: Alexander Duyck Signed-off-by: Andrew Morton Reviewed-by: David Hildenbrand Acked-by: Michael S. Tsirkin Cc: Andrea Arcangeli Cc: Dan Williams Cc: Dave Hansen Cc: Konrad Rzeszutek Wilk Cc: Luiz Capitulino Cc: Matthew Wilcox Cc: Mel Gorman Cc: Michal Hocko Cc: Nitesh Narayan Lal Cc: Oscar Salvador Cc: Pankaj Gupta Cc: Paolo Bonzini Cc: Rik van Riel Cc: Vlastimil Babka Cc: Wei Wang Cc: Yang Zhang Cc: wei qi Link: http://lkml.kernel.org/r/20200211224657.29318.68624.stgit@localhost.localdomain Signed-off-by: Linus Torvalds --- drivers/virtio/Kconfig | 1 + drivers/virtio/virtio_balloon.c | 64 +++++++++++++++++++++++++++++ include/uapi/linux/virtio_balloon.h | 1 + 3 files changed, 66 insertions(+) diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig index 078615cf2afc..4b2dd8259ff5 100644 --- a/drivers/virtio/Kconfig +++ b/drivers/virtio/Kconfig @@ -58,6 +58,7 @@ config VIRTIO_BALLOON tristate "Virtio balloon driver" depends on VIRTIO select MEMORY_BALLOON + select PAGE_REPORTING ---help--- This driver supports increasing and decreasing the amount of memory within a KVM guest. diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c index e41a62d84260..9e0177529bad 100644 --- a/drivers/virtio/virtio_balloon.c +++ b/drivers/virtio/virtio_balloon.c @@ -19,6 +19,7 @@ #include #include #include +#include /* * Balloon device works in 4K page units. So each page is pointed to by @@ -47,6 +48,7 @@ enum virtio_balloon_vq { VIRTIO_BALLOON_VQ_DEFLATE, VIRTIO_BALLOON_VQ_STATS, VIRTIO_BALLOON_VQ_FREE_PAGE, + VIRTIO_BALLOON_VQ_REPORTING, VIRTIO_BALLOON_VQ_MAX }; @@ -114,6 +116,10 @@ struct virtio_balloon { /* To register a shrinker to shrink memory upon memory pressure */ struct shrinker shrinker; + + /* Free page reporting device */ + struct virtqueue *reporting_vq; + struct page_reporting_dev_info pr_dev_info; }; static struct virtio_device_id id_table[] = { @@ -153,6 +159,33 @@ static void tell_host(struct virtio_balloon *vb, struct virtqueue *vq) } +int virtballoon_free_page_report(struct page_reporting_dev_info *pr_dev_info, + struct scatterlist *sg, unsigned int nents) +{ + struct virtio_balloon *vb = + container_of(pr_dev_info, struct virtio_balloon, pr_dev_info); + struct virtqueue *vq = vb->reporting_vq; + unsigned int unused, err; + + /* We should always be able to add these buffers to an empty queue. */ + err = virtqueue_add_inbuf(vq, sg, nents, vb, GFP_NOWAIT | __GFP_NOWARN); + + /* + * In the extremely unlikely case that something has occurred and we + * are able to trigger an error we will simply display a warning + * and exit without actually processing the pages. + */ + if (WARN_ON_ONCE(err)) + return err; + + virtqueue_kick(vq); + + /* When host has read buffer, this completes via balloon_ack */ + wait_event(vb->acked, virtqueue_get_buf(vq, &unused)); + + return 0; +} + static void set_page_pfns(struct virtio_balloon *vb, __virtio32 pfns[], struct page *page) { @@ -481,6 +514,7 @@ static int init_vqs(struct virtio_balloon *vb) names[VIRTIO_BALLOON_VQ_STATS] = NULL; callbacks[VIRTIO_BALLOON_VQ_FREE_PAGE] = NULL; names[VIRTIO_BALLOON_VQ_FREE_PAGE] = NULL; + names[VIRTIO_BALLOON_VQ_REPORTING] = NULL; if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) { names[VIRTIO_BALLOON_VQ_STATS] = "stats"; @@ -492,6 +526,11 @@ static int init_vqs(struct virtio_balloon *vb) callbacks[VIRTIO_BALLOON_VQ_FREE_PAGE] = NULL; } + if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING)) { + names[VIRTIO_BALLOON_VQ_REPORTING] = "reporting_vq"; + callbacks[VIRTIO_BALLOON_VQ_REPORTING] = balloon_ack; + } + err = vb->vdev->config->find_vqs(vb->vdev, VIRTIO_BALLOON_VQ_MAX, vqs, callbacks, names, NULL, NULL); if (err) @@ -524,6 +563,9 @@ static int init_vqs(struct virtio_balloon *vb) if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) vb->free_page_vq = vqs[VIRTIO_BALLOON_VQ_FREE_PAGE]; + if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING)) + vb->reporting_vq = vqs[VIRTIO_BALLOON_VQ_REPORTING]; + return 0; } @@ -953,12 +995,31 @@ static int virtballoon_probe(struct virtio_device *vdev) if (err) goto out_del_balloon_wq; } + + vb->pr_dev_info.report = virtballoon_free_page_report; + if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING)) { + unsigned int capacity; + + capacity = virtqueue_get_vring_size(vb->reporting_vq); + if (capacity < PAGE_REPORTING_CAPACITY) { + err = -ENOSPC; + goto out_unregister_shrinker; + } + + err = page_reporting_register(&vb->pr_dev_info); + if (err) + goto out_unregister_shrinker; + } + virtio_device_ready(vdev); if (towards_target(vb)) virtballoon_changed(vdev); return 0; +out_unregister_shrinker: + if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) + virtio_balloon_unregister_shrinker(vb); out_del_balloon_wq: if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) destroy_workqueue(vb->balloon_wq); @@ -997,6 +1058,8 @@ static void virtballoon_remove(struct virtio_device *vdev) { struct virtio_balloon *vb = vdev->priv; + if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING)) + page_reporting_unregister(&vb->pr_dev_info); if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) virtio_balloon_unregister_shrinker(vb); spin_lock_irq(&vb->stop_update_lock); @@ -1069,6 +1132,7 @@ static unsigned int features[] = { VIRTIO_BALLOON_F_DEFLATE_ON_OOM, VIRTIO_BALLOON_F_FREE_PAGE_HINT, VIRTIO_BALLOON_F_PAGE_POISON, + VIRTIO_BALLOON_F_REPORTING, }; static struct virtio_driver virtio_balloon_driver = { diff --git a/include/uapi/linux/virtio_balloon.h b/include/uapi/linux/virtio_balloon.h index a1966cd7b677..19974392d324 100644 --- a/include/uapi/linux/virtio_balloon.h +++ b/include/uapi/linux/virtio_balloon.h @@ -36,6 +36,7 @@ #define VIRTIO_BALLOON_F_DEFLATE_ON_OOM 2 /* Deflate balloon on OOM */ #define VIRTIO_BALLOON_F_FREE_PAGE_HINT 3 /* VQ to report free pages */ #define VIRTIO_BALLOON_F_PAGE_POISON 4 /* Guest is using page poisoning */ +#define VIRTIO_BALLOON_F_REPORTING 5 /* Page reporting virtqueue */ /* Size of a PFN in the balloon interface. */ #define VIRTIO_BALLOON_PFN_SHIFT 12 From 02cf8719b8cb2b474b37bcbeee4706950b3d1d3d Mon Sep 17 00:00:00 2001 From: Alexander Duyck Date: Mon, 6 Apr 2020 20:05:10 -0700 Subject: [PATCH 501/635] mm/page_reporting: rotate reported pages to the tail of the list Rather than walking over the same pages again and again to get to the pages that have yet to be reported we can save ourselves a significant amount of time by simply rotating the list so that when we have a full list of reported pages the head of the list is pointing to the next non-reported page. Doing this should save us some significant time when processing each free list. This doesn't gain us much in the standard case as all of the non-reported pages should be near the top of the list already. However in the case of page shuffling this results in a noticeable improvement. Below are the will-it-scale page_fault1 w/ THP numbers for 16 tasks with and without this patch. Without: tasks processes processes_idle threads threads_idle 16 8093776.25 0.17 5393242.00 38.20 With: tasks processes processes_idle threads threads_idle 16 8283274.75 0.17 5594261.00 38.15 Signed-off-by: Alexander Duyck Signed-off-by: Andrew Morton Acked-by: Mel Gorman Cc: Andrea Arcangeli Cc: Dan Williams Cc: Dave Hansen Cc: David Hildenbrand Cc: Konrad Rzeszutek Wilk Cc: Luiz Capitulino Cc: Matthew Wilcox Cc: Michael S. Tsirkin Cc: Michal Hocko Cc: Nitesh Narayan Lal Cc: Oscar Salvador Cc: Pankaj Gupta Cc: Paolo Bonzini Cc: Rik van Riel Cc: Vlastimil Babka Cc: Wei Wang Cc: Yang Zhang Cc: wei qi Link: http://lkml.kernel.org/r/20200211224708.29318.16862.stgit@localhost.localdomain Signed-off-by: Linus Torvalds --- mm/page_reporting.c | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/mm/page_reporting.c b/mm/page_reporting.c index 1047c6872d4f..6885e74c2367 100644 --- a/mm/page_reporting.c +++ b/mm/page_reporting.c @@ -131,17 +131,27 @@ page_reporting_cycle(struct page_reporting_dev_info *prdev, struct zone *zone, if (PageReported(page)) continue; - /* Attempt to pull page from list */ - if (!__isolate_free_page(page, order)) - break; + /* Attempt to pull page from list and place in scatterlist */ + if (*offset) { + if (!__isolate_free_page(page, order)) { + next = page; + break; + } - /* Add page to scatter list */ - --(*offset); - sg_set_page(&sgl[*offset], page, page_len, 0); + /* Add page to scatter list */ + --(*offset); + sg_set_page(&sgl[*offset], page, page_len, 0); - /* If scatterlist isn't full grab more pages */ - if (*offset) continue; + } + + /* + * Make the first non-processed page in the free list + * the new head of the free list before we release the + * zone lock. + */ + if (&page->lru != list && !list_is_first(&page->lru, list)) + list_rotate_to_front(&page->lru, list); /* release lock before waiting on report processing */ spin_unlock_irq(&zone->lock); @@ -169,6 +179,10 @@ page_reporting_cycle(struct page_reporting_dev_info *prdev, struct zone *zone, break; } + /* Rotate any leftover pages to the head of the freelist */ + if (&next->lru != list && !list_is_first(&next->lru, list)) + list_rotate_to_front(&next->lru, list); + spin_unlock_irq(&zone->lock); return err; From 43b76f298f023d32273c2b9c25dd83ae02711019 Mon Sep 17 00:00:00 2001 From: Alexander Duyck Date: Mon, 6 Apr 2020 20:05:14 -0700 Subject: [PATCH 502/635] mm/page_reporting: add budget limit on how many pages can be reported per pass In order to keep ourselves from reporting pages that are just going to be reused again in the case of heavy churn we can put a limit on how many total pages we will process per pass. Doing this will allow the worker thread to go into idle much more quickly so that we avoid competing with other threads that might be allocating or freeing pages. The logic added here will limit the worker thread to no more than one sixteenth of the total free pages in a given area per list. Once that limit is reached it will update the state so that at the end of the pass we will reschedule the worker to try again in 2 seconds when the memory churn has hopefully settled down. Again this optimization doesn't show much of a benefit in the standard case as the memory churn is minmal. However with page allocator shuffling enabled the gain is quite noticeable. Below are the results with a THP enabled version of the will-it-scale page_fault1 test showing the improvement in iterations for 16 processes or threads. Without: tasks processes processes_idle threads threads_idle 16 8283274.75 0.17 5594261.00 38.15 With: tasks processes processes_idle threads threads_idle 16 8767010.50 0.21 5791312.75 36.98 Signed-off-by: Alexander Duyck Signed-off-by: Andrew Morton Acked-by: Mel Gorman Cc: Andrea Arcangeli Cc: Dan Williams Cc: Dave Hansen Cc: David Hildenbrand Cc: Konrad Rzeszutek Wilk Cc: Luiz Capitulino Cc: Matthew Wilcox Cc: Michael S. Tsirkin Cc: Michal Hocko Cc: Nitesh Narayan Lal Cc: Oscar Salvador Cc: Pankaj Gupta Cc: Paolo Bonzini Cc: Rik van Riel Cc: Vlastimil Babka Cc: Wei Wang Cc: Yang Zhang Cc: wei qi Link: http://lkml.kernel.org/r/20200211224719.29318.72113.stgit@localhost.localdomain Signed-off-by: Linus Torvalds --- include/linux/page_reporting.h | 1 + mm/page_reporting.c | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/include/linux/page_reporting.h b/include/linux/page_reporting.h index 32355486f572..3b99e0ec24f2 100644 --- a/include/linux/page_reporting.h +++ b/include/linux/page_reporting.h @@ -5,6 +5,7 @@ #include #include +/* This value should always be a power of 2, see page_reporting_cycle() */ #define PAGE_REPORTING_CAPACITY 32 struct page_reporting_dev_info { diff --git a/mm/page_reporting.c b/mm/page_reporting.c index 6885e74c2367..3bbd471cfc81 100644 --- a/mm/page_reporting.c +++ b/mm/page_reporting.c @@ -114,6 +114,7 @@ page_reporting_cycle(struct page_reporting_dev_info *prdev, struct zone *zone, struct list_head *list = &area->free_list[mt]; unsigned int page_len = PAGE_SIZE << order; struct page *page, *next; + long budget; int err = 0; /* @@ -125,12 +126,39 @@ page_reporting_cycle(struct page_reporting_dev_info *prdev, struct zone *zone, spin_lock_irq(&zone->lock); + /* + * Limit how many calls we will be making to the page reporting + * device for this list. By doing this we avoid processing any + * given list for too long. + * + * The current value used allows us enough calls to process over a + * sixteenth of the current list plus one additional call to handle + * any pages that may have already been present from the previous + * list processed. This should result in us reporting all pages on + * an idle system in about 30 seconds. + * + * The division here should be cheap since PAGE_REPORTING_CAPACITY + * should always be a power of 2. + */ + budget = DIV_ROUND_UP(area->nr_free, PAGE_REPORTING_CAPACITY * 16); + /* loop through free list adding unreported pages to sg list */ list_for_each_entry_safe(page, next, list, lru) { /* We are going to skip over the reported pages. */ if (PageReported(page)) continue; + /* + * If we fully consumed our budget then update our + * state to indicate that we are requesting additional + * processing and exit this list. + */ + if (budget < 0) { + atomic_set(&prdev->state, PAGE_REPORTING_REQUESTED); + next = page; + break; + } + /* Attempt to pull page from list and place in scatterlist */ if (*offset) { if (!__isolate_free_page(page, order)) { @@ -146,7 +174,7 @@ page_reporting_cycle(struct page_reporting_dev_info *prdev, struct zone *zone, } /* - * Make the first non-processed page in the free list + * Make the first non-reported page in the free list * the new head of the free list before we release the * zone lock. */ @@ -162,6 +190,9 @@ page_reporting_cycle(struct page_reporting_dev_info *prdev, struct zone *zone, /* reset offset since the full list was reported */ *offset = PAGE_REPORTING_CAPACITY; + /* update budget to reflect call to report function */ + budget--; + /* reacquire zone lock and resume processing */ spin_lock_irq(&zone->lock); From 1edca85e768a0819c1bed6dc0adcfa766e28e2cd Mon Sep 17 00:00:00 2001 From: Alexander Duyck Date: Mon, 6 Apr 2020 20:05:18 -0700 Subject: [PATCH 503/635] mm/page_reporting: add free page reporting documentation Add documentation for free page reporting. Currently the only consumer is virtio-balloon, however it is possible that other drivers might make use of this so it is best to add a bit of documetation explaining at a high level how to use the API. Signed-off-by: Alexander Duyck Signed-off-by: Andrew Morton Cc: Andrea Arcangeli Cc: Dan Williams Cc: Dave Hansen Cc: David Hildenbrand Cc: Konrad Rzeszutek Wilk Cc: Luiz Capitulino Cc: Matthew Wilcox Cc: Mel Gorman Cc: Michael S. Tsirkin Cc: Michal Hocko Cc: Nitesh Narayan Lal Cc: Oscar Salvador Cc: Pankaj Gupta Cc: Paolo Bonzini Cc: Rik van Riel Cc: Vlastimil Babka Cc: Wei Wang Cc: Yang Zhang Cc: wei qi Link: http://lkml.kernel.org/r/20200211224730.29318.43815.stgit@localhost.localdomain Signed-off-by: Linus Torvalds --- Documentation/vm/free_page_reporting.rst | 40 ++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 Documentation/vm/free_page_reporting.rst diff --git a/Documentation/vm/free_page_reporting.rst b/Documentation/vm/free_page_reporting.rst new file mode 100644 index 000000000000..8c05e62d8b2b --- /dev/null +++ b/Documentation/vm/free_page_reporting.rst @@ -0,0 +1,40 @@ +.. _free_page_reporting: + +===================== +Free Page Reporting +===================== + +Free page reporting is an API by which a device can register to receive +lists of pages that are currently unused by the system. This is useful in +the case of virtualization where a guest is then able to use this data to +notify the hypervisor that it is no longer using certain pages in memory. + +For the driver, typically a balloon driver, to use of this functionality +it will allocate and initialize a page_reporting_dev_info structure. The +field within the structure it will populate is the "report" function +pointer used to process the scatterlist. It must also guarantee that it can +handle at least PAGE_REPORTING_CAPACITY worth of scatterlist entries per +call to the function. A call to page_reporting_register will register the +page reporting interface with the reporting framework assuming no other +page reporting devices are already registered. + +Once registered the page reporting API will begin reporting batches of +pages to the driver. The API will start reporting pages 2 seconds after +the interface is registered and will continue to do so 2 seconds after any +page of a sufficiently high order is freed. + +Pages reported will be stored in the scatterlist passed to the reporting +function with the final entry having the end bit set in entry nent - 1. +While pages are being processed by the report function they will not be +accessible to the allocator. Once the report function has been completed +the pages will be returned to the free area from which they were obtained. + +Prior to removing a driver that is making use of free page reporting it +is necessary to call page_reporting_unregister to have the +page_reporting_dev_info structure that is currently in use by free page +reporting removed. Doing this will prevent further reports from being +issued via the interface. If another driver or the same driver is +registered it is possible for it to resume where the previous driver had +left off in terms of reporting free pages. + +Alexander Duyck, Dec 04, 2019 From da10329cb057e49be7f099eb7864d16f7705f798 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Mon, 6 Apr 2020 20:05:22 -0700 Subject: [PATCH 504/635] virtio-balloon: switch back to OOM handler for VIRTIO_BALLOON_F_DEFLATE_ON_OOM Commit 71994620bb25 ("virtio_balloon: replace oom notifier with shrinker") changed the behavior when deflation happens automatically. Instead of deflating when called by the OOM handler, the shrinker is used. However, the balloon is not simply some other slab cache that should be shrunk when under memory pressure. The shrinker does not have a concept of priorities yet, so this behavior cannot be configured. Eventually once that is in place, we might want to switch back after doing proper testing. There was a report that this results in undesired side effects when inflating the balloon to shrink the page cache. [1] "When inflating the balloon against page cache (i.e. no free memory remains) vmscan.c will both shrink page cache, but also invoke the shrinkers -- including the balloon's shrinker. So the balloon driver allocates memory which requires reclaim, vmscan gets this memory by shrinking the balloon, and then the driver adds the memory back to the balloon. Basically a busy no-op." The name "deflate on OOM" makes it pretty clear when deflation should happen - after other approaches to reclaim memory failed, not while reclaiming. This allows to minimize the footprint of a guest - memory will only be taken out of the balloon when really needed. Keep using the shrinker for VIRTIO_BALLOON_F_FREE_PAGE_HINT, because this has no such side effects. Always register the shrinker with VIRTIO_BALLOON_F_FREE_PAGE_HINT now. We are always allowed to reuse free pages that are still to be processed by the guest. The hypervisor takes care of identifying and resolving possible races between processing a hinting request and the guest reusing a page. In contrast to pre commit 71994620bb25 ("virtio_balloon: replace oom notifier with shrinker"), don't add a module parameter to configure the number of pages to deflate on OOM. Can be re-added if really needed. Also, pay attention that leak_balloon() returns the number of 4k pages - convert it properly in virtio_balloon_oom_notify(). Testing done by Tyler for future reference: Test setup: VM with 16 CPU, 64GB RAM. Running Debian 10. We have a 42 GB file full of random bytes that we continually cat to /dev/null. This fills the page cache as the file is read. Meanwhile, we trigger the balloon to inflate, with a target size of 53 GB. This setup causes the balloon inflation to pressure the page cache as the page cache is also trying to grow. Afterwards we shrink the balloon back to zero (so total deflate == total inflate). Without this patch (kernel 4.19.0-5): Inflation never reaches the target until we stop the "cat file > /dev/null" process. Total inflation time was 542 seconds. The longest period that made no net forward progress was 315 seconds. Result of "grep balloon /proc/vmstat" after the test: balloon_inflate 154828377 balloon_deflate 154828377 With this patch (kernel 5.6.0-rc4+): Total inflation duration was 63 seconds. No deflate-queue activity occurs when pressuring the page-cache. Result of "grep balloon /proc/vmstat" after the test: balloon_inflate 12968539 balloon_deflate 12968539 Conclusion: This patch fixes the issue. In the test it reduced inflate/deflate activity by 12x, and reduced inflation time by 8.6x. But more importantly, if we hadn't killed the "cat file > /dev/null" process then, without the patch, the inflation process would never reach the target. [1] https://www.spinics.net/lists/linux-virtualization/msg40863.html Link: http://lkml.kernel.org/r/20200311135523.18512-2-david@redhat.com Fixes: 71994620bb25 ("virtio_balloon: replace oom notifier with shrinker") Signed-off-by: David Hildenbrand Reported-by: Tyler Sanderson Tested-by: Tyler Sanderson Acked-by: David Rientjes Acked-by: Michael S. Tsirkin Cc: Wei Wang Cc: Alexander Duyck Cc: David Rientjes Cc: Nadav Amit Cc: Michal Hocko Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/virtio/virtio_balloon.c | 103 +++++++++++++++----------------- 1 file changed, 47 insertions(+), 56 deletions(-) diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c index 9e0177529bad..0ef16566c3f3 100644 --- a/drivers/virtio/virtio_balloon.c +++ b/drivers/virtio/virtio_balloon.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -28,7 +29,9 @@ */ #define VIRTIO_BALLOON_PAGES_PER_PAGE (unsigned)(PAGE_SIZE >> VIRTIO_BALLOON_PFN_SHIFT) #define VIRTIO_BALLOON_ARRAY_PFNS_MAX 256 -#define VIRTBALLOON_OOM_NOTIFY_PRIORITY 80 +/* Maximum number of (4k) pages to deflate on OOM notifications. */ +#define VIRTIO_BALLOON_OOM_NR_PAGES 256 +#define VIRTIO_BALLOON_OOM_NOTIFY_PRIORITY 80 #define VIRTIO_BALLOON_FREE_PAGE_ALLOC_FLAG (__GFP_NORETRY | __GFP_NOWARN | \ __GFP_NOMEMALLOC) @@ -114,9 +117,12 @@ struct virtio_balloon { /* Memory statistics */ struct virtio_balloon_stat stats[VIRTIO_BALLOON_S_NR]; - /* To register a shrinker to shrink memory upon memory pressure */ + /* Shrinker to return free pages - VIRTIO_BALLOON_F_FREE_PAGE_HINT */ struct shrinker shrinker; + /* OOM notifier to deflate on OOM - VIRTIO_BALLOON_F_DEFLATE_ON_OOM */ + struct notifier_block oom_nb; + /* Free page reporting device */ struct virtqueue *reporting_vq; struct page_reporting_dev_info pr_dev_info; @@ -830,50 +836,13 @@ static unsigned long shrink_free_pages(struct virtio_balloon *vb, return blocks_freed * VIRTIO_BALLOON_HINT_BLOCK_PAGES; } -static unsigned long leak_balloon_pages(struct virtio_balloon *vb, - unsigned long pages_to_free) -{ - return leak_balloon(vb, pages_to_free * VIRTIO_BALLOON_PAGES_PER_PAGE) / - VIRTIO_BALLOON_PAGES_PER_PAGE; -} - -static unsigned long shrink_balloon_pages(struct virtio_balloon *vb, - unsigned long pages_to_free) -{ - unsigned long pages_freed = 0; - - /* - * One invocation of leak_balloon can deflate at most - * VIRTIO_BALLOON_ARRAY_PFNS_MAX balloon pages, so we call it - * multiple times to deflate pages till reaching pages_to_free. - */ - while (vb->num_pages && pages_freed < pages_to_free) - pages_freed += leak_balloon_pages(vb, - pages_to_free - pages_freed); - - update_balloon_size(vb); - - return pages_freed; -} - static unsigned long virtio_balloon_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) { - unsigned long pages_to_free, pages_freed = 0; struct virtio_balloon *vb = container_of(shrinker, struct virtio_balloon, shrinker); - pages_to_free = sc->nr_to_scan; - - if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) - pages_freed = shrink_free_pages(vb, pages_to_free); - - if (pages_freed >= pages_to_free) - return pages_freed; - - pages_freed += shrink_balloon_pages(vb, pages_to_free - pages_freed); - - return pages_freed; + return shrink_free_pages(vb, sc->nr_to_scan); } static unsigned long virtio_balloon_shrinker_count(struct shrinker *shrinker, @@ -881,12 +850,22 @@ static unsigned long virtio_balloon_shrinker_count(struct shrinker *shrinker, { struct virtio_balloon *vb = container_of(shrinker, struct virtio_balloon, shrinker); - unsigned long count; - count = vb->num_pages / VIRTIO_BALLOON_PAGES_PER_PAGE; - count += vb->num_free_page_blocks * VIRTIO_BALLOON_HINT_BLOCK_PAGES; + return vb->num_free_page_blocks * VIRTIO_BALLOON_HINT_BLOCK_PAGES; +} - return count; +static int virtio_balloon_oom_notify(struct notifier_block *nb, + unsigned long dummy, void *parm) +{ + struct virtio_balloon *vb = container_of(nb, + struct virtio_balloon, oom_nb); + unsigned long *freed = parm; + + *freed += leak_balloon(vb, VIRTIO_BALLOON_OOM_NR_PAGES) / + VIRTIO_BALLOON_PAGES_PER_PAGE; + update_balloon_size(vb); + + return NOTIFY_OK; } static void virtio_balloon_unregister_shrinker(struct virtio_balloon *vb) @@ -971,7 +950,23 @@ static int virtballoon_probe(struct virtio_device *vdev) VIRTIO_BALLOON_CMD_ID_STOP); spin_lock_init(&vb->free_page_list_lock); INIT_LIST_HEAD(&vb->free_page_list); + /* + * We're allowed to reuse any free pages, even if they are + * still to be processed by the host. + */ + err = virtio_balloon_register_shrinker(vb); + if (err) + goto out_del_balloon_wq; } + + if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) { + vb->oom_nb.notifier_call = virtio_balloon_oom_notify; + vb->oom_nb.priority = VIRTIO_BALLOON_OOM_NOTIFY_PRIORITY; + err = register_oom_notifier(&vb->oom_nb); + if (err < 0) + goto out_unregister_shrinker; + } + if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_PAGE_POISON)) { /* Start with poison val of 0 representing general init */ __u32 poison_val = 0; @@ -986,15 +981,6 @@ static int virtballoon_probe(struct virtio_device *vdev) virtio_cwrite(vb->vdev, struct virtio_balloon_config, poison_val, &poison_val); } - /* - * We continue to use VIRTIO_BALLOON_F_DEFLATE_ON_OOM to decide if a - * shrinker needs to be registered to relieve memory pressure. - */ - if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) { - err = virtio_balloon_register_shrinker(vb); - if (err) - goto out_del_balloon_wq; - } vb->pr_dev_info.report = virtballoon_free_page_report; if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING)) { @@ -1003,12 +989,12 @@ static int virtballoon_probe(struct virtio_device *vdev) capacity = virtqueue_get_vring_size(vb->reporting_vq); if (capacity < PAGE_REPORTING_CAPACITY) { err = -ENOSPC; - goto out_unregister_shrinker; + goto out_unregister_oom; } err = page_reporting_register(&vb->pr_dev_info); if (err) - goto out_unregister_shrinker; + goto out_unregister_oom; } virtio_device_ready(vdev); @@ -1017,8 +1003,11 @@ static int virtballoon_probe(struct virtio_device *vdev) virtballoon_changed(vdev); return 0; -out_unregister_shrinker: +out_unregister_oom: if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) + unregister_oom_notifier(&vb->oom_nb); +out_unregister_shrinker: + if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) virtio_balloon_unregister_shrinker(vb); out_del_balloon_wq: if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) @@ -1061,6 +1050,8 @@ static void virtballoon_remove(struct virtio_device *vdev) if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING)) page_reporting_unregister(&vb->pr_dev_info); if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) + unregister_oom_notifier(&vb->oom_nb); + if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) virtio_balloon_unregister_shrinker(vb); spin_lock_irq(&vb->stop_update_lock); vb->stop_update = true; From 1df319e0b4dee11436fe2ab1a0d536d3fad7cfef Mon Sep 17 00:00:00 2001 From: Shaohua Li Date: Mon, 6 Apr 2020 20:05:25 -0700 Subject: [PATCH 505/635] userfaultfd: wp: add helper for writeprotect check Patch series "userfaultfd: write protection support", v6. Overview ======== The uffd-wp work was initialized by Shaohua Li [1], and later continued by Andrea [2]. This series is based upon Andrea's latest userfaultfd tree, and it is a continuous works from both Shaohua and Andrea. Many of the follow up ideas come from Andrea too. Besides the old MISSING register mode of userfaultfd, the new uffd-wp support provides another alternative register mode called UFFDIO_REGISTER_MODE_WP that can be used to listen to not only missing page faults but also write protection page faults, or even they can be registered together. At the same time, the new feature also provides a new userfaultfd ioctl called UFFDIO_WRITEPROTECT which allows the userspace to write protect a range or memory or fixup write permission of faulted pages. Please refer to the document patch "userfaultfd: wp: UFFDIO_REGISTER_MODE_WP documentation update" for more information on the new interface and what it can do. The major workflow of an uffd-wp program should be: 1. Register a memory region with WP mode using UFFDIO_REGISTER_MODE_WP 2. Write protect part of the whole registered region using UFFDIO_WRITEPROTECT, passing in UFFDIO_WRITEPROTECT_MODE_WP to show that we want to write protect the range. 3. Start a working thread that modifies the protected pages, meanwhile listening to UFFD messages. 4. When a write is detected upon the protected range, page fault happens, a UFFD message will be generated and reported to the page fault handling thread 5. The page fault handler thread resolves the page fault using the new UFFDIO_WRITEPROTECT ioctl, but this time passing in !UFFDIO_WRITEPROTECT_MODE_WP instead showing that we want to recover the write permission. Before this operation, the fault handler thread can do anything it wants, e.g., dumps the page to a persistent storage. 6. The worker thread will continue running with the correctly applied write permission from step 5. Currently there are already two projects that are based on this new userfaultfd feature. QEMU Live Snapshot: The project provides a way to allow the QEMU hypervisor to take snapshot of VMs without stopping the VM [3]. LLNL umap library: The project provides a mmap-like interface and "allow to have an application specific buffer of pages cached from a large file, i.e. out-of-core execution using memory map" [4][5]. Before posting the patchset, this series was smoke tested against QEMU live snapshot and the LLNL umap library (by doing parallel quicksort using 128 sorting threads + 80 uffd servicing threads). My sincere thanks to Marty Mcfadden and Denis Plotnikov for the help along the way. TODO ==== - hugetlbfs/shmem support - performance - more architectures - cooperate with mprotect()-allowed processes (???) - ... References ========== [1] https://lwn.net/Articles/666187/ [2] https://git.kernel.org/pub/scm/linux/kernel/git/andrea/aa.git/log/?h=userfault [3] https://github.com/denis-plotnikov/qemu/commits/background-snapshot-kvm [4] https://github.com/LLNL/umap [5] https://llnl-umap.readthedocs.io/en/develop/ [6] https://git.kernel.org/pub/scm/linux/kernel/git/andrea/aa.git/commit/?h=userfault&id=b245ecf6cf59156966f3da6e6b674f6695a5ffa5 [7] https://lkml.org/lkml/2018/11/21/370 [8] https://lkml.org/lkml/2018/12/30/64 This patch (of 19): Add helper for writeprotect check. Will use it later. Signed-off-by: Shaohua Li Signed-off-by: Andrea Arcangeli Signed-off-by: Peter Xu Signed-off-by: Andrew Morton Reviewed-by: Jerome Glisse Reviewed-by: Mike Rapoport Cc: Rik van Riel Cc: Kirill A. Shutemov Cc: Mel Gorman Cc: Hugh Dickins Cc: Johannes Weiner Cc: Bobby Powers Cc: Brian Geffon Cc: David Hildenbrand Cc: Denis Plotnikov Cc: "Dr . David Alan Gilbert" Cc: Martin Cracauer Cc: Marty McFadden Cc: Maya Gokhale Cc: Mike Kravetz Cc: Pavel Emelyanov Link: http://lkml.kernel.org/r/20200220163112.11409-2-peterx@redhat.com Signed-off-by: Linus Torvalds --- include/linux/userfaultfd_k.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index ac9d71e24b81..5dc247af0f2e 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -52,6 +52,11 @@ static inline bool userfaultfd_missing(struct vm_area_struct *vma) return vma->vm_flags & VM_UFFD_MISSING; } +static inline bool userfaultfd_wp(struct vm_area_struct *vma) +{ + return vma->vm_flags & VM_UFFD_WP; +} + static inline bool userfaultfd_armed(struct vm_area_struct *vma) { return vma->vm_flags & (VM_UFFD_MISSING | VM_UFFD_WP); @@ -96,6 +101,11 @@ static inline bool userfaultfd_missing(struct vm_area_struct *vma) return false; } +static inline bool userfaultfd_wp(struct vm_area_struct *vma) +{ + return false; +} + static inline bool userfaultfd_armed(struct vm_area_struct *vma) { return false; From 529b930b87d997c3d231c9d8638a0bf8db569d70 Mon Sep 17 00:00:00 2001 From: Andrea Arcangeli Date: Mon, 6 Apr 2020 20:05:29 -0700 Subject: [PATCH 506/635] userfaultfd: wp: hook userfault handler to write protection fault There are several cases write protection fault happens. It could be a write to zero page, swaped page or userfault write protected page. When the fault happens, there is no way to know if userfault write protect the page before. Here we just blindly issue a userfault notification for vma with VM_UFFD_WP regardless if app write protects it yet. Application should be ready to handle such wp fault. In the swapin case, always swapin as readonly. This will cause false positive userfaults. We need to decide later if to eliminate them with a flag like soft-dirty in the swap entry (see _PAGE_SWP_SOFT_DIRTY). hugetlbfs wouldn't need to worry about swapouts but and tmpfs would be handled by a swap entry bit like anonymous memory. The main problem with no easy solution to eliminate the false positives, will be if/when userfaultfd is extended to real filesystem pagecache. When the pagecache is freed by reclaim we can't leave the radix tree pinned if the inode and in turn the radix tree is reclaimed as well. The estimation is that full accuracy and lack of false positives could be easily provided only to anonymous memory (as long as there's no fork or as long as MADV_DONTFORK is used on the userfaultfd anonymous range) tmpfs and hugetlbfs, it's most certainly worth to achieve it but in a later incremental patch. [peterx@redhat.com: don't conditionally drop FAULT_FLAG_WRITE in do_swap_page] Signed-off-by: Andrea Arcangeli Signed-off-by: Peter Xu Signed-off-by: Andrew Morton Reviewed-by: Mike Rapoport Reviewed-by: Jerome Glisse Cc: Shaohua Li Cc: Bobby Powers Cc: Brian Geffon Cc: David Hildenbrand Cc: Denis Plotnikov Cc: "Dr . David Alan Gilbert" Cc: Hugh Dickins Cc: Johannes Weiner Cc: "Kirill A . Shutemov" Cc: Martin Cracauer Cc: Marty McFadden Cc: Maya Gokhale Cc: Mel Gorman Cc: Mike Kravetz Cc: Pavel Emelyanov Cc: Rik van Riel Link: http://lkml.kernel.org/r/20200220163112.11409-3-peterx@redhat.com Signed-off-by: Linus Torvalds --- mm/memory.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mm/memory.c b/mm/memory.c index d527e0ec29c7..46aa79600ed8 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -2752,6 +2752,11 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; + if (userfaultfd_wp(vma)) { + pte_unmap_unlock(vmf->pte, vmf->ptl); + return handle_userfault(vmf, VM_UFFD_WP); + } + vmf->page = vm_normal_page(vma, vmf->address, vmf->orig_pte); if (!vmf->page) { /* @@ -3948,8 +3953,11 @@ static inline vm_fault_t create_huge_pmd(struct vm_fault *vmf) /* `inline' is required to avoid gcc 4.1.2 build error */ static inline vm_fault_t wp_huge_pmd(struct vm_fault *vmf, pmd_t orig_pmd) { - if (vma_is_anonymous(vmf->vma)) + if (vma_is_anonymous(vmf->vma)) { + if (userfaultfd_wp(vmf->vma)) + return handle_userfault(vmf, VM_UFFD_WP); return do_huge_pmd_wp_page(vmf, orig_pmd); + } if (vmf->vma->vm_ops->huge_fault) { vm_fault_t ret = vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PMD); From 5a281062af1d43d3f3956a6b429c2d727bc92603 Mon Sep 17 00:00:00 2001 From: Andrea Arcangeli Date: Mon, 6 Apr 2020 20:05:33 -0700 Subject: [PATCH 507/635] userfaultfd: wp: add WP pagetable tracking to x86 Accurate userfaultfd WP tracking is possible by tracking exactly which virtual memory ranges were writeprotected by userland. We can't relay only on the RW bit of the mapped pagetable because that information is destroyed by fork() or KSM or swap. If we were to relay on that, we'd need to stay on the safe side and generate false positive wp faults for every swapped out page. [peterx@redhat.com: append _PAGE_UFD_WP to _PAGE_CHG_MASK] Signed-off-by: Andrea Arcangeli Signed-off-by: Peter Xu Signed-off-by: Andrew Morton Reviewed-by: Jerome Glisse Reviewed-by: Mike Rapoport Cc: Bobby Powers Cc: Brian Geffon Cc: David Hildenbrand Cc: Denis Plotnikov Cc: "Dr . David Alan Gilbert" Cc: Hugh Dickins Cc: Johannes Weiner Cc: "Kirill A . Shutemov" Cc: Martin Cracauer Cc: Marty McFadden Cc: Maya Gokhale Cc: Mel Gorman Cc: Mike Kravetz Cc: Pavel Emelyanov Cc: Rik van Riel Cc: Shaohua Li Link: http://lkml.kernel.org/r/20200220163112.11409-4-peterx@redhat.com Signed-off-by: Linus Torvalds --- arch/x86/Kconfig | 1 + arch/x86/include/asm/pgtable.h | 52 ++++++++++++++++++++++++++++ arch/x86/include/asm/pgtable_64.h | 8 ++++- arch/x86/include/asm/pgtable_types.h | 12 ++++++- include/asm-generic/pgtable.h | 1 + include/asm-generic/pgtable_uffd.h | 51 +++++++++++++++++++++++++++ init/Kconfig | 5 +++ 7 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 include/asm-generic/pgtable_uffd.h diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 1edf788d301c..8d078642b4be 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -149,6 +149,7 @@ config X86 select HAVE_ARCH_TRACEHOOK select HAVE_ARCH_TRANSPARENT_HUGEPAGE select HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD if X86_64 + select HAVE_ARCH_USERFAULTFD_WP if USERFAULTFD select HAVE_ARCH_VMAP_STACK if X86_64 select HAVE_ARCH_WITHIN_STACK_FRAMES select HAVE_ASM_MODVERSIONS diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index afda66a6d325..c37e1649fb7e 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -25,6 +25,7 @@ #include #include #include +#include extern pgd_t early_top_pgt[PTRS_PER_PGD]; int __init __early_make_pgtable(unsigned long address, pmdval_t pmd); @@ -313,6 +314,23 @@ static inline pte_t pte_clear_flags(pte_t pte, pteval_t clear) return native_make_pte(v & ~clear); } +#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP +static inline int pte_uffd_wp(pte_t pte) +{ + return pte_flags(pte) & _PAGE_UFFD_WP; +} + +static inline pte_t pte_mkuffd_wp(pte_t pte) +{ + return pte_set_flags(pte, _PAGE_UFFD_WP); +} + +static inline pte_t pte_clear_uffd_wp(pte_t pte) +{ + return pte_clear_flags(pte, _PAGE_UFFD_WP); +} +#endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ + static inline pte_t pte_mkclean(pte_t pte) { return pte_clear_flags(pte, _PAGE_DIRTY); @@ -392,6 +410,23 @@ static inline pmd_t pmd_clear_flags(pmd_t pmd, pmdval_t clear) return native_make_pmd(v & ~clear); } +#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP +static inline int pmd_uffd_wp(pmd_t pmd) +{ + return pmd_flags(pmd) & _PAGE_UFFD_WP; +} + +static inline pmd_t pmd_mkuffd_wp(pmd_t pmd) +{ + return pmd_set_flags(pmd, _PAGE_UFFD_WP); +} + +static inline pmd_t pmd_clear_uffd_wp(pmd_t pmd) +{ + return pmd_clear_flags(pmd, _PAGE_UFFD_WP); +} +#endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ + static inline pmd_t pmd_mkold(pmd_t pmd) { return pmd_clear_flags(pmd, _PAGE_ACCESSED); @@ -1374,6 +1409,23 @@ static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd) #endif #endif +#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP +static inline pte_t pte_swp_mkuffd_wp(pte_t pte) +{ + return pte_set_flags(pte, _PAGE_SWP_UFFD_WP); +} + +static inline int pte_swp_uffd_wp(pte_t pte) +{ + return pte_flags(pte) & _PAGE_SWP_UFFD_WP; +} + +static inline pte_t pte_swp_clear_uffd_wp(pte_t pte) +{ + return pte_clear_flags(pte, _PAGE_SWP_UFFD_WP); +} +#endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ + #define PKRU_AD_BIT 0x1 #define PKRU_WD_BIT 0x2 #define PKRU_BITS_PER_PKEY 2 diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h index 0b6c4042942a..df1373415f11 100644 --- a/arch/x86/include/asm/pgtable_64.h +++ b/arch/x86/include/asm/pgtable_64.h @@ -189,7 +189,7 @@ extern void sync_global_pgds(unsigned long start, unsigned long end); * * | ... | 11| 10| 9|8|7|6|5| 4| 3|2| 1|0| <- bit number * | ... |SW3|SW2|SW1|G|L|D|A|CD|WT|U| W|P| <- bit names - * | TYPE (59-63) | ~OFFSET (9-58) |0|0|X|X| X| X|X|SD|0| <- swp entry + * | TYPE (59-63) | ~OFFSET (9-58) |0|0|X|X| X| X|F|SD|0| <- swp entry * * G (8) is aliased and used as a PROT_NONE indicator for * !present ptes. We need to start storing swap entries above @@ -197,9 +197,15 @@ extern void sync_global_pgds(unsigned long start, unsigned long end); * erratum where they can be incorrectly set by hardware on * non-present PTEs. * + * SD Bits 1-4 are not used in non-present format and available for + * special use described below: + * * SD (1) in swp entry is used to store soft dirty bit, which helps us * remember soft dirty over page migration * + * F (2) in swp entry is used to record when a pagetable is + * writeprotected by userfaultfd WP support. + * * Bit 7 in swp entry should be 0 because pmd_present checks not only P, * but also L and G. * diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h index 65c2ecd730c5..b6606fe6cfdf 100644 --- a/arch/x86/include/asm/pgtable_types.h +++ b/arch/x86/include/asm/pgtable_types.h @@ -32,6 +32,7 @@ #define _PAGE_BIT_SPECIAL _PAGE_BIT_SOFTW1 #define _PAGE_BIT_CPA_TEST _PAGE_BIT_SOFTW1 +#define _PAGE_BIT_UFFD_WP _PAGE_BIT_SOFTW2 /* userfaultfd wrprotected */ #define _PAGE_BIT_SOFT_DIRTY _PAGE_BIT_SOFTW3 /* software dirty tracking */ #define _PAGE_BIT_DEVMAP _PAGE_BIT_SOFTW4 @@ -100,6 +101,14 @@ #define _PAGE_SWP_SOFT_DIRTY (_AT(pteval_t, 0)) #endif +#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP +#define _PAGE_UFFD_WP (_AT(pteval_t, 1) << _PAGE_BIT_UFFD_WP) +#define _PAGE_SWP_UFFD_WP _PAGE_USER +#else +#define _PAGE_UFFD_WP (_AT(pteval_t, 0)) +#define _PAGE_SWP_UFFD_WP (_AT(pteval_t, 0)) +#endif + #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE) #define _PAGE_NX (_AT(pteval_t, 1) << _PAGE_BIT_NX) #define _PAGE_DEVMAP (_AT(u64, 1) << _PAGE_BIT_DEVMAP) @@ -118,7 +127,8 @@ */ #define _PAGE_CHG_MASK (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT | \ _PAGE_SPECIAL | _PAGE_ACCESSED | _PAGE_DIRTY | \ - _PAGE_SOFT_DIRTY | _PAGE_DEVMAP | _PAGE_ENC) + _PAGE_SOFT_DIRTY | _PAGE_DEVMAP | _PAGE_ENC | \ + _PAGE_UFFD_WP) #define _HPAGE_CHG_MASK (_PAGE_CHG_MASK | _PAGE_PSE) /* diff --git a/include/asm-generic/pgtable.h b/include/asm-generic/pgtable.h index e2e2bef07dd2..329b8c8ca703 100644 --- a/include/asm-generic/pgtable.h +++ b/include/asm-generic/pgtable.h @@ -10,6 +10,7 @@ #include #include #include +#include #if 5 - defined(__PAGETABLE_P4D_FOLDED) - defined(__PAGETABLE_PUD_FOLDED) - \ defined(__PAGETABLE_PMD_FOLDED) != CONFIG_PGTABLE_LEVELS diff --git a/include/asm-generic/pgtable_uffd.h b/include/asm-generic/pgtable_uffd.h new file mode 100644 index 000000000000..643d1bf559c2 --- /dev/null +++ b/include/asm-generic/pgtable_uffd.h @@ -0,0 +1,51 @@ +#ifndef _ASM_GENERIC_PGTABLE_UFFD_H +#define _ASM_GENERIC_PGTABLE_UFFD_H + +#ifndef CONFIG_HAVE_ARCH_USERFAULTFD_WP +static __always_inline int pte_uffd_wp(pte_t pte) +{ + return 0; +} + +static __always_inline int pmd_uffd_wp(pmd_t pmd) +{ + return 0; +} + +static __always_inline pte_t pte_mkuffd_wp(pte_t pte) +{ + return pte; +} + +static __always_inline pmd_t pmd_mkuffd_wp(pmd_t pmd) +{ + return pmd; +} + +static __always_inline pte_t pte_clear_uffd_wp(pte_t pte) +{ + return pte; +} + +static __always_inline pmd_t pmd_clear_uffd_wp(pmd_t pmd) +{ + return pmd; +} + +static __always_inline pte_t pte_swp_mkuffd_wp(pte_t pte) +{ + return pte; +} + +static __always_inline int pte_swp_uffd_wp(pte_t pte) +{ + return 0; +} + +static __always_inline pte_t pte_swp_clear_uffd_wp(pte_t pte) +{ + return pte; +} +#endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ + +#endif /* _ASM_GENERIC_PGTABLE_UFFD_H */ diff --git a/init/Kconfig b/init/Kconfig index 1c12059e0f7e..2bc0e47689d8 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1556,6 +1556,11 @@ config ADVISE_SYSCALLS applications use these syscalls, you can disable this option to save space. +config HAVE_ARCH_USERFAULTFD_WP + bool + help + Arch has userfaultfd write protection support + config MEMBARRIER bool "Enable membarrier() system call" if EXPERT default y From 55adf4de30346e0ec6b988cb9b885a5dddc954af Mon Sep 17 00:00:00 2001 From: Andrea Arcangeli Date: Mon, 6 Apr 2020 20:05:37 -0700 Subject: [PATCH 508/635] userfaultfd: wp: userfaultfd_pte/huge_pmd_wp() helpers Implement helpers methods to invoke userfaultfd wp faults more selectively: not only when a wp fault triggers on a vma with vma->vm_flags VM_UFFD_WP set, but only if the _PAGE_UFFD_WP bit is set in the pagetable too. Signed-off-by: Andrea Arcangeli Signed-off-by: Peter Xu Signed-off-by: Andrew Morton Reviewed-by: Jerome Glisse Reviewed-by: Mike Rapoport Cc: Bobby Powers Cc: Brian Geffon Cc: David Hildenbrand Cc: Denis Plotnikov Cc: "Dr . David Alan Gilbert" Cc: Hugh Dickins Cc: Johannes Weiner Cc: "Kirill A . Shutemov" Cc: Martin Cracauer Cc: Marty McFadden Cc: Maya Gokhale Cc: Mel Gorman Cc: Mike Kravetz Cc: Pavel Emelyanov Cc: Rik van Riel Cc: Shaohua Li Link: http://lkml.kernel.org/r/20200220163112.11409-5-peterx@redhat.com Signed-off-by: Linus Torvalds --- include/linux/userfaultfd_k.h | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index 5dc247af0f2e..7b91b76aac58 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -14,6 +14,8 @@ #include /* linux/include/uapi/linux/userfaultfd.h */ #include +#include +#include /* * CAREFUL: Check include/uapi/asm-generic/fcntl.h when defining @@ -57,6 +59,18 @@ static inline bool userfaultfd_wp(struct vm_area_struct *vma) return vma->vm_flags & VM_UFFD_WP; } +static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, + pte_t pte) +{ + return userfaultfd_wp(vma) && pte_uffd_wp(pte); +} + +static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, + pmd_t pmd) +{ + return userfaultfd_wp(vma) && pmd_uffd_wp(pmd); +} + static inline bool userfaultfd_armed(struct vm_area_struct *vma) { return vma->vm_flags & (VM_UFFD_MISSING | VM_UFFD_WP); @@ -106,6 +120,19 @@ static inline bool userfaultfd_wp(struct vm_area_struct *vma) return false; } +static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, + pte_t pte) +{ + return false; +} + +static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, + pmd_t pmd) +{ + return false; +} + + static inline bool userfaultfd_armed(struct vm_area_struct *vma) { return false; From 72981e0e7b609c741d7764cc920c8fec00920bd5 Mon Sep 17 00:00:00 2001 From: Andrea Arcangeli Date: Mon, 6 Apr 2020 20:05:41 -0700 Subject: [PATCH 509/635] userfaultfd: wp: add UFFDIO_COPY_MODE_WP This allows UFFDIO_COPY to map pages write-protected. [peterx@redhat.com: switch to VM_WARN_ON_ONCE in mfill_atomic_pte; add brackets around "dst_vma->vm_flags & VM_WRITE"; fix wordings in comments and commit messages] Signed-off-by: Andrea Arcangeli Signed-off-by: Peter Xu Signed-off-by: Andrew Morton Reviewed-by: Jerome Glisse Reviewed-by: Mike Rapoport Cc: Bobby Powers Cc: Brian Geffon Cc: David Hildenbrand Cc: Denis Plotnikov Cc: "Dr . David Alan Gilbert" Cc: Hugh Dickins Cc: Johannes Weiner Cc: "Kirill A . Shutemov" Cc: Martin Cracauer Cc: Marty McFadden Cc: Maya Gokhale Cc: Mel Gorman Cc: Mike Kravetz Cc: Pavel Emelyanov Cc: Rik van Riel Cc: Shaohua Li Link: http://lkml.kernel.org/r/20200220163112.11409-6-peterx@redhat.com Signed-off-by: Linus Torvalds --- fs/userfaultfd.c | 5 +++-- include/linux/userfaultfd_k.h | 2 +- include/uapi/linux/userfaultfd.h | 13 ++++++------ mm/userfaultfd.c | 36 ++++++++++++++++++++++---------- 4 files changed, 36 insertions(+), 20 deletions(-) diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 703c1c3faa6e..c49bef505775 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -1724,11 +1724,12 @@ static int userfaultfd_copy(struct userfaultfd_ctx *ctx, ret = -EINVAL; if (uffdio_copy.src + uffdio_copy.len <= uffdio_copy.src) goto out; - if (uffdio_copy.mode & ~UFFDIO_COPY_MODE_DONTWAKE) + if (uffdio_copy.mode & ~(UFFDIO_COPY_MODE_DONTWAKE|UFFDIO_COPY_MODE_WP)) goto out; if (mmget_not_zero(ctx->mm)) { ret = mcopy_atomic(ctx->mm, uffdio_copy.dst, uffdio_copy.src, - uffdio_copy.len, &ctx->mmap_changing); + uffdio_copy.len, &ctx->mmap_changing, + uffdio_copy.mode); mmput(ctx->mm); } else { return -ESRCH; diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index 7b91b76aac58..dcd33172b728 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -36,7 +36,7 @@ extern vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason); extern ssize_t mcopy_atomic(struct mm_struct *dst_mm, unsigned long dst_start, unsigned long src_start, unsigned long len, - bool *mmap_changing); + bool *mmap_changing, __u64 mode); extern ssize_t mfill_zeropage(struct mm_struct *dst_mm, unsigned long dst_start, unsigned long len, diff --git a/include/uapi/linux/userfaultfd.h b/include/uapi/linux/userfaultfd.h index 48f1a7c2f1f0..340f23bc251d 100644 --- a/include/uapi/linux/userfaultfd.h +++ b/include/uapi/linux/userfaultfd.h @@ -203,13 +203,14 @@ struct uffdio_copy { __u64 dst; __u64 src; __u64 len; - /* - * There will be a wrprotection flag later that allows to map - * pages wrprotected on the fly. And such a flag will be - * available if the wrprotection ioctl are implemented for the - * range according to the uffdio_register.ioctls. - */ #define UFFDIO_COPY_MODE_DONTWAKE ((__u64)1<<0) + /* + * UFFDIO_COPY_MODE_WP will map the page write protected on + * the fly. UFFDIO_COPY_MODE_WP is available only if the + * write protected ioctl is implemented for the range + * according to the uffdio_register.ioctls. + */ +#define UFFDIO_COPY_MODE_WP ((__u64)1<<1) __u64 mode; /* diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index bd96855f3961..05dbbcafdcc0 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -53,7 +53,8 @@ static int mcopy_atomic_pte(struct mm_struct *dst_mm, struct vm_area_struct *dst_vma, unsigned long dst_addr, unsigned long src_addr, - struct page **pagep) + struct page **pagep, + bool wp_copy) { struct mem_cgroup *memcg; pte_t _dst_pte, *dst_pte; @@ -99,9 +100,9 @@ static int mcopy_atomic_pte(struct mm_struct *dst_mm, if (mem_cgroup_try_charge(page, dst_mm, GFP_KERNEL, &memcg, false)) goto out_release; - _dst_pte = mk_pte(page, dst_vma->vm_page_prot); - if (dst_vma->vm_flags & VM_WRITE) - _dst_pte = pte_mkwrite(pte_mkdirty(_dst_pte)); + _dst_pte = pte_mkdirty(mk_pte(page, dst_vma->vm_page_prot)); + if ((dst_vma->vm_flags & VM_WRITE) && !wp_copy) + _dst_pte = pte_mkwrite(_dst_pte); dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl); if (dst_vma->vm_file) { @@ -415,7 +416,8 @@ static __always_inline ssize_t mfill_atomic_pte(struct mm_struct *dst_mm, unsigned long dst_addr, unsigned long src_addr, struct page **page, - bool zeropage) + bool zeropage, + bool wp_copy) { ssize_t err; @@ -432,11 +434,13 @@ static __always_inline ssize_t mfill_atomic_pte(struct mm_struct *dst_mm, if (!(dst_vma->vm_flags & VM_SHARED)) { if (!zeropage) err = mcopy_atomic_pte(dst_mm, dst_pmd, dst_vma, - dst_addr, src_addr, page); + dst_addr, src_addr, page, + wp_copy); else err = mfill_zeropage_pte(dst_mm, dst_pmd, dst_vma, dst_addr); } else { + VM_WARN_ON_ONCE(wp_copy); if (!zeropage) err = shmem_mcopy_atomic_pte(dst_mm, dst_pmd, dst_vma, dst_addr, @@ -454,7 +458,8 @@ static __always_inline ssize_t __mcopy_atomic(struct mm_struct *dst_mm, unsigned long src_start, unsigned long len, bool zeropage, - bool *mmap_changing) + bool *mmap_changing, + __u64 mode) { struct vm_area_struct *dst_vma; ssize_t err; @@ -462,6 +467,7 @@ static __always_inline ssize_t __mcopy_atomic(struct mm_struct *dst_mm, unsigned long src_addr, dst_addr; long copied; struct page *page; + bool wp_copy; /* * Sanitize the command parameters: @@ -507,6 +513,14 @@ retry: dst_vma->vm_flags & VM_SHARED)) goto out_unlock; + /* + * validate 'mode' now that we know the dst_vma: don't allow + * a wrprotect copy if the userfaultfd didn't register as WP. + */ + wp_copy = mode & UFFDIO_COPY_MODE_WP; + if (wp_copy && !(dst_vma->vm_flags & VM_UFFD_WP)) + goto out_unlock; + /* * If this is a HUGETLB vma, pass off to appropriate routine */ @@ -562,7 +576,7 @@ retry: BUG_ON(pmd_trans_huge(*dst_pmd)); err = mfill_atomic_pte(dst_mm, dst_pmd, dst_vma, dst_addr, - src_addr, &page, zeropage); + src_addr, &page, zeropage, wp_copy); cond_resched(); if (unlikely(err == -ENOENT)) { @@ -609,14 +623,14 @@ out: ssize_t mcopy_atomic(struct mm_struct *dst_mm, unsigned long dst_start, unsigned long src_start, unsigned long len, - bool *mmap_changing) + bool *mmap_changing, __u64 mode) { return __mcopy_atomic(dst_mm, dst_start, src_start, len, false, - mmap_changing); + mmap_changing, mode); } ssize_t mfill_zeropage(struct mm_struct *dst_mm, unsigned long start, unsigned long len, bool *mmap_changing) { - return __mcopy_atomic(dst_mm, start, 0, len, true, mmap_changing); + return __mcopy_atomic(dst_mm, start, 0, len, true, mmap_changing, 0); } From 58705444c45b3ca987b03bd9beb41bbbe41ae439 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Mon, 6 Apr 2020 20:05:45 -0700 Subject: [PATCH 510/635] mm: merge parameters for change_protection() change_protection() was used by either the NUMA or mprotect() code, there's one parameter for each of the callers (dirty_accountable and prot_numa). Further, these parameters are passed along the calls: - change_protection_range() - change_p4d_range() - change_pud_range() - change_pmd_range() - ... Now we introduce a flag for change_protect() and all these helpers to replace these parameters. Then we can avoid passing multiple parameters multiple times along the way. More importantly, it'll greatly simplify the work if we want to introduce any new parameters to change_protection(). In the follow up patches, a new parameter for userfaultfd write protection will be introduced. No functional change at all. Signed-off-by: Peter Xu Signed-off-by: Andrew Morton Reviewed-by: Jerome Glisse Cc: Andrea Arcangeli Cc: Bobby Powers Cc: Brian Geffon Cc: David Hildenbrand Cc: Denis Plotnikov Cc: "Dr . David Alan Gilbert" Cc: Hugh Dickins Cc: Johannes Weiner Cc: "Kirill A . Shutemov" Cc: Martin Cracauer Cc: Marty McFadden Cc: Maya Gokhale Cc: Mel Gorman Cc: Mike Kravetz Cc: Mike Rapoport Cc: Pavel Emelyanov Cc: Rik van Riel Cc: Shaohua Li Link: http://lkml.kernel.org/r/20200220163112.11409-7-peterx@redhat.com Signed-off-by: Linus Torvalds --- include/linux/huge_mm.h | 2 +- include/linux/mm.h | 14 +++++++++++++- mm/huge_memory.c | 3 ++- mm/mempolicy.c | 2 +- mm/mprotect.c | 29 ++++++++++++++++------------- 5 files changed, 33 insertions(+), 17 deletions(-) diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index f2df2247026a..cfbb0a87c5f0 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -46,7 +46,7 @@ extern bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, pmd_t *old_pmd, pmd_t *new_pmd); extern int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr, pgprot_t newprot, - int prot_numa); + unsigned long cp_flags); vm_fault_t vmf_insert_pfn_pmd_prot(struct vm_fault *vmf, pfn_t pfn, pgprot_t pgprot, bool write); diff --git a/include/linux/mm.h b/include/linux/mm.h index be49e371e4b5..c7d87ff5027b 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1771,9 +1771,21 @@ extern unsigned long move_page_tables(struct vm_area_struct *vma, unsigned long old_addr, struct vm_area_struct *new_vma, unsigned long new_addr, unsigned long len, bool need_rmap_locks); + +/* + * Flags used by change_protection(). For now we make it a bitmap so + * that we can pass in multiple flags just like parameters. However + * for now all the callers are only use one of the flags at the same + * time. + */ +/* Whether we should allow dirty bit accounting */ +#define MM_CP_DIRTY_ACCT (1UL << 0) +/* Whether this protection change is for NUMA hints */ +#define MM_CP_PROT_NUMA (1UL << 1) + extern unsigned long change_protection(struct vm_area_struct *vma, unsigned long start, unsigned long end, pgprot_t newprot, - int dirty_accountable, int prot_numa); + unsigned long cp_flags); extern int mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev, unsigned long start, unsigned long end, unsigned long newflags); diff --git a/mm/huge_memory.c b/mm/huge_memory.c index c1e7c71db1e6..dc12249af6df 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1979,13 +1979,14 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, * - HPAGE_PMD_NR is protections changed and TLB flush necessary */ int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, - unsigned long addr, pgprot_t newprot, int prot_numa) + unsigned long addr, pgprot_t newprot, unsigned long cp_flags) { struct mm_struct *mm = vma->vm_mm; spinlock_t *ptl; pmd_t entry; bool preserve_write; int ret; + bool prot_numa = cp_flags & MM_CP_PROT_NUMA; ptl = __pmd_trans_huge_lock(pmd, vma); if (!ptl) diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 037e5f548118..145be04b7108 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -627,7 +627,7 @@ unsigned long change_prot_numa(struct vm_area_struct *vma, { int nr_updated; - nr_updated = change_protection(vma, addr, end, PAGE_NONE, 0, 1); + nr_updated = change_protection(vma, addr, end, PAGE_NONE, MM_CP_PROT_NUMA); if (nr_updated) count_vm_numa_events(NUMA_PTE_UPDATES, nr_updated); diff --git a/mm/mprotect.c b/mm/mprotect.c index 0fee14b39416..046e0889e65f 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -37,12 +37,14 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr, unsigned long end, pgprot_t newprot, - int dirty_accountable, int prot_numa) + unsigned long cp_flags) { pte_t *pte, oldpte; spinlock_t *ptl; unsigned long pages = 0; int target_node = NUMA_NO_NODE; + bool dirty_accountable = cp_flags & MM_CP_DIRTY_ACCT; + bool prot_numa = cp_flags & MM_CP_PROT_NUMA; /* * Can be called with only the mmap_sem for reading by @@ -188,7 +190,7 @@ static inline int pmd_none_or_clear_bad_unless_trans_huge(pmd_t *pmd) static inline unsigned long change_pmd_range(struct vm_area_struct *vma, pud_t *pud, unsigned long addr, unsigned long end, - pgprot_t newprot, int dirty_accountable, int prot_numa) + pgprot_t newprot, unsigned long cp_flags) { pmd_t *pmd; unsigned long next; @@ -229,7 +231,7 @@ static inline unsigned long change_pmd_range(struct vm_area_struct *vma, __split_huge_pmd(vma, pmd, addr, false, NULL); } else { int nr_ptes = change_huge_pmd(vma, pmd, addr, - newprot, prot_numa); + newprot, cp_flags); if (nr_ptes) { if (nr_ptes == HPAGE_PMD_NR) { @@ -244,7 +246,7 @@ static inline unsigned long change_pmd_range(struct vm_area_struct *vma, /* fall through, the trans huge pmd just split */ } this_pages = change_pte_range(vma, pmd, addr, next, newprot, - dirty_accountable, prot_numa); + cp_flags); pages += this_pages; next: cond_resched(); @@ -260,7 +262,7 @@ next: static inline unsigned long change_pud_range(struct vm_area_struct *vma, p4d_t *p4d, unsigned long addr, unsigned long end, - pgprot_t newprot, int dirty_accountable, int prot_numa) + pgprot_t newprot, unsigned long cp_flags) { pud_t *pud; unsigned long next; @@ -272,7 +274,7 @@ static inline unsigned long change_pud_range(struct vm_area_struct *vma, if (pud_none_or_clear_bad(pud)) continue; pages += change_pmd_range(vma, pud, addr, next, newprot, - dirty_accountable, prot_numa); + cp_flags); } while (pud++, addr = next, addr != end); return pages; @@ -280,7 +282,7 @@ static inline unsigned long change_pud_range(struct vm_area_struct *vma, static inline unsigned long change_p4d_range(struct vm_area_struct *vma, pgd_t *pgd, unsigned long addr, unsigned long end, - pgprot_t newprot, int dirty_accountable, int prot_numa) + pgprot_t newprot, unsigned long cp_flags) { p4d_t *p4d; unsigned long next; @@ -292,7 +294,7 @@ static inline unsigned long change_p4d_range(struct vm_area_struct *vma, if (p4d_none_or_clear_bad(p4d)) continue; pages += change_pud_range(vma, p4d, addr, next, newprot, - dirty_accountable, prot_numa); + cp_flags); } while (p4d++, addr = next, addr != end); return pages; @@ -300,7 +302,7 @@ static inline unsigned long change_p4d_range(struct vm_area_struct *vma, static unsigned long change_protection_range(struct vm_area_struct *vma, unsigned long addr, unsigned long end, pgprot_t newprot, - int dirty_accountable, int prot_numa) + unsigned long cp_flags) { struct mm_struct *mm = vma->vm_mm; pgd_t *pgd; @@ -317,7 +319,7 @@ static unsigned long change_protection_range(struct vm_area_struct *vma, if (pgd_none_or_clear_bad(pgd)) continue; pages += change_p4d_range(vma, pgd, addr, next, newprot, - dirty_accountable, prot_numa); + cp_flags); } while (pgd++, addr = next, addr != end); /* Only flush the TLB if we actually modified any entries: */ @@ -330,14 +332,15 @@ static unsigned long change_protection_range(struct vm_area_struct *vma, unsigned long change_protection(struct vm_area_struct *vma, unsigned long start, unsigned long end, pgprot_t newprot, - int dirty_accountable, int prot_numa) + unsigned long cp_flags) { unsigned long pages; if (is_vm_hugetlb_page(vma)) pages = hugetlb_change_protection(vma, start, end, newprot); else - pages = change_protection_range(vma, start, end, newprot, dirty_accountable, prot_numa); + pages = change_protection_range(vma, start, end, newprot, + cp_flags); return pages; } @@ -459,7 +462,7 @@ success: vma_set_page_prot(vma); change_protection(vma, start, end, vma->vm_page_prot, - dirty_accountable, 0); + dirty_accountable ? MM_CP_DIRTY_ACCT : 0); /* * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major From 292924b260247483a58916f6d3550d8c92f32f55 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Mon, 6 Apr 2020 20:05:49 -0700 Subject: [PATCH 511/635] userfaultfd: wp: apply _PAGE_UFFD_WP bit Firstly, introduce two new flags MM_CP_UFFD_WP[_RESOLVE] for change_protection() when used with uffd-wp and make sure the two new flags are exclusively used. Then, - For MM_CP_UFFD_WP: apply the _PAGE_UFFD_WP bit and remove _PAGE_RW when a range of memory is write protected by uffd - For MM_CP_UFFD_WP_RESOLVE: remove the _PAGE_UFFD_WP bit and recover _PAGE_RW when write protection is resolved from userspace And use this new interface in mwriteprotect_range() to replace the old MM_CP_DIRTY_ACCT. Do this change for both PTEs and huge PMDs. Then we can start to identify which PTE/PMD is write protected by general (e.g., COW or soft dirty tracking), and which is for userfaultfd-wp. Since we should keep the _PAGE_UFFD_WP when doing pte_modify(), add it into _PAGE_CHG_MASK as well. Meanwhile, since we have this new bit, we can be even more strict when detecting uffd-wp page faults in either do_wp_page() or wp_huge_pmd(). After we're with _PAGE_UFFD_WP, a special case is when a page is both protected by the general COW logic and also userfault-wp. Here the userfault-wp will have higher priority and will be handled first. Only after the uffd-wp bit is cleared on the PTE/PMD will we continue to handle the general COW. These are the steps on what will happen with such a page: 1. CPU accesses write protected shared page (so both protected by general COW and uffd-wp), blocked by uffd-wp first because in do_wp_page we'll handle uffd-wp first, so it has higher priority than general COW. 2. Uffd service thread receives the request, do UFFDIO_WRITEPROTECT to remove the uffd-wp bit upon the PTE/PMD. However here we still keep the write bit cleared. Notify the blocked CPU. 3. The blocked CPU resumes the page fault process with a fault retry, during retry it'll notice it was not with the uffd-wp bit this time but it is still write protected by general COW, then it'll go though the COW path in the fault handler, copy the page, apply write bit where necessary, and retry again. 4. The CPU will be able to access this page with write bit set. Suggested-by: Andrea Arcangeli Signed-off-by: Peter Xu Signed-off-by: Andrew Morton Cc: Brian Geffon Cc: Pavel Emelyanov Cc: Mike Kravetz Cc: David Hildenbrand Cc: Martin Cracauer Cc: Mel Gorman Cc: Bobby Powers Cc: Mike Rapoport Cc: "Kirill A . Shutemov" Cc: Maya Gokhale Cc: Johannes Weiner Cc: Marty McFadden Cc: Denis Plotnikov Cc: Hugh Dickins Cc: "Dr . David Alan Gilbert" Cc: Jerome Glisse Cc: Rik van Riel Cc: Shaohua Li Link: http://lkml.kernel.org/r/20200220163112.11409-8-peterx@redhat.com Signed-off-by: Linus Torvalds --- include/linux/mm.h | 5 +++++ mm/huge_memory.c | 18 +++++++++++++++++- mm/memory.c | 4 ++-- mm/mprotect.c | 17 +++++++++++++++++ mm/userfaultfd.c | 8 ++++++-- 5 files changed, 47 insertions(+), 5 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index c7d87ff5027b..e2f938c5a9d8 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1782,6 +1782,11 @@ extern unsigned long move_page_tables(struct vm_area_struct *vma, #define MM_CP_DIRTY_ACCT (1UL << 0) /* Whether this protection change is for NUMA hints */ #define MM_CP_PROT_NUMA (1UL << 1) +/* Whether this change is for write protecting */ +#define MM_CP_UFFD_WP (1UL << 2) /* do wp */ +#define MM_CP_UFFD_WP_RESOLVE (1UL << 3) /* Resolve wp */ +#define MM_CP_UFFD_WP_ALL (MM_CP_UFFD_WP | \ + MM_CP_UFFD_WP_RESOLVE) extern unsigned long change_protection(struct vm_area_struct *vma, unsigned long start, unsigned long end, pgprot_t newprot, diff --git a/mm/huge_memory.c b/mm/huge_memory.c index dc12249af6df..425339491677 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1987,6 +1987,8 @@ int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, bool preserve_write; int ret; bool prot_numa = cp_flags & MM_CP_PROT_NUMA; + bool uffd_wp = cp_flags & MM_CP_UFFD_WP; + bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; ptl = __pmd_trans_huge_lock(pmd, vma); if (!ptl) @@ -2053,6 +2055,17 @@ int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, entry = pmd_modify(entry, newprot); if (preserve_write) entry = pmd_mk_savedwrite(entry); + if (uffd_wp) { + entry = pmd_wrprotect(entry); + entry = pmd_mkuffd_wp(entry); + } else if (uffd_wp_resolve) { + /* + * Leave the write bit to be handled by PF interrupt + * handler, then things like COW could be properly + * handled. + */ + entry = pmd_clear_uffd_wp(entry); + } ret = HPAGE_PMD_NR; set_pmd_at(mm, addr, pmd, entry); BUG_ON(vma_is_anonymous(vma) && !preserve_write && pmd_write(entry)); @@ -2201,7 +2214,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, struct page *page; pgtable_t pgtable; pmd_t old_pmd, _pmd; - bool young, write, soft_dirty, pmd_migration = false; + bool young, write, soft_dirty, pmd_migration = false, uffd_wp = false; unsigned long addr; int i; @@ -2283,6 +2296,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, write = pmd_write(old_pmd); young = pmd_young(old_pmd); soft_dirty = pmd_soft_dirty(old_pmd); + uffd_wp = pmd_uffd_wp(old_pmd); } VM_BUG_ON_PAGE(!page_count(page), page); page_ref_add(page, HPAGE_PMD_NR - 1); @@ -2316,6 +2330,8 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, entry = pte_mkold(entry); if (soft_dirty) entry = pte_mksoft_dirty(entry); + if (uffd_wp) + entry = pte_mkuffd_wp(entry); } pte = pte_offset_map(&_pmd, addr); BUG_ON(!pte_none(*pte)); diff --git a/mm/memory.c b/mm/memory.c index 46aa79600ed8..f35821b43c1b 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -2752,7 +2752,7 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; - if (userfaultfd_wp(vma)) { + if (userfaultfd_pte_wp(vma, *vmf->pte)) { pte_unmap_unlock(vmf->pte, vmf->ptl); return handle_userfault(vmf, VM_UFFD_WP); } @@ -3954,7 +3954,7 @@ static inline vm_fault_t create_huge_pmd(struct vm_fault *vmf) static inline vm_fault_t wp_huge_pmd(struct vm_fault *vmf, pmd_t orig_pmd) { if (vma_is_anonymous(vmf->vma)) { - if (userfaultfd_wp(vmf->vma)) + if (userfaultfd_huge_pmd_wp(vmf->vma, orig_pmd)) return handle_userfault(vmf, VM_UFFD_WP); return do_huge_pmd_wp_page(vmf, orig_pmd); } diff --git a/mm/mprotect.c b/mm/mprotect.c index 046e0889e65f..e4fa41a24bec 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -45,6 +45,8 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd, int target_node = NUMA_NO_NODE; bool dirty_accountable = cp_flags & MM_CP_DIRTY_ACCT; bool prot_numa = cp_flags & MM_CP_PROT_NUMA; + bool uffd_wp = cp_flags & MM_CP_UFFD_WP; + bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; /* * Can be called with only the mmap_sem for reading by @@ -116,6 +118,19 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd, if (preserve_write) ptent = pte_mk_savedwrite(ptent); + if (uffd_wp) { + ptent = pte_wrprotect(ptent); + ptent = pte_mkuffd_wp(ptent); + } else if (uffd_wp_resolve) { + /* + * Leave the write bit to be handled + * by PF interrupt handler, then + * things like COW could be properly + * handled. + */ + ptent = pte_clear_uffd_wp(ptent); + } + /* Avoid taking write faults for known dirty pages */ if (dirty_accountable && pte_dirty(ptent) && (pte_soft_dirty(ptent) || @@ -336,6 +351,8 @@ unsigned long change_protection(struct vm_area_struct *vma, unsigned long start, { unsigned long pages; + BUG_ON((cp_flags & MM_CP_UFFD_WP_ALL) == MM_CP_UFFD_WP_ALL); + if (is_vm_hugetlb_page(vma)) pages = hugetlb_change_protection(vma, start, end, newprot); else diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 05dbbcafdcc0..7d6ab05be019 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -101,8 +101,12 @@ static int mcopy_atomic_pte(struct mm_struct *dst_mm, goto out_release; _dst_pte = pte_mkdirty(mk_pte(page, dst_vma->vm_page_prot)); - if ((dst_vma->vm_flags & VM_WRITE) && !wp_copy) - _dst_pte = pte_mkwrite(_dst_pte); + if (dst_vma->vm_flags & VM_WRITE) { + if (wp_copy) + _dst_pte = pte_mkuffd_wp(_dst_pte); + else + _dst_pte = pte_mkwrite(_dst_pte); + } dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl); if (dst_vma->vm_file) { From b569a1760782f3da03ff718d61f74163dea599ff Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Mon, 6 Apr 2020 20:05:53 -0700 Subject: [PATCH 512/635] userfaultfd: wp: drop _PAGE_UFFD_WP properly when fork UFFD_EVENT_FORK support for uffd-wp should be already there, except that we should clean the uffd-wp bit if uffd fork event is not enabled. Detect that to avoid _PAGE_UFFD_WP being set even if the VMA is not being tracked by VM_UFFD_WP. Do this for both small PTEs and huge PMDs. Signed-off-by: Peter Xu Signed-off-by: Andrew Morton Reviewed-by: Jerome Glisse Reviewed-by: Mike Rapoport Cc: Andrea Arcangeli Cc: Bobby Powers Cc: Brian Geffon Cc: David Hildenbrand Cc: Denis Plotnikov Cc: "Dr . David Alan Gilbert" Cc: Hugh Dickins Cc: Johannes Weiner Cc: "Kirill A . Shutemov" Cc: Martin Cracauer Cc: Marty McFadden Cc: Maya Gokhale Cc: Mel Gorman Cc: Mike Kravetz Cc: Pavel Emelyanov Cc: Rik van Riel Cc: Shaohua Li Link: http://lkml.kernel.org/r/20200220163112.11409-9-peterx@redhat.com Signed-off-by: Linus Torvalds --- mm/huge_memory.c | 8 ++++++++ mm/memory.c | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 425339491677..8164787cd51f 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1044,6 +1044,14 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, ret = -EAGAIN; pmd = *src_pmd; + /* + * Make sure the _PAGE_UFFD_WP bit is cleared if the new VMA + * does not have the VM_UFFD_WP, which means that the uffd + * fork event is not enabled. + */ + if (!(vma->vm_flags & VM_UFFD_WP)) + pmd = pmd_clear_uffd_wp(pmd); + #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION if (unlikely(is_swap_pmd(pmd))) { swp_entry_t entry = pmd_to_swp_entry(pmd); diff --git a/mm/memory.c b/mm/memory.c index f35821b43c1b..f8b1969669b7 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -785,6 +785,14 @@ copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, pte = pte_mkclean(pte); pte = pte_mkold(pte); + /* + * Make sure the _PAGE_UFFD_WP bit is cleared if the new VMA + * does not have the VM_UFFD_WP, which means that the uffd + * fork event is not enabled. + */ + if (!(vm_flags & VM_UFFD_WP)) + pte = pte_clear_uffd_wp(pte); + page = vm_normal_page(vma, addr, pte); if (page) { get_page(page); From 2e3d5dc508cf001c4fb2d15515ebe6f30df88f76 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Mon, 6 Apr 2020 20:05:57 -0700 Subject: [PATCH 513/635] userfaultfd: wp: add pmd_swp_*uffd_wp() helpers Adding these missing helpers for uffd-wp operations with pmd swap/migration entries. Signed-off-by: Peter Xu Signed-off-by: Andrew Morton Reviewed-by: Jerome Glisse Reviewed-by: Mike Rapoport Cc: Andrea Arcangeli Cc: Bobby Powers Cc: Brian Geffon Cc: David Hildenbrand Cc: Denis Plotnikov Cc: "Dr . David Alan Gilbert" Cc: Hugh Dickins Cc: Johannes Weiner Cc: "Kirill A . Shutemov" Cc: Martin Cracauer Cc: Marty McFadden Cc: Maya Gokhale Cc: Mel Gorman Cc: Mike Kravetz Cc: Pavel Emelyanov Cc: Rik van Riel Cc: Shaohua Li Link: http://lkml.kernel.org/r/20200220163112.11409-10-peterx@redhat.com Signed-off-by: Linus Torvalds --- arch/x86/include/asm/pgtable.h | 15 +++++++++++++++ include/asm-generic/pgtable_uffd.h | 15 +++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index c37e1649fb7e..28838d790191 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -1424,6 +1424,21 @@ static inline pte_t pte_swp_clear_uffd_wp(pte_t pte) { return pte_clear_flags(pte, _PAGE_SWP_UFFD_WP); } + +static inline pmd_t pmd_swp_mkuffd_wp(pmd_t pmd) +{ + return pmd_set_flags(pmd, _PAGE_SWP_UFFD_WP); +} + +static inline int pmd_swp_uffd_wp(pmd_t pmd) +{ + return pmd_flags(pmd) & _PAGE_SWP_UFFD_WP; +} + +static inline pmd_t pmd_swp_clear_uffd_wp(pmd_t pmd) +{ + return pmd_clear_flags(pmd, _PAGE_SWP_UFFD_WP); +} #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ #define PKRU_AD_BIT 0x1 diff --git a/include/asm-generic/pgtable_uffd.h b/include/asm-generic/pgtable_uffd.h index 643d1bf559c2..828966d4c281 100644 --- a/include/asm-generic/pgtable_uffd.h +++ b/include/asm-generic/pgtable_uffd.h @@ -46,6 +46,21 @@ static __always_inline pte_t pte_swp_clear_uffd_wp(pte_t pte) { return pte; } + +static inline pmd_t pmd_swp_mkuffd_wp(pmd_t pmd) +{ + return pmd; +} + +static inline int pmd_swp_uffd_wp(pmd_t pmd) +{ + return 0; +} + +static inline pmd_t pmd_swp_clear_uffd_wp(pmd_t pmd) +{ + return pmd; +} #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ #endif /* _ASM_GENERIC_PGTABLE_UFFD_H */ From f45ec5ff16a75f96dac8c89862d75f1d8739efd4 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Mon, 6 Apr 2020 20:06:01 -0700 Subject: [PATCH 514/635] userfaultfd: wp: support swap and page migration For either swap and page migration, we all use the bit 2 of the entry to identify whether this entry is uffd write-protected. It plays a similar role as the existing soft dirty bit in swap entries but only for keeping the uffd-wp tracking for a specific PTE/PMD. Something special here is that when we want to recover the uffd-wp bit from a swap/migration entry to the PTE bit we'll also need to take care of the _PAGE_RW bit and make sure it's cleared, otherwise even with the _PAGE_UFFD_WP bit we can't trap it at all. In change_pte_range() we do nothing for uffd if the PTE is a swap entry. That can lead to data mismatch if the page that we are going to write protect is swapped out when sending the UFFDIO_WRITEPROTECT. This patch also applies/removes the uffd-wp bit even for the swap entries. Signed-off-by: Peter Xu Signed-off-by: Andrew Morton Cc: Andrea Arcangeli Cc: Bobby Powers Cc: Brian Geffon Cc: David Hildenbrand Cc: Denis Plotnikov Cc: "Dr . David Alan Gilbert" Cc: Hugh Dickins Cc: Jerome Glisse Cc: Johannes Weiner Cc: "Kirill A . Shutemov" Cc: Martin Cracauer Cc: Marty McFadden Cc: Maya Gokhale Cc: Mel Gorman Cc: Mike Kravetz Cc: Mike Rapoport Cc: Pavel Emelyanov Cc: Rik van Riel Cc: Shaohua Li Link: http://lkml.kernel.org/r/20200220163112.11409-11-peterx@redhat.com Signed-off-by: Linus Torvalds --- include/linux/swapops.h | 2 ++ mm/huge_memory.c | 3 +++ mm/memory.c | 8 ++++++++ mm/migrate.c | 6 ++++++ mm/mprotect.c | 28 +++++++++++++++++----------- mm/rmap.c | 6 ++++++ 6 files changed, 42 insertions(+), 11 deletions(-) diff --git a/include/linux/swapops.h b/include/linux/swapops.h index 877fd239b6ff..9a6f06de183b 100644 --- a/include/linux/swapops.h +++ b/include/linux/swapops.h @@ -68,6 +68,8 @@ static inline swp_entry_t pte_to_swp_entry(pte_t pte) if (pte_swp_soft_dirty(pte)) pte = pte_swp_clear_soft_dirty(pte); + if (pte_swp_uffd_wp(pte)) + pte = pte_swp_clear_uffd_wp(pte); arch_entry = __pte_to_swp_entry(pte); return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry)); } diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 8164787cd51f..6ecd1045113b 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2297,6 +2297,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, write = is_write_migration_entry(entry); young = false; soft_dirty = pmd_swp_soft_dirty(old_pmd); + uffd_wp = pmd_swp_uffd_wp(old_pmd); } else { page = pmd_page(old_pmd); if (pmd_dirty(old_pmd)) @@ -2329,6 +2330,8 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, entry = swp_entry_to_pte(swp_entry); if (soft_dirty) entry = pte_swp_mksoft_dirty(entry); + if (uffd_wp) + entry = pte_swp_mkuffd_wp(entry); } else { entry = mk_pte(page + i, READ_ONCE(vma->vm_page_prot)); entry = maybe_mkwrite(entry, vma); diff --git a/mm/memory.c b/mm/memory.c index f8b1969669b7..8ac9af73e9d2 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -733,6 +733,8 @@ copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, pte = swp_entry_to_pte(entry); if (pte_swp_soft_dirty(*src_pte)) pte = pte_swp_mksoft_dirty(pte); + if (pte_swp_uffd_wp(*src_pte)) + pte = pte_swp_mkuffd_wp(pte); set_pte_at(src_mm, addr, src_pte, pte); } } else if (is_device_private_entry(entry)) { @@ -762,6 +764,8 @@ copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, is_cow_mapping(vm_flags)) { make_device_private_entry_read(&entry); pte = swp_entry_to_pte(entry); + if (pte_swp_uffd_wp(*src_pte)) + pte = pte_swp_mkuffd_wp(pte); set_pte_at(src_mm, addr, src_pte, pte); } } @@ -3098,6 +3102,10 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) flush_icache_page(vma, page); if (pte_swp_soft_dirty(vmf->orig_pte)) pte = pte_mksoft_dirty(pte); + if (pte_swp_uffd_wp(vmf->orig_pte)) { + pte = pte_mkuffd_wp(pte); + pte = pte_wrprotect(pte); + } set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte); arch_do_swap_page(vma->vm_mm, vma, vmf->address, pte, vmf->orig_pte); vmf->orig_pte = pte; diff --git a/mm/migrate.c b/mm/migrate.c index c1412e04975e..7160c1556f79 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -243,11 +243,15 @@ static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma, entry = pte_to_swp_entry(*pvmw.pte); if (is_write_migration_entry(entry)) pte = maybe_mkwrite(pte, vma); + else if (pte_swp_uffd_wp(*pvmw.pte)) + pte = pte_mkuffd_wp(pte); if (unlikely(is_zone_device_page(new))) { if (is_device_private_page(new)) { entry = make_device_private_entry(new, pte_write(pte)); pte = swp_entry_to_pte(entry); + if (pte_swp_uffd_wp(*pvmw.pte)) + pte = pte_mkuffd_wp(pte); } } @@ -2338,6 +2342,8 @@ again: swp_pte = swp_entry_to_pte(entry); if (pte_soft_dirty(pte)) swp_pte = pte_swp_mksoft_dirty(swp_pte); + if (pte_uffd_wp(pte)) + swp_pte = pte_swp_mkuffd_wp(swp_pte); set_pte_at(mm, addr, ptep, swp_pte); /* diff --git a/mm/mprotect.c b/mm/mprotect.c index e4fa41a24bec..1d823b050329 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -139,11 +139,11 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd, } ptep_modify_prot_commit(vma, addr, pte, oldpte, ptent); pages++; - } else if (IS_ENABLED(CONFIG_MIGRATION)) { + } else if (is_swap_pte(oldpte)) { swp_entry_t entry = pte_to_swp_entry(oldpte); + pte_t newpte; if (is_write_migration_entry(entry)) { - pte_t newpte; /* * A protection check is difficult so * just be safe and disable write @@ -152,22 +152,28 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd, newpte = swp_entry_to_pte(entry); if (pte_swp_soft_dirty(oldpte)) newpte = pte_swp_mksoft_dirty(newpte); - set_pte_at(vma->vm_mm, addr, pte, newpte); - - pages++; - } - - if (is_write_device_private_entry(entry)) { - pte_t newpte; - + if (pte_swp_uffd_wp(oldpte)) + newpte = pte_swp_mkuffd_wp(newpte); + } else if (is_write_device_private_entry(entry)) { /* * We do not preserve soft-dirtiness. See * copy_one_pte() for explanation. */ make_device_private_entry_read(&entry); newpte = swp_entry_to_pte(entry); - set_pte_at(vma->vm_mm, addr, pte, newpte); + if (pte_swp_uffd_wp(oldpte)) + newpte = pte_swp_mkuffd_wp(newpte); + } else { + newpte = oldpte; + } + if (uffd_wp) + newpte = pte_swp_mkuffd_wp(newpte); + else if (uffd_wp_resolve) + newpte = pte_swp_clear_uffd_wp(newpte); + + if (!pte_same(oldpte, newpte)) { + set_pte_at(vma->vm_mm, addr, pte, newpte); pages++; } } diff --git a/mm/rmap.c b/mm/rmap.c index 374a9bfdbffa..ed8889bf4ede 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1502,6 +1502,8 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, swp_pte = swp_entry_to_pte(entry); if (pte_soft_dirty(pteval)) swp_pte = pte_swp_mksoft_dirty(swp_pte); + if (pte_uffd_wp(pteval)) + swp_pte = pte_swp_mkuffd_wp(swp_pte); set_pte_at(mm, pvmw.address, pvmw.pte, swp_pte); /* * No need to invalidate here it will synchronize on @@ -1601,6 +1603,8 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, swp_pte = swp_entry_to_pte(entry); if (pte_soft_dirty(pteval)) swp_pte = pte_swp_mksoft_dirty(swp_pte); + if (pte_uffd_wp(pteval)) + swp_pte = pte_swp_mkuffd_wp(swp_pte); set_pte_at(mm, address, pvmw.pte, swp_pte); /* * No need to invalidate here it will synchronize on @@ -1667,6 +1671,8 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, swp_pte = swp_entry_to_pte(entry); if (pte_soft_dirty(pteval)) swp_pte = pte_swp_mksoft_dirty(swp_pte); + if (pte_uffd_wp(pteval)) + swp_pte = pte_swp_mkuffd_wp(swp_pte); set_pte_at(mm, address, pvmw.pte, swp_pte); /* Invalidate as we cleared the pte */ mmu_notifier_invalidate_range(mm, address, From e1e267c7928fe387e5e1cffeafb0de2d0473663a Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Mon, 6 Apr 2020 20:06:04 -0700 Subject: [PATCH 515/635] khugepaged: skip collapse if uffd-wp detected Don't collapse the huge PMD if there is any userfault write protected small PTEs. The problem is that the write protection is in small page granularity and there's no way to keep all these write protection information if the small pages are going to be merged into a huge PMD. The same thing needs to be considered for swap entries and migration entries. So do the check as well disregarding khugepaged_max_ptes_swap. Signed-off-by: Peter Xu Signed-off-by: Andrew Morton Reviewed-by: Jerome Glisse Reviewed-by: Mike Rapoport Cc: Andrea Arcangeli Cc: Bobby Powers Cc: Brian Geffon Cc: David Hildenbrand Cc: Denis Plotnikov Cc: "Dr . David Alan Gilbert" Cc: Hugh Dickins Cc: Johannes Weiner Cc: "Kirill A . Shutemov" Cc: Martin Cracauer Cc: Marty McFadden Cc: Maya Gokhale Cc: Mel Gorman Cc: Mike Kravetz Cc: Pavel Emelyanov Cc: Rik van Riel Cc: Shaohua Li Link: http://lkml.kernel.org/r/20200220163112.11409-12-peterx@redhat.com Signed-off-by: Linus Torvalds --- include/trace/events/huge_memory.h | 1 + mm/khugepaged.c | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/include/trace/events/huge_memory.h b/include/trace/events/huge_memory.h index d82a0f4e824d..70e32ff096ec 100644 --- a/include/trace/events/huge_memory.h +++ b/include/trace/events/huge_memory.h @@ -13,6 +13,7 @@ EM( SCAN_PMD_NULL, "pmd_null") \ EM( SCAN_EXCEED_NONE_PTE, "exceed_none_pte") \ EM( SCAN_PTE_NON_PRESENT, "pte_non_present") \ + EM( SCAN_PTE_UFFD_WP, "pte_uffd_wp") \ EM( SCAN_PAGE_RO, "no_writable_page") \ EM( SCAN_LACK_REFERENCED_PAGE, "lack_referenced_page") \ EM( SCAN_PAGE_NULL, "page_null") \ diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 3afc1e2d7a55..99d77ffb79c2 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -29,6 +29,7 @@ enum scan_result { SCAN_PMD_NULL, SCAN_EXCEED_NONE_PTE, SCAN_PTE_NON_PRESENT, + SCAN_PTE_UFFD_WP, SCAN_PAGE_RO, SCAN_LACK_REFERENCED_PAGE, SCAN_PAGE_NULL, @@ -1137,6 +1138,15 @@ static int khugepaged_scan_pmd(struct mm_struct *mm, pte_t pteval = *_pte; if (is_swap_pte(pteval)) { if (++unmapped <= khugepaged_max_ptes_swap) { + /* + * Always be strict with uffd-wp + * enabled swap entries. Please see + * comment below for pte_uffd_wp(). + */ + if (pte_swp_uffd_wp(pteval)) { + result = SCAN_PTE_UFFD_WP; + goto out_unmap; + } continue; } else { result = SCAN_EXCEED_SWAP_PTE; @@ -1156,6 +1166,19 @@ static int khugepaged_scan_pmd(struct mm_struct *mm, result = SCAN_PTE_NON_PRESENT; goto out_unmap; } + if (pte_uffd_wp(pteval)) { + /* + * Don't collapse the page if any of the small + * PTEs are armed with uffd write protection. + * Here we can also mark the new huge pmd as + * write protected if any of the small ones is + * marked but that could bring uknown + * userfault messages that falls outside of + * the registered range. So, just be simple. + */ + result = SCAN_PTE_UFFD_WP; + goto out_unmap; + } if (pte_write(pteval)) writable = true; From ffd05793963a44bd119311df3c02b191982574ee Mon Sep 17 00:00:00 2001 From: Shaohua Li Date: Mon, 6 Apr 2020 20:06:09 -0700 Subject: [PATCH 516/635] userfaultfd: wp: support write protection for userfault vma range Add API to enable/disable writeprotect a vma range. Unlike mprotect, this doesn't split/merge vmas. [peterx@redhat.com: - use the helper to find VMA; - return -ENOENT if not found to match mcopy case; - use the new MM_CP_UFFD_WP* flags for change_protection - check against mmap_changing for failures - replace find_dst_vma with vma_find_uffd] Signed-off-by: Shaohua Li Signed-off-by: Andrea Arcangeli Signed-off-by: Peter Xu Signed-off-by: Andrew Morton Reviewed-by: Jerome Glisse Reviewed-by: Mike Rapoport Cc: Andrea Arcangeli Cc: Rik van Riel Cc: Kirill A. Shutemov Cc: Mel Gorman Cc: Hugh Dickins Cc: Johannes Weiner Cc: Bobby Powers Cc: Brian Geffon Cc: David Hildenbrand Cc: Denis Plotnikov Cc: "Dr . David Alan Gilbert" Cc: Martin Cracauer Cc: Marty McFadden Cc: Maya Gokhale Cc: Mike Kravetz Cc: Pavel Emelyanov Link: http://lkml.kernel.org/r/20200220163112.11409-13-peterx@redhat.com Signed-off-by: Linus Torvalds --- include/linux/userfaultfd_k.h | 3 ++ mm/userfaultfd.c | 54 +++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index dcd33172b728..a8e5f3ea9bb2 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -41,6 +41,9 @@ extern ssize_t mfill_zeropage(struct mm_struct *dst_mm, unsigned long dst_start, unsigned long len, bool *mmap_changing); +extern int mwriteprotect_range(struct mm_struct *dst_mm, + unsigned long start, unsigned long len, + bool enable_wp, bool *mmap_changing); /* mm helpers */ static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 7d6ab05be019..512576e171ce 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -638,3 +638,57 @@ ssize_t mfill_zeropage(struct mm_struct *dst_mm, unsigned long start, { return __mcopy_atomic(dst_mm, start, 0, len, true, mmap_changing, 0); } + +int mwriteprotect_range(struct mm_struct *dst_mm, unsigned long start, + unsigned long len, bool enable_wp, bool *mmap_changing) +{ + struct vm_area_struct *dst_vma; + pgprot_t newprot; + int err; + + /* + * Sanitize the command parameters: + */ + BUG_ON(start & ~PAGE_MASK); + BUG_ON(len & ~PAGE_MASK); + + /* Does the address range wrap, or is the span zero-sized? */ + BUG_ON(start + len <= start); + + down_read(&dst_mm->mmap_sem); + + /* + * If memory mappings are changing because of non-cooperative + * operation (e.g. mremap) running in parallel, bail out and + * request the user to retry later + */ + err = -EAGAIN; + if (mmap_changing && READ_ONCE(*mmap_changing)) + goto out_unlock; + + err = -ENOENT; + dst_vma = find_dst_vma(dst_mm, start, len); + /* + * Make sure the vma is not shared, that the dst range is + * both valid and fully within a single existing vma. + */ + if (!dst_vma || (dst_vma->vm_flags & VM_SHARED)) + goto out_unlock; + if (!userfaultfd_wp(dst_vma)) + goto out_unlock; + if (!vma_is_anonymous(dst_vma)) + goto out_unlock; + + if (enable_wp) + newprot = vm_get_page_prot(dst_vma->vm_flags & ~(VM_WRITE)); + else + newprot = vm_get_page_prot(dst_vma->vm_flags); + + change_protection(dst_vma, start, start + len, newprot, + enable_wp ? MM_CP_UFFD_WP : MM_CP_UFFD_WP_RESOLVE); + + err = 0; +out_unlock: + up_read(&dst_mm->mmap_sem); + return err; +} From 63b2d4174c4ad1f40b48d7138e71bcb564c1fe03 Mon Sep 17 00:00:00 2001 From: Andrea Arcangeli Date: Mon, 6 Apr 2020 20:06:12 -0700 Subject: [PATCH 517/635] userfaultfd: wp: add the writeprotect API to userfaultfd ioctl Introduce the new uffd-wp APIs for userspace. Firstly, we'll allow to do UFFDIO_REGISTER with write protection tracking using the new UFFDIO_REGISTER_MODE_WP flag. Note that this flag can co-exist with the existing UFFDIO_REGISTER_MODE_MISSING, in which case the userspace program can not only resolve missing page faults, and at the same time tracking page data changes along the way. Secondly, we introduced the new UFFDIO_WRITEPROTECT API to do page level write protection tracking. Note that we will need to register the memory region with UFFDIO_REGISTER_MODE_WP before that. [peterx@redhat.com: write up the commit message] [peterx@redhat.com: remove useless block, write commit message, check against VM_MAYWRITE rather than VM_WRITE when register] Signed-off-by: Andrea Arcangeli Signed-off-by: Peter Xu Signed-off-by: Andrew Morton Reviewed-by: Jerome Glisse Cc: Bobby Powers Cc: Brian Geffon Cc: David Hildenbrand Cc: Denis Plotnikov Cc: "Dr . David Alan Gilbert" Cc: Hugh Dickins Cc: Johannes Weiner Cc: "Kirill A . Shutemov" Cc: Martin Cracauer Cc: Marty McFadden Cc: Maya Gokhale Cc: Mel Gorman Cc: Mike Kravetz Cc: Mike Rapoport Cc: Pavel Emelyanov Cc: Rik van Riel Cc: Shaohua Li Link: http://lkml.kernel.org/r/20200220163112.11409-14-peterx@redhat.com Signed-off-by: Linus Torvalds --- fs/userfaultfd.c | 82 +++++++++++++++++++++++++------- include/uapi/linux/userfaultfd.h | 23 +++++++++ 2 files changed, 89 insertions(+), 16 deletions(-) diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index c49bef505775..59e9e399fddb 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -314,8 +314,11 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, if (!pmd_present(_pmd)) goto out; - if (pmd_trans_huge(_pmd)) + if (pmd_trans_huge(_pmd)) { + if (!pmd_write(_pmd) && (reason & VM_UFFD_WP)) + ret = true; goto out; + } /* * the pmd is stable (as in !pmd_trans_unstable) so we can re-read it @@ -328,6 +331,8 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, */ if (pte_none(*pte)) ret = true; + if (!pte_write(*pte) && (reason & VM_UFFD_WP)) + ret = true; pte_unmap(pte); out: @@ -1287,10 +1292,13 @@ static __always_inline int validate_range(struct mm_struct *mm, return 0; } -static inline bool vma_can_userfault(struct vm_area_struct *vma) +static inline bool vma_can_userfault(struct vm_area_struct *vma, + unsigned long vm_flags) { - return vma_is_anonymous(vma) || is_vm_hugetlb_page(vma) || - vma_is_shmem(vma); + /* FIXME: add WP support to hugetlbfs and shmem */ + return vma_is_anonymous(vma) || + ((is_vm_hugetlb_page(vma) || vma_is_shmem(vma)) && + !(vm_flags & VM_UFFD_WP)); } static int userfaultfd_register(struct userfaultfd_ctx *ctx, @@ -1322,15 +1330,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, vm_flags = 0; if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MISSING) vm_flags |= VM_UFFD_MISSING; - if (uffdio_register.mode & UFFDIO_REGISTER_MODE_WP) { + if (uffdio_register.mode & UFFDIO_REGISTER_MODE_WP) vm_flags |= VM_UFFD_WP; - /* - * FIXME: remove the below error constraint by - * implementing the wprotect tracking mode. - */ - ret = -EINVAL; - goto out; - } ret = validate_range(mm, &uffdio_register.range.start, uffdio_register.range.len); @@ -1380,7 +1381,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, /* check not compatible vmas */ ret = -EINVAL; - if (!vma_can_userfault(cur)) + if (!vma_can_userfault(cur, vm_flags)) goto out_unlock; /* @@ -1408,6 +1409,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, if (end & (vma_hpagesize - 1)) goto out_unlock; } + if ((vm_flags & VM_UFFD_WP) && !(cur->vm_flags & VM_MAYWRITE)) + goto out_unlock; /* * Check that this vma isn't already owned by a @@ -1437,7 +1440,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, do { cond_resched(); - BUG_ON(!vma_can_userfault(vma)); + BUG_ON(!vma_can_userfault(vma, vm_flags)); BUG_ON(vma->vm_userfaultfd_ctx.ctx && vma->vm_userfaultfd_ctx.ctx != ctx); WARN_ON(!(vma->vm_flags & VM_MAYWRITE)); @@ -1575,7 +1578,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, * provides for more strict behavior to notice * unregistration errors. */ - if (!vma_can_userfault(cur)) + if (!vma_can_userfault(cur, cur->vm_flags)) goto out_unlock; found = true; @@ -1589,7 +1592,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, do { cond_resched(); - BUG_ON(!vma_can_userfault(vma)); + BUG_ON(!vma_can_userfault(vma, vma->vm_flags)); /* * Nothing to do: this vma is already registered into this @@ -1802,6 +1805,50 @@ out: return ret; } +static int userfaultfd_writeprotect(struct userfaultfd_ctx *ctx, + unsigned long arg) +{ + int ret; + struct uffdio_writeprotect uffdio_wp; + struct uffdio_writeprotect __user *user_uffdio_wp; + struct userfaultfd_wake_range range; + + if (READ_ONCE(ctx->mmap_changing)) + return -EAGAIN; + + user_uffdio_wp = (struct uffdio_writeprotect __user *) arg; + + if (copy_from_user(&uffdio_wp, user_uffdio_wp, + sizeof(struct uffdio_writeprotect))) + return -EFAULT; + + ret = validate_range(ctx->mm, &uffdio_wp.range.start, + uffdio_wp.range.len); + if (ret) + return ret; + + if (uffdio_wp.mode & ~(UFFDIO_WRITEPROTECT_MODE_DONTWAKE | + UFFDIO_WRITEPROTECT_MODE_WP)) + return -EINVAL; + if ((uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_WP) && + (uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_DONTWAKE)) + return -EINVAL; + + ret = mwriteprotect_range(ctx->mm, uffdio_wp.range.start, + uffdio_wp.range.len, uffdio_wp.mode & + UFFDIO_WRITEPROTECT_MODE_WP, + &ctx->mmap_changing); + if (ret) + return ret; + + if (!(uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_DONTWAKE)) { + range.start = uffdio_wp.range.start; + range.len = uffdio_wp.range.len; + wake_userfault(ctx, &range); + } + return ret; +} + static inline unsigned int uffd_ctx_features(__u64 user_features) { /* @@ -1883,6 +1930,9 @@ static long userfaultfd_ioctl(struct file *file, unsigned cmd, case UFFDIO_ZEROPAGE: ret = userfaultfd_zeropage(ctx, arg); break; + case UFFDIO_WRITEPROTECT: + ret = userfaultfd_writeprotect(ctx, arg); + break; } return ret; } diff --git a/include/uapi/linux/userfaultfd.h b/include/uapi/linux/userfaultfd.h index 340f23bc251d..95c4a160e5f8 100644 --- a/include/uapi/linux/userfaultfd.h +++ b/include/uapi/linux/userfaultfd.h @@ -52,6 +52,7 @@ #define _UFFDIO_WAKE (0x02) #define _UFFDIO_COPY (0x03) #define _UFFDIO_ZEROPAGE (0x04) +#define _UFFDIO_WRITEPROTECT (0x06) #define _UFFDIO_API (0x3F) /* userfaultfd ioctl ids */ @@ -68,6 +69,8 @@ struct uffdio_copy) #define UFFDIO_ZEROPAGE _IOWR(UFFDIO, _UFFDIO_ZEROPAGE, \ struct uffdio_zeropage) +#define UFFDIO_WRITEPROTECT _IOWR(UFFDIO, _UFFDIO_WRITEPROTECT, \ + struct uffdio_writeprotect) /* read() structure */ struct uffd_msg { @@ -232,4 +235,24 @@ struct uffdio_zeropage { __s64 zeropage; }; +struct uffdio_writeprotect { + struct uffdio_range range; +/* + * UFFDIO_WRITEPROTECT_MODE_WP: set the flag to write protect a range, + * unset the flag to undo protection of a range which was previously + * write protected. + * + * UFFDIO_WRITEPROTECT_MODE_DONTWAKE: set the flag to avoid waking up + * any wait thread after the operation succeeds. + * + * NOTE: Write protecting a region (WP=1) is unrelated to page faults, + * therefore DONTWAKE flag is meaningless with WP=1. Removing write + * protection (WP=0) in response to a page fault wakes the faulting + * task unless DONTWAKE is set. + */ +#define UFFDIO_WRITEPROTECT_MODE_WP ((__u64)1<<0) +#define UFFDIO_WRITEPROTECT_MODE_DONTWAKE ((__u64)1<<1) + __u64 mode; +}; + #endif /* _LINUX_USERFAULTFD_H */ From e06f1e1dd4998ffc9da37f580703b55a93fc4de4 Mon Sep 17 00:00:00 2001 From: Shaohua Li Date: Mon, 6 Apr 2020 20:06:16 -0700 Subject: [PATCH 518/635] userfaultfd: wp: enabled write protection in userfaultfd API Now it's safe to enable write protection in userfaultfd API Signed-off-by: Shaohua Li Signed-off-by: Andrea Arcangeli Signed-off-by: Peter Xu Signed-off-by: Andrew Morton Reviewed-by: Jerome Glisse Reviewed-by: Mike Rapoport Cc: Andrea Arcangeli Cc: Rik van Riel Cc: Kirill A. Shutemov Cc: Mel Gorman Cc: Hugh Dickins Cc: Johannes Weiner Cc: Bobby Powers Cc: Brian Geffon Cc: David Hildenbrand Cc: Denis Plotnikov Cc: "Dr . David Alan Gilbert" Cc: Martin Cracauer Cc: Marty McFadden Cc: Maya Gokhale Cc: Mike Kravetz Cc: Pavel Emelyanov Link: http://lkml.kernel.org/r/20200220163112.11409-15-peterx@redhat.com Signed-off-by: Linus Torvalds --- include/uapi/linux/userfaultfd.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/include/uapi/linux/userfaultfd.h b/include/uapi/linux/userfaultfd.h index 95c4a160e5f8..e7e98bde221f 100644 --- a/include/uapi/linux/userfaultfd.h +++ b/include/uapi/linux/userfaultfd.h @@ -19,7 +19,8 @@ * means the userland is reading). */ #define UFFD_API ((__u64)0xAA) -#define UFFD_API_FEATURES (UFFD_FEATURE_EVENT_FORK | \ +#define UFFD_API_FEATURES (UFFD_FEATURE_PAGEFAULT_FLAG_WP | \ + UFFD_FEATURE_EVENT_FORK | \ UFFD_FEATURE_EVENT_REMAP | \ UFFD_FEATURE_EVENT_REMOVE | \ UFFD_FEATURE_EVENT_UNMAP | \ @@ -34,7 +35,8 @@ #define UFFD_API_RANGE_IOCTLS \ ((__u64)1 << _UFFDIO_WAKE | \ (__u64)1 << _UFFDIO_COPY | \ - (__u64)1 << _UFFDIO_ZEROPAGE) + (__u64)1 << _UFFDIO_ZEROPAGE | \ + (__u64)1 << _UFFDIO_WRITEPROTECT) #define UFFD_API_RANGE_IOCTLS_BASIC \ ((__u64)1 << _UFFDIO_WAKE | \ (__u64)1 << _UFFDIO_COPY) From 23080e2783ba45208795faad124eab0e0bdae85a Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Mon, 6 Apr 2020 20:06:20 -0700 Subject: [PATCH 519/635] userfaultfd: wp: don't wake up when doing write protect It does not make sense to try to wake up any waiting thread when we're write-protecting a memory region. Only wake up when resolving a write protected page fault. Signed-off-by: Peter Xu Signed-off-by: Andrew Morton Reviewed-by: Mike Rapoport Cc: Andrea Arcangeli Cc: Bobby Powers Cc: Brian Geffon Cc: David Hildenbrand Cc: Denis Plotnikov Cc: "Dr . David Alan Gilbert" Cc: Hugh Dickins Cc: Jerome Glisse Cc: Johannes Weiner Cc: "Kirill A . Shutemov" Cc: Martin Cracauer Cc: Marty McFadden Cc: Maya Gokhale Cc: Mel Gorman Cc: Mike Kravetz Cc: Pavel Emelyanov Cc: Rik van Riel Cc: Shaohua Li Link: http://lkml.kernel.org/r/20200220163112.11409-16-peterx@redhat.com Signed-off-by: Linus Torvalds --- fs/userfaultfd.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 59e9e399fddb..6e878d23547c 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -1812,6 +1812,7 @@ static int userfaultfd_writeprotect(struct userfaultfd_ctx *ctx, struct uffdio_writeprotect uffdio_wp; struct uffdio_writeprotect __user *user_uffdio_wp; struct userfaultfd_wake_range range; + bool mode_wp, mode_dontwake; if (READ_ONCE(ctx->mmap_changing)) return -EAGAIN; @@ -1830,18 +1831,20 @@ static int userfaultfd_writeprotect(struct userfaultfd_ctx *ctx, if (uffdio_wp.mode & ~(UFFDIO_WRITEPROTECT_MODE_DONTWAKE | UFFDIO_WRITEPROTECT_MODE_WP)) return -EINVAL; - if ((uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_WP) && - (uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_DONTWAKE)) + + mode_wp = uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_WP; + mode_dontwake = uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_DONTWAKE; + + if (mode_wp && mode_dontwake) return -EINVAL; ret = mwriteprotect_range(ctx->mm, uffdio_wp.range.start, - uffdio_wp.range.len, uffdio_wp.mode & - UFFDIO_WRITEPROTECT_MODE_WP, + uffdio_wp.range.len, mode_wp, &ctx->mmap_changing); if (ret) return ret; - if (!(uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_DONTWAKE)) { + if (!mode_wp && !mode_dontwake) { range.start = uffdio_wp.range.start; range.len = uffdio_wp.range.len; wake_userfault(ctx, &range); From 57e5d4f278b9522646b49a3a97ebf5f2b8f9d4cf Mon Sep 17 00:00:00 2001 From: Martin Cracauer Date: Mon, 6 Apr 2020 20:06:24 -0700 Subject: [PATCH 520/635] userfaultfd: wp: UFFDIO_REGISTER_MODE_WP documentation update Add documentation about the write protection support. [peterx@redhat.com: rewrite in rst format; fixups here and there] Signed-off-by: Martin Cracauer Signed-off-by: Andrea Arcangeli Signed-off-by: Peter Xu Signed-off-by: Andrew Morton Reviewed-by: Jerome Glisse Reviewed-by: Mike Rapoport Cc: Bobby Powers Cc: Brian Geffon Cc: David Hildenbrand Cc: Denis Plotnikov Cc: "Dr . David Alan Gilbert" Cc: Hugh Dickins Cc: Johannes Weiner Cc: "Kirill A . Shutemov" Cc: Marty McFadden Cc: Maya Gokhale Cc: Mel Gorman Cc: Mike Kravetz Cc: Pavel Emelyanov Cc: Rik van Riel Cc: Shaohua Li Link: http://lkml.kernel.org/r/20200220163112.11409-17-peterx@redhat.com Signed-off-by: Linus Torvalds --- Documentation/admin-guide/mm/userfaultfd.rst | 51 ++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/Documentation/admin-guide/mm/userfaultfd.rst b/Documentation/admin-guide/mm/userfaultfd.rst index 5048cf661a8a..c30176e67900 100644 --- a/Documentation/admin-guide/mm/userfaultfd.rst +++ b/Documentation/admin-guide/mm/userfaultfd.rst @@ -108,6 +108,57 @@ UFFDIO_COPY. They're atomic as in guaranteeing that nothing can see an half copied page since it'll keep userfaulting until the copy has finished. +Notes: + +- If you requested UFFDIO_REGISTER_MODE_MISSING when registering then + you must provide some kind of page in your thread after reading from + the uffd. You must provide either UFFDIO_COPY or UFFDIO_ZEROPAGE. + The normal behavior of the OS automatically providing a zero page on + an annonymous mmaping is not in place. + +- None of the page-delivering ioctls default to the range that you + registered with. You must fill in all fields for the appropriate + ioctl struct including the range. + +- You get the address of the access that triggered the missing page + event out of a struct uffd_msg that you read in the thread from the + uffd. You can supply as many pages as you want with UFFDIO_COPY or + UFFDIO_ZEROPAGE. Keep in mind that unless you used DONTWAKE then + the first of any of those IOCTLs wakes up the faulting thread. + +- Be sure to test for all errors including (pollfd[0].revents & + POLLERR). This can happen, e.g. when ranges supplied were + incorrect. + +Write Protect Notifications +--------------------------- + +This is equivalent to (but faster than) using mprotect and a SIGSEGV +signal handler. + +Firstly you need to register a range with UFFDIO_REGISTER_MODE_WP. +Instead of using mprotect(2) you use ioctl(uffd, UFFDIO_WRITEPROTECT, +struct *uffdio_writeprotect) while mode = UFFDIO_WRITEPROTECT_MODE_WP +in the struct passed in. The range does not default to and does not +have to be identical to the range you registered with. You can write +protect as many ranges as you like (inside the registered range). +Then, in the thread reading from uffd the struct will have +msg.arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WP set. Now you send +ioctl(uffd, UFFDIO_WRITEPROTECT, struct *uffdio_writeprotect) again +while pagefault.mode does not have UFFDIO_WRITEPROTECT_MODE_WP set. +This wakes up the thread which will continue to run with writes. This +allows you to do the bookkeeping about the write in the uffd reading +thread before the ioctl. + +If you registered with both UFFDIO_REGISTER_MODE_MISSING and +UFFDIO_REGISTER_MODE_WP then you need to think about the sequence in +which you supply a page and undo write protect. Note that there is a +difference between writes into a WP area and into a !WP area. The +former will have UFFD_PAGEFAULT_FLAG_WP set, the latter +UFFD_PAGEFAULT_FLAG_WRITE. The latter did not fail on protection but +you still need to supply a page when UFFDIO_REGISTER_MODE_MISSING was +used. + QEMU/KVM ======== From 14819305e09fe4fda546f0dfa12134c8e5366616 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Mon, 6 Apr 2020 20:06:29 -0700 Subject: [PATCH 521/635] userfaultfd: wp: declare _UFFDIO_WRITEPROTECT conditionally Only declare _UFFDIO_WRITEPROTECT if the user specified UFFDIO_REGISTER_MODE_WP and if all the checks passed. Then when the user registers regions with shmem/hugetlbfs we won't expose the new ioctl to them. Even with complete anonymous memory range, we'll only expose the new WP ioctl bit if the register mode has MODE_WP. Signed-off-by: Peter Xu Signed-off-by: Andrew Morton Reviewed-by: Mike Rapoport Cc: Andrea Arcangeli Cc: Bobby Powers Cc: Brian Geffon Cc: David Hildenbrand Cc: Denis Plotnikov Cc: "Dr . David Alan Gilbert" Cc: Hugh Dickins Cc: Jerome Glisse Cc: Johannes Weiner Cc: "Kirill A . Shutemov" Cc: Martin Cracauer Cc: Marty McFadden Cc: Maya Gokhale Cc: Mel Gorman Cc: Mike Kravetz Cc: Pavel Emelyanov Cc: Rik van Riel Cc: Shaohua Li Link: http://lkml.kernel.org/r/20200220163112.11409-18-peterx@redhat.com Signed-off-by: Linus Torvalds --- fs/userfaultfd.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 6e878d23547c..e39fdec8a0b0 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -1495,14 +1495,24 @@ out_unlock: up_write(&mm->mmap_sem); mmput(mm); if (!ret) { + __u64 ioctls_out; + + ioctls_out = basic_ioctls ? UFFD_API_RANGE_IOCTLS_BASIC : + UFFD_API_RANGE_IOCTLS; + + /* + * Declare the WP ioctl only if the WP mode is + * specified and all checks passed with the range + */ + if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_WP)) + ioctls_out &= ~((__u64)1 << _UFFDIO_WRITEPROTECT); + /* * Now that we scanned all vmas we can already tell * userland which ioctls methods are guaranteed to * succeed on this range. */ - if (put_user(basic_ioctls ? UFFD_API_RANGE_IOCTLS_BASIC : - UFFD_API_RANGE_IOCTLS, - &user_uffdio_register->ioctls)) + if (put_user(ioctls_out, &user_uffdio_register->ioctls)) ret = -EFAULT; } out: From 5c8aed6c1b95c3c6de68bd2814611d5d54da5057 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Mon, 6 Apr 2020 20:06:32 -0700 Subject: [PATCH 522/635] userfaultfd: selftests: refactor statistics Introduce uffd_stats structure for statistics of the self test, at the same time refactor the code to always pass in the uffd_stats for either read() or poll() typed fault handling threads instead of using two different ways to return the statistic results. No functional change. With the new structure, it's very easy to introduce new statistics. Signed-off-by: Peter Xu Signed-off-by: Andrew Morton Reviewed-by: Mike Rapoport Cc: Andrea Arcangeli Cc: Bobby Powers Cc: Brian Geffon Cc: David Hildenbrand Cc: Denis Plotnikov Cc: "Dr . David Alan Gilbert" Cc: Hugh Dickins Cc: Jerome Glisse Cc: Johannes Weiner Cc: "Kirill A . Shutemov" Cc: Martin Cracauer Cc: Marty McFadden Cc: Maya Gokhale Cc: Mel Gorman Cc: Mike Kravetz Cc: Pavel Emelyanov Cc: Rik van Riel Cc: Shaohua Li Link: http://lkml.kernel.org/r/20200220163112.11409-19-peterx@redhat.com Signed-off-by: Linus Torvalds --- tools/testing/selftests/vm/userfaultfd.c | 76 +++++++++++++++--------- 1 file changed, 49 insertions(+), 27 deletions(-) diff --git a/tools/testing/selftests/vm/userfaultfd.c b/tools/testing/selftests/vm/userfaultfd.c index d3362777a425..3911a9ccb0bb 100644 --- a/tools/testing/selftests/vm/userfaultfd.c +++ b/tools/testing/selftests/vm/userfaultfd.c @@ -86,6 +86,12 @@ static char *area_src, *area_src_alias, *area_dst, *area_dst_alias; static char *zeropage; pthread_attr_t attr; +/* Userfaultfd test statistics */ +struct uffd_stats { + int cpu; + unsigned long missing_faults; +}; + /* pthread_mutex_t starts at page offset 0 */ #define area_mutex(___area, ___nr) \ ((pthread_mutex_t *) ((___area) + (___nr)*page_size)) @@ -125,6 +131,17 @@ static void usage(void) exit(1); } +static void uffd_stats_reset(struct uffd_stats *uffd_stats, + unsigned long n_cpus) +{ + int i; + + for (i = 0; i < n_cpus; i++) { + uffd_stats[i].cpu = i; + uffd_stats[i].missing_faults = 0; + } +} + static int anon_release_pages(char *rel_area) { int ret = 0; @@ -467,8 +484,8 @@ static int uffd_read_msg(int ufd, struct uffd_msg *msg) return 0; } -/* Return 1 if page fault handled by us; otherwise 0 */ -static int uffd_handle_page_fault(struct uffd_msg *msg) +static void uffd_handle_page_fault(struct uffd_msg *msg, + struct uffd_stats *stats) { unsigned long offset; @@ -483,18 +500,19 @@ static int uffd_handle_page_fault(struct uffd_msg *msg) offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst; offset &= ~(page_size-1); - return copy_page(uffd, offset); + if (copy_page(uffd, offset)) + stats->missing_faults++; } static void *uffd_poll_thread(void *arg) { - unsigned long cpu = (unsigned long) arg; + struct uffd_stats *stats = (struct uffd_stats *)arg; + unsigned long cpu = stats->cpu; struct pollfd pollfd[2]; struct uffd_msg msg; struct uffdio_register uffd_reg; int ret; char tmp_chr; - unsigned long userfaults = 0; pollfd[0].fd = uffd; pollfd[0].events = POLLIN; @@ -524,7 +542,7 @@ static void *uffd_poll_thread(void *arg) msg.event), exit(1); break; case UFFD_EVENT_PAGEFAULT: - userfaults += uffd_handle_page_fault(&msg); + uffd_handle_page_fault(&msg, stats); break; case UFFD_EVENT_FORK: close(uffd); @@ -543,28 +561,27 @@ static void *uffd_poll_thread(void *arg) break; } } - return (void *)userfaults; + + return NULL; } pthread_mutex_t uffd_read_mutex = PTHREAD_MUTEX_INITIALIZER; static void *uffd_read_thread(void *arg) { - unsigned long *this_cpu_userfaults; + struct uffd_stats *stats = (struct uffd_stats *)arg; struct uffd_msg msg; - this_cpu_userfaults = (unsigned long *) arg; - *this_cpu_userfaults = 0; - pthread_mutex_unlock(&uffd_read_mutex); /* from here cancellation is ok */ for (;;) { if (uffd_read_msg(uffd, &msg)) continue; - (*this_cpu_userfaults) += uffd_handle_page_fault(&msg); + uffd_handle_page_fault(&msg, stats); } - return (void *)NULL; + + return NULL; } static void *background_thread(void *arg) @@ -580,13 +597,12 @@ static void *background_thread(void *arg) return NULL; } -static int stress(unsigned long *userfaults) +static int stress(struct uffd_stats *uffd_stats) { unsigned long cpu; pthread_t locking_threads[nr_cpus]; pthread_t uffd_threads[nr_cpus]; pthread_t background_threads[nr_cpus]; - void **_userfaults = (void **) userfaults; finished = 0; for (cpu = 0; cpu < nr_cpus; cpu++) { @@ -595,12 +611,13 @@ static int stress(unsigned long *userfaults) return 1; if (bounces & BOUNCE_POLL) { if (pthread_create(&uffd_threads[cpu], &attr, - uffd_poll_thread, (void *)cpu)) + uffd_poll_thread, + (void *)&uffd_stats[cpu])) return 1; } else { if (pthread_create(&uffd_threads[cpu], &attr, uffd_read_thread, - &_userfaults[cpu])) + (void *)&uffd_stats[cpu])) return 1; pthread_mutex_lock(&uffd_read_mutex); } @@ -637,7 +654,8 @@ static int stress(unsigned long *userfaults) fprintf(stderr, "pipefd write error\n"); return 1; } - if (pthread_join(uffd_threads[cpu], &_userfaults[cpu])) + if (pthread_join(uffd_threads[cpu], + (void *)&uffd_stats[cpu])) return 1; } else { if (pthread_cancel(uffd_threads[cpu])) @@ -908,11 +926,11 @@ static int userfaultfd_events_test(void) { struct uffdio_register uffdio_register; unsigned long expected_ioctls; - unsigned long userfaults; pthread_t uffd_mon; int err, features; pid_t pid; char c; + struct uffd_stats stats = { 0 }; printf("testing events (fork, remap, remove): "); fflush(stdout); @@ -939,7 +957,7 @@ static int userfaultfd_events_test(void) "unexpected missing ioctl for anon memory\n"), exit(1); - if (pthread_create(&uffd_mon, &attr, uffd_poll_thread, NULL)) + if (pthread_create(&uffd_mon, &attr, uffd_poll_thread, &stats)) perror("uffd_poll_thread create"), exit(1); pid = fork(); @@ -955,13 +973,13 @@ static int userfaultfd_events_test(void) if (write(pipefd[1], &c, sizeof(c)) != sizeof(c)) perror("pipe write"), exit(1); - if (pthread_join(uffd_mon, (void **)&userfaults)) + if (pthread_join(uffd_mon, NULL)) return 1; close(uffd); - printf("userfaults: %ld\n", userfaults); + printf("userfaults: %ld\n", stats.missing_faults); - return userfaults != nr_pages; + return stats.missing_faults != nr_pages; } static int userfaultfd_sig_test(void) @@ -973,6 +991,7 @@ static int userfaultfd_sig_test(void) int err, features; pid_t pid; char c; + struct uffd_stats stats = { 0 }; printf("testing signal delivery: "); fflush(stdout); @@ -1004,7 +1023,7 @@ static int userfaultfd_sig_test(void) if (uffd_test_ops->release_pages(area_dst)) return 1; - if (pthread_create(&uffd_mon, &attr, uffd_poll_thread, NULL)) + if (pthread_create(&uffd_mon, &attr, uffd_poll_thread, &stats)) perror("uffd_poll_thread create"), exit(1); pid = fork(); @@ -1030,6 +1049,7 @@ static int userfaultfd_sig_test(void) close(uffd); return userfaults != 0; } + static int userfaultfd_stress(void) { void *area; @@ -1038,7 +1058,7 @@ static int userfaultfd_stress(void) struct uffdio_register uffdio_register; unsigned long cpu; int err; - unsigned long userfaults[nr_cpus]; + struct uffd_stats uffd_stats[nr_cpus]; uffd_test_ops->allocate_area((void **)&area_src); if (!area_src) @@ -1167,8 +1187,10 @@ static int userfaultfd_stress(void) if (uffd_test_ops->release_pages(area_dst)) return 1; + uffd_stats_reset(uffd_stats, nr_cpus); + /* bounce pass */ - if (stress(userfaults)) + if (stress(uffd_stats)) return 1; /* unregister */ @@ -1211,7 +1233,7 @@ static int userfaultfd_stress(void) printf("userfaults:"); for (cpu = 0; cpu < nr_cpus; cpu++) - printf(" %lu", userfaults[cpu]); + printf(" %lu", uffd_stats[cpu].missing_faults); printf("\n"); } From 9b12488a7711b9aa2d0915f6949a8ad2069eb072 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Mon, 6 Apr 2020 20:06:36 -0700 Subject: [PATCH 523/635] userfaultfd: selftests: add write-protect test Add uffd tests for write protection. Instead of introducing new tests for it, let's simply squashing uffd-wp tests into existing uffd-missing test cases. Changes are: (1) Bouncing tests We do the write-protection in two ways during the bouncing test: - By using UFFDIO_COPY_MODE_WP when resolving MISSING pages: then we'll make sure for each bounce process every single page will be at least fault twice: once for MISSING, once for WP. - By direct call UFFDIO_WRITEPROTECT on existing faulted memories: To further torture the explicit page protection procedures of uffd-wp, we split each bounce procedure into two halves (in the background thread): the first half will be MISSING+WP for each page as explained above. After the first half, we write protect the faulted region in the background thread to make sure at least half of the pages will be write protected again which is the first half to test the new UFFDIO_WRITEPROTECT call. Then we continue with the 2nd half, which will contain both MISSING and WP faulting tests for the 2nd half and WP-only faults from the 1st half. (2) Event/Signal test Mostly previous tests but will do MISSING+WP for each page. For sigbus-mode test we'll need to provide standalone path to handle the write protection faults. For all tests, do statistics as well for uffd-wp pages. Signed-off-by: Peter Xu Signed-off-by: Andrew Morton Cc: Andrea Arcangeli Cc: Bobby Powers Cc: Brian Geffon Cc: David Hildenbrand Cc: Denis Plotnikov Cc: "Dr . David Alan Gilbert" Cc: Hugh Dickins Cc: Jerome Glisse Cc: Johannes Weiner Cc: "Kirill A . Shutemov" Cc: Martin Cracauer Cc: Marty McFadden Cc: Maya Gokhale Cc: Mel Gorman Cc: Mike Kravetz Cc: Mike Rapoport Cc: Pavel Emelyanov Cc: Rik van Riel Cc: Shaohua Li Link: http://lkml.kernel.org/r/20200220163112.11409-20-peterx@redhat.com Signed-off-by: Linus Torvalds --- tools/testing/selftests/vm/userfaultfd.c | 157 +++++++++++++++++++---- 1 file changed, 133 insertions(+), 24 deletions(-) diff --git a/tools/testing/selftests/vm/userfaultfd.c b/tools/testing/selftests/vm/userfaultfd.c index 3911a9ccb0bb..61e5cfeb1350 100644 --- a/tools/testing/selftests/vm/userfaultfd.c +++ b/tools/testing/selftests/vm/userfaultfd.c @@ -54,6 +54,7 @@ #include #include #include +#include #include "../kselftest.h" @@ -76,6 +77,8 @@ static int test_type; #define ALARM_INTERVAL_SECS 10 static volatile bool test_uffdio_copy_eexist = true; static volatile bool test_uffdio_zeropage_eexist = true; +/* Whether to test uffd write-protection */ +static bool test_uffdio_wp = false; static bool map_shared; static int huge_fd; @@ -90,6 +93,7 @@ pthread_attr_t attr; struct uffd_stats { int cpu; unsigned long missing_faults; + unsigned long wp_faults; }; /* pthread_mutex_t starts at page offset 0 */ @@ -139,9 +143,29 @@ static void uffd_stats_reset(struct uffd_stats *uffd_stats, for (i = 0; i < n_cpus; i++) { uffd_stats[i].cpu = i; uffd_stats[i].missing_faults = 0; + uffd_stats[i].wp_faults = 0; } } +static void uffd_stats_report(struct uffd_stats *stats, int n_cpus) +{ + int i; + unsigned long long miss_total = 0, wp_total = 0; + + for (i = 0; i < n_cpus; i++) { + miss_total += stats[i].missing_faults; + wp_total += stats[i].wp_faults; + } + + printf("userfaults: %llu missing (", miss_total); + for (i = 0; i < n_cpus; i++) + printf("%lu+", stats[i].missing_faults); + printf("\b), %llu wp (", wp_total); + for (i = 0; i < n_cpus; i++) + printf("%lu+", stats[i].wp_faults); + printf("\b)\n"); +} + static int anon_release_pages(char *rel_area) { int ret = 0; @@ -262,10 +286,15 @@ struct uffd_test_ops { void (*alias_mapping)(__u64 *start, size_t len, unsigned long offset); }; -#define ANON_EXPECTED_IOCTLS ((1 << _UFFDIO_WAKE) | \ +#define SHMEM_EXPECTED_IOCTLS ((1 << _UFFDIO_WAKE) | \ (1 << _UFFDIO_COPY) | \ (1 << _UFFDIO_ZEROPAGE)) +#define ANON_EXPECTED_IOCTLS ((1 << _UFFDIO_WAKE) | \ + (1 << _UFFDIO_COPY) | \ + (1 << _UFFDIO_ZEROPAGE) | \ + (1 << _UFFDIO_WRITEPROTECT)) + static struct uffd_test_ops anon_uffd_test_ops = { .expected_ioctls = ANON_EXPECTED_IOCTLS, .allocate_area = anon_allocate_area, @@ -274,7 +303,7 @@ static struct uffd_test_ops anon_uffd_test_ops = { }; static struct uffd_test_ops shmem_uffd_test_ops = { - .expected_ioctls = ANON_EXPECTED_IOCTLS, + .expected_ioctls = SHMEM_EXPECTED_IOCTLS, .allocate_area = shmem_allocate_area, .release_pages = shmem_release_pages, .alias_mapping = noop_alias_mapping, @@ -298,6 +327,21 @@ static int my_bcmp(char *str1, char *str2, size_t n) return 0; } +static void wp_range(int ufd, __u64 start, __u64 len, bool wp) +{ + struct uffdio_writeprotect prms = { 0 }; + + /* Write protection page faults */ + prms.range.start = start; + prms.range.len = len; + /* Undo write-protect, do wakeup after that */ + prms.mode = wp ? UFFDIO_WRITEPROTECT_MODE_WP : 0; + + if (ioctl(ufd, UFFDIO_WRITEPROTECT, &prms)) + fprintf(stderr, "clear WP failed for address 0x%Lx\n", + start), exit(1); +} + static void *locking_thread(void *arg) { unsigned long cpu = (unsigned long) arg; @@ -436,7 +480,10 @@ static int __copy_page(int ufd, unsigned long offset, bool retry) uffdio_copy.dst = (unsigned long) area_dst + offset; uffdio_copy.src = (unsigned long) area_src + offset; uffdio_copy.len = page_size; - uffdio_copy.mode = 0; + if (test_uffdio_wp) + uffdio_copy.mode = UFFDIO_COPY_MODE_WP; + else + uffdio_copy.mode = 0; uffdio_copy.copy = 0; if (ioctl(ufd, UFFDIO_COPY, &uffdio_copy)) { /* real retval in ufdio_copy.copy */ @@ -493,15 +540,21 @@ static void uffd_handle_page_fault(struct uffd_msg *msg, fprintf(stderr, "unexpected msg event %u\n", msg->event), exit(1); - if (bounces & BOUNCE_VERIFY && - msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE) - fprintf(stderr, "unexpected write fault\n"), exit(1); + if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WP) { + wp_range(uffd, msg->arg.pagefault.address, page_size, false); + stats->wp_faults++; + } else { + /* Missing page faults */ + if (bounces & BOUNCE_VERIFY && + msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE) + fprintf(stderr, "unexpected write fault\n"), exit(1); - offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst; - offset &= ~(page_size-1); + offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst; + offset &= ~(page_size-1); - if (copy_page(uffd, offset)) - stats->missing_faults++; + if (copy_page(uffd, offset)) + stats->missing_faults++; + } } static void *uffd_poll_thread(void *arg) @@ -587,11 +640,30 @@ static void *uffd_read_thread(void *arg) static void *background_thread(void *arg) { unsigned long cpu = (unsigned long) arg; - unsigned long page_nr; + unsigned long page_nr, start_nr, mid_nr, end_nr; - for (page_nr = cpu * nr_pages_per_cpu; - page_nr < (cpu+1) * nr_pages_per_cpu; - page_nr++) + start_nr = cpu * nr_pages_per_cpu; + end_nr = (cpu+1) * nr_pages_per_cpu; + mid_nr = (start_nr + end_nr) / 2; + + /* Copy the first half of the pages */ + for (page_nr = start_nr; page_nr < mid_nr; page_nr++) + copy_page_retry(uffd, page_nr * page_size); + + /* + * If we need to test uffd-wp, set it up now. Then we'll have + * at least the first half of the pages mapped already which + * can be write-protected for testing + */ + if (test_uffdio_wp) + wp_range(uffd, (unsigned long)area_dst + start_nr * page_size, + nr_pages_per_cpu * page_size, true); + + /* + * Continue the 2nd half of the page copying, handling write + * protection faults if any + */ + for (page_nr = mid_nr; page_nr < end_nr; page_nr++) copy_page_retry(uffd, page_nr * page_size); return NULL; @@ -753,17 +825,31 @@ static int faulting_process(int signal_test) } for (nr = 0; nr < split_nr_pages; nr++) { + int steps = 1; + unsigned long offset = nr * page_size; + if (signal_test) { if (sigsetjmp(*sigbuf, 1) != 0) { - if (nr == lastnr) { + if (steps == 1 && nr == lastnr) { fprintf(stderr, "Signal repeated\n"); return 1; } lastnr = nr; if (signal_test == 1) { - if (copy_page(uffd, nr * page_size)) - signalled++; + if (steps == 1) { + /* This is a MISSING request */ + steps++; + if (copy_page(uffd, offset)) + signalled++; + } else { + /* This is a WP request */ + assert(steps == 2); + wp_range(uffd, + (__u64)area_dst + + offset, + page_size, false); + } } else { signalled++; continue; @@ -776,8 +862,13 @@ static int faulting_process(int signal_test) fprintf(stderr, "nr %lu memory corruption %Lu %Lu\n", nr, count, - count_verify[nr]), exit(1); - } + count_verify[nr]); + } + /* + * Trigger write protection if there is by writting + * the same value back. + */ + *area_count(area_dst, nr) = count; } if (signal_test) @@ -799,6 +890,11 @@ static int faulting_process(int signal_test) nr, count, count_verify[nr]), exit(1); } + /* + * Trigger write protection if there is by writting + * the same value back. + */ + *area_count(area_dst, nr) = count; } if (uffd_test_ops->release_pages(area_dst)) @@ -902,6 +998,8 @@ static int userfaultfd_zeropage_test(void) uffdio_register.range.start = (unsigned long) area_dst; uffdio_register.range.len = nr_pages * page_size; uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING; + if (test_uffdio_wp) + uffdio_register.mode |= UFFDIO_REGISTER_MODE_WP; if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register)) fprintf(stderr, "register failure\n"), exit(1); @@ -947,6 +1045,8 @@ static int userfaultfd_events_test(void) uffdio_register.range.start = (unsigned long) area_dst; uffdio_register.range.len = nr_pages * page_size; uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING; + if (test_uffdio_wp) + uffdio_register.mode |= UFFDIO_REGISTER_MODE_WP; if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register)) fprintf(stderr, "register failure\n"), exit(1); @@ -977,7 +1077,8 @@ static int userfaultfd_events_test(void) return 1; close(uffd); - printf("userfaults: %ld\n", stats.missing_faults); + + uffd_stats_report(&stats, 1); return stats.missing_faults != nr_pages; } @@ -1007,6 +1108,8 @@ static int userfaultfd_sig_test(void) uffdio_register.range.start = (unsigned long) area_dst; uffdio_register.range.len = nr_pages * page_size; uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING; + if (test_uffdio_wp) + uffdio_register.mode |= UFFDIO_REGISTER_MODE_WP; if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register)) fprintf(stderr, "register failure\n"), exit(1); @@ -1139,6 +1242,8 @@ static int userfaultfd_stress(void) uffdio_register.range.start = (unsigned long) area_dst; uffdio_register.range.len = nr_pages * page_size; uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING; + if (test_uffdio_wp) + uffdio_register.mode |= UFFDIO_REGISTER_MODE_WP; if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register)) { fprintf(stderr, "register failure\n"); return 1; @@ -1193,6 +1298,11 @@ static int userfaultfd_stress(void) if (stress(uffd_stats)) return 1; + /* Clear all the write protections if there is any */ + if (test_uffdio_wp) + wp_range(uffd, (unsigned long)area_dst, + nr_pages * page_size, false); + /* unregister */ if (ioctl(uffd, UFFDIO_UNREGISTER, &uffdio_register.range)) { fprintf(stderr, "unregister failure\n"); @@ -1231,10 +1341,7 @@ static int userfaultfd_stress(void) area_src_alias = area_dst_alias; area_dst_alias = tmp_area; - printf("userfaults:"); - for (cpu = 0; cpu < nr_cpus; cpu++) - printf(" %lu", uffd_stats[cpu].missing_faults); - printf("\n"); + uffd_stats_report(uffd_stats, nr_cpus); } if (err) @@ -1274,6 +1381,8 @@ static void set_test_type(const char *type) if (!strcmp(type, "anon")) { test_type = TEST_ANON; uffd_test_ops = &anon_uffd_test_ops; + /* Only enable write-protect test for anonymous test */ + test_uffdio_wp = true; } else if (!strcmp(type, "hugetlb")) { test_type = TEST_HUGETLB; uffd_test_ops = &hugetlb_uffd_test_ops; From 68c3a6ac65f675b4b783635787fa0ed896f5b3d5 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Mon, 6 Apr 2020 20:06:40 -0700 Subject: [PATCH 524/635] drivers/base/memory.c: drop section_count Patch series "mm: drop superfluous section checks when onlining/offlining". Let's drop some superfluous section checks on the onlining/offlining path. This patch (of 3): Since commit c5e79ef561b0 ("mm/memory_hotplug.c: don't allow to online/offline memory blocks with holes") we have a generic check in offline_pages() that disallows offlining memory blocks with holes. Memory blocks with missing sections are just another variant of these type of blocks. We can stop checking (and especially storing) present sections. A proper error message is now printed why offlining failed. section_count was initially introduced in commit 07681215975e ("Driver core: Add section count to memory_block struct") in order to detect when it is okay to remove a memory block. It was used in commit 26bbe7ef6d5c ("drivers/base/memory.c: prohibit offlining of memory blocks with missing sections") to disallow offlining memory blocks with missing sections. As we refactored creation/removal of memory devices and have a proper check for holes in place, we can drop the section_count. This also removes a leftover comment regarding the mem_sysfs_mutex, which was removed in commit 848e19ad3c33 ("drivers/base/memory.c: drop the mem_sysfs_mutex"). Signed-off-by: David Hildenbrand Signed-off-by: Andrew Morton Cc: Greg Kroah-Hartman Cc: "Rafael J. Wysocki" Cc: Michal Hocko Cc: Dan Williams Cc: Pavel Tatashin Cc: Anshuman Khandual Link: http://lkml.kernel.org/r/20200127110424.5757-2-david@redhat.com Signed-off-by: Linus Torvalds --- drivers/base/memory.c | 17 +++-------------- include/linux/memory.h | 1 - 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/drivers/base/memory.c b/drivers/base/memory.c index 4086718f6876..086997212dbb 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -267,10 +267,6 @@ static int memory_subsys_offline(struct device *dev) if (mem->state == MEM_OFFLINE) return 0; - /* Can't offline block with non-present sections */ - if (mem->section_count != sections_per_block) - return -EINVAL; - return memory_block_change_state(mem, MEM_OFFLINE, MEM_ONLINE); } @@ -627,7 +623,7 @@ static int init_memory_block(struct memory_block **memory, static int add_memory_block(unsigned long base_section_nr) { - int ret, section_count = 0; + int section_count = 0; struct memory_block *mem; unsigned long nr; @@ -638,12 +634,8 @@ static int add_memory_block(unsigned long base_section_nr) if (section_count == 0) return 0; - ret = init_memory_block(&mem, base_memory_block_id(base_section_nr), - MEM_ONLINE); - if (ret) - return ret; - mem->section_count = section_count; - return 0; + return init_memory_block(&mem, base_memory_block_id(base_section_nr), + MEM_ONLINE); } static void unregister_memory(struct memory_block *memory) @@ -679,7 +671,6 @@ int create_memory_block_devices(unsigned long start, unsigned long size) ret = init_memory_block(&mem, block_id, MEM_OFFLINE); if (ret) break; - mem->section_count = sections_per_block; } if (ret) { end_block_id = block_id; @@ -688,7 +679,6 @@ int create_memory_block_devices(unsigned long start, unsigned long size) mem = find_memory_block_by_id(block_id); if (WARN_ON_ONCE(!mem)) continue; - mem->section_count = 0; unregister_memory(mem); } } @@ -717,7 +707,6 @@ void remove_memory_block_devices(unsigned long start, unsigned long size) mem = find_memory_block_by_id(block_id); if (WARN_ON_ONCE(!mem)) continue; - mem->section_count = 0; unregister_memory_block_under_nodes(mem); unregister_memory(mem); } diff --git a/include/linux/memory.h b/include/linux/memory.h index 0b8d791b6669..439a89e758d8 100644 --- a/include/linux/memory.h +++ b/include/linux/memory.h @@ -26,7 +26,6 @@ struct memory_block { unsigned long start_section_nr; unsigned long state; /* serialized by the dev->lock */ - int section_count; /* serialized by mem_sysfs_mutex */ int online_type; /* for passing data to online routine */ int phys_device; /* to which fru does this belong? */ struct device dev; From fada9ae3edeb4175e0f0ac9b369333806dcffbaa Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Mon, 6 Apr 2020 20:06:43 -0700 Subject: [PATCH 525/635] drivers/base/memory.c: drop pages_correctly_probed() pages_correctly_probed() is a leftover from ancient times. It dates back to commit 3947be1969a9 ("[PATCH] memory hotplug: sysfs and add/remove functions"), where Pg_reserved checks were added as a sfety net: /* * The probe routines leave the pages reserved, just * as the bootmem code does. Make sure they're still * that way. */ The checks were refactored quite a bit over the years, especially in commit b77eab7079d9 ("mm/memory_hotplug: optimize probe routine"), where checks for present, valid, and online sections were added. Hotplugged memory is added via add_memory(), which will create the full memmap for the hotplugged memory, and mark all sections valid and present. Only full memory blocks are onlined/offlined, so we also cannot have an inconsistency in that regard (especially, memory blocks with some sections being online and some being offline). 1. Boot memory always starts online. Since commit c5e79ef561b0 ("mm/memory_hotplug.c: don't allow to online/offline memory blocks with holes") we disallow to offline any memory with holes. Therefore, we never online memory with holes. Present and validity checks are superfluous. 2. Only complete memory blocks are onlined/offlined (and especially, the state - online or offline - is stored for whole memory blocks). Besides the core, only arch/powerpc/platforms/powernv/memtrace.c manually calls offline_pages() and fiddels with memory block states. But it also only offlines complete memory blocks. 3. To make any of these conditions trigger, something would have to be terribly messed up in the core. (e.g., online/offline only some sections of a memory block). 4. Memory unplug properly makes sure that all sysfs attributes were removed (and therefore, that all threads left the sysfs handlers). We don't have to worry about zombie devices at this point. 5. The valid_section_nr(section_nr) check is actually dead code, as it would never have been reached due to the WARN_ON_ONCE(!pfn_valid(pfn)). No wonder we haven't seen any of these errors in a long time (or even ever, according to my search). Let's just get rid of them. Now, all checks that could hinder onlining and offlining are completely contained in online_pages()/offline_pages(). Signed-off-by: David Hildenbrand Signed-off-by: Andrew Morton Cc: Greg Kroah-Hartman Cc: "Rafael J. Wysocki" Cc: Andrew Morton Cc: Michal Hocko Cc: Dan Williams Cc: Pavel Tatashin Cc: Anshuman Khandual Link: http://lkml.kernel.org/r/20200127110424.5757-3-david@redhat.com Signed-off-by: Linus Torvalds --- drivers/base/memory.c | 42 ------------------------------------------ 1 file changed, 42 deletions(-) diff --git a/drivers/base/memory.c b/drivers/base/memory.c index 086997212dbb..96c80dfaac90 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -144,45 +144,6 @@ int memory_notify(unsigned long val, void *v) return blocking_notifier_call_chain(&memory_chain, val, v); } -/* - * The probe routines leave the pages uninitialized, just as the bootmem code - * does. Make sure we do not access them, but instead use only information from - * within sections. - */ -static bool pages_correctly_probed(unsigned long start_pfn) -{ - unsigned long section_nr = pfn_to_section_nr(start_pfn); - unsigned long section_nr_end = section_nr + sections_per_block; - unsigned long pfn = start_pfn; - - /* - * memmap between sections is not contiguous except with - * SPARSEMEM_VMEMMAP. We lookup the page once per section - * and assume memmap is contiguous within each section - */ - for (; section_nr < section_nr_end; section_nr++) { - if (WARN_ON_ONCE(!pfn_valid(pfn))) - return false; - - if (!present_section_nr(section_nr)) { - pr_warn("section %ld pfn[%lx, %lx) not present\n", - section_nr, pfn, pfn + PAGES_PER_SECTION); - return false; - } else if (!valid_section_nr(section_nr)) { - pr_warn("section %ld pfn[%lx, %lx) no valid memmap\n", - section_nr, pfn, pfn + PAGES_PER_SECTION); - return false; - } else if (online_section_nr(section_nr)) { - pr_warn("section %ld pfn[%lx, %lx) is already online\n", - section_nr, pfn, pfn + PAGES_PER_SECTION); - return false; - } - pfn += PAGES_PER_SECTION; - } - - return true; -} - /* * MEMORY_HOTPLUG depends on SPARSEMEM in mm/Kconfig, so it is * OK to have direct references to sparsemem variables in here. @@ -199,9 +160,6 @@ memory_block_action(unsigned long start_section_nr, unsigned long action, switch (action) { case MEM_ONLINE: - if (!pages_correctly_probed(start_pfn)) - return -EBUSY; - ret = online_pages(start_pfn, nr_pages, online_type, nid); break; case MEM_OFFLINE: From dccacf8def2b718528754867b58b37734b59b0d7 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Mon, 6 Apr 2020 20:06:47 -0700 Subject: [PATCH 526/635] mm/page_ext.c: drop pfn_present() check when onlining Since commit c5e79ef561b0 ("mm/memory_hotplug.c: don't allow to online/offline memory blocks with holes") we disallow to offline any memory with holes. As all boot memory is online and hotplugged memory cannot contain holes, we never online memory with holes. This present check can be dropped. Signed-off-by: David Hildenbrand Signed-off-by: Andrew Morton Cc: Michal Hocko Cc: Anshuman Khandual Cc: Dan Williams Cc: Greg Kroah-Hartman Cc: Pavel Tatashin Cc: "Rafael J. Wysocki" Link: http://lkml.kernel.org/r/20200127110424.5757-4-david@redhat.com Signed-off-by: Linus Torvalds --- mm/page_ext.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mm/page_ext.c b/mm/page_ext.c index 08ded037f89f..a3616f7a0e9e 100644 --- a/mm/page_ext.c +++ b/mm/page_ext.c @@ -303,11 +303,8 @@ static int __meminit online_page_ext(unsigned long start_pfn, VM_BUG_ON(!node_state(nid, N_ONLINE)); } - for (pfn = start; !fail && pfn < end; pfn += PAGES_PER_SECTION) { - if (!pfn_in_present_section(pfn)) - continue; + for (pfn = start; !fail && pfn < end; pfn += PAGES_PER_SECTION) fail = init_section_page_ext(pfn, nid); - } if (!fail) return 0; From f3cd4c865b8ad61ff6dc61feba759a98ca06f039 Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Mon, 6 Apr 2020 20:06:50 -0700 Subject: [PATCH 527/635] mm/memory_hotplug.c: only respect mem= parameter during boot stage In commit 357b4da50a62 ("x86: respect memory size limiting via mem= parameter") a global varialbe max_mem_size is added to store the value parsed from 'mem= ', then checked when memory region is added. This truly stops those DIMMs from being added into system memory during boot-time. However, it also limits the later memory hotplug functionality. Any DIMM can't be hotplugged any more if its region is beyond the max_mem_size. We will get errors like: [ 216.387164] acpi PNP0C80:02: add_memory failed [ 216.389301] acpi PNP0C80:02: acpi_memory_enable_device() error [ 216.392187] acpi PNP0C80:02: Enumeration failure This will cause issue in a known use case where 'mem=' is added to the hypervisor. The memory that lies after 'mem=' boundary will be assigned to KVM guests. After commit 357b4da50a62 merged, memory can't be extended dynamically if system memory on hypervisor is not sufficient. So fix it by also checking if it's during boot-time restricting to add memory. Otherwise, skip the restriction. And also add this use case to document of 'mem=' kernel parameter. Fixes: 357b4da50a62 ("x86: respect memory size limiting via mem= parameter") Signed-off-by: Baoquan He Signed-off-by: Andrew Morton Reviewed-by: Juergen Gross Acked-by: Michal Hocko Cc: Ingo Molnar Cc: William Kucharski Cc: David Hildenbrand Cc: Balbir Singh Link: http://lkml.kernel.org/r/20200204050643.20925-1-bhe@redhat.com Signed-off-by: Linus Torvalds --- Documentation/admin-guide/kernel-parameters.txt | 13 +++++++++++-- mm/memory_hotplug.c | 8 +++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 2d31d811a52d..86aae1fa099a 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -2573,13 +2573,22 @@ For details see: Documentation/admin-guide/hw-vuln/mds.rst mem=nn[KMG] [KNL,BOOT] Force usage of a specific amount of memory - Amount of memory to be used when the kernel is not able - to see the whole system memory or for test. + Amount of memory to be used in cases as follows: + + 1 for test; + 2 when the kernel is not able to see the whole system memory; + 3 memory that lies after 'mem=' boundary is excluded from + the hypervisor, then assigned to KVM guests. + [X86] Work as limiting max address. Use together with memmap= to avoid physical address space collisions. Without memmap= PCI devices could be placed at addresses belonging to unused RAM. + Note that this only takes effects during boot time since + in above case 3, memory may need be hot added after boot + if system memory of hypervisor is not sufficient. + mem=nopentium [BUGS=X86-32] Disable usage of 4MB pages for kernel memory. diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 005eab3411e5..d6f813cc12c1 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -105,7 +105,13 @@ static struct resource *register_memory_resource(u64 start, u64 size) unsigned long flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; char *resource_name = "System RAM"; - if (start + size > max_mem_size) + /* + * Make sure value parsed from 'mem=' only restricts memory adding + * while booting, so that memory hotplug won't be impacted. Please + * refer to document of 'mem=' in kernel-parameters.txt for more + * details. + */ + if (start + size > max_mem_size && system_state < SYSTEM_RUNNING) return ERR_PTR(-E2BIG); /* From a11b9419243b9c5c7c5778d623926d9a12f68d15 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Mon, 6 Apr 2020 20:06:53 -0700 Subject: [PATCH 528/635] mm/memory_hotplug.c: simplify calculation of number of pages in __remove_pages() In commit 52fb87c81f11 ("mm/memory_hotplug: cleanup __remove_pages()"), we cleaned up __remove_pages(), and introduced a shorter variant to calculate the number of pages to the next section boundary. Turns out we can make this calculation easier to read. We always want to have the number of pages (> 0) to the next section boundary, starting from the current pfn. We'll clean up __remove_pages() in a follow-up patch and directly make use of this computation. Suggested-by: Segher Boessenkool Signed-off-by: David Hildenbrand Signed-off-by: Andrew Morton Reviewed-by: Baoquan He Reviewed-by: Wei Yang Cc: Oscar Salvador Cc: Michal Hocko Cc: Dan Williams Link: http://lkml.kernel.org/r/20200228095819.10750-2-david@redhat.com Signed-off-by: Linus Torvalds --- mm/memory_hotplug.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index d6f813cc12c1..df748b783418 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -534,7 +534,8 @@ void __remove_pages(unsigned long pfn, unsigned long nr_pages, for (; pfn < end_pfn; pfn += cur_nr_pages) { cond_resched(); /* Select all remaining pages up to the next section boundary */ - cur_nr_pages = min(end_pfn - pfn, -(pfn | PAGE_SECTION_MASK)); + cur_nr_pages = min(end_pfn - pfn, + SECTION_ALIGN_UP(pfn + 1) - pfn); __remove_section(pfn, cur_nr_pages, map_offset, altmap); map_offset = 0; } From 6cdd0b30a920b35d901c8ca6b82e9ca4f44f54d6 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Mon, 6 Apr 2020 20:06:56 -0700 Subject: [PATCH 529/635] mm/memory_hotplug.c: cleanup __add_pages() Let's drop the basically unused section stuff and simplify. The logic now matches the logic in __remove_pages(). Signed-off-by: David Hildenbrand Signed-off-by: Andrew Morton Reviewed-by: Baoquan He Reviewed-by: Wei Yang Cc: Segher Boessenkool Cc: Oscar Salvador Cc: Michal Hocko Cc: Dan Williams Link: http://lkml.kernel.org/r/20200228095819.10750-3-david@redhat.com Signed-off-by: Linus Torvalds --- mm/memory_hotplug.c | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index df748b783418..04cb87ec39d0 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -307,8 +307,9 @@ static int check_hotplug_memory_addressable(unsigned long pfn, int __ref __add_pages(int nid, unsigned long pfn, unsigned long nr_pages, struct mhp_restrictions *restrictions) { + const unsigned long end_pfn = pfn + nr_pages; + unsigned long cur_nr_pages; int err; - unsigned long nr, start_sec, end_sec; struct vmem_altmap *altmap = restrictions->altmap; err = check_hotplug_memory_addressable(pfn, nr_pages); @@ -331,18 +332,13 @@ int __ref __add_pages(int nid, unsigned long pfn, unsigned long nr_pages, if (err) return err; - start_sec = pfn_to_section_nr(pfn); - end_sec = pfn_to_section_nr(pfn + nr_pages - 1); - for (nr = start_sec; nr <= end_sec; nr++) { - unsigned long pfns; - - pfns = min(nr_pages, PAGES_PER_SECTION - - (pfn & ~PAGE_SECTION_MASK)); - err = sparse_add_section(nid, pfn, pfns, altmap); + for (; pfn < end_pfn; pfn += cur_nr_pages) { + /* Select all remaining pages up to the next section boundary */ + cur_nr_pages = min(end_pfn - pfn, + SECTION_ALIGN_UP(pfn + 1) - pfn); + err = sparse_add_section(nid, pfn, cur_nr_pages, altmap); if (err) break; - pfn += pfns; - nr_pages -= pfns; cond_resched(); } vmemmap_populate_print_last(); From 5d87255cadde243763ca22b35e01312550114167 Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Mon, 6 Apr 2020 20:07:00 -0700 Subject: [PATCH 530/635] mm/sparse.c: introduce new function fill_subsection_map() Patch series "mm/hotplug: Only use subsection map for VMEMMAP", v4. Memory sub-section hotplug was added to fix the issue that nvdimm could be mapped at non-section aligned starting address. A subsection map is added into struct mem_section_usage to implement it. However, config ZONE_DEVICE depends on SPARSEMEM_VMEMMAP. It means subsection map only makes sense when SPARSEMEM_VMEMMAP enabled. For the classic sparse, subsection map is meaningless and confusing. About the classic sparse which doesn't support subsection hotplug, Dan said it's more because the effort and maintenance burden outweighs the benefit. Besides, the current 64 bit ARCHes all enable SPARSEMEM_VMEMMAP_ENABLE by default. This patch (of 5): Factor out the code that fills the subsection map from section_activate() into fill_subsection_map(), this makes section_activate() cleaner and easier to follow. Signed-off-by: Baoquan He Signed-off-by: Andrew Morton Reviewed-by: Wei Yang Reviewed-by: David Hildenbrand Acked-by: Pankaj Gupta Cc: Dan Williams Cc: Michal Hocko Link: http://lkml.kernel.org/r/20200312124414.439-2-bhe@redhat.com Signed-off-by: Linus Torvalds --- mm/sparse.c | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/mm/sparse.c b/mm/sparse.c index f1af4d4ee80b..51965d56cd39 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -777,24 +777,15 @@ static void section_deactivate(unsigned long pfn, unsigned long nr_pages, ms->section_mem_map = (unsigned long)NULL; } -static struct page * __meminit section_activate(int nid, unsigned long pfn, - unsigned long nr_pages, struct vmem_altmap *altmap) +static int fill_subsection_map(unsigned long pfn, unsigned long nr_pages) { - DECLARE_BITMAP(map, SUBSECTIONS_PER_SECTION) = { 0 }; struct mem_section *ms = __pfn_to_section(pfn); - struct mem_section_usage *usage = NULL; + DECLARE_BITMAP(map, SUBSECTIONS_PER_SECTION) = { 0 }; unsigned long *subsection_map; - struct page *memmap; int rc = 0; subsection_mask_set(map, pfn, nr_pages); - if (!ms->usage) { - usage = kzalloc(mem_section_usage_size(), GFP_KERNEL); - if (!usage) - return ERR_PTR(-ENOMEM); - ms->usage = usage; - } subsection_map = &ms->usage->subsection_map[0]; if (bitmap_empty(map, SUBSECTIONS_PER_SECTION)) @@ -805,6 +796,25 @@ static struct page * __meminit section_activate(int nid, unsigned long pfn, bitmap_or(subsection_map, map, subsection_map, SUBSECTIONS_PER_SECTION); + return rc; +} + +static struct page * __meminit section_activate(int nid, unsigned long pfn, + unsigned long nr_pages, struct vmem_altmap *altmap) +{ + struct mem_section *ms = __pfn_to_section(pfn); + struct mem_section_usage *usage = NULL; + struct page *memmap; + int rc = 0; + + if (!ms->usage) { + usage = kzalloc(mem_section_usage_size(), GFP_KERNEL); + if (!usage) + return ERR_PTR(-ENOMEM); + ms->usage = usage; + } + + rc = fill_subsection_map(pfn, nr_pages); if (rc) { if (usage) ms->usage = NULL; From 37bc15020a96035cb157be5864b958672fe02d7c Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Mon, 6 Apr 2020 20:07:03 -0700 Subject: [PATCH 531/635] mm/sparse.c: introduce a new function clear_subsection_map() Factor out the code which clear subsection map of one memory region from section_deactivate() into clear_subsection_map(). And also add helper function is_subsection_map_empty() to check if the current subsection map is empty or not. Signed-off-by: Baoquan He Signed-off-by: Andrew Morton Reviewed-by: David Hildenbrand Acked-by: Pankaj Gupta Cc: Dan Williams Cc: Michal Hocko Cc: Wei Yang Link: http://lkml.kernel.org/r/20200312124414.439-3-bhe@redhat.com Signed-off-by: Linus Torvalds --- mm/sparse.c | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/mm/sparse.c b/mm/sparse.c index 51965d56cd39..01204c3b4649 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -705,15 +705,11 @@ static void free_map_bootmem(struct page *memmap) } #endif /* CONFIG_SPARSEMEM_VMEMMAP */ -static void section_deactivate(unsigned long pfn, unsigned long nr_pages, - struct vmem_altmap *altmap) +static int clear_subsection_map(unsigned long pfn, unsigned long nr_pages) { DECLARE_BITMAP(map, SUBSECTIONS_PER_SECTION) = { 0 }; DECLARE_BITMAP(tmp, SUBSECTIONS_PER_SECTION) = { 0 }; struct mem_section *ms = __pfn_to_section(pfn); - bool section_is_early = early_section(ms); - struct page *memmap = NULL; - bool empty; unsigned long *subsection_map = ms->usage ? &ms->usage->subsection_map[0] : NULL; @@ -724,8 +720,28 @@ static void section_deactivate(unsigned long pfn, unsigned long nr_pages, if (WARN(!subsection_map || !bitmap_equal(tmp, map, SUBSECTIONS_PER_SECTION), "section already deactivated (%#lx + %ld)\n", pfn, nr_pages)) - return; + return -EINVAL; + bitmap_xor(subsection_map, map, subsection_map, SUBSECTIONS_PER_SECTION); + return 0; +} + +static bool is_subsection_map_empty(struct mem_section *ms) +{ + return bitmap_empty(&ms->usage->subsection_map[0], + SUBSECTIONS_PER_SECTION); +} + +static void section_deactivate(unsigned long pfn, unsigned long nr_pages, + struct vmem_altmap *altmap) +{ + struct mem_section *ms = __pfn_to_section(pfn); + bool section_is_early = early_section(ms); + struct page *memmap = NULL; + bool empty; + + if (clear_subsection_map(pfn, nr_pages)) + return; /* * There are 3 cases to handle across two configurations * (SPARSEMEM_VMEMMAP={y,n}): @@ -743,8 +759,7 @@ static void section_deactivate(unsigned long pfn, unsigned long nr_pages, * * For 2/ and 3/ the SPARSEMEM_VMEMMAP={y,n} cases are unified */ - bitmap_xor(subsection_map, map, subsection_map, SUBSECTIONS_PER_SECTION); - empty = bitmap_empty(subsection_map, SUBSECTIONS_PER_SECTION); + empty = is_subsection_map_empty(ms); if (empty) { unsigned long section_nr = pfn_to_section_nr(pfn); From 0a9f9f62316606ee827fa3318e95a1c489d9acf5 Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Mon, 6 Apr 2020 20:07:06 -0700 Subject: [PATCH 532/635] mm/sparse.c: only use subsection map in VMEMMAP case Currently, to support subsection aligned memory region adding for pmem, subsection map is added to track which subsection is present. However, config ZONE_DEVICE depends on SPARSEMEM_VMEMMAP. It means subsection map only makes sense when SPARSEMEM_VMEMMAP enabled. For the classic sparse, it's meaningless. Even worse, it may confuse people when checking code related to the classic sparse. About the classic sparse which doesn't support subsection hotplug, Dan said it's more because the effort and maintenance burden outweighs the benefit. Besides, the current 64 bit ARCHes all enable SPARSEMEM_VMEMMAP_ENABLE by default. Combining the above reasons, no need to provide subsection map and the relevant handling for the classic sparse. Let's remove them. Signed-off-by: Baoquan He Signed-off-by: Andrew Morton Reviewed-by: David Hildenbrand Cc: Dan Williams Cc: Michal Hocko Cc: Pankaj Gupta Cc: Wei Yang Link: http://lkml.kernel.org/r/20200312124414.439-4-bhe@redhat.com Signed-off-by: Linus Torvalds --- include/linux/mmzone.h | 2 ++ mm/sparse.c | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 42b77d3b68e8..f3f264826423 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -1143,7 +1143,9 @@ static inline unsigned long section_nr_to_pfn(unsigned long sec) #define SUBSECTION_ALIGN_DOWN(pfn) ((pfn) & PAGE_SUBSECTION_MASK) struct mem_section_usage { +#ifdef CONFIG_SPARSEMEM_VMEMMAP DECLARE_BITMAP(subsection_map, SUBSECTIONS_PER_SECTION); +#endif /* See declaration of similar field in struct zone */ unsigned long pageblock_flags[0]; }; diff --git a/mm/sparse.c b/mm/sparse.c index 01204c3b4649..095ecf5bb6d3 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -209,6 +209,7 @@ static inline unsigned long first_present_section_nr(void) return next_present_section_nr(-1); } +#ifdef CONFIG_SPARSEMEM_VMEMMAP static void subsection_mask_set(unsigned long *map, unsigned long pfn, unsigned long nr_pages) { @@ -243,6 +244,11 @@ void __init subsection_map_init(unsigned long pfn, unsigned long nr_pages) nr_pages -= pfns; } } +#else +void __init subsection_map_init(unsigned long pfn, unsigned long nr_pages) +{ +} +#endif /* Record a memory area against a node. */ void __init memory_present(int nid, unsigned long start, unsigned long end) @@ -705,6 +711,7 @@ static void free_map_bootmem(struct page *memmap) } #endif /* CONFIG_SPARSEMEM_VMEMMAP */ +#ifdef CONFIG_SPARSEMEM_VMEMMAP static int clear_subsection_map(unsigned long pfn, unsigned long nr_pages) { DECLARE_BITMAP(map, SUBSECTIONS_PER_SECTION) = { 0 }; @@ -731,6 +738,17 @@ static bool is_subsection_map_empty(struct mem_section *ms) return bitmap_empty(&ms->usage->subsection_map[0], SUBSECTIONS_PER_SECTION); } +#else +static int clear_subsection_map(unsigned long pfn, unsigned long nr_pages) +{ + return 0; +} + +static bool is_subsection_map_empty(struct mem_section *ms) +{ + return true; +} +#endif static void section_deactivate(unsigned long pfn, unsigned long nr_pages, struct vmem_altmap *altmap) @@ -792,6 +810,7 @@ static void section_deactivate(unsigned long pfn, unsigned long nr_pages, ms->section_mem_map = (unsigned long)NULL; } +#ifdef CONFIG_SPARSEMEM_VMEMMAP static int fill_subsection_map(unsigned long pfn, unsigned long nr_pages) { struct mem_section *ms = __pfn_to_section(pfn); @@ -813,6 +832,12 @@ static int fill_subsection_map(unsigned long pfn, unsigned long nr_pages) return rc; } +#else +static int fill_subsection_map(unsigned long pfn, unsigned long nr_pages) +{ + return 0; +} +#endif static struct page * __meminit section_activate(int nid, unsigned long pfn, unsigned long nr_pages, struct vmem_altmap *altmap) From 95a5a34dfe22be11bc4af58854585e90124b1db0 Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Mon, 6 Apr 2020 20:07:09 -0700 Subject: [PATCH 533/635] mm/sparse.c: add note about only VMEMMAP supporting sub-section hotplug And tell check_pfn_span() gating the porper alignment and size of hot added memory region. And also move the code comments from inside section_deactivate() to being above it. The code comments are reasonable for the whole function, and the moving makes code cleaner. Signed-off-by: Baoquan He Signed-off-by: Andrew Morton Reviewed-by: David Hildenbrand Acked-by: Michal Hocko Cc: Dan Williams Cc: Pankaj Gupta Cc: Wei Yang Link: http://lkml.kernel.org/r/20200312124414.439-5-bhe@redhat.com Signed-off-by: Linus Torvalds --- mm/sparse.c | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/mm/sparse.c b/mm/sparse.c index 095ecf5bb6d3..9d43fde1f630 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -750,6 +750,22 @@ static bool is_subsection_map_empty(struct mem_section *ms) } #endif +/* + * To deactivate a memory region, there are 3 cases to handle across + * two configurations (SPARSEMEM_VMEMMAP={y,n}): + * + * 1. deactivation of a partial hot-added section (only possible in + * the SPARSEMEM_VMEMMAP=y case). + * a) section was present at memory init. + * b) section was hot-added post memory init. + * 2. deactivation of a complete hot-added section. + * 3. deactivation of a complete section from memory init. + * + * For 1, when subsection_map does not empty we will not be freeing the + * usage map, but still need to free the vmemmap range. + * + * For 2 and 3, the SPARSEMEM_VMEMMAP={y,n} cases are unified + */ static void section_deactivate(unsigned long pfn, unsigned long nr_pages, struct vmem_altmap *altmap) { @@ -760,23 +776,7 @@ static void section_deactivate(unsigned long pfn, unsigned long nr_pages, if (clear_subsection_map(pfn, nr_pages)) return; - /* - * There are 3 cases to handle across two configurations - * (SPARSEMEM_VMEMMAP={y,n}): - * - * 1/ deactivation of a partial hot-added section (only possible - * in the SPARSEMEM_VMEMMAP=y case). - * a/ section was present at memory init - * b/ section was hot-added post memory init - * 2/ deactivation of a complete hot-added section - * 3/ deactivation of a complete section from memory init - * - * For 1/, when subsection_map does not empty we will not be - * freeing the usage map, but still need to free the vmemmap - * range. - * - * For 2/ and 3/ the SPARSEMEM_VMEMMAP={y,n} cases are unified - */ + empty = is_subsection_map_empty(ms); if (empty) { unsigned long section_nr = pfn_to_section_nr(pfn); @@ -890,6 +890,10 @@ static struct page * __meminit section_activate(int nid, unsigned long pfn, * * This is only intended for hotplug. * + * Note that only VMEMMAP supports sub-section aligned hotplug, + * the proper alignment and size are gated by check_pfn_span(). + * + * * Return: * * 0 - On success. * * -EEXIST - Section has been present. From 6ecb0fc61290e16217a8f6165225b53b193b337c Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Mon, 6 Apr 2020 20:07:13 -0700 Subject: [PATCH 534/635] mm/sparse.c: move subsection_map related functions together No functional change. [bhe@redhat.com: move functions into CONFIG_MEMORY_HOTPLUG ifdeffery scope] Link: http://lkml.kernel.org/r/20200316045804.GC3486@MiWiFi-R3L-srv Signed-off-by: Baoquan He Signed-off-by: Andrew Morton Cc: Michal Hocko Cc: David Hildenbrand Cc: Wei Yang Cc: Dan Williams Cc: Pankaj Gupta Cc: Stephen Rothwell Link: http://lkml.kernel.org/r/20200312124414.439-6-bhe@redhat.com Signed-off-by: Linus Torvalds --- mm/sparse.c | 114 +++++++++++++++++++++++++--------------------------- 1 file changed, 55 insertions(+), 59 deletions(-) diff --git a/mm/sparse.c b/mm/sparse.c index 9d43fde1f630..1aee5a481571 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -666,6 +666,55 @@ static void free_map_bootmem(struct page *memmap) vmemmap_free(start, end, NULL); } + +static int clear_subsection_map(unsigned long pfn, unsigned long nr_pages) +{ + DECLARE_BITMAP(map, SUBSECTIONS_PER_SECTION) = { 0 }; + DECLARE_BITMAP(tmp, SUBSECTIONS_PER_SECTION) = { 0 }; + struct mem_section *ms = __pfn_to_section(pfn); + unsigned long *subsection_map = ms->usage + ? &ms->usage->subsection_map[0] : NULL; + + subsection_mask_set(map, pfn, nr_pages); + if (subsection_map) + bitmap_and(tmp, map, subsection_map, SUBSECTIONS_PER_SECTION); + + if (WARN(!subsection_map || !bitmap_equal(tmp, map, SUBSECTIONS_PER_SECTION), + "section already deactivated (%#lx + %ld)\n", + pfn, nr_pages)) + return -EINVAL; + + bitmap_xor(subsection_map, map, subsection_map, SUBSECTIONS_PER_SECTION); + return 0; +} + +static bool is_subsection_map_empty(struct mem_section *ms) +{ + return bitmap_empty(&ms->usage->subsection_map[0], + SUBSECTIONS_PER_SECTION); +} + +static int fill_subsection_map(unsigned long pfn, unsigned long nr_pages) +{ + struct mem_section *ms = __pfn_to_section(pfn); + DECLARE_BITMAP(map, SUBSECTIONS_PER_SECTION) = { 0 }; + unsigned long *subsection_map; + int rc = 0; + + subsection_mask_set(map, pfn, nr_pages); + + subsection_map = &ms->usage->subsection_map[0]; + + if (bitmap_empty(map, SUBSECTIONS_PER_SECTION)) + rc = -EINVAL; + else if (bitmap_intersects(map, subsection_map, SUBSECTIONS_PER_SECTION)) + rc = -EEXIST; + else + bitmap_or(subsection_map, map, subsection_map, + SUBSECTIONS_PER_SECTION); + + return rc; +} #else struct page * __meminit populate_section_memmap(unsigned long pfn, unsigned long nr_pages, int nid, struct vmem_altmap *altmap) @@ -709,36 +758,7 @@ static void free_map_bootmem(struct page *memmap) put_page_bootmem(page); } } -#endif /* CONFIG_SPARSEMEM_VMEMMAP */ -#ifdef CONFIG_SPARSEMEM_VMEMMAP -static int clear_subsection_map(unsigned long pfn, unsigned long nr_pages) -{ - DECLARE_BITMAP(map, SUBSECTIONS_PER_SECTION) = { 0 }; - DECLARE_BITMAP(tmp, SUBSECTIONS_PER_SECTION) = { 0 }; - struct mem_section *ms = __pfn_to_section(pfn); - unsigned long *subsection_map = ms->usage - ? &ms->usage->subsection_map[0] : NULL; - - subsection_mask_set(map, pfn, nr_pages); - if (subsection_map) - bitmap_and(tmp, map, subsection_map, SUBSECTIONS_PER_SECTION); - - if (WARN(!subsection_map || !bitmap_equal(tmp, map, SUBSECTIONS_PER_SECTION), - "section already deactivated (%#lx + %ld)\n", - pfn, nr_pages)) - return -EINVAL; - - bitmap_xor(subsection_map, map, subsection_map, SUBSECTIONS_PER_SECTION); - return 0; -} - -static bool is_subsection_map_empty(struct mem_section *ms) -{ - return bitmap_empty(&ms->usage->subsection_map[0], - SUBSECTIONS_PER_SECTION); -} -#else static int clear_subsection_map(unsigned long pfn, unsigned long nr_pages) { return 0; @@ -748,7 +768,12 @@ static bool is_subsection_map_empty(struct mem_section *ms) { return true; } -#endif + +static int fill_subsection_map(unsigned long pfn, unsigned long nr_pages) +{ + return 0; +} +#endif /* CONFIG_SPARSEMEM_VMEMMAP */ /* * To deactivate a memory region, there are 3 cases to handle across @@ -810,35 +835,6 @@ static void section_deactivate(unsigned long pfn, unsigned long nr_pages, ms->section_mem_map = (unsigned long)NULL; } -#ifdef CONFIG_SPARSEMEM_VMEMMAP -static int fill_subsection_map(unsigned long pfn, unsigned long nr_pages) -{ - struct mem_section *ms = __pfn_to_section(pfn); - DECLARE_BITMAP(map, SUBSECTIONS_PER_SECTION) = { 0 }; - unsigned long *subsection_map; - int rc = 0; - - subsection_mask_set(map, pfn, nr_pages); - - subsection_map = &ms->usage->subsection_map[0]; - - if (bitmap_empty(map, SUBSECTIONS_PER_SECTION)) - rc = -EINVAL; - else if (bitmap_intersects(map, subsection_map, SUBSECTIONS_PER_SECTION)) - rc = -EEXIST; - else - bitmap_or(subsection_map, map, subsection_map, - SUBSECTIONS_PER_SECTION); - - return rc; -} -#else -static int fill_subsection_map(unsigned long pfn, unsigned long nr_pages) -{ - return 0; -} -#endif - static struct page * __meminit section_activate(int nid, unsigned long pfn, unsigned long nr_pages, struct vmem_altmap *altmap) { From 956f8b445061667c3545baa24778f890d1d522f4 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Mon, 6 Apr 2020 20:07:16 -0700 Subject: [PATCH 535/635] drivers/base/memory: rename MMOP_ONLINE_KEEP to MMOP_ONLINE Patch series "mm/memory_hotplug: allow to specify a default online_type", v3. Distributions nowadays use udev rules ([1] [2]) to specify if and how to online hotplugged memory. The rules seem to get more complex with many special cases. Due to the various special cases, CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE cannot be used. All memory hotplug is handled via udev rules. Every time we hotplug memory, the udev rule will come to the same conclusion. Especially Hyper-V (but also soon virtio-mem) add a lot of memory in separate memory blocks and wait for memory to get onlined by user space before continuing to add more memory blocks (to not add memory faster than it is getting onlined). This of course slows down the whole memory hotplug process. To make the job of distributions easier and to avoid udev rules that get more and more complicated, let's extend the mechanism provided by - /sys/devices/system/memory/auto_online_blocks - "memhp_default_state=" on the kernel cmdline to be able to specify also "online_movable" as well as "online_kernel" === Example /usr/libexec/config-memhotplug === #!/bin/bash VIRT=`systemd-detect-virt --vm` ARCH=`uname -p` sense_virtio_mem() { if [ -d "/sys/bus/virtio/drivers/virtio_mem/" ]; then DEVICES=`find /sys/bus/virtio/drivers/virtio_mem/ -maxdepth 1 -type l | wc -l` if [ $DEVICES != "0" ]; then return 0 fi fi return 1 } if [ ! -e "/sys/devices/system/memory/auto_online_blocks" ]; then echo "Memory hotplug configuration support missing in the kernel" exit 1 fi if grep "memhp_default_state=" /proc/cmdline > /dev/null; then echo "Memory hotplug configuration overridden in kernel cmdline (memhp_default_state=)" exit 1 fi if [ $VIRT == "microsoft" ]; then echo "Detected Hyper-V on $ARCH" # Hyper-V wants all memory in ZONE_NORMAL ONLINE_TYPE="online_kernel" elif sense_virtio_mem; then echo "Detected virtio-mem on $ARCH" # virtio-mem wants all memory in ZONE_NORMAL ONLINE_TYPE="online_kernel" elif [ $ARCH == "s390x" ] || [ $ARCH == "s390" ]; then echo "Detected $ARCH" # standby memory should not be onlined automatically ONLINE_TYPE="offline" elif [ $ARCH == "ppc64" ] || [ $ARCH == "ppc64le" ]; then echo "Detected" $ARCH # PPC64 onlines all hotplugged memory right from the kernel ONLINE_TYPE="offline" elif [ $VIRT == "none" ]; then echo "Detected bare-metal on $ARCH" # Bare metal users expect hotplugged memory to be unpluggable. We assume # that ZONE imbalances on such enterpise servers cannot happen and is # properly documented ONLINE_TYPE="online_movable" else # TODO: Hypervisors that want to unplug DIMMs and can guarantee that ZONE # imbalances won't happen echo "Detected $VIRT on $ARCH" # Usually, ballooning is used in virtual environments, so memory should go to # ZONE_NORMAL. However, sometimes "movable_node" is relevant. ONLINE_TYPE="online" fi echo "Selected online_type:" $ONLINE_TYPE # Configure what to do with memory that will be hotplugged in the future echo $ONLINE_TYPE 2>/dev/null > /sys/devices/system/memory/auto_online_blocks if [ $? != "0" ]; then echo "Memory hotplug cannot be configured (e.g., old kernel or missing permissions)" # A backup udev rule should handle old kernels if necessary exit 1 fi # Process all already pluggedd blocks (e.g., DIMMs, but also Hyper-V or virtio-mem) if [ $ONLINE_TYPE != "offline" ]; then for MEMORY in /sys/devices/system/memory/memory*; do STATE=`cat $MEMORY/state` if [ $STATE == "offline" ]; then echo $ONLINE_TYPE > $MEMORY/state fi done fi === Example /usr/lib/systemd/system/config-memhotplug.service === [Unit] Description=Configure memory hotplug behavior DefaultDependencies=no Conflicts=shutdown.target Before=sysinit.target shutdown.target After=systemd-modules-load.service ConditionPathExists=|/sys/devices/system/memory/auto_online_blocks [Service] ExecStart=/usr/libexec/config-memhotplug Type=oneshot TimeoutSec=0 RemainAfterExit=yes [Install] WantedBy=sysinit.target === Example modification to the 40-redhat.rules [2] === : diff --git a/40-redhat.rules b/40-redhat.rules-new : index 2c690e5..168fd03 100644 : --- a/40-redhat.rules : +++ b/40-redhat.rules-new : @@ -6,6 +6,9 @@ SUBSYSTEM=="cpu", ACTION=="add", TEST=="online", ATTR{online}=="0", ATTR{online} : # Memory hotadd request : SUBSYSTEM!="memory", GOTO="memory_hotplug_end" : ACTION!="add", GOTO="memory_hotplug_end" : +# memory hotplug behavior configured : +PROGRAM=="grep online /sys/devices/system/memory/auto_online_blocks", GOTO="memory_hotplug_end" : + : PROGRAM="/bin/uname -p", RESULT=="s390*", GOTO="memory_hotplug_end" : : ENV{.state}="online" === [1] https://github.com/lnykryn/systemd-rhel/pull/281 [2] https://github.com/lnykryn/systemd-rhel/blob/staging/rules/40-redhat.rules This patch (of 8): The name is misleading and it's not really clear what is "kept". Let's just name it like the online_type name we expose to user space ("online"). Add some documentation to the types. Signed-off-by: David Hildenbrand Signed-off-by: Andrew Morton Reviewed-by: Wei Yang Reviewed-by: Baoquan He Acked-by: Pankaj Gupta Cc: Greg Kroah-Hartman Cc: Michal Hocko Cc: Oscar Salvador Cc: "Rafael J. Wysocki" Cc: Wei Yang Cc: Vitaly Kuznetsov Cc: Yumei Huang Cc: Igor Mammedov Cc: Eduardo Habkost Cc: Benjamin Herrenschmidt Cc: Haiyang Zhang Cc: K. Y. Srinivasan Cc: Michael Ellerman (powerpc) Cc: Paul Mackerras Cc: Stephen Hemminger Cc: Wei Liu Link: http://lkml.kernel.org/r/20200319131221.14044-1-david@redhat.com Link: http://lkml.kernel.org/r/20200317104942.11178-2-david@redhat.com Signed-off-by: Linus Torvalds --- drivers/base/memory.c | 9 +++++---- include/linux/memory_hotplug.h | 6 +++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/drivers/base/memory.c b/drivers/base/memory.c index 96c80dfaac90..156b89b14fcc 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -208,7 +208,7 @@ static int memory_subsys_online(struct device *dev) * attribute and need to set the online_type. */ if (mem->online_type < 0) - mem->online_type = MMOP_ONLINE_KEEP; + mem->online_type = MMOP_ONLINE; ret = memory_block_change_state(mem, MEM_ONLINE, MEM_OFFLINE); @@ -243,7 +243,7 @@ static ssize_t state_store(struct device *dev, struct device_attribute *attr, else if (sysfs_streq(buf, "online_movable")) online_type = MMOP_ONLINE_MOVABLE; else if (sysfs_streq(buf, "online")) - online_type = MMOP_ONLINE_KEEP; + online_type = MMOP_ONLINE; else if (sysfs_streq(buf, "offline")) online_type = MMOP_OFFLINE; else { @@ -254,7 +254,7 @@ static ssize_t state_store(struct device *dev, struct device_attribute *attr, switch (online_type) { case MMOP_ONLINE_KERNEL: case MMOP_ONLINE_MOVABLE: - case MMOP_ONLINE_KEEP: + case MMOP_ONLINE: /* mem->online_type is protected by device_hotplug_lock */ mem->online_type = online_type; ret = device_online(&mem->dev); @@ -334,7 +334,8 @@ static ssize_t valid_zones_show(struct device *dev, } nid = mem->nid; - default_zone = zone_for_pfn_range(MMOP_ONLINE_KEEP, nid, start_pfn, nr_pages); + default_zone = zone_for_pfn_range(MMOP_ONLINE, nid, start_pfn, + nr_pages); strcat(buf, default_zone->name); print_allowed_zone(buf, nid, start_pfn, nr_pages, MMOP_ONLINE_KERNEL, diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index f4d59155f3d4..261dbf010d5d 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -47,9 +47,13 @@ enum { /* Types for control the zone type of onlined and offlined memory */ enum { + /* Offline the memory. */ MMOP_OFFLINE = -1, - MMOP_ONLINE_KEEP, + /* Online the memory. Zone depends, see default_zone_for_pfn(). */ + MMOP_ONLINE, + /* Online the memory to ZONE_NORMAL. */ MMOP_ONLINE_KERNEL, + /* Online the memory to ZONE_MOVABLE. */ MMOP_ONLINE_MOVABLE, }; From efc978ad0e05ed6401c7854811750bf55b67f4b9 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Mon, 6 Apr 2020 20:07:20 -0700 Subject: [PATCH 536/635] drivers/base/memory: map MMOP_OFFLINE to 0 Historically, we used the value -1. Just treat 0 as the special case now. Clarify a comment (which was wrong, when we come via device_online() the first time, the online_type would have been 0 / MEM_ONLINE). The default is now always MMOP_OFFLINE. This removes the last user of the manual "-1", which didn't use the enum value. This is a preparation to use the online_type as an array index. Signed-off-by: David Hildenbrand Signed-off-by: Andrew Morton Reviewed-by: Wei Yang Reviewed-by: Baoquan He Acked-by: Michal Hocko Acked-by: Pankaj Gupta Cc: Greg Kroah-Hartman Cc: Oscar Salvador Cc: "Rafael J. Wysocki" Cc: Wei Yang Cc: Benjamin Herrenschmidt Cc: Eduardo Habkost Cc: Haiyang Zhang Cc: Igor Mammedov Cc: "K. Y. Srinivasan" Cc: Michael Ellerman Cc: Paul Mackerras Cc: Stephen Hemminger Cc: Vitaly Kuznetsov Cc: Wei Liu Cc: Yumei Huang Link: http://lkml.kernel.org/r/20200317104942.11178-3-david@redhat.com Signed-off-by: Linus Torvalds --- drivers/base/memory.c | 11 ++++------- include/linux/memory_hotplug.h | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/drivers/base/memory.c b/drivers/base/memory.c index 156b89b14fcc..f65f3d53dc64 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -203,17 +203,14 @@ static int memory_subsys_online(struct device *dev) return 0; /* - * If we are called from state_store(), online_type will be - * set >= 0 Otherwise we were called from the device online - * attribute and need to set the online_type. + * When called via device_online() without configuring the online_type, + * we want to default to MMOP_ONLINE. */ - if (mem->online_type < 0) + if (mem->online_type == MMOP_OFFLINE) mem->online_type = MMOP_ONLINE; ret = memory_block_change_state(mem, MEM_ONLINE, MEM_OFFLINE); - - /* clear online_type */ - mem->online_type = -1; + mem->online_type = MMOP_OFFLINE; return ret; } diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index 261dbf010d5d..c2e06ed5e0e9 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -48,7 +48,7 @@ enum { /* Types for control the zone type of onlined and offlined memory */ enum { /* Offline the memory. */ - MMOP_OFFLINE = -1, + MMOP_OFFLINE = 0, /* Online the memory. Zone depends, see default_zone_for_pfn(). */ MMOP_ONLINE, /* Online the memory to ZONE_NORMAL. */ From 4dc8207bfd45799525f882e1039e63e9438d605e Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Mon, 6 Apr 2020 20:07:24 -0700 Subject: [PATCH 537/635] drivers/base/memory: store mapping between MMOP_* and string in an array Let's use a simple array which we can reuse soon. While at it, move the string->mmop conversion out of the device hotplug lock. Signed-off-by: David Hildenbrand Signed-off-by: Andrew Morton Reviewed-by: Wei Yang Reviewed-by: Baoquan He Acked-by: Michal Hocko Acked-by: Pankaj Gupta Cc: Greg Kroah-Hartman Cc: Oscar Salvador Cc: "Rafael J. Wysocki" Cc: Wei Yang Cc: Benjamin Herrenschmidt Cc: Eduardo Habkost Cc: Haiyang Zhang Cc: Igor Mammedov Cc: "K. Y. Srinivasan" Cc: Michael Ellerman Cc: Paul Mackerras Cc: Stephen Hemminger Cc: Vitaly Kuznetsov Cc: Wei Liu Cc: Yumei Huang Link: http://lkml.kernel.org/r/20200317104942.11178-4-david@redhat.com Signed-off-by: Linus Torvalds --- drivers/base/memory.c | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/drivers/base/memory.c b/drivers/base/memory.c index f65f3d53dc64..1c90bdf60d85 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -27,6 +27,24 @@ #define MEMORY_CLASS_NAME "memory" +static const char *const online_type_to_str[] = { + [MMOP_OFFLINE] = "offline", + [MMOP_ONLINE] = "online", + [MMOP_ONLINE_KERNEL] = "online_kernel", + [MMOP_ONLINE_MOVABLE] = "online_movable", +}; + +static int memhp_online_type_from_str(const char *str) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(online_type_to_str); i++) { + if (sysfs_streq(str, online_type_to_str[i])) + return i; + } + return -EINVAL; +} + #define to_memory_block(dev) container_of(dev, struct memory_block, dev) static int sections_per_block; @@ -228,26 +246,17 @@ static int memory_subsys_offline(struct device *dev) static ssize_t state_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { + const int online_type = memhp_online_type_from_str(buf); struct memory_block *mem = to_memory_block(dev); - int ret, online_type; + int ret; + + if (online_type < 0) + return -EINVAL; ret = lock_device_hotplug_sysfs(); if (ret) return ret; - if (sysfs_streq(buf, "online_kernel")) - online_type = MMOP_ONLINE_KERNEL; - else if (sysfs_streq(buf, "online_movable")) - online_type = MMOP_ONLINE_MOVABLE; - else if (sysfs_streq(buf, "online")) - online_type = MMOP_ONLINE; - else if (sysfs_streq(buf, "offline")) - online_type = MMOP_OFFLINE; - else { - ret = -EINVAL; - goto err; - } - switch (online_type) { case MMOP_ONLINE_KERNEL: case MMOP_ONLINE_MOVABLE: @@ -263,7 +272,6 @@ static ssize_t state_store(struct device *dev, struct device_attribute *attr, ret = -EINVAL; /* should never happen */ } -err: unlock_device_hotplug(); if (ret < 0) From ed7f9fec8c8f8227ebd1fb69feda60ce4a7df61f Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Mon, 6 Apr 2020 20:07:28 -0700 Subject: [PATCH 538/635] powernv/memtrace: always online added memory blocks Let's always try to online the re-added memory blocks. In case add_memory() already onlined the added memory blocks, the first device_online() call will fail and stop processing the remaining memory blocks. This avoids manually having to check memhp_auto_online. Note: PPC always onlines all hotplugged memory directly from the kernel as well - something that is handled by user space on other architectures. Signed-off-by: David Hildenbrand Signed-off-by: Andrew Morton Reviewed-by: Wei Yang Reviewed-by: Baoquan He Acked-by: Michal Hocko Cc: Benjamin Herrenschmidt Cc: Paul Mackerras Cc: Michael Ellerman Cc: Greg Kroah-Hartman Cc: Oscar Salvador Cc: "Rafael J. Wysocki" Cc: Eduardo Habkost Cc: Haiyang Zhang Cc: Igor Mammedov Cc: "K. Y. Srinivasan" Cc: Stephen Hemminger Cc: Vitaly Kuznetsov Cc: Wei Liu Cc: Yumei Huang Link: http://lkml.kernel.org/r/20200317104942.11178-5-david@redhat.com Signed-off-by: Linus Torvalds --- arch/powerpc/platforms/powernv/memtrace.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/arch/powerpc/platforms/powernv/memtrace.c b/arch/powerpc/platforms/powernv/memtrace.c index d6d64f8718e6..13b369d2cc45 100644 --- a/arch/powerpc/platforms/powernv/memtrace.c +++ b/arch/powerpc/platforms/powernv/memtrace.c @@ -231,16 +231,10 @@ static int memtrace_online(void) continue; } - /* - * If kernel isn't compiled with the auto online option - * we need to online the memory ourselves. - */ - if (!memhp_auto_online) { - lock_device_hotplug(); - walk_memory_blocks(ent->start, ent->size, NULL, - online_mem_block); - unlock_device_hotplug(); - } + lock_device_hotplug(); + walk_memory_blocks(ent->start, ent->size, NULL, + online_mem_block); + unlock_device_hotplug(); /* * Memory was added successfully so clean up references to it From bc58ebd506c369c26337cf6b1a400af1a25c989c Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Mon, 6 Apr 2020 20:07:32 -0700 Subject: [PATCH 539/635] hv_balloon: don't check for memhp_auto_online manually We get the MEM_ONLINE notifier call if memory is added right from the kernel via add_memory() or later from user space. Let's get rid of the "ha_waiting" flag - the wait event has an inbuilt mechanism (->done) for that. Initialize the wait event only once and reinitialize before adding memory. Unconditionally call complete() and wait_for_completion_timeout(). If there are no waiters, complete() will only increment ->done - which will be reset by reinit_completion(). If complete() has already been called, wait_for_completion_timeout() will not wait. There is still the chance for a small race between concurrent reinit_completion() and complete(). If complete() wins, we would not wait - which is tolerable (and the race exists in current code as well). Note: We only wait for "some" memory to get onlined, which seems to be good enough for now. [akpm@linux-foundation.org: register_memory_notifier() after init_completion(), per David] Signed-off-by: David Hildenbrand Signed-off-by: Andrew Morton Reviewed-by: Vitaly Kuznetsov Reviewed-by: Baoquan He Cc: "K. Y. Srinivasan" Cc: Haiyang Zhang Cc: Stephen Hemminger Cc: Wei Liu Cc: Oscar Salvador Cc: "Rafael J. Wysocki" Cc: Wei Yang Cc: Benjamin Herrenschmidt Cc: Eduardo Habkost Cc: Greg Kroah-Hartman Cc: Igor Mammedov Cc: Michael Ellerman Cc: Paul Mackerras Cc: Yumei Huang Link: http://lkml.kernel.org/r/20200317104942.11178-6-david@redhat.com Signed-off-by: Linus Torvalds --- drivers/hv/hv_balloon.c | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/drivers/hv/hv_balloon.c b/drivers/hv/hv_balloon.c index a02ce43d778d..32e3bc0aa665 100644 --- a/drivers/hv/hv_balloon.c +++ b/drivers/hv/hv_balloon.c @@ -533,7 +533,6 @@ struct hv_dynmem_device { * State to synchronize hot-add. */ struct completion ol_waitevent; - bool ha_waiting; /* * This thread handles hot-add * requests from the host as well as notifying @@ -634,10 +633,7 @@ static int hv_memory_notifier(struct notifier_block *nb, unsigned long val, switch (val) { case MEM_ONLINE: case MEM_CANCEL_ONLINE: - if (dm_device.ha_waiting) { - dm_device.ha_waiting = false; - complete(&dm_device.ol_waitevent); - } + complete(&dm_device.ol_waitevent); break; case MEM_OFFLINE: @@ -726,8 +722,7 @@ static void hv_mem_hot_add(unsigned long start, unsigned long size, has->covered_end_pfn += processed_pfn; spin_unlock_irqrestore(&dm_device.ha_lock, flags); - init_completion(&dm_device.ol_waitevent); - dm_device.ha_waiting = !memhp_auto_online; + reinit_completion(&dm_device.ol_waitevent); nid = memory_add_physaddr_to_nid(PFN_PHYS(start_pfn)); ret = add_memory(nid, PFN_PHYS((start_pfn)), @@ -753,15 +748,14 @@ static void hv_mem_hot_add(unsigned long start, unsigned long size, } /* - * Wait for the memory block to be onlined when memory onlining - * is done outside of kernel (memhp_auto_online). Since the hot - * add has succeeded, it is ok to proceed even if the pages in - * the hot added region have not been "onlined" within the - * allowed time. + * Wait for memory to get onlined. If the kernel onlined the + * memory when adding it, this will return directly. Otherwise, + * it will wait for user space to online the memory. This helps + * to avoid adding memory faster than it is getting onlined. As + * adding succeeded, it is ok to proceed even if the memory was + * not onlined in time. */ - if (dm_device.ha_waiting) - wait_for_completion_timeout(&dm_device.ol_waitevent, - 5*HZ); + wait_for_completion_timeout(&dm_device.ol_waitevent, 5 * HZ); post_status(&dm_device); } } @@ -1706,6 +1700,7 @@ static int balloon_probe(struct hv_device *dev, #ifdef CONFIG_MEMORY_HOTPLUG set_online_page_callback(&hv_online_page); + init_completion(&dm_device.ol_waitevent); register_memory_notifier(&hv_memory_nb); #endif From 5a04af1322f0124c6e89870ca4e69d5d0f00b4f8 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Mon, 6 Apr 2020 20:07:36 -0700 Subject: [PATCH 540/635] mm/memory_hotplug: unexport memhp_auto_online All in-tree users except the mm-core are gone. Let's drop the export. Signed-off-by: David Hildenbrand Signed-off-by: Andrew Morton Reviewed-by: Wei Yang Reviewed-by: Baoquan He Acked-by: Michal Hocko Acked-by: Pankaj Gupta Cc: Oscar Salvador Cc: "Rafael J. Wysocki" Cc: Benjamin Herrenschmidt Cc: Eduardo Habkost Cc: Greg Kroah-Hartman Cc: Haiyang Zhang Cc: Igor Mammedov Cc: "K. Y. Srinivasan" Cc: Michael Ellerman Cc: Paul Mackerras Cc: Stephen Hemminger Cc: Vitaly Kuznetsov Cc: Wei Liu Cc: Yumei Huang Link: http://lkml.kernel.org/r/20200317104942.11178-7-david@redhat.com Signed-off-by: Linus Torvalds --- mm/memory_hotplug.c | 1 - 1 file changed, 1 deletion(-) diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 04cb87ec39d0..9691cbb4383e 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -71,7 +71,6 @@ bool memhp_auto_online; #else bool memhp_auto_online = true; #endif -EXPORT_SYMBOL_GPL(memhp_auto_online); static int __init setup_memhp_default_state(char *str) { From 862919e568356cc36288a11b42cd88ec3a7100e9 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Mon, 6 Apr 2020 20:07:40 -0700 Subject: [PATCH 541/635] mm/memory_hotplug: convert memhp_auto_online to store an online_type ... and rename it to memhp_default_online_type. This is a preparation for more detailed default online behavior. Signed-off-by: David Hildenbrand Signed-off-by: Andrew Morton Reviewed-by: Wei Yang Reviewed-by: Baoquan He Acked-by: Michal Hocko Acked-by: Pankaj Gupta Cc: Greg Kroah-Hartman Cc: Oscar Salvador Cc: "Rafael J. Wysocki" Cc: Wei Yang Cc: Benjamin Herrenschmidt Cc: Eduardo Habkost Cc: Haiyang Zhang Cc: Igor Mammedov Cc: "K. Y. Srinivasan" Cc: Michael Ellerman Cc: Paul Mackerras Cc: Stephen Hemminger Cc: Vitaly Kuznetsov Cc: Wei Liu Cc: Yumei Huang Link: http://lkml.kernel.org/r/20200317104942.11178-8-david@redhat.com Signed-off-by: Linus Torvalds --- drivers/base/memory.c | 10 ++++------ include/linux/memory_hotplug.h | 3 ++- mm/memory_hotplug.c | 11 ++++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/drivers/base/memory.c b/drivers/base/memory.c index 1c90bdf60d85..7d2f829d00d7 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -378,10 +378,8 @@ static DEVICE_ATTR_RO(block_size_bytes); static ssize_t auto_online_blocks_show(struct device *dev, struct device_attribute *attr, char *buf) { - if (memhp_auto_online) - return sprintf(buf, "online\n"); - else - return sprintf(buf, "offline\n"); + return sprintf(buf, "%s\n", + online_type_to_str[memhp_default_online_type]); } static ssize_t auto_online_blocks_store(struct device *dev, @@ -389,9 +387,9 @@ static ssize_t auto_online_blocks_store(struct device *dev, const char *buf, size_t count) { if (sysfs_streq(buf, "online")) - memhp_auto_online = true; + memhp_default_online_type = MMOP_ONLINE; else if (sysfs_streq(buf, "offline")) - memhp_auto_online = false; + memhp_default_online_type = MMOP_OFFLINE; else return -EINVAL; diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index c2e06ed5e0e9..c6e090b34c4b 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -117,7 +117,8 @@ extern int arch_add_memory(int nid, u64 start, u64 size, struct mhp_restrictions *restrictions); extern u64 max_mem_size; -extern bool memhp_auto_online; +/* Default online_type (MMOP_*) when new memory blocks are added. */ +extern int memhp_default_online_type; /* If movable_node boot option specified */ extern bool movable_node_enabled; static inline bool movable_node_is_enabled(void) diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 9691cbb4383e..9436f7e6257a 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -67,17 +67,17 @@ void put_online_mems(void) bool movable_node_enabled = false; #ifndef CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE -bool memhp_auto_online; +int memhp_default_online_type = MMOP_OFFLINE; #else -bool memhp_auto_online = true; +int memhp_default_online_type = MMOP_ONLINE; #endif static int __init setup_memhp_default_state(char *str) { if (!strcmp(str, "online")) - memhp_auto_online = true; + memhp_default_online_type = MMOP_ONLINE; else if (!strcmp(str, "offline")) - memhp_auto_online = false; + memhp_default_online_type = MMOP_OFFLINE; return 1; } @@ -990,6 +990,7 @@ static int check_hotplug_memory_range(u64 start, u64 size) static int online_memory_block(struct memory_block *mem, void *arg) { + mem->online_type = memhp_default_online_type; return device_online(&mem->dev); } @@ -1062,7 +1063,7 @@ int __ref add_memory_resource(int nid, struct resource *res) mem_hotplug_done(); /* online pages if requested */ - if (memhp_auto_online) + if (memhp_default_online_type != MMOP_OFFLINE) walk_memory_blocks(start, size, NULL, online_memory_block); return ret; From 5f47adf762b78cae97de58d9ff01d2d44db09467 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Mon, 6 Apr 2020 20:07:44 -0700 Subject: [PATCH 542/635] mm/memory_hotplug: allow to specify a default online_type For now, distributions implement advanced udev rules to essentially - Don't online any hotplugged memory (s390x) - Online all memory to ZONE_NORMAL (e.g., most virt environments like hyperv) - Online all memory to ZONE_MOVABLE in case the zone imbalance is taken care of (e.g., bare metal, special virt environments) In summary: All memory is usually onlined the same way, however, the kernel always has to ask user space to come up with the same answer. E.g., Hyper-V always waits for a memory block to get onlined before continuing, otherwise it might end up adding memory faster than onlining it, which can result in strange OOM situations. This waiting slows down adding of a bigger amount of memory. Let's allow to specify a default online_type, not just "online" and "offline". This allows distributions to configure the default online_type when booting up and be done with it. We can now specify "offline", "online", "online_movable" and "online_kernel" via - "memhp_default_state=" on the kernel cmdline - /sys/devices/system/memory/auto_online_blocks just like we are able to specify for a single memory block via /sys/devices/system/memory/memoryX/state Signed-off-by: David Hildenbrand Signed-off-by: Andrew Morton Reviewed-by: Wei Yang Reviewed-by: Baoquan He Acked-by: Michal Hocko Acked-by: Pankaj Gupta Cc: Greg Kroah-Hartman Cc: Oscar Salvador Cc: "Rafael J. Wysocki" Cc: Wei Yang Cc: Benjamin Herrenschmidt Cc: Eduardo Habkost Cc: Haiyang Zhang Cc: Igor Mammedov Cc: "K. Y. Srinivasan" Cc: Michael Ellerman Cc: Paul Mackerras Cc: Stephen Hemminger Cc: Vitaly Kuznetsov Cc: Wei Liu Cc: Yumei Huang Link: http://lkml.kernel.org/r/20200317104942.11178-9-david@redhat.com Signed-off-by: Linus Torvalds --- drivers/base/memory.c | 11 +++++------ include/linux/memory_hotplug.h | 2 ++ mm/memory_hotplug.c | 8 ++++---- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/drivers/base/memory.c b/drivers/base/memory.c index 7d2f829d00d7..dbec3a05590a 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -34,7 +34,7 @@ static const char *const online_type_to_str[] = { [MMOP_ONLINE_MOVABLE] = "online_movable", }; -static int memhp_online_type_from_str(const char *str) +int memhp_online_type_from_str(const char *str) { int i; @@ -386,13 +386,12 @@ static ssize_t auto_online_blocks_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - if (sysfs_streq(buf, "online")) - memhp_default_online_type = MMOP_ONLINE; - else if (sysfs_streq(buf, "offline")) - memhp_default_online_type = MMOP_OFFLINE; - else + const int online_type = memhp_online_type_from_str(buf); + + if (online_type < 0) return -EINVAL; + memhp_default_online_type = online_type; return count; } diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index c6e090b34c4b..ef55115320fb 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -117,6 +117,8 @@ extern int arch_add_memory(int nid, u64 start, u64 size, struct mhp_restrictions *restrictions); extern u64 max_mem_size; +extern int memhp_online_type_from_str(const char *str); + /* Default online_type (MMOP_*) when new memory blocks are added. */ extern int memhp_default_online_type; /* If movable_node boot option specified */ diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 9436f7e6257a..2fb78c5ebaf3 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -74,10 +74,10 @@ int memhp_default_online_type = MMOP_ONLINE; static int __init setup_memhp_default_state(char *str) { - if (!strcmp(str, "online")) - memhp_default_online_type = MMOP_ONLINE; - else if (!strcmp(str, "offline")) - memhp_default_online_type = MMOP_OFFLINE; + const int online_type = memhp_online_type_from_str(str); + + if (online_type >= 0) + memhp_default_online_type = online_type; return 1; } From 104049017b774036fa971722a202a17e6585a200 Mon Sep 17 00:00:00 2001 From: chenqiwu Date: Mon, 6 Apr 2020 20:07:48 -0700 Subject: [PATCH 543/635] mm/memory_hotplug.c: use __pfn_to_section() instead of open-coding Use __pfn_to_section() API instead of open-coding for better code readability. Signed-off-by: chenqiwu Signed-off-by: Andrew Morton Acked-by: David Hildenbrand Link: http://lkml.kernel.org/r/1584345134-16671-1-git-send-email-qiwuchen55@gmail.com Signed-off-by: Linus Torvalds --- mm/memory_hotplug.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 2fb78c5ebaf3..635e8e286598 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -495,7 +495,7 @@ static void __remove_section(unsigned long pfn, unsigned long nr_pages, unsigned long map_offset, struct vmem_altmap *altmap) { - struct mem_section *ms = __nr_to_section(pfn_to_section_nr(pfn)); + struct mem_section *ms = __pfn_to_section(pfn); if (WARN_ON_ONCE(!valid_section(ms))) return; From 27d80fa24326b7b33c8ee7527843776e5df808a7 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Mon, 6 Apr 2020 20:07:51 -0700 Subject: [PATCH 544/635] mm/shmem.c: distribute switch variables for initialization Variables declared in a switch statement before any case statements cannot be automatically initialized with compiler instrumentation (as they are not part of any execution flow). With GCC's proposed automatic stack variable initialization feature, this triggers a warning (and they don't get initialized). Clang's automatic stack variable initialization (via CONFIG_INIT_STACK_ALL=y) doesn't throw a warning, but it also doesn't initialize such variables[1]. Note that these warnings (or silent skipping) happen before the dead-store elimination optimization phase, so even when the automatic initializations are later elided in favor of direct initializations, the warnings remain. To avoid these problems, move such variables into the "case" where they're used or lift them up into the main function body. mm/shmem.c: In function `shmem_getpage_gfp': mm/shmem.c:1816:10: warning: statement will never be executed [-Wswitch-unreachable] 1816 | loff_t i_size; | ^~~~~~ [1] https://bugs.llvm.org/show_bug.cgi?id=44916 Signed-off-by: Kees Cook Signed-off-by: Andrew Morton Reviewed-by: Andrew Morton Cc: Hugh Dickins Cc: Alexander Potapenko Link: http://lkml.kernel.org/r/20200220062312.69165-1-keescook@chromium.org Signed-off-by: Linus Torvalds --- mm/shmem.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/mm/shmem.c b/mm/shmem.c index 2c255f383608..97448a46a4dc 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1812,17 +1812,20 @@ repeat: if (shmem_huge == SHMEM_HUGE_FORCE) goto alloc_huge; switch (sbinfo->huge) { - loff_t i_size; - pgoff_t off; case SHMEM_HUGE_NEVER: goto alloc_nohuge; - case SHMEM_HUGE_WITHIN_SIZE: + case SHMEM_HUGE_WITHIN_SIZE: { + loff_t i_size; + pgoff_t off; + off = round_up(index, HPAGE_PMD_NR); i_size = round_up(i_size_read(inode), PAGE_SIZE); if (i_size >= HPAGE_PMD_SIZE && i_size >> PAGE_SHIFT >= off) goto alloc_huge; - /* fallthrough */ + + fallthrough; + } case SHMEM_HUGE_ADVISE: if (sgp_huge == SGP_HUGE) goto alloc_huge; From 343c3d7f0927e000427fae5e361aa560f83dd5b5 Mon Sep 17 00:00:00 2001 From: Mateusz Nosek Date: Mon, 6 Apr 2020 20:07:54 -0700 Subject: [PATCH 545/635] mm/shmem.c: clean code by removing unnecessary assignment Previously 0 was assigned to variable 'error' but the variable was never read before reassignemnt later. So the assignment can be removed. Signed-off-by: Mateusz Nosek Signed-off-by: Andrew Morton Reviewed-by: Matthew Wilcox (Oracle) Acked-by: Pankaj Gupta Cc: Hugh Dickins Link: http://lkml.kernel.org/r/20200301152832.24595-1-mateusznosek0@gmail.com Signed-off-by: Linus Torvalds --- mm/shmem.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/mm/shmem.c b/mm/shmem.c index 97448a46a4dc..e23fea40767e 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -3120,12 +3120,9 @@ static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *s error = security_inode_init_security(inode, dir, &dentry->d_name, shmem_initxattrs, NULL); - if (error) { - if (error != -EOPNOTSUPP) { - iput(inode); - return error; - } - error = 0; + if (error && error != -EOPNOTSUPP) { + iput(inode); + return error; } inode->i_size = len-1; From 71725ed10c40696dc6bdccf8e225815dcef24dba Mon Sep 17 00:00:00 2001 From: Hugh Dickins Date: Mon, 6 Apr 2020 20:07:57 -0700 Subject: [PATCH 546/635] mm: huge tmpfs: try to split_huge_page() when punching hole Yang Shi writes: Currently, when truncating a shmem file, if the range is partly in a THP (start or end is in the middle of THP), the pages actually will just get cleared rather than being freed, unless the range covers the whole THP. Even though all the subpages are truncated (randomly or sequentially), the THP may still be kept in page cache. This might be fine for some usecases which prefer preserving THP, but balloon inflation is handled in base page size. So when using shmem THP as memory backend, QEMU inflation actually doesn't work as expected since it doesn't free memory. But the inflation usecase really needs to get the memory freed. (Anonymous THP will also not get freed right away, but will be freed eventually when all subpages are unmapped: whereas shmem THP still stays in page cache.) Split THP right away when doing partial hole punch, and if split fails just clear the page so that read of the punched area will return zeroes. Hugh Dickins adds: Our earlier "team of pages" huge tmpfs implementation worked in the way that Yang Shi proposes; and we have been using this patch to continue to split the huge page when hole-punched or truncated, since converting over to the compound page implementation. Although huge tmpfs gives out huge pages when available, if the user specifically asks to truncate or punch a hole (perhaps to free memory, perhaps to reduce the memcg charge), then the filesystem should do so as best it can, splitting the huge page. That is not always possible: any additional reference to the huge page prevents split_huge_page() from succeeding, so the result can be flaky. But in practice it works successfully enough that we've not seen any problem from that. Add shmem_punch_compound() to encapsulate the decision of when a split is needed, and doing the split if so. Using this simplifies the flow in shmem_undo_range(); and the first (trylock) pass does not need to do any page clearing on failure, because the second pass will either succeed or do that clearing. Following the example of zero_user_segment() when clearing a partial page, add flush_dcache_page() and set_page_dirty() when clearing a hole - though I'm not certain that either is needed. But: split_huge_page() would be sure to fail if shmem_undo_range()'s pagevec holds further references to the huge page. The easiest way to fix that is for find_get_entries() to return early, as soon as it has put one compound head or tail into the pagevec. At first this felt like a hack; but on examination, this convention better suits all its callers - or will do, if the slight one-page-per-pagevec slowdown in shmem_unlock_mapping() and shmem_seek_hole_data() is transformed into a 512-page-per-pagevec speedup by checking for compound pages there. Signed-off-by: Hugh Dickins Signed-off-by: Andrew Morton Cc: Yang Shi Cc: Alexander Duyck Cc: "Michael S. Tsirkin" Cc: David Hildenbrand Cc: "Kirill A. Shutemov" Cc: Matthew Wilcox Cc: Andrea Arcangeli Link: http://lkml.kernel.org/r/alpine.LSU.2.11.2002261959020.10801@eggly.anvils Signed-off-by: Linus Torvalds --- mm/filemap.c | 14 +++++++- mm/shmem.c | 98 +++++++++++++++++++++++----------------------------- mm/swap.c | 4 +++ 3 files changed, 60 insertions(+), 56 deletions(-) diff --git a/mm/filemap.c b/mm/filemap.c index 0fbdc8e30dd2..23a051a7ef0f 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -1693,6 +1693,11 @@ EXPORT_SYMBOL(pagecache_get_page); * Any shadow entries of evicted pages, or swap entries from * shmem/tmpfs, are included in the returned array. * + * If it finds a Transparent Huge Page, head or tail, find_get_entries() + * stops at that page: the caller is likely to have a better way to handle + * the compound page as a whole, and then skip its extent, than repeatedly + * calling find_get_entries() to return all its tails. + * * Return: the number of pages and shadow entries which were found. */ unsigned find_get_entries(struct address_space *mapping, @@ -1724,8 +1729,15 @@ unsigned find_get_entries(struct address_space *mapping, /* Has the page moved or been split? */ if (unlikely(page != xas_reload(&xas))) goto put_page; - page = find_subpage(page, xas.xa_index); + /* + * Terminate early on finding a THP, to allow the caller to + * handle it all at once; but continue if this is hugetlbfs. + */ + if (PageTransHuge(page) && !PageHuge(page)) { + page = find_subpage(page, xas.xa_index); + nr_entries = ret + 1; + } export: indices[ret] = xas.xa_index; entries[ret] = page; diff --git a/mm/shmem.c b/mm/shmem.c index e23fea40767e..8d0aba1cc155 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -788,6 +788,32 @@ void shmem_unlock_mapping(struct address_space *mapping) } } +/* + * Check whether a hole-punch or truncation needs to split a huge page, + * returning true if no split was required, or the split has been successful. + * + * Eviction (or truncation to 0 size) should never need to split a huge page; + * but in rare cases might do so, if shmem_undo_range() failed to trylock on + * head, and then succeeded to trylock on tail. + * + * A split can only succeed when there are no additional references on the + * huge page: so the split below relies upon find_get_entries() having stopped + * when it found a subpage of the huge page, without getting further references. + */ +static bool shmem_punch_compound(struct page *page, pgoff_t start, pgoff_t end) +{ + if (!PageTransCompound(page)) + return true; + + /* Just proceed to delete a huge page wholly within the range punched */ + if (PageHead(page) && + page->index >= start && page->index + HPAGE_PMD_NR <= end) + return true; + + /* Try to split huge page, so we can truly punch the hole or truncate */ + return split_huge_page(page) >= 0; +} + /* * Remove range of pages and swap entries from page cache, and free them. * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate. @@ -838,31 +864,11 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, if (!trylock_page(page)) continue; - if (PageTransTail(page)) { - /* Middle of THP: zero out the page */ - clear_highpage(page); - unlock_page(page); - continue; - } else if (PageTransHuge(page)) { - if (index == round_down(end, HPAGE_PMD_NR)) { - /* - * Range ends in the middle of THP: - * zero out the page - */ - clear_highpage(page); - unlock_page(page); - continue; - } - index += HPAGE_PMD_NR - 1; - i += HPAGE_PMD_NR - 1; - } - - if (!unfalloc || !PageUptodate(page)) { - VM_BUG_ON_PAGE(PageTail(page), page); - if (page_mapping(page) == mapping) { - VM_BUG_ON_PAGE(PageWriteback(page), page); + if ((!unfalloc || !PageUptodate(page)) && + page_mapping(page) == mapping) { + VM_BUG_ON_PAGE(PageWriteback(page), page); + if (shmem_punch_compound(page, start, end)) truncate_inode_page(mapping, page); - } } unlock_page(page); } @@ -936,43 +942,25 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, lock_page(page); - if (PageTransTail(page)) { - /* Middle of THP: zero out the page */ - clear_highpage(page); - unlock_page(page); - /* - * Partial thp truncate due 'start' in middle - * of THP: don't need to look on these pages - * again on !pvec.nr restart. - */ - if (index != round_down(end, HPAGE_PMD_NR)) - start++; - continue; - } else if (PageTransHuge(page)) { - if (index == round_down(end, HPAGE_PMD_NR)) { - /* - * Range ends in the middle of THP: - * zero out the page - */ - clear_highpage(page); - unlock_page(page); - continue; - } - index += HPAGE_PMD_NR - 1; - i += HPAGE_PMD_NR - 1; - } - if (!unfalloc || !PageUptodate(page)) { - VM_BUG_ON_PAGE(PageTail(page), page); - if (page_mapping(page) == mapping) { - VM_BUG_ON_PAGE(PageWriteback(page), page); - truncate_inode_page(mapping, page); - } else { + if (page_mapping(page) != mapping) { /* Page was replaced by swap: retry */ unlock_page(page); index--; break; } + VM_BUG_ON_PAGE(PageWriteback(page), page); + if (shmem_punch_compound(page, start, end)) + truncate_inode_page(mapping, page); + else { + /* Wipe the page and don't get stuck */ + clear_highpage(page); + flush_dcache_page(page); + set_page_dirty(page); + if (index < + round_up(start, HPAGE_PMD_NR)) + start = index + 1; + } } unlock_page(page); } diff --git a/mm/swap.c b/mm/swap.c index 18505990c3b1..bf9a79fed62d 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -1004,6 +1004,10 @@ void __pagevec_lru_add(struct pagevec *pvec) * ascending indexes. There may be holes in the indices due to * not-present entries. * + * Only one subpage of a Transparent Huge Page is returned in one call: + * allowing truncate_inode_pages_range() to evict the whole THP without + * cycling through a pagevec of extra references. + * * pagevec_lookup_entries() returns the number of entries which were * found. */ From 4708f31885a0d3e41a03c39501df697e73406f6a Mon Sep 17 00:00:00 2001 From: Palmer Dabbelt Date: Mon, 6 Apr 2020 20:08:00 -0700 Subject: [PATCH 547/635] mm: prevent a warning when casting void* -> enum I recently build the RISC-V port with LLVM trunk, which has introduced a new warning when casting from a pointer to an enum of a smaller size. This patch simply casts to a long in the middle to stop the warning. I'd be surprised this is the only one in the kernel, but it's the only one I saw. Signed-off-by: Palmer Dabbelt Signed-off-by: Andrew Morton Reviewed-by: Andrew Morton Link: http://lkml.kernel.org/r/20200227211741.83165-1-palmer@dabbelt.com Signed-off-by: Linus Torvalds --- mm/rmap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/rmap.c b/mm/rmap.c index ed8889bf4ede..f79a206b271a 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1372,7 +1372,7 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, struct page *subpage; bool ret = true; struct mmu_notifier_range range; - enum ttu_flags flags = (enum ttu_flags)arg; + enum ttu_flags flags = (enum ttu_flags)(long)arg; /* munlock has nothing to gain from examining un-locked vmas */ if ((flags & TTU_MUNLOCK) && !(vma->vm_flags & VM_LOCKED)) From bb8b93b5b65130138d3c80f3136af721863f561a Mon Sep 17 00:00:00 2001 From: "Maciej S. Szmigiero" Date: Mon, 6 Apr 2020 20:08:03 -0700 Subject: [PATCH 548/635] mm/zswap: allow setting default status, compressor and allocator in Kconfig The compressed cache for swap pages (zswap) currently needs from 1 to 3 extra kernel command line parameters in order to make it work: it has to be enabled by adding a "zswap.enabled=1" command line parameter and if one wants a different compressor or pool allocator than the default lzo / zbud combination then these choices also need to be specified on the kernel command line in additional parameters. Using a different compressor and allocator for zswap is actually pretty common as guides often recommend using the lz4 / z3fold pair instead of the default one. In such case it is also necessary to remember to enable the appropriate compression algorithm and pool allocator in the kernel config manually. Let's avoid the need for adding these kernel command line parameters and automatically pull in the dependencies for the selected compressor algorithm and pool allocator by adding an appropriate default switches to Kconfig. The default values for these options match what the code was using previously as its defaults. Signed-off-by: Maciej S. Szmigiero Signed-off-by: Andrew Morton Reviewed-by: Vitaly Wool Link: http://lkml.kernel.org/r/20200202000112.456103-1-mail@maciej.szmigiero.name Signed-off-by: Linus Torvalds --- Documentation/vm/zswap.rst | 20 ++++--- mm/Kconfig | 118 ++++++++++++++++++++++++++++++++++++- mm/zswap.c | 24 ++++---- 3 files changed, 141 insertions(+), 21 deletions(-) diff --git a/Documentation/vm/zswap.rst b/Documentation/vm/zswap.rst index 61f6185188cd..f8c6a79d7c70 100644 --- a/Documentation/vm/zswap.rst +++ b/Documentation/vm/zswap.rst @@ -35,9 +35,11 @@ Zswap evicts pages from compressed cache on an LRU basis to the backing swap device when the compressed pool reaches its size limit. This requirement had been identified in prior community discussions. -Zswap is disabled by default but can be enabled at boot time by setting -the ``enabled`` attribute to 1 at boot time. ie: ``zswap.enabled=1``. Zswap -can also be enabled and disabled at runtime using the sysfs interface. +Whether Zswap is enabled at the boot time depends on whether +the ``CONFIG_ZSWAP_DEFAULT_ON`` Kconfig option is enabled or not. +This setting can then be overridden by providing the kernel command line +``zswap.enabled=`` option, for example ``zswap.enabled=0``. +Zswap can also be enabled and disabled at runtime using the sysfs interface. An example command to enable zswap at runtime, assuming sysfs is mounted at ``/sys``, is:: @@ -64,9 +66,10 @@ allocation in zpool is not directly accessible by address. Rather, a handle is returned by the allocation routine and that handle must be mapped before being accessed. The compressed memory pool grows on demand and shrinks as compressed pages are freed. The pool is not preallocated. By default, a zpool -of type zbud is created, but it can be selected at boot time by -setting the ``zpool`` attribute, e.g. ``zswap.zpool=zbud``. It can -also be changed at runtime using the sysfs ``zpool`` attribute, e.g.:: +of type selected in ``CONFIG_ZSWAP_ZPOOL_DEFAULT`` Kconfig option is created, +but it can be overridden at boot time by setting the ``zpool`` attribute, +e.g. ``zswap.zpool=zbud``. It can also be changed at runtime using the sysfs +``zpool`` attribute, e.g.:: echo zbud > /sys/module/zswap/parameters/zpool @@ -97,8 +100,9 @@ controlled policy: * max_pool_percent - The maximum percentage of memory that the compressed pool can occupy. -The default compressor is lzo, but it can be selected at boot time by -setting the ``compressor`` attribute, e.g. ``zswap.compressor=lzo``. +The default compressor is selected in ``CONFIG_ZSWAP_COMPRESSOR_DEFAULT`` +Kconfig option, but it can be overridden at boot time by setting the +``compressor`` attribute, e.g. ``zswap.compressor=lzo``. It can also be changed at runtime using the sysfs "compressor" attribute, e.g.:: diff --git a/mm/Kconfig b/mm/Kconfig index d286dc54458c..691021492e78 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -533,7 +533,6 @@ config MEM_SOFT_DIRTY config ZSWAP bool "Compressed cache for swap pages (EXPERIMENTAL)" depends on FRONTSWAP && CRYPTO=y - select CRYPTO_LZO select ZPOOL help A lightweight compressed cache for swap pages. It takes @@ -549,6 +548,123 @@ config ZSWAP they have not be fully explored on the large set of potential configurations and workloads that exist. +choice + prompt "Compressed cache for swap pages default compressor" + depends on ZSWAP + default ZSWAP_COMPRESSOR_DEFAULT_LZO + help + Selects the default compression algorithm for the compressed cache + for swap pages. + + For an overview what kind of performance can be expected from + a particular compression algorithm please refer to the benchmarks + available at the following LWN page: + https://lwn.net/Articles/751795/ + + If in doubt, select 'LZO'. + + The selection made here can be overridden by using the kernel + command line 'zswap.compressor=' option. + +config ZSWAP_COMPRESSOR_DEFAULT_DEFLATE + bool "Deflate" + select CRYPTO_DEFLATE + help + Use the Deflate algorithm as the default compression algorithm. + +config ZSWAP_COMPRESSOR_DEFAULT_LZO + bool "LZO" + select CRYPTO_LZO + help + Use the LZO algorithm as the default compression algorithm. + +config ZSWAP_COMPRESSOR_DEFAULT_842 + bool "842" + select CRYPTO_842 + help + Use the 842 algorithm as the default compression algorithm. + +config ZSWAP_COMPRESSOR_DEFAULT_LZ4 + bool "LZ4" + select CRYPTO_LZ4 + help + Use the LZ4 algorithm as the default compression algorithm. + +config ZSWAP_COMPRESSOR_DEFAULT_LZ4HC + bool "LZ4HC" + select CRYPTO_LZ4HC + help + Use the LZ4HC algorithm as the default compression algorithm. + +config ZSWAP_COMPRESSOR_DEFAULT_ZSTD + bool "zstd" + select CRYPTO_ZSTD + help + Use the zstd algorithm as the default compression algorithm. +endchoice + +config ZSWAP_COMPRESSOR_DEFAULT + string + depends on ZSWAP + default "deflate" if ZSWAP_COMPRESSOR_DEFAULT_DEFLATE + default "lzo" if ZSWAP_COMPRESSOR_DEFAULT_LZO + default "842" if ZSWAP_COMPRESSOR_DEFAULT_842 + default "lz4" if ZSWAP_COMPRESSOR_DEFAULT_LZ4 + default "lz4hc" if ZSWAP_COMPRESSOR_DEFAULT_LZ4HC + default "zstd" if ZSWAP_COMPRESSOR_DEFAULT_ZSTD + default "" + +choice + prompt "Compressed cache for swap pages default allocator" + depends on ZSWAP + default ZSWAP_ZPOOL_DEFAULT_ZBUD + help + Selects the default allocator for the compressed cache for + swap pages. + The default is 'zbud' for compatibility, however please do + read the description of each of the allocators below before + making a right choice. + + The selection made here can be overridden by using the kernel + command line 'zswap.zpool=' option. + +config ZSWAP_ZPOOL_DEFAULT_ZBUD + bool "zbud" + select ZBUD + help + Use the zbud allocator as the default allocator. + +config ZSWAP_ZPOOL_DEFAULT_Z3FOLD + bool "z3fold" + select Z3FOLD + help + Use the z3fold allocator as the default allocator. + +config ZSWAP_ZPOOL_DEFAULT_ZSMALLOC + bool "zsmalloc" + select ZSMALLOC + help + Use the zsmalloc allocator as the default allocator. +endchoice + +config ZSWAP_ZPOOL_DEFAULT + string + depends on ZSWAP + default "zbud" if ZSWAP_ZPOOL_DEFAULT_ZBUD + default "z3fold" if ZSWAP_ZPOOL_DEFAULT_Z3FOLD + default "zsmalloc" if ZSWAP_ZPOOL_DEFAULT_ZSMALLOC + default "" + +config ZSWAP_DEFAULT_ON + bool "Enable the compressed cache for swap pages by default" + depends on ZSWAP + help + If selected, the compressed cache for swap pages will be enabled + at boot, otherwise it will be disabled. + + The selection made here can be overridden by using the kernel + command line 'zswap.enabled=' option. + config ZPOOL tristate "Common API for compressed memory storage" help diff --git a/mm/zswap.c b/mm/zswap.c index 55094e63b72d..fbb782924ccc 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -77,8 +77,8 @@ static bool zswap_pool_reached_full; #define ZSWAP_PARAM_UNSET "" -/* Enable/disable zswap (disabled by default) */ -static bool zswap_enabled; +/* Enable/disable zswap */ +static bool zswap_enabled = IS_ENABLED(CONFIG_ZSWAP_DEFAULT_ON); static int zswap_enabled_param_set(const char *, const struct kernel_param *); static struct kernel_param_ops zswap_enabled_param_ops = { @@ -88,8 +88,7 @@ static struct kernel_param_ops zswap_enabled_param_ops = { module_param_cb(enabled, &zswap_enabled_param_ops, &zswap_enabled, 0644); /* Crypto compressor to use */ -#define ZSWAP_COMPRESSOR_DEFAULT "lzo" -static char *zswap_compressor = ZSWAP_COMPRESSOR_DEFAULT; +static char *zswap_compressor = CONFIG_ZSWAP_COMPRESSOR_DEFAULT; static int zswap_compressor_param_set(const char *, const struct kernel_param *); static struct kernel_param_ops zswap_compressor_param_ops = { @@ -101,8 +100,7 @@ module_param_cb(compressor, &zswap_compressor_param_ops, &zswap_compressor, 0644); /* Compressed storage zpool to use */ -#define ZSWAP_ZPOOL_DEFAULT "zbud" -static char *zswap_zpool_type = ZSWAP_ZPOOL_DEFAULT; +static char *zswap_zpool_type = CONFIG_ZSWAP_ZPOOL_DEFAULT; static int zswap_zpool_param_set(const char *, const struct kernel_param *); static struct kernel_param_ops zswap_zpool_param_ops = { .set = zswap_zpool_param_set, @@ -599,11 +597,12 @@ static __init struct zswap_pool *__zswap_pool_create_fallback(void) bool has_comp, has_zpool; has_comp = crypto_has_comp(zswap_compressor, 0, 0); - if (!has_comp && strcmp(zswap_compressor, ZSWAP_COMPRESSOR_DEFAULT)) { + if (!has_comp && strcmp(zswap_compressor, + CONFIG_ZSWAP_COMPRESSOR_DEFAULT)) { pr_err("compressor %s not available, using default %s\n", - zswap_compressor, ZSWAP_COMPRESSOR_DEFAULT); + zswap_compressor, CONFIG_ZSWAP_COMPRESSOR_DEFAULT); param_free_charp(&zswap_compressor); - zswap_compressor = ZSWAP_COMPRESSOR_DEFAULT; + zswap_compressor = CONFIG_ZSWAP_COMPRESSOR_DEFAULT; has_comp = crypto_has_comp(zswap_compressor, 0, 0); } if (!has_comp) { @@ -614,11 +613,12 @@ static __init struct zswap_pool *__zswap_pool_create_fallback(void) } has_zpool = zpool_has_pool(zswap_zpool_type); - if (!has_zpool && strcmp(zswap_zpool_type, ZSWAP_ZPOOL_DEFAULT)) { + if (!has_zpool && strcmp(zswap_zpool_type, + CONFIG_ZSWAP_ZPOOL_DEFAULT)) { pr_err("zpool %s not available, using default %s\n", - zswap_zpool_type, ZSWAP_ZPOOL_DEFAULT); + zswap_zpool_type, CONFIG_ZSWAP_ZPOOL_DEFAULT); param_free_charp(&zswap_zpool_type); - zswap_zpool_type = ZSWAP_ZPOOL_DEFAULT; + zswap_zpool_type = CONFIG_ZSWAP_ZPOOL_DEFAULT; has_zpool = zpool_has_pool(zswap_zpool_type); } if (!has_zpool) { From 77337edee7598d82fb5acf66cb91a5b3f0c46add Mon Sep 17 00:00:00 2001 From: Jules Irenge Date: Mon, 6 Apr 2020 20:08:06 -0700 Subject: [PATCH 549/635] mm/compaction: add missing annotation for compact_lock_irqsave Sparse reports a warning at compact_lock_irqsave() warning: context imbalance in compact_lock_irqsave() - wrong count at exit The root cause is the missing annotation at compact_lock_irqsave() Add the missing __acquires(lock) annotation. Signed-off-by: Jules Irenge Signed-off-by: Andrew Morton Link: http://lkml.kernel.org/r/20200214204741.94112-6-jbi.octave@gmail.com Signed-off-by: Linus Torvalds --- mm/compaction.c | 1 + 1 file changed, 1 insertion(+) diff --git a/mm/compaction.c b/mm/compaction.c index 46af63eb8212..46f0fcc93081 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -481,6 +481,7 @@ static bool test_and_set_skip(struct compact_control *cc, struct page *page, */ static bool compact_lock_irqsave(spinlock_t *lock, unsigned long *flags, struct compact_control *cc) + __acquires(lock) { /* Track if the lock is contended in async mode */ if (cc->mode == MIGRATE_ASYNC && !cc->contended) { From 1b2a1e7bb9ce9936d6839c1023737ccff7889dee Mon Sep 17 00:00:00 2001 From: Jules Irenge Date: Mon, 6 Apr 2020 20:08:09 -0700 Subject: [PATCH 550/635] mm/hugetlb: add missing annotation for gather_surplus_pages() Sparse reports a warning at gather_surplus_pages() warning: context imbalance in hugetlb_cow() - unexpected unlock The root cause is the missing annotation at gather_surplus_pages() Add the missing __must_hold(&hugetlb_lock) Signed-off-by: Jules Irenge Signed-off-by: Andrew Morton Reviewed-by: Mike Kravetz Link: http://lkml.kernel.org/r/20200214204741.94112-7-jbi.octave@gmail.com Signed-off-by: Linus Torvalds --- mm/hugetlb.c | 1 + 1 file changed, 1 insertion(+) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index f9ea1e5197b4..f5fb53fdfa02 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -2010,6 +2010,7 @@ struct page *alloc_huge_page_vma(struct hstate *h, struct vm_area_struct *vma, * of size 'delta'. */ static int gather_surplus_pages(struct hstate *h, int delta) + __must_hold(&hugetlb_lock) { struct list_head surplus_list; struct page *page, *tmp; From 959a7e136d52bff429c774a4ed451b094706116b Mon Sep 17 00:00:00 2001 From: Jules Irenge Date: Mon, 6 Apr 2020 20:08:12 -0700 Subject: [PATCH 551/635] mm/mempolicy: add missing annotation for queue_pages_pmd() Sparse reports a warning at queue_pages_pmd() context imbalance in queue_pages_pmd() - unexpected unlock The root cause is the missing annotation at queue_pages_pmd() Add the missing __releases(ptl) Signed-off-by: Jules Irenge Signed-off-by: Andrew Morton Link: http://lkml.kernel.org/r/20200214204741.94112-8-jbi.octave@gmail.com Signed-off-by: Linus Torvalds --- mm/mempolicy.c | 1 + 1 file changed, 1 insertion(+) diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 145be04b7108..c4118bb508ec 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -442,6 +442,7 @@ static inline bool queue_pages_required(struct page *page, */ static int queue_pages_pmd(pmd_t *pmd, spinlock_t *ptl, unsigned long addr, unsigned long end, struct mm_walk *walk) + __releases(ptl) { int ret = 0; struct page *page; From 31364c2e168b48c8596d4d43bad9bff70e21976b Mon Sep 17 00:00:00 2001 From: Jules Irenge Date: Mon, 6 Apr 2020 20:08:15 -0700 Subject: [PATCH 552/635] mm/slub: add missing annotation for get_map() Sparse reports a warning at get_map()() warning: context imbalance in get_map() - wrong count at exit The root cause is the missing annotation at get_map() Add the missing __acquires(&object_map_lock) annotation Signed-off-by: Jules Irenge Signed-off-by: Andrew Morton Link: http://lkml.kernel.org/r/20200214204741.94112-9-jbi.octave@gmail.com Signed-off-by: Linus Torvalds --- mm/slub.c | 1 + 1 file changed, 1 insertion(+) diff --git a/mm/slub.c b/mm/slub.c index 3098e0cf2899..c6d603648c79 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -449,6 +449,7 @@ static DEFINE_SPINLOCK(object_map_lock); * not vanish from under us. */ static unsigned long *get_map(struct kmem_cache *s, struct page *page) + __acquires(&object_map_lock) { void *p; void *addr = page_address(page); From 81aba9e06ba8cc1e2494258bc4b48875af522f32 Mon Sep 17 00:00:00 2001 From: Jules Irenge Date: Mon, 6 Apr 2020 20:08:18 -0700 Subject: [PATCH 553/635] mm/slub: add missing annotation for put_map() Sparse reports a warning at put_map()() warning: context imbalance in put_map() - unexpected unlock The root cause is the missing annotation at put_map() Add the missing __releases(&object_map_lock) annotation Signed-off-by: Jules Irenge Signed-off-by: Andrew Morton Link: http://lkml.kernel.org/r/20200214204741.94112-10-jbi.octave@gmail.com Signed-off-by: Linus Torvalds --- mm/slub.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/slub.c b/mm/slub.c index c6d603648c79..332d4b459a90 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -466,7 +466,7 @@ static unsigned long *get_map(struct kmem_cache *s, struct page *page) return object_map; } -static void put_map(unsigned long *map) +static void put_map(unsigned long *map) __releases(&object_map_lock) { VM_BUG_ON(map != object_map); lockdep_assert_held(&object_map_lock); From cfc451cfdf1dde36eada652a4b16d9b1524c5932 Mon Sep 17 00:00:00 2001 From: Jules Irenge Date: Mon, 6 Apr 2020 20:08:21 -0700 Subject: [PATCH 554/635] mm/zsmalloc: add missing annotation for migrate_read_lock() Sparse reports a warning at migrate_read_lock()() warning: context imbalance in migrate_read_lock() - wrong count at exit The root cause is the missing annotation at migrate_read_lock() Add the missing __acquires(&zspage->lock) annotation Signed-off-by: Jules Irenge Signed-off-by: Andrew Morton Acked-by: Minchan Kim Link: http://lkml.kernel.org/r/20200214204741.94112-11-jbi.octave@gmail.com Signed-off-by: Linus Torvalds --- mm/zsmalloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index 22d17ecfe7df..da70817b4ed8 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -1833,7 +1833,7 @@ static void migrate_lock_init(struct zspage *zspage) rwlock_init(&zspage->lock); } -static void migrate_read_lock(struct zspage *zspage) +static void migrate_read_lock(struct zspage *zspage) __acquires(&zspage->lock) { read_lock(&zspage->lock); } From 8a374cccee8cdfa2902bb7a07a10671ffc1a72c1 Mon Sep 17 00:00:00 2001 From: Jules Irenge Date: Mon, 6 Apr 2020 20:08:24 -0700 Subject: [PATCH 555/635] mm/zsmalloc: add missing annotation for migrate_read_unlock() Sparse reports a warning at migrate_read_unlock()() warning: context imbalance in migrate_read_unlock() - unexpected unlock The root cause is the missing annotation at migrate_read_unlock() Add the missing __releases(&zspage->lock) annotation Signed-off-by: Jules Irenge Signed-off-by: Andrew Morton Acked-by: Minchan Kim Link: http://lkml.kernel.org/r/20200214204741.94112-12-jbi.octave@gmail.com Signed-off-by: Linus Torvalds --- mm/zsmalloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index da70817b4ed8..2eab424c8c67 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -1838,7 +1838,7 @@ static void migrate_read_lock(struct zspage *zspage) __acquires(&zspage->lock) read_lock(&zspage->lock); } -static void migrate_read_unlock(struct zspage *zspage) +static void migrate_read_unlock(struct zspage *zspage) __releases(&zspage->lock) { read_unlock(&zspage->lock); } From 70c7ec95bece1bd01e25fbc4f17c9262445de417 Mon Sep 17 00:00:00 2001 From: Jules Irenge Date: Mon, 6 Apr 2020 20:08:27 -0700 Subject: [PATCH 556/635] mm/zsmalloc: add missing annotation for pin_tag() Sparse reports a warning at pin_tag()() warning: context imbalance in pin_tag() - wrong count at exit The root cause is the missing annotation at pin_tag() Add the missing __acquires(bitlock) annotation Signed-off-by: Jules Irenge Signed-off-by: Andrew Morton Acked-by: Minchan Kim Link: http://lkml.kernel.org/r/20200214204741.94112-13-jbi.octave@gmail.com Signed-off-by: Linus Torvalds --- mm/zsmalloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index 2eab424c8c67..7bac76ae11b3 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -891,7 +891,7 @@ static inline int trypin_tag(unsigned long handle) return bit_spin_trylock(HANDLE_PIN_BIT, (unsigned long *)handle); } -static void pin_tag(unsigned long handle) +static void pin_tag(unsigned long handle) __acquires(bitlock) { bit_spin_lock(HANDLE_PIN_BIT, (unsigned long *)handle); } From bc22b18b1f805dd785c408060f21aecb4c500ea5 Mon Sep 17 00:00:00 2001 From: Jules Irenge Date: Mon, 6 Apr 2020 20:08:30 -0700 Subject: [PATCH 557/635] mm/zsmalloc: add missing annotation for unpin_tag() Sparse reports a warning at unpin_tag()() warning: context imbalance in unpin_tag() - unexpected unlock The root cause is the missing annotation at unpin_tag() Add the missing __releases(bitlock) annotation Signed-off-by: Jules Irenge Signed-off-by: Andrew Morton Acked-by: Minchan Kim Link: http://lkml.kernel.org/r/20200214204741.94112-14-jbi.octave@gmail.com Signed-off-by: Linus Torvalds --- mm/zsmalloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index 7bac76ae11b3..2aa2d524a343 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -896,7 +896,7 @@ static void pin_tag(unsigned long handle) __acquires(bitlock) bit_spin_lock(HANDLE_PIN_BIT, (unsigned long *)handle); } -static void unpin_tag(unsigned long handle) +static void unpin_tag(unsigned long handle) __releases(bitlock) { bit_spin_unlock(HANDLE_PIN_BIT, (unsigned long *)handle); } From 552657b7b3343851916fde7e4fd6bfb6516d2bcb Mon Sep 17 00:00:00 2001 From: chenqiwu Date: Mon, 6 Apr 2020 20:08:33 -0700 Subject: [PATCH 558/635] mm: fix ambiguous comments for better code readability The parameter of remap_pfn_range() @pfn passed from the caller is actually a page-frame number converted by corresponding physical address of kernel memory, the original comment is ambiguous that may mislead the users. Meanwhile, there is an ambiguous typo "VMM" in the comment of vm_area_struct. So fixing them will make the code more readable. Signed-off-by: chenqiwu Signed-off-by: Andrew Morton Reviewed-by: Andrew Morton Link: http://lkml.kernel.org/r/1583026921-15279-1-git-send-email-qiwuchen55@gmail.com Signed-off-by: Linus Torvalds --- include/linux/mm_types.h | 4 ++-- mm/memory.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index dd555e6d23f3..4aba6c0c2ba8 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -289,8 +289,8 @@ struct vm_userfaultfd_ctx {}; #endif /* CONFIG_USERFAULTFD */ /* - * This struct defines a memory VMM memory area. There is one of these - * per VM-area/task. A VM area is any part of the process virtual memory + * This struct describes a virtual memory area. There is one of these + * per VM-area/task. A VM area is any part of the process virtual memory * space that has a special rule for the page-fault handlers (ie a shared * library, the executable area etc). */ diff --git a/mm/memory.c b/mm/memory.c index 8ac9af73e9d2..19874d133a66 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1952,7 +1952,7 @@ static inline int remap_p4d_range(struct mm_struct *mm, pgd_t *pgd, * @vma: user vma to map to * @addr: target user address to start at * @pfn: page frame number of kernel physical memory address - * @size: size of map area + * @size: size of mapping area * @prot: page protection flags for this mapping * * Note: this is only safe if the mm semaphore is held when called. From e46b893dd113253c2ff032aed22d9df404c5511d Mon Sep 17 00:00:00 2001 From: Mateusz Nosek Date: Mon, 6 Apr 2020 20:08:36 -0700 Subject: [PATCH 559/635] mm/mm_init.c: clean code. Use BUILD_BUG_ON when comparing compile time constant MAX_ZONELISTS is a compile time constant, so it should be compared using BUILD_BUG_ON not BUG_ON. Signed-off-by: Mateusz Nosek Signed-off-by: Andrew Morton Reviewed-by: Andrew Morton Reviewed-by: Wei Yang Link: http://lkml.kernel.org/r/20200228224617.11343-1-mateusznosek0@gmail.com Signed-off-by: Linus Torvalds --- mm/mm_init.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index 5c918388de99..7da6991d9435 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -37,7 +37,7 @@ void __init mminit_verify_zonelist(void) struct zonelist *zonelist; int i, listid, zoneid; - BUG_ON(MAX_ZONELISTS > 2); + BUILD_BUG_ON(MAX_ZONELISTS > 2); for (i = 0; i < MAX_ZONELISTS * MAX_NR_ZONES; i++) { /* Identify the zone and nodelist */ From e4a9bc58969abc695a6ebb06d801a99c1bafc001 Mon Sep 17 00:00:00 2001 From: Joe Perches Date: Mon, 6 Apr 2020 20:08:39 -0700 Subject: [PATCH 560/635] mm: use fallthrough; Convert the various /* fallthrough */ comments to the pseudo-keyword fallthrough; Done via script: https://lore.kernel.org/lkml/b56602fcf79f849e733e7b521bb0e17895d390fa.1582230379.git.joe@perches.com/ Signed-off-by: Joe Perches Signed-off-by: Andrew Morton Reviewed-by: Gustavo A. R. Silva Link: http://lkml.kernel.org/r/f62fea5d10eb0ccfc05d87c242a620c261219b66.camel@perches.com Signed-off-by: Linus Torvalds --- mm/gup.c | 2 +- mm/hugetlb_cgroup.c | 6 +++--- mm/ksm.c | 3 +-- mm/list_lru.c | 2 +- mm/memcontrol.c | 2 +- mm/mempolicy.c | 3 --- mm/mmap.c | 5 ++--- mm/shmem.c | 2 +- mm/zsmalloc.c | 2 +- 9 files changed, 11 insertions(+), 16 deletions(-) diff --git a/mm/gup.c b/mm/gup.c index b185377c38b7..a21230569520 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -1102,7 +1102,7 @@ retry: goto retry; case -EBUSY: ret = 0; - /* FALLTHRU */ + fallthrough; case -EFAULT: case -ENOMEM: case -EHWPOISON: diff --git a/mm/hugetlb_cgroup.c b/mm/hugetlb_cgroup.c index c2d7ae6cabd1..aabf65d4d91b 100644 --- a/mm/hugetlb_cgroup.c +++ b/mm/hugetlb_cgroup.c @@ -467,14 +467,14 @@ static int hugetlb_cgroup_read_u64_max(struct seq_file *seq, void *v) switch (MEMFILE_ATTR(cft->private)) { case RES_RSVD_USAGE: counter = &h_cg->rsvd_hugepage[idx]; - /* Fall through. */ + fallthrough; case RES_USAGE: val = (u64)page_counter_read(counter); seq_printf(seq, "%llu\n", val * PAGE_SIZE); break; case RES_RSVD_LIMIT: counter = &h_cg->rsvd_hugepage[idx]; - /* Fall through. */ + fallthrough; case RES_LIMIT: val = (u64)counter->max; if (val == limit) @@ -514,7 +514,7 @@ static ssize_t hugetlb_cgroup_write(struct kernfs_open_file *of, switch (MEMFILE_ATTR(of_cft(of)->private)) { case RES_RSVD_LIMIT: rsvd = true; - /* Fall through. */ + fallthrough; case RES_LIMIT: mutex_lock(&hugetlb_limit_mutex); ret = page_counter_set_max( diff --git a/mm/ksm.c b/mm/ksm.c index 363ec8189561..a558da9e7177 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -2813,8 +2813,7 @@ static int ksm_memory_callback(struct notifier_block *self, */ ksm_check_stable_tree(mn->start_pfn, mn->start_pfn + mn->nr_pages); - /* fallthrough */ - + fallthrough; case MEM_CANCEL_OFFLINE: mutex_lock(&ksm_thread_mutex); ksm_run &= ~KSM_RUN_OFFLINE; diff --git a/mm/list_lru.c b/mm/list_lru.c index 8de5e3784ee4..4d5294c39bba 100644 --- a/mm/list_lru.c +++ b/mm/list_lru.c @@ -223,7 +223,7 @@ restart: switch (ret) { case LRU_REMOVED_RETRY: assert_spin_locked(&nlru->lock); - /* fall through */ + fallthrough; case LRU_REMOVED: isolated++; nlru->nr_items--; diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 9965a77d53e5..05b4ec2c6499 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5813,7 +5813,7 @@ retry: switch (get_mctgt_type(vma, addr, ptent, &target)) { case MC_TARGET_DEVICE: device = true; - /* fall through */ + fallthrough; case MC_TARGET_PAGE: page = target.page; /* diff --git a/mm/mempolicy.c b/mm/mempolicy.c index c4118bb508ec..b4a039d779b0 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -881,7 +881,6 @@ static void get_policy_nodemask(struct mempolicy *p, nodemask_t *nodes) switch (p->mode) { case MPOL_BIND: - /* Fall through */ case MPOL_INTERLEAVE: *nodes = p->v.nodes; break; @@ -2066,7 +2065,6 @@ bool init_nodemask_of_mempolicy(nodemask_t *mask) break; case MPOL_BIND: - /* Fall through */ case MPOL_INTERLEAVE: *mask = mempolicy->v.nodes; break; @@ -2333,7 +2331,6 @@ bool __mpol_equal(struct mempolicy *a, struct mempolicy *b) switch (a->mode) { case MPOL_BIND: - /* Fall through */ case MPOL_INTERLEAVE: return !!nodes_equal(a->v.nodes, b->v.nodes); case MPOL_PREFERRED: diff --git a/mm/mmap.c b/mm/mmap.c index aa09429d5888..8d77dbbb80fe 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1460,7 +1460,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, * with MAP_SHARED to preserve backward compatibility. */ flags &= LEGACY_MAP_MASK; - /* fall through */ + fallthrough; case MAP_SHARED_VALIDATE: if (flags & ~flags_mask) return -EOPNOTSUPP; @@ -1487,8 +1487,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, vm_flags |= VM_SHARED | VM_MAYSHARE; if (!(file->f_mode & FMODE_WRITE)) vm_flags &= ~(VM_MAYWRITE | VM_SHARED); - - /* fall through */ + fallthrough; case MAP_PRIVATE: if (!(file->f_mode & FMODE_READ)) return -EACCES; diff --git a/mm/shmem.c b/mm/shmem.c index 8d0aba1cc155..d722eb830317 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -3996,7 +3996,7 @@ bool shmem_huge_enabled(struct vm_area_struct *vma) if (i_size >= HPAGE_PMD_SIZE && i_size >> PAGE_SHIFT >= off) return true; - /* fall through */ + fallthrough; case SHMEM_HUGE_ADVISE: /* TODO: implement fadvise() hints */ return (vma->vm_flags & VM_HUGEPAGE); diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index 2aa2d524a343..2f836a2b993f 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -424,7 +424,7 @@ static void *zs_zpool_map(void *pool, unsigned long handle, case ZPOOL_MM_WO: zs_mm = ZS_MM_WO; break; - case ZPOOL_MM_RW: /* fall through */ + case ZPOOL_MM_RW: default: zs_mm = ZS_MM_RW; break; From 3f3673d7d324d872d9d8ddb73b3e5e47fbf12e0d Mon Sep 17 00:00:00 2001 From: Steven Price Date: Mon, 6 Apr 2020 20:08:43 -0700 Subject: [PATCH 561/635] include/linux/swapops.h: correct guards for non_swap_entry() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If CONFIG_DEVICE_PRIVATE is defined, but neither CONFIG_MEMORY_FAILURE nor CONFIG_MIGRATION, then non_swap_entry() will return 0, meaning that the condition (non_swap_entry(entry) && is_device_private_entry(entry)) in zap_pte_range() will never be true even if the entry is a device private one. Equally any other code depending on non_swap_entry() will not function as expected. I originally spotted this just by looking at the code, I haven't actually observed any problems. Looking a bit more closely it appears that actually this situation (currently at least) cannot occur: DEVICE_PRIVATE depends on ZONE_DEVICE ZONE_DEVICE depends on MEMORY_HOTREMOVE MEMORY_HOTREMOVE depends on MIGRATION Fixes: 5042db43cc26 ("mm/ZONE_DEVICE: new type of ZONE_DEVICE for unaddressable memory") Signed-off-by: Steven Price Signed-off-by: Andrew Morton Cc: Jérôme Glisse Cc: Arnd Bergmann Cc: Dan Williams Cc: John Hubbard Link: http://lkml.kernel.org/r/20200305130550.22693-1-steven.price@arm.com Signed-off-by: Linus Torvalds --- include/linux/swapops.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/linux/swapops.h b/include/linux/swapops.h index 9a6f06de183b..d9b7c9132c2f 100644 --- a/include/linux/swapops.h +++ b/include/linux/swapops.h @@ -350,7 +350,8 @@ static inline void num_poisoned_pages_inc(void) } #endif -#if defined(CONFIG_MEMORY_FAILURE) || defined(CONFIG_MIGRATION) +#if defined(CONFIG_MEMORY_FAILURE) || defined(CONFIG_MIGRATION) || \ + defined(CONFIG_DEVICE_PRIVATE) static inline int non_swap_entry(swp_entry_t entry) { return swp_type(entry) >= MAX_SWAPFILES; From 1d90b6491014ead775146726b81a78ed993c3188 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Mon, 6 Apr 2020 20:08:46 -0700 Subject: [PATCH 562/635] include/linux/memremap.h: remove stale comments Fixes: 80a72d0af05a ("memremap: remove the data field in struct dev_pagemap") Fixes: fdc029b19dfd ("memremap: remove the dev field in struct dev_pagemap") Signed-off-by: Ira Weiny Signed-off-by: Andrew Morton Reviewed-by: Christoph Hellwig Cc: Jason Gunthorpe Cc: Dan Williams Link: http://lkml.kernel.org/r/20200316213205.145333-1-ira.weiny@intel.com Signed-off-by: Linus Torvalds --- include/linux/memremap.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/include/linux/memremap.h b/include/linux/memremap.h index 60d97e8fd3c0..8b37c4c9222c 100644 --- a/include/linux/memremap.h +++ b/include/linux/memremap.h @@ -98,8 +98,6 @@ struct dev_pagemap_ops { * @ref: reference count that pins the devm_memremap_pages() mapping * @internal_ref: internal reference if @ref is not provided by the caller * @done: completion for @internal_ref - * @dev: host device of the mapping for debug - * @data: private data pointer for page_free() * @type: memory type: see MEMORY_* in memory_hotplug.h * @flags: PGMAP_* flags to specify defailed behavior * @ops: method table From 1386f7a3bfa6245a69f1fa39786be54a73889ff7 Mon Sep 17 00:00:00 2001 From: Mateusz Nosek Date: Mon, 6 Apr 2020 20:08:49 -0700 Subject: [PATCH 563/635] mm/dmapool.c: micro-optimisation remove unnecessary branch Previously there was a check if 'size' is aligned to 'align' and if not then it was aligned. This check was expensive as both branch and division are expensive instructions in most architectures. 'ALIGN' function on already aligned value will not change it, and as it is cheaper than branch + division it can be executed all the time and branch can be removed. Signed-off-by: Mateusz Nosek Signed-off-by: Andrew Morton Reviewed-by: Andrew Morton Link: http://lkml.kernel.org/r/20200320173317.26408-1-mateusznosek0@gmail.com Signed-off-by: Linus Torvalds --- mm/dmapool.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mm/dmapool.c b/mm/dmapool.c index fe5d33060415..f9fb9bbd733e 100644 --- a/mm/dmapool.c +++ b/mm/dmapool.c @@ -144,9 +144,7 @@ struct dma_pool *dma_pool_create(const char *name, struct device *dev, else if (size < 4) size = 4; - if ((size % align) != 0) - size = ALIGN(size, align); - + size = ALIGN(size, align); allocation = max_t(size_t, size, PAGE_SIZE); if (!boundary) From 6218d740ac1bc723d57900b865d3e52b83550c2b Mon Sep 17 00:00:00 2001 From: Waiman Long Date: Mon, 6 Apr 2020 20:08:52 -0700 Subject: [PATCH 564/635] mm: remove dummy struct bootmem_data/bootmem_data_t Both bootmem_data and bootmem_data_t structures are no longer defined. Remove the dummy forward declarations. Signed-off-by: Waiman Long Signed-off-by: Andrew Morton Reviewed-by: Baoquan He Acked-by: Mike Rapoport Link: http://lkml.kernel.org/r/20200326022617.26208-1-longman@redhat.com Signed-off-by: Linus Torvalds --- arch/alpha/include/asm/mmzone.h | 2 -- include/linux/mmzone.h | 1 - 2 files changed, 3 deletions(-) diff --git a/arch/alpha/include/asm/mmzone.h b/arch/alpha/include/asm/mmzone.h index 7ee144f484f1..9b521c857436 100644 --- a/arch/alpha/include/asm/mmzone.h +++ b/arch/alpha/include/asm/mmzone.h @@ -8,8 +8,6 @@ #include -struct bootmem_data_t; /* stupid forward decl. */ - /* * Following are macros that are specific to this numa platform. */ diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index f3f264826423..e9892bf9eba9 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -664,7 +664,6 @@ struct deferred_split { * Memory statistics and page replacement data structures are maintained on a * per-zone basis. */ -struct bootmem_data; typedef struct pglist_data { struct zone node_zones[MAX_NR_ZONES]; struct zonelist node_zonelists[MAX_ZONELISTS]; From 904f394e2e9fd362dc78d58646b85c4cf6c87cac Mon Sep 17 00:00:00 2001 From: Jules Irenge Date: Mon, 6 Apr 2020 20:08:55 -0700 Subject: [PATCH 565/635] fs/proc/inode.c: annotate close_pdeo() for sparse Fix sparse locking imbalance warning: warning: context imbalance in close_pdeo() - unexpected unlock Signed-off-by: Jules Irenge Signed-off-by: Alexey Dobriyan Signed-off-by: Andrew Morton Link: http://lkml.kernel.org/r/20200227201538.GA30462@avx2 Signed-off-by: Linus Torvalds --- fs/proc/inode.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/proc/inode.c b/fs/proc/inode.c index 1e730ea1dcd6..05d31c464bee 100644 --- a/fs/proc/inode.c +++ b/fs/proc/inode.c @@ -202,6 +202,7 @@ static void unuse_pde(struct proc_dir_entry *pde) /* pde is locked on entry, unlocked on exit */ static void close_pdeo(struct proc_dir_entry *pde, struct pde_opener *pdeo) + __releases(&pde->pde_unload_lock) { /* * close() (proc_reg_release()) can't delete an entry and proceed: From d919b33dafb3e222d23671b2bb06d119aede625f Mon Sep 17 00:00:00 2001 From: Alexey Dobriyan Date: Mon, 6 Apr 2020 20:09:01 -0700 Subject: [PATCH 566/635] proc: faster open/read/close with "permanent" files Now that "struct proc_ops" exist we can start putting there stuff which could not fly with VFS "struct file_operations"... Most of fs/proc/inode.c file is dedicated to make open/read/.../close reliable in the event of disappearing /proc entries which usually happens if module is getting removed. Files like /proc/cpuinfo which never disappear simply do not need such protection. Save 2 atomic ops, 1 allocation, 1 free per open/read/close sequence for such "permanent" files. Enable "permanent" flag for /proc/cpuinfo /proc/kmsg /proc/modules /proc/slabinfo /proc/stat /proc/sysvipc/* /proc/swaps More will come once I figure out foolproof way to prevent out module authors from marking their stuff "permanent" for performance reasons when it is not. This should help with scalability: benchmark is "read /proc/cpuinfo R times by N threads scattered over the system". N R t, s (before) t, s (after) ----------------------------------------------------- 64 4096 1.582458 1.530502 -3.2% 256 4096 6.371926 6.125168 -3.9% 1024 4096 25.64888 24.47528 -4.6% Benchmark source: #include #include #include #include #include #include #include #include const int NR_CPUS = sysconf(_SC_NPROCESSORS_ONLN); int N; const char *filename; int R; int xxx = 0; int glue(int n) { cpu_set_t m; CPU_ZERO(&m); CPU_SET(n, &m); return sched_setaffinity(0, sizeof(cpu_set_t), &m); } void f(int n) { glue(n % NR_CPUS); while (*(volatile int *)&xxx == 0) { } for (int i = 0; i < R; i++) { int fd = open(filename, O_RDONLY); char buf[4096]; ssize_t rv = read(fd, buf, sizeof(buf)); asm volatile ("" :: "g" (rv)); close(fd); } } int main(int argc, char *argv[]) { if (argc < 4) { std::cerr << "usage: " << argv[0] << ' ' << "N /proc/filename R "; return 1; } N = atoi(argv[1]); filename = argv[2]; R = atoi(argv[3]); for (int i = 0; i < NR_CPUS; i++) { if (glue(i) == 0) break; } std::vector T; T.reserve(N); for (int i = 0; i < N; i++) { T.emplace_back(f, i); } auto t0 = std::chrono::system_clock::now(); { *(volatile int *)&xxx = 1; for (auto& t: T) { t.join(); } } auto t1 = std::chrono::system_clock::now(); std::chrono::duration dt = t1 - t0; std::cout << dt.count() << ' '; return 0; } P.S.: Explicit randomization marker is added because adding non-function pointer will silently disable structure layout randomization. [akpm@linux-foundation.org: coding style fixes] Reported-by: kbuild test robot Reported-by: Dan Carpenter Signed-off-by: Alexey Dobriyan Signed-off-by: Andrew Morton Cc: Al Viro Cc: Joe Perches Link: http://lkml.kernel.org/r/20200222201539.GA22576@avx2 Signed-off-by: Linus Torvalds --- fs/proc/cpuinfo.c | 1 + fs/proc/generic.c | 31 ++++++- fs/proc/inode.c | 187 +++++++++++++++++++++++++++++----------- fs/proc/internal.h | 6 ++ fs/proc/kmsg.c | 1 + fs/proc/stat.c | 1 + include/linux/proc_fs.h | 17 +++- ipc/util.c | 1 + kernel/module.c | 1 + mm/slab_common.c | 1 + mm/swapfile.c | 1 + 11 files changed, 194 insertions(+), 54 deletions(-) diff --git a/fs/proc/cpuinfo.c b/fs/proc/cpuinfo.c index c1dea9b8222e..d0989a443c77 100644 --- a/fs/proc/cpuinfo.c +++ b/fs/proc/cpuinfo.c @@ -17,6 +17,7 @@ static int cpuinfo_open(struct inode *inode, struct file *file) } static const struct proc_ops cpuinfo_proc_ops = { + .proc_flags = PROC_ENTRY_PERMANENT, .proc_open = cpuinfo_open, .proc_read = seq_read, .proc_lseek = seq_lseek, diff --git a/fs/proc/generic.c b/fs/proc/generic.c index 3faed94e4b65..4ed6dabdf6ff 100644 --- a/fs/proc/generic.c +++ b/fs/proc/generic.c @@ -531,6 +531,12 @@ struct proc_dir_entry *proc_create_reg(const char *name, umode_t mode, return p; } +static inline void pde_set_flags(struct proc_dir_entry *pde) +{ + if (pde->proc_ops->proc_flags & PROC_ENTRY_PERMANENT) + pde->flags |= PROC_ENTRY_PERMANENT; +} + struct proc_dir_entry *proc_create_data(const char *name, umode_t mode, struct proc_dir_entry *parent, const struct proc_ops *proc_ops, void *data) @@ -541,6 +547,7 @@ struct proc_dir_entry *proc_create_data(const char *name, umode_t mode, if (!p) return NULL; p->proc_ops = proc_ops; + pde_set_flags(p); return proc_register(parent, p); } EXPORT_SYMBOL(proc_create_data); @@ -572,6 +579,7 @@ static int proc_seq_release(struct inode *inode, struct file *file) } static const struct proc_ops proc_seq_ops = { + /* not permanent -- can call into arbitrary seq_operations */ .proc_open = proc_seq_open, .proc_read = seq_read, .proc_lseek = seq_lseek, @@ -602,6 +610,7 @@ static int proc_single_open(struct inode *inode, struct file *file) } static const struct proc_ops proc_single_ops = { + /* not permanent -- can call into arbitrary ->single_show */ .proc_open = proc_single_open, .proc_read = seq_read, .proc_lseek = seq_lseek, @@ -662,9 +671,13 @@ void remove_proc_entry(const char *name, struct proc_dir_entry *parent) de = pde_subdir_find(parent, fn, len); if (de) { - rb_erase(&de->subdir_node, &parent->subdir); - if (S_ISDIR(de->mode)) { - parent->nlink--; + if (unlikely(pde_is_permanent(de))) { + WARN(1, "removing permanent /proc entry '%s'", de->name); + de = NULL; + } else { + rb_erase(&de->subdir_node, &parent->subdir); + if (S_ISDIR(de->mode)) + parent->nlink--; } } write_unlock(&proc_subdir_lock); @@ -700,12 +713,24 @@ int remove_proc_subtree(const char *name, struct proc_dir_entry *parent) write_unlock(&proc_subdir_lock); return -ENOENT; } + if (unlikely(pde_is_permanent(root))) { + write_unlock(&proc_subdir_lock); + WARN(1, "removing permanent /proc entry '%s/%s'", + root->parent->name, root->name); + return -EINVAL; + } rb_erase(&root->subdir_node, &parent->subdir); de = root; while (1) { next = pde_subdir_first(de); if (next) { + if (unlikely(pde_is_permanent(root))) { + write_unlock(&proc_subdir_lock); + WARN(1, "removing permanent /proc entry '%s/%s'", + next->parent->name, next->name); + return -EINVAL; + } rb_erase(&next->subdir_node, &de->subdir); de = next; continue; diff --git a/fs/proc/inode.c b/fs/proc/inode.c index 05d31c464bee..fb4cace9ea41 100644 --- a/fs/proc/inode.c +++ b/fs/proc/inode.c @@ -259,114 +259,192 @@ void proc_entry_rundown(struct proc_dir_entry *de) spin_unlock(&de->pde_unload_lock); } +static loff_t pde_lseek(struct proc_dir_entry *pde, struct file *file, loff_t offset, int whence) +{ + typeof_member(struct proc_ops, proc_lseek) lseek; + + lseek = pde->proc_ops->proc_lseek; + if (!lseek) + lseek = default_llseek; + return lseek(file, offset, whence); +} + static loff_t proc_reg_llseek(struct file *file, loff_t offset, int whence) { struct proc_dir_entry *pde = PDE(file_inode(file)); loff_t rv = -EINVAL; - if (use_pde(pde)) { - typeof_member(struct proc_ops, proc_lseek) lseek; - lseek = pde->proc_ops->proc_lseek; - if (!lseek) - lseek = default_llseek; - rv = lseek(file, offset, whence); + if (pde_is_permanent(pde)) { + return pde_lseek(pde, file, offset, whence); + } else if (use_pde(pde)) { + rv = pde_lseek(pde, file, offset, whence); unuse_pde(pde); } return rv; } +static ssize_t pde_read(struct proc_dir_entry *pde, struct file *file, char __user *buf, size_t count, loff_t *ppos) +{ + typeof_member(struct proc_ops, proc_read) read; + + read = pde->proc_ops->proc_read; + if (read) + return read(file, buf, count, ppos); + return -EIO; +} + static ssize_t proc_reg_read(struct file *file, char __user *buf, size_t count, loff_t *ppos) { struct proc_dir_entry *pde = PDE(file_inode(file)); ssize_t rv = -EIO; - if (use_pde(pde)) { - typeof_member(struct proc_ops, proc_read) read; - read = pde->proc_ops->proc_read; - if (read) - rv = read(file, buf, count, ppos); + if (pde_is_permanent(pde)) { + return pde_read(pde, file, buf, count, ppos); + } else if (use_pde(pde)) { + rv = pde_read(pde, file, buf, count, ppos); unuse_pde(pde); } return rv; } +static ssize_t pde_write(struct proc_dir_entry *pde, struct file *file, const char __user *buf, size_t count, loff_t *ppos) +{ + typeof_member(struct proc_ops, proc_write) write; + + write = pde->proc_ops->proc_write; + if (write) + return write(file, buf, count, ppos); + return -EIO; +} + static ssize_t proc_reg_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { struct proc_dir_entry *pde = PDE(file_inode(file)); ssize_t rv = -EIO; - if (use_pde(pde)) { - typeof_member(struct proc_ops, proc_write) write; - write = pde->proc_ops->proc_write; - if (write) - rv = write(file, buf, count, ppos); + if (pde_is_permanent(pde)) { + return pde_write(pde, file, buf, count, ppos); + } else if (use_pde(pde)) { + rv = pde_write(pde, file, buf, count, ppos); unuse_pde(pde); } return rv; } +static __poll_t pde_poll(struct proc_dir_entry *pde, struct file *file, struct poll_table_struct *pts) +{ + typeof_member(struct proc_ops, proc_poll) poll; + + poll = pde->proc_ops->proc_poll; + if (poll) + return poll(file, pts); + return DEFAULT_POLLMASK; +} + static __poll_t proc_reg_poll(struct file *file, struct poll_table_struct *pts) { struct proc_dir_entry *pde = PDE(file_inode(file)); __poll_t rv = DEFAULT_POLLMASK; - if (use_pde(pde)) { - typeof_member(struct proc_ops, proc_poll) poll; - poll = pde->proc_ops->proc_poll; - if (poll) - rv = poll(file, pts); + if (pde_is_permanent(pde)) { + return pde_poll(pde, file, pts); + } else if (use_pde(pde)) { + rv = pde_poll(pde, file, pts); unuse_pde(pde); } return rv; } +static long pde_ioctl(struct proc_dir_entry *pde, struct file *file, unsigned int cmd, unsigned long arg) +{ + typeof_member(struct proc_ops, proc_ioctl) ioctl; + + ioctl = pde->proc_ops->proc_ioctl; + if (ioctl) + return ioctl(file, cmd, arg); + return -ENOTTY; +} + static long proc_reg_unlocked_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { struct proc_dir_entry *pde = PDE(file_inode(file)); long rv = -ENOTTY; - if (use_pde(pde)) { - typeof_member(struct proc_ops, proc_ioctl) ioctl; - ioctl = pde->proc_ops->proc_ioctl; - if (ioctl) - rv = ioctl(file, cmd, arg); + if (pde_is_permanent(pde)) { + return pde_ioctl(pde, file, cmd, arg); + } else if (use_pde(pde)) { + rv = pde_ioctl(pde, file, cmd, arg); unuse_pde(pde); } return rv; } #ifdef CONFIG_COMPAT +static long pde_compat_ioctl(struct proc_dir_entry *pde, struct file *file, unsigned int cmd, unsigned long arg) +{ + typeof_member(struct proc_ops, proc_compat_ioctl) compat_ioctl; + + compat_ioctl = pde->proc_ops->proc_compat_ioctl; + if (compat_ioctl) + return compat_ioctl(file, cmd, arg); + return -ENOTTY; +} + static long proc_reg_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { struct proc_dir_entry *pde = PDE(file_inode(file)); long rv = -ENOTTY; - if (use_pde(pde)) { - typeof_member(struct proc_ops, proc_compat_ioctl) compat_ioctl; - - compat_ioctl = pde->proc_ops->proc_compat_ioctl; - if (compat_ioctl) - rv = compat_ioctl(file, cmd, arg); + if (pde_is_permanent(pde)) { + return pde_compat_ioctl(pde, file, cmd, arg); + } else if (use_pde(pde)) { + rv = pde_compat_ioctl(pde, file, cmd, arg); unuse_pde(pde); } return rv; } #endif +static int pde_mmap(struct proc_dir_entry *pde, struct file *file, struct vm_area_struct *vma) +{ + typeof_member(struct proc_ops, proc_mmap) mmap; + + mmap = pde->proc_ops->proc_mmap; + if (mmap) + return mmap(file, vma); + return -EIO; +} + static int proc_reg_mmap(struct file *file, struct vm_area_struct *vma) { struct proc_dir_entry *pde = PDE(file_inode(file)); int rv = -EIO; - if (use_pde(pde)) { - typeof_member(struct proc_ops, proc_mmap) mmap; - mmap = pde->proc_ops->proc_mmap; - if (mmap) - rv = mmap(file, vma); + if (pde_is_permanent(pde)) { + return pde_mmap(pde, file, vma); + } else if (use_pde(pde)) { + rv = pde_mmap(pde, file, vma); unuse_pde(pde); } return rv; } +static unsigned long +pde_get_unmapped_area(struct proc_dir_entry *pde, struct file *file, unsigned long orig_addr, + unsigned long len, unsigned long pgoff, + unsigned long flags) +{ + typeof_member(struct proc_ops, proc_get_unmapped_area) get_area; + + get_area = pde->proc_ops->proc_get_unmapped_area; +#ifdef CONFIG_MMU + if (!get_area) + get_area = current->mm->get_unmapped_area; +#endif + if (get_area) + return get_area(file, orig_addr, len, pgoff, flags); + return orig_addr; +} + static unsigned long proc_reg_get_unmapped_area(struct file *file, unsigned long orig_addr, unsigned long len, unsigned long pgoff, @@ -375,19 +453,10 @@ proc_reg_get_unmapped_area(struct file *file, unsigned long orig_addr, struct proc_dir_entry *pde = PDE(file_inode(file)); unsigned long rv = -EIO; - if (use_pde(pde)) { - typeof_member(struct proc_ops, proc_get_unmapped_area) get_area; - - get_area = pde->proc_ops->proc_get_unmapped_area; -#ifdef CONFIG_MMU - if (!get_area) - get_area = current->mm->get_unmapped_area; -#endif - - if (get_area) - rv = get_area(file, orig_addr, len, pgoff, flags); - else - rv = orig_addr; + if (pde_is_permanent(pde)) { + return pde_get_unmapped_area(pde, file, orig_addr, len, pgoff, flags); + } else if (use_pde(pde)) { + rv = pde_get_unmapped_area(pde, file, orig_addr, len, pgoff, flags); unuse_pde(pde); } return rv; @@ -401,6 +470,13 @@ static int proc_reg_open(struct inode *inode, struct file *file) typeof_member(struct proc_ops, proc_release) release; struct pde_opener *pdeo; + if (pde_is_permanent(pde)) { + open = pde->proc_ops->proc_open; + if (open) + rv = open(inode, file); + return rv; + } + /* * Ensure that * 1) PDE's ->release hook will be called no matter what @@ -450,6 +526,17 @@ static int proc_reg_release(struct inode *inode, struct file *file) { struct proc_dir_entry *pde = PDE(inode); struct pde_opener *pdeo; + + if (pde_is_permanent(pde)) { + typeof_member(struct proc_ops, proc_release) release; + + release = pde->proc_ops->proc_release; + if (release) { + return release(inode, file); + } + return 0; + } + spin_lock(&pde->pde_unload_lock); list_for_each_entry(pdeo, &pde->pde_openers, lh) { if (pdeo->file == file) { diff --git a/fs/proc/internal.h b/fs/proc/internal.h index 9e294f0290e5..917cc85e3466 100644 --- a/fs/proc/internal.h +++ b/fs/proc/internal.h @@ -61,6 +61,7 @@ struct proc_dir_entry { struct rb_node subdir_node; char *name; umode_t mode; + u8 flags; u8 namelen; char inline_name[]; } __randomize_layout; @@ -73,6 +74,11 @@ struct proc_dir_entry { 0) #define SIZEOF_PDE_INLINE_NAME (SIZEOF_PDE - sizeof(struct proc_dir_entry)) +static inline bool pde_is_permanent(const struct proc_dir_entry *pde) +{ + return pde->flags & PROC_ENTRY_PERMANENT; +} + extern struct kmem_cache *proc_dir_entry_cache; void pde_free(struct proc_dir_entry *pde); diff --git a/fs/proc/kmsg.c b/fs/proc/kmsg.c index ec1b7d2fb773..b38ad552887f 100644 --- a/fs/proc/kmsg.c +++ b/fs/proc/kmsg.c @@ -50,6 +50,7 @@ static __poll_t kmsg_poll(struct file *file, poll_table *wait) static const struct proc_ops kmsg_proc_ops = { + .proc_flags = PROC_ENTRY_PERMANENT, .proc_read = kmsg_read, .proc_poll = kmsg_poll, .proc_open = kmsg_open, diff --git a/fs/proc/stat.c b/fs/proc/stat.c index 0449edf460f5..46b3293015fe 100644 --- a/fs/proc/stat.c +++ b/fs/proc/stat.c @@ -224,6 +224,7 @@ static int stat_open(struct inode *inode, struct file *file) } static const struct proc_ops stat_proc_ops = { + .proc_flags = PROC_ENTRY_PERMANENT, .proc_open = stat_open, .proc_read = seq_read, .proc_lseek = seq_lseek, diff --git a/include/linux/proc_fs.h b/include/linux/proc_fs.h index 40a7982b7285..45c05fd9c99d 100644 --- a/include/linux/proc_fs.h +++ b/include/linux/proc_fs.h @@ -5,6 +5,7 @@ #ifndef _LINUX_PROC_FS_H #define _LINUX_PROC_FS_H +#include #include #include @@ -12,7 +13,21 @@ struct proc_dir_entry; struct seq_file; struct seq_operations; +enum { + /* + * All /proc entries using this ->proc_ops instance are never removed. + * + * If in doubt, ignore this flag. + */ +#ifdef MODULE + PROC_ENTRY_PERMANENT = 0U, +#else + PROC_ENTRY_PERMANENT = 1U << 0, +#endif +}; + struct proc_ops { + unsigned int proc_flags; int (*proc_open)(struct inode *, struct file *); ssize_t (*proc_read)(struct file *, char __user *, size_t, loff_t *); ssize_t (*proc_write)(struct file *, const char __user *, size_t, loff_t *); @@ -25,7 +40,7 @@ struct proc_ops { #endif int (*proc_mmap)(struct file *, struct vm_area_struct *); unsigned long (*proc_get_unmapped_area)(struct file *, unsigned long, unsigned long, unsigned long, unsigned long); -}; +} __randomize_layout; #ifdef CONFIG_PROC_FS diff --git a/ipc/util.c b/ipc/util.c index fe61df53775a..97638eb2d7cb 100644 --- a/ipc/util.c +++ b/ipc/util.c @@ -885,6 +885,7 @@ static int sysvipc_proc_release(struct inode *inode, struct file *file) } static const struct proc_ops sysvipc_proc_ops = { + .proc_flags = PROC_ENTRY_PERMANENT, .proc_open = sysvipc_proc_open, .proc_read = seq_read, .proc_lseek = seq_lseek, diff --git a/kernel/module.c b/kernel/module.c index 33569a01d6e1..3447f3b74870 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -4355,6 +4355,7 @@ static int modules_open(struct inode *inode, struct file *file) } static const struct proc_ops modules_proc_ops = { + .proc_flags = PROC_ENTRY_PERMANENT, .proc_open = modules_open, .proc_read = seq_read, .proc_lseek = seq_lseek, diff --git a/mm/slab_common.c b/mm/slab_common.c index 5282f881d2f5..93ec4a574d8d 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -1581,6 +1581,7 @@ static int slabinfo_open(struct inode *inode, struct file *file) } static const struct proc_ops slabinfo_proc_ops = { + .proc_flags = PROC_ENTRY_PERMANENT, .proc_open = slabinfo_open, .proc_read = seq_read, .proc_write = slabinfo_write, diff --git a/mm/swapfile.c b/mm/swapfile.c index 273a923c275c..5871a2aa86a5 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -2797,6 +2797,7 @@ static int swaps_open(struct inode *inode, struct file *file) } static const struct proc_ops swaps_proc_ops = { + .proc_flags = PROC_ENTRY_PERMANENT, .proc_open = swaps_open, .proc_read = seq_read, .proc_lseek = seq_lseek, From 5c5ab9714c2225d50119e397c537ad5e568f268b Mon Sep 17 00:00:00 2001 From: Alexey Dobriyan Date: Mon, 6 Apr 2020 20:09:05 -0700 Subject: [PATCH 567/635] proc: speed up /proc/*/statm top(1) reads all /proc/*/statm files but kernel threads will always have zeros. Print those zeroes directly without going through seq_put_decimal_ull(). Speed up reading /proc/2/statm (which is kthreadd) is like 3%. My system has more kernel threads than normal processes after booting KDE. Signed-off-by: Alexey Dobriyan Signed-off-by: Andrew Morton Link: http://lkml.kernel.org/r/20200307154435.GA2788@avx2 Signed-off-by: Linus Torvalds --- fs/proc/array.c | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/fs/proc/array.c b/fs/proc/array.c index 5efaf3708ec6..8e16f14bb05a 100644 --- a/fs/proc/array.c +++ b/fs/proc/array.c @@ -635,28 +635,35 @@ int proc_tgid_stat(struct seq_file *m, struct pid_namespace *ns, int proc_pid_statm(struct seq_file *m, struct pid_namespace *ns, struct pid *pid, struct task_struct *task) { - unsigned long size = 0, resident = 0, shared = 0, text = 0, data = 0; struct mm_struct *mm = get_task_mm(task); if (mm) { + unsigned long size; + unsigned long resident = 0; + unsigned long shared = 0; + unsigned long text = 0; + unsigned long data = 0; + size = task_statm(mm, &shared, &text, &data, &resident); mmput(mm); - } - /* - * For quick read, open code by putting numbers directly - * expected format is - * seq_printf(m, "%lu %lu %lu %lu 0 %lu 0\n", - * size, resident, shared, text, data); - */ - seq_put_decimal_ull(m, "", size); - seq_put_decimal_ull(m, " ", resident); - seq_put_decimal_ull(m, " ", shared); - seq_put_decimal_ull(m, " ", text); - seq_put_decimal_ull(m, " ", 0); - seq_put_decimal_ull(m, " ", data); - seq_put_decimal_ull(m, " ", 0); - seq_putc(m, '\n'); + /* + * For quick read, open code by putting numbers directly + * expected format is + * seq_printf(m, "%lu %lu %lu %lu 0 %lu 0\n", + * size, resident, shared, text, data); + */ + seq_put_decimal_ull(m, "", size); + seq_put_decimal_ull(m, " ", resident); + seq_put_decimal_ull(m, " ", shared); + seq_put_decimal_ull(m, " ", text); + seq_put_decimal_ull(m, " ", 0); + seq_put_decimal_ull(m, " ", data); + seq_put_decimal_ull(m, " ", 0); + seq_putc(m, '\n'); + } else { + seq_write(m, "0 0 0 0 0 0 0\n", 14); + } return 0; } From d07ded611e46d276d41dafcbd6abc9dd7ab51a7f Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Mon, 6 Apr 2020 20:09:08 -0700 Subject: [PATCH 568/635] proc: inline vma_stop into m_stop Instead of calling vma_stop() from m_start() and m_next(), do its work in m_stop(). Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Alexey Dobriyan Signed-off-by: Andrew Morton Link: http://lkml.kernel.org/r/20200317193201.9924-1-adobriyan@gmail.com Signed-off-by: Linus Torvalds --- fs/proc/task_mmu.c | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 3ba9ae83bff5..4b19a46d351f 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -123,15 +123,6 @@ static void release_task_mempolicy(struct proc_maps_private *priv) } #endif -static void vma_stop(struct proc_maps_private *priv) -{ - struct mm_struct *mm = priv->mm; - - release_task_mempolicy(priv); - up_read(&mm->mmap_sem); - mmput(mm); -} - static struct vm_area_struct * m_next_vma(struct proc_maps_private *priv, struct vm_area_struct *vma) { @@ -163,11 +154,16 @@ static void *m_start(struct seq_file *m, loff_t *ppos) return ERR_PTR(-ESRCH); mm = priv->mm; - if (!mm || !mmget_not_zero(mm)) + if (!mm || !mmget_not_zero(mm)) { + put_task_struct(priv->task); + priv->task = NULL; return NULL; + } if (down_read_killable(&mm->mmap_sem)) { mmput(mm); + put_task_struct(priv->task); + priv->task = NULL; return ERR_PTR(-EINTR); } @@ -195,7 +191,6 @@ static void *m_start(struct seq_file *m, loff_t *ppos) if (pos == mm->map_count && priv->tail_vma) return priv->tail_vma; - vma_stop(priv); return NULL; } @@ -206,21 +201,22 @@ static void *m_next(struct seq_file *m, void *v, loff_t *pos) (*pos)++; next = m_next_vma(priv, v); - if (!next) - vma_stop(priv); return next; } static void m_stop(struct seq_file *m, void *v) { struct proc_maps_private *priv = m->private; + struct mm_struct *mm = priv->mm; - if (!IS_ERR_OR_NULL(v)) - vma_stop(priv); - if (priv->task) { - put_task_struct(priv->task); - priv->task = NULL; - } + if (!priv->task) + return; + + release_task_mempolicy(priv); + up_read(&mm->mmap_sem); + mmput(mm); + put_task_struct(priv->task); + priv->task = NULL; } static int proc_maps_open(struct inode *inode, struct file *file, From c2e88d22e8ea698c183c6c36485c134742b99838 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Mon, 6 Apr 2020 20:09:11 -0700 Subject: [PATCH 569/635] proc: remove m_cache_vma Instead of setting m->version in the show method, set it in m_next(), where it should be. Also remove the fallback code for failing to find a vma, or version being zero. Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Alexey Dobriyan Signed-off-by: Andrew Morton Link: http://lkml.kernel.org/r/20200317193201.9924-2-adobriyan@gmail.com Signed-off-by: Linus Torvalds --- fs/proc/task_mmu.c | 38 ++++++-------------------------------- 1 file changed, 6 insertions(+), 32 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 4b19a46d351f..981085b32708 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -131,21 +131,14 @@ m_next_vma(struct proc_maps_private *priv, struct vm_area_struct *vma) return vma->vm_next ?: priv->tail_vma; } -static void m_cache_vma(struct seq_file *m, struct vm_area_struct *vma) -{ - if (m->count < m->size) /* vma is copied successfully */ - m->version = m_next_vma(m->private, vma) ? vma->vm_end : -1UL; -} - static void *m_start(struct seq_file *m, loff_t *ppos) { struct proc_maps_private *priv = m->private; unsigned long last_addr = m->version; struct mm_struct *mm; struct vm_area_struct *vma; - unsigned int pos = *ppos; - /* See m_cache_vma(). Zero at the start or after lseek. */ + /* See m_next(). Zero at the start or after lseek. */ if (last_addr == -1UL) return NULL; @@ -170,28 +163,11 @@ static void *m_start(struct seq_file *m, loff_t *ppos) hold_task_mempolicy(priv); priv->tail_vma = get_gate_vma(mm); - if (last_addr) { - vma = find_vma(mm, last_addr - 1); - if (vma && vma->vm_start <= last_addr) - vma = m_next_vma(priv, vma); - if (vma) - return vma; - } - - m->version = 0; - if (pos < mm->map_count) { - for (vma = mm->mmap; pos; pos--) { - m->version = vma->vm_start; - vma = vma->vm_next; - } + vma = find_vma(mm, last_addr); + if (vma) return vma; - } - /* we do not bother to update m->version in this case */ - if (pos == mm->map_count && priv->tail_vma) - return priv->tail_vma; - - return NULL; + return priv->tail_vma; } static void *m_next(struct seq_file *m, void *v, loff_t *pos) @@ -201,6 +177,8 @@ static void *m_next(struct seq_file *m, void *v, loff_t *pos) (*pos)++; next = m_next_vma(priv, v); + m->version = next ? next->vm_start : -1UL; + return next; } @@ -359,7 +337,6 @@ done: static int show_map(struct seq_file *m, void *v) { show_map_vma(m, v); - m_cache_vma(m, v); return 0; } @@ -843,8 +820,6 @@ static int show_smap(struct seq_file *m, void *v) seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma)); show_smap_vma_flags(m, vma); - m_cache_vma(m, vma); - return 0; } @@ -1883,7 +1858,6 @@ static int show_numa_map(struct seq_file *m, void *v) seq_printf(m, " kernelpagesize_kB=%lu", vma_kernel_pagesize(vma) >> 10); out: seq_putc(m, '\n'); - m_cache_vma(m, vma); return 0; } From 4781f2c3abddc400cfdb6ed18c2bf655d402efb5 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Mon, 6 Apr 2020 20:09:14 -0700 Subject: [PATCH 570/635] proc: use ppos instead of m->version The ppos is a private cursor, just like m->version. Use the canonical cursor, not a special one. Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Alexey Dobriyan Signed-off-by: Andrew Morton Link: http://lkml.kernel.org/r/20200317193201.9924-3-adobriyan@gmail.com Signed-off-by: Linus Torvalds --- fs/proc/task_mmu.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 981085b32708..9419c6972fcd 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -134,7 +134,7 @@ m_next_vma(struct proc_maps_private *priv, struct vm_area_struct *vma) static void *m_start(struct seq_file *m, loff_t *ppos) { struct proc_maps_private *priv = m->private; - unsigned long last_addr = m->version; + unsigned long last_addr = *ppos; struct mm_struct *mm; struct vm_area_struct *vma; @@ -170,14 +170,13 @@ static void *m_start(struct seq_file *m, loff_t *ppos) return priv->tail_vma; } -static void *m_next(struct seq_file *m, void *v, loff_t *pos) +static void *m_next(struct seq_file *m, void *v, loff_t *ppos) { struct proc_maps_private *priv = m->private; struct vm_area_struct *next; - (*pos)++; next = m_next_vma(priv, v); - m->version = next ? next->vm_start : -1UL; + *ppos = next ? next->vm_start : -1UL; return next; } From b829a0f0f2f2094c1e40637259c44b854e6ebe96 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Mon, 6 Apr 2020 20:09:17 -0700 Subject: [PATCH 571/635] seq_file: remove m->version The process maps file was the only user of version (introduced back in 2005). Now that it uses ppos instead, we can remove it. Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Alexey Dobriyan Signed-off-by: Andrew Morton Link: http://lkml.kernel.org/r/20200317193201.9924-4-adobriyan@gmail.com Signed-off-by: Linus Torvalds --- fs/seq_file.c | 28 ---------------------------- include/linux/seq_file.h | 1 - 2 files changed, 29 deletions(-) diff --git a/fs/seq_file.c b/fs/seq_file.c index 1600034a929b..79781ebd2145 100644 --- a/fs/seq_file.c +++ b/fs/seq_file.c @@ -67,13 +67,6 @@ int seq_open(struct file *file, const struct seq_operations *op) // to the lifetime of the file. p->file = file; - /* - * Wrappers around seq_open(e.g. swaps_open) need to be - * aware of this. If they set f_version themselves, they - * should call seq_open first and then set f_version. - */ - file->f_version = 0; - /* * seq_files support lseek() and pread(). They do not implement * write() at all, but we clear FMODE_PWRITE here for historical @@ -94,7 +87,6 @@ static int traverse(struct seq_file *m, loff_t offset) int error = 0; void *p; - m->version = 0; m->index = 0; m->count = m->from = 0; if (!offset) @@ -160,26 +152,12 @@ ssize_t seq_read(struct file *file, char __user *buf, size_t size, loff_t *ppos) mutex_lock(&m->lock); - /* - * seq_file->op->..m_start/m_stop/m_next may do special actions - * or optimisations based on the file->f_version, so we want to - * pass the file->f_version to those methods. - * - * seq_file->version is just copy of f_version, and seq_file - * methods can treat it simply as file version. - * It is copied in first and copied out after all operations. - * It is convenient to have it as part of structure to avoid the - * need of passing another argument to all the seq_file methods. - */ - m->version = file->f_version; - /* * if request is to read from zero offset, reset iterator to first * record as it might have been already advanced by previous requests */ if (*ppos == 0) { m->index = 0; - m->version = 0; m->count = 0; } @@ -190,7 +168,6 @@ ssize_t seq_read(struct file *file, char __user *buf, size_t size, loff_t *ppos) if (err) { /* With prejudice... */ m->read_pos = 0; - m->version = 0; m->index = 0; m->count = 0; goto Done; @@ -243,7 +220,6 @@ ssize_t seq_read(struct file *file, char __user *buf, size_t size, loff_t *ppos) m->buf = seq_buf_alloc(m->size <<= 1); if (!m->buf) goto Enomem; - m->version = 0; p = m->op->start(m, &m->index); } m->op->stop(m, p); @@ -287,7 +263,6 @@ Done: *ppos += copied; m->read_pos += copied; } - file->f_version = m->version; mutex_unlock(&m->lock); return copied; Enomem: @@ -313,7 +288,6 @@ loff_t seq_lseek(struct file *file, loff_t offset, int whence) loff_t retval = -EINVAL; mutex_lock(&m->lock); - m->version = file->f_version; switch (whence) { case SEEK_CUR: offset += file->f_pos; @@ -329,7 +303,6 @@ loff_t seq_lseek(struct file *file, loff_t offset, int whence) /* with extreme prejudice... */ file->f_pos = 0; m->read_pos = 0; - m->version = 0; m->index = 0; m->count = 0; } else { @@ -340,7 +313,6 @@ loff_t seq_lseek(struct file *file, loff_t offset, int whence) file->f_pos = offset; } } - file->f_version = m->version; mutex_unlock(&m->lock); return retval; } diff --git a/include/linux/seq_file.h b/include/linux/seq_file.h index 770c2bf3aa43..1672cf6f7614 100644 --- a/include/linux/seq_file.h +++ b/include/linux/seq_file.h @@ -21,7 +21,6 @@ struct seq_file { size_t pad_until; loff_t index; loff_t read_pos; - u64 version; struct mutex lock; const struct seq_operations *op; int poll_event; From fad955009c2ba49e2eac98926ad5cce48b777e7e Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Mon, 6 Apr 2020 20:09:20 -0700 Subject: [PATCH 572/635] proc: inline m_next_vma into m_next It's clearer to just put this inline. Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Alexey Dobriyan Signed-off-by: Andrew Morton Link: http://lkml.kernel.org/r/20200317193201.9924-5-adobriyan@gmail.com Signed-off-by: Linus Torvalds --- fs/proc/task_mmu.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 9419c6972fcd..8d382d4ec067 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -123,14 +123,6 @@ static void release_task_mempolicy(struct proc_maps_private *priv) } #endif -static struct vm_area_struct * -m_next_vma(struct proc_maps_private *priv, struct vm_area_struct *vma) -{ - if (vma == priv->tail_vma) - return NULL; - return vma->vm_next ?: priv->tail_vma; -} - static void *m_start(struct seq_file *m, loff_t *ppos) { struct proc_maps_private *priv = m->private; @@ -173,9 +165,15 @@ static void *m_start(struct seq_file *m, loff_t *ppos) static void *m_next(struct seq_file *m, void *v, loff_t *ppos) { struct proc_maps_private *priv = m->private; - struct vm_area_struct *next; + struct vm_area_struct *next, *vma = v; + + if (vma == priv->tail_vma) + next = NULL; + else if (vma->vm_next) + next = vma->vm_next; + else + next = priv->tail_vma; - next = m_next_vma(priv, v); *ppos = next ? next->vm_start : -1UL; return next; From 06e85c7e9a1c1356038936566fc23f7c0d363b96 Mon Sep 17 00:00:00 2001 From: Michal Simek Date: Mon, 6 Apr 2020 20:09:23 -0700 Subject: [PATCH 573/635] asm-generic: fix unistd_32.h generation format Generated files are also checked by sparse that's why add newline to remove sparse (C=1) warning. The issue was found on Microblaze and reported like this: ./arch/microblaze/include/generated/uapi/asm/unistd_32.h:438:45: warning: no newline at end of file Mips and PowerPC have it already but let's align with style used by m68k. Signed-off-by: Michal Simek Signed-off-by: Andrew Morton Reviewed-by: Stefan Asserhall Acked-by: Max Filippov (xtensa) Cc: Arnd Bergmann Cc: Max Filippov Cc: Benjamin Herrenschmidt Cc: Chris Zankel Cc: David S. Miller Cc: Fenghua Yu Cc: Helge Deller Cc: Ivan Kokshaysky Cc: James Bottomley Cc: Matt Turner Cc: Michael Ellerman Cc: Paul Burton Cc: Paul Mackerras Cc: Ralf Baechle Cc: Rich Felker Cc: Richard Henderson Cc: Tony Luck Cc: Yoshinori Sato Link: http://lkml.kernel.org/r/4d32ab4e1fb2edb691d2e1687e8fb303c09fd023.1581504803.git.michal.simek@xilinx.com Signed-off-by: Linus Torvalds --- arch/alpha/kernel/syscalls/syscallhdr.sh | 2 +- arch/ia64/kernel/syscalls/syscallhdr.sh | 2 +- arch/microblaze/kernel/syscalls/syscallhdr.sh | 2 +- arch/mips/kernel/syscalls/syscallhdr.sh | 3 +-- arch/parisc/kernel/syscalls/syscallhdr.sh | 2 +- arch/powerpc/kernel/syscalls/syscallhdr.sh | 3 +-- arch/sh/kernel/syscalls/syscallhdr.sh | 2 +- arch/sparc/kernel/syscalls/syscallhdr.sh | 2 +- arch/xtensa/kernel/syscalls/syscallhdr.sh | 2 +- 9 files changed, 9 insertions(+), 11 deletions(-) diff --git a/arch/alpha/kernel/syscalls/syscallhdr.sh b/arch/alpha/kernel/syscalls/syscallhdr.sh index e5b99bd2e5e7..1780e861492a 100644 --- a/arch/alpha/kernel/syscalls/syscallhdr.sh +++ b/arch/alpha/kernel/syscalls/syscallhdr.sh @@ -32,5 +32,5 @@ grep -E "^[0-9A-Fa-fXx]+[[:space:]]+${my_abis}" "$in" | sort -n | ( printf "#define __NR_syscalls\t%s\n" "${nxt}" printf "#endif\n" printf "\n" - printf "#endif /* %s */" "${fileguard}" + printf "#endif /* %s */\n" "${fileguard}" ) > "$out" diff --git a/arch/ia64/kernel/syscalls/syscallhdr.sh b/arch/ia64/kernel/syscalls/syscallhdr.sh index 0c2d2c748565..f407b6e53283 100644 --- a/arch/ia64/kernel/syscalls/syscallhdr.sh +++ b/arch/ia64/kernel/syscalls/syscallhdr.sh @@ -32,5 +32,5 @@ grep -E "^[0-9A-Fa-fXx]+[[:space:]]+${my_abis}" "$in" | sort -n | ( printf "#define __NR_syscalls\t%s\n" "${nxt}" printf "#endif\n" printf "\n" - printf "#endif /* %s */" "${fileguard}" + printf "#endif /* %s */\n" "${fileguard}" ) > "$out" diff --git a/arch/microblaze/kernel/syscalls/syscallhdr.sh b/arch/microblaze/kernel/syscalls/syscallhdr.sh index 2e9062a926a3..a914854f8d9f 100644 --- a/arch/microblaze/kernel/syscalls/syscallhdr.sh +++ b/arch/microblaze/kernel/syscalls/syscallhdr.sh @@ -32,5 +32,5 @@ grep -E "^[0-9A-Fa-fXx]+[[:space:]]+${my_abis}" "$in" | sort -n | ( printf "#define __NR_syscalls\t%s\n" "${nxt}" printf "#endif\n" printf "\n" - printf "#endif /* %s */" "${fileguard}" + printf "#endif /* %s */\n" "${fileguard}" ) > "$out" diff --git a/arch/mips/kernel/syscalls/syscallhdr.sh b/arch/mips/kernel/syscalls/syscallhdr.sh index d2bcfa8f4d1a..2e241e713a7d 100644 --- a/arch/mips/kernel/syscalls/syscallhdr.sh +++ b/arch/mips/kernel/syscalls/syscallhdr.sh @@ -32,6 +32,5 @@ grep -E "^[0-9A-Fa-fXx]+[[:space:]]+${my_abis}" "$in" | sort -n | ( printf "#define __NR_syscalls\t%s\n" "${nxt}" printf "#endif\n" printf "\n" - printf "#endif /* %s */" "${fileguard}" - printf "\n" + printf "#endif /* %s */\n" "${fileguard}" ) > "$out" diff --git a/arch/parisc/kernel/syscalls/syscallhdr.sh b/arch/parisc/kernel/syscalls/syscallhdr.sh index 50242b747d7c..730db288fe54 100644 --- a/arch/parisc/kernel/syscalls/syscallhdr.sh +++ b/arch/parisc/kernel/syscalls/syscallhdr.sh @@ -32,5 +32,5 @@ grep -E "^[0-9A-Fa-fXx]+[[:space:]]+${my_abis}" "$in" | sort -n | ( printf "#define __NR_syscalls\t%s\n" "${nxt}" printf "#endif\n" printf "\n" - printf "#endif /* %s */" "${fileguard}" + printf "#endif /* %s */\n" "${fileguard}" ) > "$out" diff --git a/arch/powerpc/kernel/syscalls/syscallhdr.sh b/arch/powerpc/kernel/syscalls/syscallhdr.sh index c0a9a32937f1..02d6751f3be3 100644 --- a/arch/powerpc/kernel/syscalls/syscallhdr.sh +++ b/arch/powerpc/kernel/syscalls/syscallhdr.sh @@ -32,6 +32,5 @@ grep -E "^[0-9A-Fa-fXx]+[[:space:]]+${my_abis}" "$in" | sort -n | ( printf "#define __NR_syscalls\t%s\n" "${nxt}" printf "#endif\n" printf "\n" - printf "#endif /* %s */" "${fileguard}" - printf "\n" + printf "#endif /* %s */\n" "${fileguard}" ) > "$out" diff --git a/arch/sh/kernel/syscalls/syscallhdr.sh b/arch/sh/kernel/syscalls/syscallhdr.sh index 1de0334e577f..4c0519861e97 100644 --- a/arch/sh/kernel/syscalls/syscallhdr.sh +++ b/arch/sh/kernel/syscalls/syscallhdr.sh @@ -32,5 +32,5 @@ grep -E "^[0-9A-Fa-fXx]+[[:space:]]+${my_abis}" "$in" | sort -n | ( printf "#define __NR_syscalls\t%s\n" "${nxt}" printf "#endif\n" printf "\n" - printf "#endif /* %s */" "${fileguard}" + printf "#endif /* %s */\n" "${fileguard}" ) > "$out" diff --git a/arch/sparc/kernel/syscalls/syscallhdr.sh b/arch/sparc/kernel/syscalls/syscallhdr.sh index 626b5740a9f1..cf50a75cc0bb 100644 --- a/arch/sparc/kernel/syscalls/syscallhdr.sh +++ b/arch/sparc/kernel/syscalls/syscallhdr.sh @@ -32,5 +32,5 @@ grep -E "^[0-9A-Fa-fXx]+[[:space:]]+${my_abis}" "$in" | sort -n | ( printf "#define __NR_syscalls\t%s\n" "${nxt}" printf "#endif\n" printf "\n" - printf "#endif /* %s */" "${fileguard}" + printf "#endif /* %s */\n" "${fileguard}" ) > "$out" diff --git a/arch/xtensa/kernel/syscalls/syscallhdr.sh b/arch/xtensa/kernel/syscalls/syscallhdr.sh index d37db641ca31..eebfb8a8ace6 100644 --- a/arch/xtensa/kernel/syscalls/syscallhdr.sh +++ b/arch/xtensa/kernel/syscalls/syscallhdr.sh @@ -32,5 +32,5 @@ grep -E "^[0-9A-Fa-fXx]+[[:space:]]+${my_abis}" "$in" | sort -n | ( printf "#define __NR_syscalls\t%s\n" "${nxt}" printf "#endif\n" printf "\n" - printf "#endif /* %s */" "${fileguard}" + printf "#endif /* %s */\n" "${fileguard}" ) > "$out" From 63174f61dfaef58dc0e813eaf6602636794f8942 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Mon, 6 Apr 2020 20:09:27 -0700 Subject: [PATCH 574/635] kernel/extable.c: use address-of operator on section symbols Clang warns: ../kernel/extable.c:37:52: warning: array comparison always evaluates to a constant [-Wtautological-compare] if (main_extable_sort_needed && __stop___ex_table > __start___ex_table) { ^ 1 warning generated. These are not true arrays, they are linker defined symbols, which are just addresses. Using the address of operator silences the warning and does not change the resulting assembly with either clang/ld.lld or gcc/ld (tested with diff + objdump -Dr). Suggested-by: Nick Desaulniers Signed-off-by: Nathan Chancellor Signed-off-by: Andrew Morton Reviewed-by: Andrew Morton Link: https://github.com/ClangBuiltLinux/linux/issues/892 Link: http://lkml.kernel.org/r/20200219202036.45702-1-natechancellor@gmail.com Signed-off-by: Linus Torvalds --- kernel/extable.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/extable.c b/kernel/extable.c index 7681f87e89dd..b0ea5eb0c3b4 100644 --- a/kernel/extable.c +++ b/kernel/extable.c @@ -34,7 +34,8 @@ u32 __initdata __visible main_extable_sort_needed = 1; /* Sort the kernel's built-in exception table */ void __init sort_main_extable(void) { - if (main_extable_sort_needed && __stop___ex_table > __start___ex_table) { + if (main_extable_sort_needed && + &__stop___ex_table > &__start___ex_table) { pr_notice("Sorting __ex_table...\n"); sort_extable(__start___ex_table, __stop___ex_table); } From 12a5b00a5366467e95a25f55da751794e5fe6788 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Mon, 6 Apr 2020 20:09:30 -0700 Subject: [PATCH 575/635] sparc,x86: vdso: remove meaningless undefining CONFIG_OPTIMIZE_INLINING The code, #undef CONFIG_OPTIMIZE_INLINING, is not working as expected because is parsed before vclock_gettime.c since 28128c61e08e ("kconfig.h: Include compiler types to avoid missed struct attributes"). Since then, is included really early by using the '-include' option. So, you cannot negate the decision of in this way. You can confirm it by checking the pre-processed code, like this: $ make arch/x86/entry/vdso/vdso32/vclock_gettime.i There is no difference with/without CONFIG_CC_OPTIMIZE_FOR_SIZE. It is about two years since 28128c61e08e. Nobody has reported a problem (or, nobody has even noticed the fact that this code is not working). It is ugly and unreliable to attempt to undefine a CONFIG option from C files, and anyway the inlining heuristic is up to the compiler. Just remove the broken code. Signed-off-by: Masahiro Yamada Signed-off-by: Andrew Morton Reviewed-by: Nathan Chancellor Acked-by: Miguel Ojeda Cc: Arnd Bergmann Cc: Ingo Molnar Cc: Thomas Gleixner Cc: Masahiro Yamada Cc: Borislav Petkov Cc: "H. Peter Anvin" Cc: David Miller Link: http://lkml.kernel.org/r/20200220110807.32534-1-masahiroy@kernel.org Signed-off-by: Linus Torvalds --- arch/sparc/vdso/vdso32/vclock_gettime.c | 4 ---- arch/x86/entry/vdso/vdso32/vclock_gettime.c | 4 ---- 2 files changed, 8 deletions(-) diff --git a/arch/sparc/vdso/vdso32/vclock_gettime.c b/arch/sparc/vdso/vdso32/vclock_gettime.c index 026abb3b826c..d7f99e6745ea 100644 --- a/arch/sparc/vdso/vdso32/vclock_gettime.c +++ b/arch/sparc/vdso/vdso32/vclock_gettime.c @@ -4,10 +4,6 @@ #define BUILD_VDSO32 -#ifndef CONFIG_CC_OPTIMIZE_FOR_SIZE -#undef CONFIG_OPTIMIZE_INLINING -#endif - #ifdef CONFIG_SPARC64 /* diff --git a/arch/x86/entry/vdso/vdso32/vclock_gettime.c b/arch/x86/entry/vdso/vdso32/vclock_gettime.c index 1e82bd43286c..84a4a73f77f7 100644 --- a/arch/x86/entry/vdso/vdso32/vclock_gettime.c +++ b/arch/x86/entry/vdso/vdso32/vclock_gettime.c @@ -1,10 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #define BUILD_VDSO32 -#ifndef CONFIG_CC_OPTIMIZE_FOR_SIZE -#undef CONFIG_OPTIMIZE_INLINING -#endif - #ifdef CONFIG_X86_64 /* From 889b3c1245de48ed0cacf7aebb25c489d3e4a3e9 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Mon, 6 Apr 2020 20:09:33 -0700 Subject: [PATCH 576/635] compiler: remove CONFIG_OPTIMIZE_INLINING entirely Commit ac7c3e4ff401 ("compiler: enable CONFIG_OPTIMIZE_INLINING forcibly") made this always-on option. We released v5.4 and v5.5 including that commit. Remove the CONFIG option and clean up the code now. Signed-off-by: Masahiro Yamada Signed-off-by: Andrew Morton Reviewed-by: Miguel Ojeda Reviewed-by: Nathan Chancellor Cc: Arnd Bergmann Cc: Borislav Petkov Cc: David Miller Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: Thomas Gleixner Link: http://lkml.kernel.org/r/20200220110807.32534-2-masahiroy@kernel.org Signed-off-by: Linus Torvalds --- arch/x86/configs/i386_defconfig | 1 - arch/x86/configs/x86_64_defconfig | 1 - include/linux/compiler_types.h | 11 +---------- kernel/configs/tiny.config | 1 - lib/Kconfig.debug | 12 ------------ 5 files changed, 1 insertion(+), 25 deletions(-) diff --git a/arch/x86/configs/i386_defconfig b/arch/x86/configs/i386_defconfig index ab8b30cb978e..550904591e94 100644 --- a/arch/x86/configs/i386_defconfig +++ b/arch/x86/configs/i386_defconfig @@ -285,7 +285,6 @@ CONFIG_EARLY_PRINTK_DBGP=y CONFIG_DEBUG_STACKOVERFLOW=y # CONFIG_DEBUG_RODATA_TEST is not set CONFIG_DEBUG_BOOT_PARAMS=y -CONFIG_OPTIMIZE_INLINING=y CONFIG_SECURITY=y CONFIG_SECURITY_NETWORK=y CONFIG_SECURITY_SELINUX=y diff --git a/arch/x86/configs/x86_64_defconfig b/arch/x86/configs/x86_64_defconfig index 2d196cb49084..614961009075 100644 --- a/arch/x86/configs/x86_64_defconfig +++ b/arch/x86/configs/x86_64_defconfig @@ -282,7 +282,6 @@ CONFIG_EARLY_PRINTK_DBGP=y CONFIG_DEBUG_STACKOVERFLOW=y # CONFIG_DEBUG_RODATA_TEST is not set CONFIG_DEBUG_BOOT_PARAMS=y -CONFIG_OPTIMIZE_INLINING=y CONFIG_UNWINDER_ORC=y CONFIG_SECURITY=y CONFIG_SECURITY_NETWORK=y diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h index 72393a8c1a6c..e970f97a7fcb 100644 --- a/include/linux/compiler_types.h +++ b/include/linux/compiler_types.h @@ -129,22 +129,13 @@ struct ftrace_likely_data { #define __compiler_offsetof(a, b) __builtin_offsetof(a, b) /* - * Force always-inline if the user requests it so via the .config. * Prefer gnu_inline, so that extern inline functions do not emit an * externally visible function. This makes extern inline behave as per gnu89 * semantics rather than c99. This prevents multiple symbol definition errors * of extern inline functions at link time. * A lot of inline functions can cause havoc with function tracing. - * Do not use __always_inline here, since currently it expands to inline again - * (which would break users of __always_inline). */ -#if !defined(CONFIG_OPTIMIZE_INLINING) -#define inline inline __attribute__((__always_inline__)) __gnu_inline \ - __inline_maybe_unused notrace -#else -#define inline inline __gnu_inline \ - __inline_maybe_unused notrace -#endif +#define inline inline __gnu_inline __inline_maybe_unused notrace /* * gcc provides both __inline__ and __inline as alternate spellings of diff --git a/kernel/configs/tiny.config b/kernel/configs/tiny.config index 7fa0c4ae6394..8a44b93da0f3 100644 --- a/kernel/configs/tiny.config +++ b/kernel/configs/tiny.config @@ -6,7 +6,6 @@ CONFIG_CC_OPTIMIZE_FOR_SIZE=y CONFIG_KERNEL_XZ=y # CONFIG_KERNEL_LZO is not set # CONFIG_KERNEL_LZ4 is not set -CONFIG_OPTIMIZE_INLINING=y # CONFIG_SLAB is not set # CONFIG_SLUB is not set CONFIG_SLOB=y diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index d1398cef3b18..7f9a89847b65 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -305,18 +305,6 @@ config HEADERS_INSTALL user-space program samples. It is also needed by some features such as uapi header sanity checks. -config OPTIMIZE_INLINING - def_bool y - help - This option determines if the kernel forces gcc to inline the functions - developers have marked 'inline'. Doing so takes away freedom from gcc to - do what it thinks is best, which is desirable for the gcc 3.x series of - compilers. The gcc 4.x series have a rewritten inlining algorithm and - enabling this option will generate a smaller kernel there. Hopefully - this algorithm is so good that allowing gcc 4.x and above to make the - decision will become the default in the future. Until then this option - is there to test gcc for this. - config DEBUG_SECTION_MISMATCH bool "Enable full Section mismatch analysis" help From af9c5d2e3b355854ff0e4acfbfbfadcd5198a349 Mon Sep 17 00:00:00 2001 From: Vegard Nossum Date: Mon, 6 Apr 2020 20:09:37 -0700 Subject: [PATCH 577/635] compiler.h: fix error in BUILD_BUG_ON() reporting compiletime_assert() uses __LINE__ to create a unique function name. This means that if you have more than one BUILD_BUG_ON() in the same source line (which can happen if they appear e.g. in a macro), then the error message from the compiler might output the wrong condition. For this source file: #include #define macro() \ BUILD_BUG_ON(1); \ BUILD_BUG_ON(0); void foo() { macro(); } gcc would output: ./include/linux/compiler.h:350:38: error: call to `__compiletime_assert_9' declared with attribute error: BUILD_BUG_ON failed: 0 _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__) However, it was not the BUILD_BUG_ON(0) that failed, so it should say 1 instead of 0. With this patch, we use __COUNTER__ instead of __LINE__, so each BUILD_BUG_ON() gets a different function name and the correct condition is printed: ./include/linux/compiler.h:350:38: error: call to `__compiletime_assert_0' declared with attribute error: BUILD_BUG_ON failed: 1 _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__) Signed-off-by: Vegard Nossum Signed-off-by: Andrew Morton Reviewed-by: Masahiro Yamada Reviewed-by: Daniel Santos Cc: Rasmus Villemoes Cc: Ian Abbott Cc: Joe Perches Link: http://lkml.kernel.org/r/20200331112637.25047-1-vegard.nossum@oracle.com Signed-off-by: Linus Torvalds --- include/linux/compiler.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/compiler.h b/include/linux/compiler.h index 5e88e7e33abe..034b0a644efc 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h @@ -347,7 +347,7 @@ static inline void *offset_to_ptr(const int *off) * compiler has support to do so. */ #define compiletime_assert(condition, msg) \ - _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__) + _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__) #define compiletime_assert_atomic_type(t) \ compiletime_assert(__native_word(t), \ From 6680125ea5a2a5a69eeb8f7dcd89d0a1d23998ce Mon Sep 17 00:00:00 2001 From: Joe Perches Date: Mon, 6 Apr 2020 20:09:40 -0700 Subject: [PATCH 578/635] MAINTAINERS: list the section entries in the preferred order The MAINTAINERS file header has never shown a preferred order for the section entries but scripts/parse-maintainers.pl added a preferred order with commit 61f741645a35 ("parse-maintainers: Add section pattern sorting") Commit 5cdbec108fd2 ("parse-maintainers: Do not sort section content by default") changed the preferred order to be a bit more sensible. Update the MAINTAINERS section description block to use this preferred section entry ordering. Add a slightly better description for the N: entry too. Signed-off-by: Joe Perches Signed-off-by: Andrew Morton Reviewed-by: Andy Shevchenko Link: http://lkml.kernel.org/r/5aa5aad6fb1678230c260337dc066cd449a2bf32.camel@perches.com Signed-off-by: Linus Torvalds --- MAINTAINERS | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index 534a8dc4f84a..9271068bde63 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -77,21 +77,13 @@ Tips for patch submitters 8. Happy hacking. -Descriptions of section entries -------------------------------- +Descriptions of section entries and preferred order +--------------------------------------------------- M: *Mail* patches to: FullName R: Designated *Reviewer*: FullName These reviewers should be CCed on patches. L: *Mailing list* that is relevant to this area - W: *Web-page* with status/info - B: URI for where to file *bugs*. A web-page with detailed bug - filing info, a direct bug tracker link, or a mailto: URI. - C: URI for *chat* protocol, server and channel where developers - usually hang out, for example irc://server/channel. - Q: *Patchwork* web based patch tracking system site - T: *SCM* tree type and location. - Type is one of: git, hg, quilt, stgit, topgit S: *Status*, one of the following: Supported: Someone is actually paid to look after this. Maintained: Someone actually looks after it. @@ -102,30 +94,39 @@ Descriptions of section entries Obsolete: Old code. Something tagged obsolete generally means it has been replaced by a better system and you should be using that. + W: *Web-page* with status/info + Q: *Patchwork* web based patch tracking system site + B: URI for where to file *bugs*. A web-page with detailed bug + filing info, a direct bug tracker link, or a mailto: URI. + C: URI for *chat* protocol, server and channel where developers + usually hang out, for example irc://server/channel. P: Subsystem Profile document for more details submitting patches to the given subsystem. This is either an in-tree file, or a URI. See Documentation/maintainer/maintainer-entry-profile.rst for details. + T: *SCM* tree type and location. + Type is one of: git, hg, quilt, stgit, topgit F: *Files* and directories wildcard patterns. A trailing slash includes all files and subdirectory files. F: drivers/net/ all files in and below drivers/net F: drivers/net/* all files in drivers/net, but not below F: */net/* all files in "any top level directory"/net One pattern per line. Multiple F: lines acceptable. - N: Files and directories *Regex* patterns. - N: [^a-z]tegra all files whose path contains the word tegra - One pattern per line. Multiple N: lines acceptable. - scripts/get_maintainer.pl has different behavior for files that - match F: pattern and matches of N: patterns. By default, - get_maintainer will not look at git log history when an F: pattern - match occurs. When an N: match occurs, git log history is used - to also notify the people that have git commit signatures. X: *Excluded* files and directories that are NOT maintained, same rules as F:. Files exclusions are tested before file matches. Can be useful for excluding a specific subdirectory, for instance: F: net/ X: net/ipv6/ matches all files in and below net excluding net/ipv6/ + N: Files and directories *Regex* patterns. + N: [^a-z]tegra all files whose path contains tegra + (not including files like integrator) + One pattern per line. Multiple N: lines acceptable. + scripts/get_maintainer.pl has different behavior for files that + match F: pattern and matches of N: patterns. By default, + get_maintainer will not look at git log history when an F: pattern + match occurs. When an N: match occurs, git log history is used + to also notify the people that have git commit signatures. K: *Content regex* (perl extended) pattern match in a patch or file. For instance: K: of_get_profile From f80ac98a641a03097cbc9fdfd4b6a41a8dd3b7ae Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Mon, 6 Apr 2020 20:09:43 -0700 Subject: [PATCH 579/635] bitops: always inline sign extension helpers With CONFIG_CC_OPTIMIZE_FOR_SIZE, objtool reports: drivers/gpu/drm/i915/gem/i915_gem_execbuffer.o: warning: objtool: i915_gem_execbuffer2_ioctl()+0x5b7: call to gen8_canonical_addr() with UACCESS enabled This means i915_gem_execbuffer2_ioctl() is calling gen8_canonical_addr() from the user_access_begin/end critical region (i.e, with SMAP disabled). While it's probably harmless in this case, in general we like to avoid extra function calls in SMAP-disabled regions because it can open up inadvertent security holes. Fix the warning by changing the sign extension helpers to __always_inline. This convinces GCC to inline gen8_canonical_addr(). The sign extension functions are trivial anyway, so it makes sense to always inline them. With my test optimize-for-size-based config, this actually shrinks the text size of i915_gem_execbuffer.o by 45 bytes -- and no change for vmlinux. Reported-by: Randy Dunlap Signed-off-by: Josh Poimboeuf Signed-off-by: Andrew Morton Cc: Peter Zijlstra Cc: Al Viro Cc: Chris Wilson Link: http://lkml.kernel.org/r/740179324b2b18b750b16295c48357f00b5fa9ed.1582982020.git.jpoimboe@redhat.com Signed-off-by: Linus Torvalds --- include/linux/bitops.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/bitops.h b/include/linux/bitops.h index 47f54b459c26..9acf654f0b19 100644 --- a/include/linux/bitops.h +++ b/include/linux/bitops.h @@ -162,7 +162,7 @@ static inline __u8 ror8(__u8 word, unsigned int shift) * * This is safe to use for 16- and 8-bit types as well. */ -static inline __s32 sign_extend32(__u32 value, int index) +static __always_inline __s32 sign_extend32(__u32 value, int index) { __u8 shift = 31 - index; return (__s32)(value << shift) >> shift; @@ -173,7 +173,7 @@ static inline __s32 sign_extend32(__u32 value, int index) * @value: value to sign extend * @index: 0 based bit index (0<=index<64) to sign bit */ -static inline __s64 sign_extend64(__u64 value, int index) +static __always_inline __s64 sign_extend64(__u64 value, int index) { __u8 shift = 63 - index; return (__s64)(value << shift) >> shift; From 30428ef5d1e8caf78639cc70a802f1cb7b1cec04 Mon Sep 17 00:00:00 2001 From: Konstantin Khlebnikov Date: Mon, 6 Apr 2020 20:09:47 -0700 Subject: [PATCH 580/635] lib/test_lockup: test module to generate lockups CONFIG_TEST_LOCKUP=m adds module "test_lockup" that helps to make sure that watchdogs and lockup detectors are working properly. Depending on module parameters test_lockup could emulate soft or hard lockup, "hung task", hold arbitrary lock, allocate bunch of pages. Also it could generate series of lockups with cooling-down periods, in this way it could be used as "ping" for locks or page allocator. Loop checks signals between iteration thus could be stopped by ^C. # modinfo test_lockup ... parm: time_secs:lockup time in seconds, default 0 (uint) parm: time_nsecs:nanoseconds part of lockup time, default 0 (uint) parm: cooldown_secs:cooldown time between iterations in seconds, default 0 (uint) parm: cooldown_nsecs:nanoseconds part of cooldown, default 0 (uint) parm: iterations:lockup iterations, default 1 (uint) parm: all_cpus:trigger lockup at all cpus at once (bool) parm: state:wait in 'R' running (default), 'D' uninterruptible, 'K' killable, 'S' interruptible state (charp) parm: use_hrtimer:use high-resolution timer for sleeping (bool) parm: iowait:account sleep time as iowait (bool) parm: lock_read:lock read-write locks for read (bool) parm: lock_single:acquire locks only at one cpu (bool) parm: reacquire_locks:release and reacquire locks/irq/preempt between iterations (bool) parm: touch_softlockup:touch soft-lockup watchdog between iterations (bool) parm: touch_hardlockup:touch hard-lockup watchdog between iterations (bool) parm: call_cond_resched:call cond_resched() between iterations (bool) parm: measure_lock_wait:measure lock wait time (bool) parm: lock_wait_threshold:print lock wait time longer than this in nanoseconds, default off (ulong) parm: disable_irq:disable interrupts: generate hard-lockups (bool) parm: disable_softirq:disable bottom-half irq handlers (bool) parm: disable_preempt:disable preemption: generate soft-lockups (bool) parm: lock_rcu:grab rcu_read_lock: generate rcu stalls (bool) parm: lock_mmap_sem:lock mm->mmap_sem: block procfs interfaces (bool) parm: lock_rwsem_ptr:lock rw_semaphore at address (ulong) parm: lock_mutex_ptr:lock mutex at address (ulong) parm: lock_spinlock_ptr:lock spinlock at address (ulong) parm: lock_rwlock_ptr:lock rwlock at address (ulong) parm: alloc_pages_nr:allocate and free pages under locks (uint) parm: alloc_pages_order:page order to allocate (uint) parm: alloc_pages_gfp:allocate pages with this gfp_mask, default GFP_KERNEL (uint) parm: alloc_pages_atomic:allocate pages with GFP_ATOMIC (bool) parm: reallocate_pages:free and allocate pages between iterations (bool) Parameters for locking by address are unsafe and taints kernel. With CONFIG_DEBUG_SPINLOCK=y they at least check magics for embedded spinlocks. Examples: task hang in D-state: modprobe test_lockup time_secs=1 iterations=60 state=D task hang in io-wait D-state: modprobe test_lockup time_secs=1 iterations=60 state=D iowait softlockup: modprobe test_lockup time_secs=1 iterations=60 state=R hardlockup: modprobe test_lockup time_secs=1 iterations=60 state=R disable_irq system-wide hardlockup: modprobe test_lockup time_secs=1 iterations=60 state=R \ disable_irq all_cpus rcu stall: modprobe test_lockup time_secs=1 iterations=60 state=R \ lock_rcu touch_softlockup lock mmap_sem / block procfs interfaces: modprobe test_lockup time_secs=1 iterations=60 state=S lock_mmap_sem lock tasklist_lock for read / block forks: TASKLIST_LOCK=$(awk '$3 == "tasklist_lock" {print "0x"$1}' /proc/kallsyms) modprobe test_lockup time_secs=1 iterations=60 state=R \ disable_irq lock_read lock_rwlock_ptr=$TASKLIST_LOCK lock namespace_sem / block vfs mount operations: NAMESPACE_SEM=$(awk '$3 == "namespace_sem" {print "0x"$1}' /proc/kallsyms) modprobe test_lockup time_secs=1 iterations=60 state=S \ lock_rwsem_ptr=$NAMESPACE_SEM lock cgroup mutex / block cgroup operations: CGROUP_MUTEX=$(awk '$3 == "cgroup_mutex" {print "0x"$1}' /proc/kallsyms) modprobe test_lockup time_secs=1 iterations=60 state=S \ lock_mutex_ptr=$CGROUP_MUTEX ping cgroup_mutex every second and measure maximum lock wait time: modprobe test_lockup cooldown_secs=1 iterations=60 state=S \ lock_mutex_ptr=$CGROUP_MUTEX reacquire_locks measure_lock_wait [linux@roeck-us.net: rename disable_irq to fix build error] Link: http://lkml.kernel.org/r/20200317133614.23152-1-linux@roeck-us.net Signed-off-by: Konstantin Khlebnikov Signed-off-by: Guenter Roeck Signed-off-by: Andrew Morton Cc: Sasha Levin Cc: Petr Mladek Cc: Kees Cook Cc: Peter Zijlstra Cc: Greg Kroah-Hartman Cc: Steven Rostedt Cc: Sergey Senozhatsky Cc: Dmitry Monakhov Cc: Guenter Roeck Link: http://lkml.kernel.org/r/158132859146.2797.525923171323227836.stgit@buzz Signed-off-by: Linus Torvalds --- lib/Kconfig.debug | 12 + lib/Makefile | 1 + lib/test_lockup.c | 554 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 567 insertions(+) create mode 100644 lib/test_lockup.c diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 7f9a89847b65..ddcf000022ae 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -976,6 +976,18 @@ config WQ_WATCHDOG state. This can be configured through kernel parameter "workqueue.watchdog_thresh" and its sysfs counterpart. +config TEST_LOCKUP + tristate "Test module to generate lockups" + help + This builds the "test_lockup" module that helps to make sure + that watchdogs and lockup detectors are working properly. + + Depending on module parameters it could emulate soft or hard + lockup, "hung task", or locking arbitrary lock for a long time. + Also it could generate series of lockups with cooling-down periods. + + If unsure, say N. + endmenu # "Debug lockups and hangs" menu "Scheduler Debugging" diff --git a/lib/Makefile b/lib/Makefile index 09a8acb0cf92..0fd125c4ad07 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -90,6 +90,7 @@ obj-$(CONFIG_TEST_OBJAGG) += test_objagg.o obj-$(CONFIG_TEST_STACKINIT) += test_stackinit.o obj-$(CONFIG_TEST_BLACKHOLE_DEV) += test_blackhole_dev.o obj-$(CONFIG_TEST_MEMINIT) += test_meminit.o +obj-$(CONFIG_TEST_LOCKUP) += test_lockup.o obj-$(CONFIG_TEST_LIVEPATCH) += livepatch/ diff --git a/lib/test_lockup.c b/lib/test_lockup.c new file mode 100644 index 000000000000..9e8b8a0be9af --- /dev/null +++ b/lib/test_lockup.c @@ -0,0 +1,554 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Test module to generate lockups + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static unsigned int time_secs; +module_param(time_secs, uint, 0600); +MODULE_PARM_DESC(time_secs, "lockup time in seconds, default 0"); + +static unsigned int time_nsecs; +module_param(time_nsecs, uint, 0600); +MODULE_PARM_DESC(time_nsecs, "nanoseconds part of lockup time, default 0"); + +static unsigned int cooldown_secs; +module_param(cooldown_secs, uint, 0600); +MODULE_PARM_DESC(cooldown_secs, "cooldown time between iterations in seconds, default 0"); + +static unsigned int cooldown_nsecs; +module_param(cooldown_nsecs, uint, 0600); +MODULE_PARM_DESC(cooldown_nsecs, "nanoseconds part of cooldown, default 0"); + +static unsigned int iterations = 1; +module_param(iterations, uint, 0600); +MODULE_PARM_DESC(iterations, "lockup iterations, default 1"); + +static bool all_cpus; +module_param(all_cpus, bool, 0400); +MODULE_PARM_DESC(all_cpus, "trigger lockup at all cpus at once"); + +static int wait_state; +static char *state = "R"; +module_param(state, charp, 0400); +MODULE_PARM_DESC(state, "wait in 'R' running (default), 'D' uninterruptible, 'K' killable, 'S' interruptible state"); + +static bool use_hrtimer; +module_param(use_hrtimer, bool, 0400); +MODULE_PARM_DESC(use_hrtimer, "use high-resolution timer for sleeping"); + +static bool iowait; +module_param(iowait, bool, 0400); +MODULE_PARM_DESC(iowait, "account sleep time as iowait"); + +static bool lock_read; +module_param(lock_read, bool, 0400); +MODULE_PARM_DESC(lock_read, "lock read-write locks for read"); + +static bool lock_single; +module_param(lock_single, bool, 0400); +MODULE_PARM_DESC(lock_single, "acquire locks only at one cpu"); + +static bool reacquire_locks; +module_param(reacquire_locks, bool, 0400); +MODULE_PARM_DESC(reacquire_locks, "release and reacquire locks/irq/preempt between iterations"); + +static bool touch_softlockup; +module_param(touch_softlockup, bool, 0600); +MODULE_PARM_DESC(touch_softlockup, "touch soft-lockup watchdog between iterations"); + +static bool touch_hardlockup; +module_param(touch_hardlockup, bool, 0600); +MODULE_PARM_DESC(touch_hardlockup, "touch hard-lockup watchdog between iterations"); + +static bool call_cond_resched; +module_param(call_cond_resched, bool, 0600); +MODULE_PARM_DESC(call_cond_resched, "call cond_resched() between iterations"); + +static bool measure_lock_wait; +module_param(measure_lock_wait, bool, 0400); +MODULE_PARM_DESC(measure_lock_wait, "measure lock wait time"); + +static unsigned long lock_wait_threshold = ULONG_MAX; +module_param(lock_wait_threshold, ulong, 0400); +MODULE_PARM_DESC(lock_wait_threshold, "print lock wait time longer than this in nanoseconds, default off"); + +static bool test_disable_irq; +module_param_named(disable_irq, test_disable_irq, bool, 0400); +MODULE_PARM_DESC(disable_irq, "disable interrupts: generate hard-lockups"); + +static bool disable_softirq; +module_param(disable_softirq, bool, 0400); +MODULE_PARM_DESC(disable_softirq, "disable bottom-half irq handlers"); + +static bool disable_preempt; +module_param(disable_preempt, bool, 0400); +MODULE_PARM_DESC(disable_preempt, "disable preemption: generate soft-lockups"); + +static bool lock_rcu; +module_param(lock_rcu, bool, 0400); +MODULE_PARM_DESC(lock_rcu, "grab rcu_read_lock: generate rcu stalls"); + +static bool lock_mmap_sem; +module_param(lock_mmap_sem, bool, 0400); +MODULE_PARM_DESC(lock_mmap_sem, "lock mm->mmap_sem: block procfs interfaces"); + +static unsigned long lock_rwsem_ptr; +module_param_unsafe(lock_rwsem_ptr, ulong, 0400); +MODULE_PARM_DESC(lock_rwsem_ptr, "lock rw_semaphore at address"); + +static unsigned long lock_mutex_ptr; +module_param_unsafe(lock_mutex_ptr, ulong, 0400); +MODULE_PARM_DESC(lock_mutex_ptr, "lock mutex at address"); + +static unsigned long lock_spinlock_ptr; +module_param_unsafe(lock_spinlock_ptr, ulong, 0400); +MODULE_PARM_DESC(lock_spinlock_ptr, "lock spinlock at address"); + +static unsigned long lock_rwlock_ptr; +module_param_unsafe(lock_rwlock_ptr, ulong, 0400); +MODULE_PARM_DESC(lock_rwlock_ptr, "lock rwlock at address"); + +static unsigned int alloc_pages_nr; +module_param_unsafe(alloc_pages_nr, uint, 0600); +MODULE_PARM_DESC(alloc_pages_nr, "allocate and free pages under locks"); + +static unsigned int alloc_pages_order; +module_param(alloc_pages_order, uint, 0400); +MODULE_PARM_DESC(alloc_pages_order, "page order to allocate"); + +static gfp_t alloc_pages_gfp = GFP_KERNEL; +module_param_unsafe(alloc_pages_gfp, uint, 0400); +MODULE_PARM_DESC(alloc_pages_gfp, "allocate pages with this gfp_mask, default GFP_KERNEL"); + +static bool alloc_pages_atomic; +module_param(alloc_pages_atomic, bool, 0400); +MODULE_PARM_DESC(alloc_pages_atomic, "allocate pages with GFP_ATOMIC"); + +static bool reallocate_pages; +module_param(reallocate_pages, bool, 0400); +MODULE_PARM_DESC(reallocate_pages, "free and allocate pages between iterations"); + +static atomic_t alloc_pages_failed = ATOMIC_INIT(0); + +static atomic64_t max_lock_wait = ATOMIC64_INIT(0); + +static struct task_struct *main_task; +static int master_cpu; + +static void test_lock(bool master, bool verbose) +{ + u64 uninitialized_var(wait_start); + + if (measure_lock_wait) + wait_start = local_clock(); + + if (lock_mutex_ptr && master) { + if (verbose) + pr_notice("lock mutex %ps\n", (void *)lock_mutex_ptr); + mutex_lock((struct mutex *)lock_mutex_ptr); + } + + if (lock_rwsem_ptr && master) { + if (verbose) + pr_notice("lock rw_semaphore %ps\n", + (void *)lock_rwsem_ptr); + if (lock_read) + down_read((struct rw_semaphore *)lock_rwsem_ptr); + else + down_write((struct rw_semaphore *)lock_rwsem_ptr); + } + + if (lock_mmap_sem && master) { + if (verbose) + pr_notice("lock mmap_sem pid=%d\n", main_task->pid); + if (lock_read) + down_read(&main_task->mm->mmap_sem); + else + down_write(&main_task->mm->mmap_sem); + } + + if (test_disable_irq) + local_irq_disable(); + + if (disable_softirq) + local_bh_disable(); + + if (disable_preempt) + preempt_disable(); + + if (lock_rcu) + rcu_read_lock(); + + if (lock_spinlock_ptr && master) { + if (verbose) + pr_notice("lock spinlock %ps\n", + (void *)lock_spinlock_ptr); + spin_lock((spinlock_t *)lock_spinlock_ptr); + } + + if (lock_rwlock_ptr && master) { + if (verbose) + pr_notice("lock rwlock %ps\n", + (void *)lock_rwlock_ptr); + if (lock_read) + read_lock((rwlock_t *)lock_rwlock_ptr); + else + write_lock((rwlock_t *)lock_rwlock_ptr); + } + + if (measure_lock_wait) { + s64 cur_wait = local_clock() - wait_start; + s64 max_wait = atomic64_read(&max_lock_wait); + + do { + if (cur_wait < max_wait) + break; + max_wait = atomic64_cmpxchg(&max_lock_wait, + max_wait, cur_wait); + } while (max_wait != cur_wait); + + if (cur_wait > lock_wait_threshold) + pr_notice_ratelimited("lock wait %lld ns\n", cur_wait); + } +} + +static void test_unlock(bool master, bool verbose) +{ + if (lock_rwlock_ptr && master) { + if (lock_read) + read_unlock((rwlock_t *)lock_rwlock_ptr); + else + write_unlock((rwlock_t *)lock_rwlock_ptr); + if (verbose) + pr_notice("unlock rwlock %ps\n", + (void *)lock_rwlock_ptr); + } + + if (lock_spinlock_ptr && master) { + spin_unlock((spinlock_t *)lock_spinlock_ptr); + if (verbose) + pr_notice("unlock spinlock %ps\n", + (void *)lock_spinlock_ptr); + } + + if (lock_rcu) + rcu_read_unlock(); + + if (disable_preempt) + preempt_enable(); + + if (disable_softirq) + local_bh_enable(); + + if (test_disable_irq) + local_irq_enable(); + + if (lock_mmap_sem && master) { + if (lock_read) + up_read(&main_task->mm->mmap_sem); + else + up_write(&main_task->mm->mmap_sem); + if (verbose) + pr_notice("unlock mmap_sem pid=%d\n", main_task->pid); + } + + if (lock_rwsem_ptr && master) { + if (lock_read) + up_read((struct rw_semaphore *)lock_rwsem_ptr); + else + up_write((struct rw_semaphore *)lock_rwsem_ptr); + if (verbose) + pr_notice("unlock rw_semaphore %ps\n", + (void *)lock_rwsem_ptr); + } + + if (lock_mutex_ptr && master) { + mutex_unlock((struct mutex *)lock_mutex_ptr); + if (verbose) + pr_notice("unlock mutex %ps\n", + (void *)lock_mutex_ptr); + } +} + +static void test_alloc_pages(struct list_head *pages) +{ + struct page *page; + unsigned int i; + + for (i = 0; i < alloc_pages_nr; i++) { + page = alloc_pages(alloc_pages_gfp, alloc_pages_order); + if (!page) { + atomic_inc(&alloc_pages_failed); + break; + } + list_add(&page->lru, pages); + } +} + +static void test_free_pages(struct list_head *pages) +{ + struct page *page, *next; + + list_for_each_entry_safe(page, next, pages, lru) + __free_pages(page, alloc_pages_order); + INIT_LIST_HEAD(pages); +} + +static void test_wait(unsigned int secs, unsigned int nsecs) +{ + if (wait_state == TASK_RUNNING) { + if (secs) + mdelay(secs * MSEC_PER_SEC); + if (nsecs) + ndelay(nsecs); + return; + } + + __set_current_state(wait_state); + if (use_hrtimer) { + ktime_t time; + + time = ns_to_ktime((u64)secs * NSEC_PER_SEC + nsecs); + schedule_hrtimeout(&time, HRTIMER_MODE_REL); + } else { + schedule_timeout(secs * HZ + nsecs_to_jiffies(nsecs)); + } +} + +static void test_lockup(bool master) +{ + u64 lockup_start = local_clock(); + unsigned int iter = 0; + LIST_HEAD(pages); + + pr_notice("Start on CPU%d\n", raw_smp_processor_id()); + + test_lock(master, true); + + test_alloc_pages(&pages); + + while (iter++ < iterations && !signal_pending(main_task)) { + + if (iowait) + current->in_iowait = 1; + + test_wait(time_secs, time_nsecs); + + if (iowait) + current->in_iowait = 0; + + if (reallocate_pages) + test_free_pages(&pages); + + if (reacquire_locks) + test_unlock(master, false); + + if (touch_softlockup) + touch_softlockup_watchdog(); + + if (touch_hardlockup) + touch_nmi_watchdog(); + + if (call_cond_resched) + cond_resched(); + + test_wait(cooldown_secs, cooldown_nsecs); + + if (reacquire_locks) + test_lock(master, false); + + if (reallocate_pages) + test_alloc_pages(&pages); + } + + pr_notice("Finish on CPU%d in %lld ns\n", raw_smp_processor_id(), + local_clock() - lockup_start); + + test_free_pages(&pages); + + test_unlock(master, true); +} + +DEFINE_PER_CPU(struct work_struct, test_works); + +static void test_work_fn(struct work_struct *work) +{ + test_lockup(!lock_single || + work == per_cpu_ptr(&test_works, master_cpu)); +} + +static bool test_kernel_ptr(unsigned long addr, int size) +{ + void *ptr = (void *)addr; + char buf; + + if (!addr) + return false; + + /* should be at least readable kernel address */ + if (access_ok(ptr, 1) || + access_ok(ptr + size - 1, 1) || + probe_kernel_address(ptr, buf) || + probe_kernel_address(ptr + size - 1, buf)) { + pr_err("invalid kernel ptr: %#lx\n", addr); + return true; + } + + return false; +} + +static bool __maybe_unused test_magic(unsigned long addr, int offset, + unsigned int expected) +{ + void *ptr = (void *)addr + offset; + unsigned int magic = 0; + + if (!addr) + return false; + + if (probe_kernel_address(ptr, magic) || magic != expected) { + pr_err("invalid magic at %#lx + %#x = %#x, expected %#x\n", + addr, offset, magic, expected); + return true; + } + + return false; +} + +static int __init test_lockup_init(void) +{ + u64 test_start = local_clock(); + + main_task = current; + + switch (state[0]) { + case 'S': + wait_state = TASK_INTERRUPTIBLE; + break; + case 'D': + wait_state = TASK_UNINTERRUPTIBLE; + break; + case 'K': + wait_state = TASK_KILLABLE; + break; + case 'R': + wait_state = TASK_RUNNING; + break; + default: + pr_err("unknown state=%s\n", state); + return -EINVAL; + } + + if (alloc_pages_atomic) + alloc_pages_gfp = GFP_ATOMIC; + + if (test_kernel_ptr(lock_spinlock_ptr, sizeof(spinlock_t)) || + test_kernel_ptr(lock_rwlock_ptr, sizeof(rwlock_t)) || + test_kernel_ptr(lock_mutex_ptr, sizeof(struct mutex)) || + test_kernel_ptr(lock_rwsem_ptr, sizeof(struct rw_semaphore))) + return -EINVAL; + +#ifdef CONFIG_DEBUG_SPINLOCK + if (test_magic(lock_spinlock_ptr, + offsetof(spinlock_t, rlock.magic), + SPINLOCK_MAGIC) || + test_magic(lock_rwlock_ptr, + offsetof(rwlock_t, magic), + RWLOCK_MAGIC) || + test_magic(lock_mutex_ptr, + offsetof(struct mutex, wait_lock.rlock.magic), + SPINLOCK_MAGIC) || + test_magic(lock_rwsem_ptr, + offsetof(struct rw_semaphore, wait_lock.magic), + SPINLOCK_MAGIC)) + return -EINVAL; +#endif + + if ((wait_state != TASK_RUNNING || + (call_cond_resched && !reacquire_locks) || + (alloc_pages_nr && gfpflags_allow_blocking(alloc_pages_gfp))) && + (test_disable_irq || disable_softirq || disable_preempt || + lock_rcu || lock_spinlock_ptr || lock_rwlock_ptr)) { + pr_err("refuse to sleep in atomic context\n"); + return -EINVAL; + } + + if (lock_mmap_sem && !main_task->mm) { + pr_err("no mm to lock mmap_sem\n"); + return -EINVAL; + } + + pr_notice("START pid=%d time=%u +%u ns cooldown=%u +%u ns iteraions=%u state=%s %s%s%s%s%s%s%s%s%s%s%s\n", + main_task->pid, time_secs, time_nsecs, + cooldown_secs, cooldown_nsecs, iterations, state, + all_cpus ? "all_cpus " : "", + iowait ? "iowait " : "", + test_disable_irq ? "disable_irq " : "", + disable_softirq ? "disable_softirq " : "", + disable_preempt ? "disable_preempt " : "", + lock_rcu ? "lock_rcu " : "", + lock_read ? "lock_read " : "", + touch_softlockup ? "touch_softlockup " : "", + touch_hardlockup ? "touch_hardlockup " : "", + call_cond_resched ? "call_cond_resched " : "", + reacquire_locks ? "reacquire_locks " : ""); + + if (alloc_pages_nr) + pr_notice("ALLOCATE PAGES nr=%u order=%u gfp=%pGg %s\n", + alloc_pages_nr, alloc_pages_order, &alloc_pages_gfp, + reallocate_pages ? "reallocate_pages " : ""); + + if (all_cpus) { + unsigned int cpu; + + cpus_read_lock(); + + preempt_disable(); + master_cpu = smp_processor_id(); + for_each_online_cpu(cpu) { + INIT_WORK(per_cpu_ptr(&test_works, cpu), test_work_fn); + queue_work_on(cpu, system_highpri_wq, + per_cpu_ptr(&test_works, cpu)); + } + preempt_enable(); + + for_each_online_cpu(cpu) + flush_work(per_cpu_ptr(&test_works, cpu)); + + cpus_read_unlock(); + } else { + test_lockup(true); + } + + if (measure_lock_wait) + pr_notice("Maximum lock wait: %lld ns\n", + atomic64_read(&max_lock_wait)); + + if (alloc_pages_nr) + pr_notice("Page allocation failed %u times\n", + atomic_read(&alloc_pages_failed)); + + pr_notice("FINISH in %llu ns\n", local_clock() - test_start); + + if (signal_pending(main_task)) + return -EINTR; + + return -EAGAIN; +} +module_init(test_lockup_init); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Konstantin Khlebnikov "); +MODULE_DESCRIPTION("Test module to generate lockups"); From ad3f434b87e7d2be4f2f5d602d33f070823f4d48 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Mon, 6 Apr 2020 20:09:50 -0700 Subject: [PATCH 581/635] lib/test_lockup.c: fix spelling mistake "iteraions" -> "iterations" There is a spelling mistake in a pr_notice message. Fix it. Signed-off-by: Colin Ian King Signed-off-by: Andrew Morton Cc: Greg Kroah-Hartman Cc: Guenter Roeck Cc: Kees Cook Cc: Konstantin Khlebnikov Cc: Peter Zijlstra Cc: Petr Mladek Cc: Sasha Levin Cc: Sergey Senozhatsky Cc: Steven Rostedt Link: http://lkml.kernel.org/r/20200221155145.79522-1-colin.king@canonical.com Signed-off-by: Linus Torvalds --- lib/test_lockup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/test_lockup.c b/lib/test_lockup.c index 9e8b8a0be9af..83683ec1f429 100644 --- a/lib/test_lockup.c +++ b/lib/test_lockup.c @@ -490,7 +490,7 @@ static int __init test_lockup_init(void) return -EINVAL; } - pr_notice("START pid=%d time=%u +%u ns cooldown=%u +%u ns iteraions=%u state=%s %s%s%s%s%s%s%s%s%s%s%s\n", + pr_notice("START pid=%d time=%u +%u ns cooldown=%u +%u ns iterations=%u state=%s %s%s%s%s%s%s%s%s%s%s%s\n", main_task->pid, time_secs, time_nsecs, cooldown_secs, cooldown_nsecs, iterations, state, all_cpus ? "all_cpus " : "", From aecd42df6d399346fd8e5551dd8878338f214ec1 Mon Sep 17 00:00:00 2001 From: Konstantin Khlebnikov Date: Mon, 6 Apr 2020 20:09:54 -0700 Subject: [PATCH 582/635] lib/test_lockup.c: add parameters for locking generic vfs locks file_path= defines file or directory to open lock_inode=Y set lock_rwsem_ptr to inode->i_rwsem lock_mapping=Y set lock_rwsem_ptr to mapping->i_mmap_rwsem lock_sb_umount=Y set lock_rwsem_ptr to sb->s_umount This gives safe and simple way to see how system reacts to contention of common vfs locks and how syscalls depend on them directly or indirectly. For example to block s_umount for 60 seconds: # modprobe test_lockup file_path=. lock_sb_umount time_secs=60 state=S This is useful for checking/testing scalability issues like this: https://lore.kernel.org/lkml/158497590858.7371.9311902565121473436.stgit@buzz/ Signed-off-by: Konstantin Khlebnikov Signed-off-by: Andrew Morton Cc: Colin Ian King Cc: Greg Kroah-Hartman Cc: Guenter Roeck Cc: Kees Cook Cc: Peter Zijlstra Cc: Petr Mladek Cc: Sasha Levin Cc: Sergey Senozhatsky Cc: Steven Rostedt Link: http://lkml.kernel.org/r/158498153964.5621.83061779039255681.stgit@buzz Signed-off-by: Linus Torvalds --- lib/test_lockup.c | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/lib/test_lockup.c b/lib/test_lockup.c index 83683ec1f429..ea09ca335b21 100644 --- a/lib/test_lockup.c +++ b/lib/test_lockup.c @@ -14,6 +14,7 @@ #include #include #include +#include static unsigned int time_secs; module_param(time_secs, uint, 0600); @@ -140,6 +141,24 @@ static bool reallocate_pages; module_param(reallocate_pages, bool, 0400); MODULE_PARM_DESC(reallocate_pages, "free and allocate pages between iterations"); +struct file *test_file; +struct inode *test_inode; +static char test_file_path[256]; +module_param_string(file_path, test_file_path, sizeof(test_file_path), 0400); +MODULE_PARM_DESC(file_path, "file path to test"); + +static bool test_lock_inode; +module_param_named(lock_inode, test_lock_inode, bool, 0400); +MODULE_PARM_DESC(lock_inode, "lock file -> inode -> i_rwsem"); + +static bool test_lock_mapping; +module_param_named(lock_mapping, test_lock_mapping, bool, 0400); +MODULE_PARM_DESC(lock_mapping, "lock file -> mapping -> i_mmap_rwsem"); + +static bool test_lock_sb_umount; +module_param_named(lock_sb_umount, test_lock_sb_umount, bool, 0400); +MODULE_PARM_DESC(lock_sb_umount, "lock file -> sb -> s_umount"); + static atomic_t alloc_pages_failed = ATOMIC_INIT(0); static atomic64_t max_lock_wait = ATOMIC64_INIT(0); @@ -490,6 +509,29 @@ static int __init test_lockup_init(void) return -EINVAL; } + if (test_file_path[0]) { + test_file = filp_open(test_file_path, O_RDONLY, 0); + if (IS_ERR(test_file)) { + pr_err("cannot find file_path\n"); + return -EINVAL; + } + test_inode = file_inode(test_file); + } else if (test_lock_inode || + test_lock_mapping || + test_lock_sb_umount) { + pr_err("no file to lock\n"); + return -EINVAL; + } + + if (test_lock_inode && test_inode) + lock_rwsem_ptr = (unsigned long)&test_inode->i_rwsem; + + if (test_lock_mapping && test_file && test_file->f_mapping) + lock_rwsem_ptr = (unsigned long)&test_file->f_mapping->i_mmap_rwsem; + + if (test_lock_sb_umount && test_inode) + lock_rwsem_ptr = (unsigned long)&test_inode->i_sb->s_umount; + pr_notice("START pid=%d time=%u +%u ns cooldown=%u +%u ns iterations=%u state=%s %s%s%s%s%s%s%s%s%s%s%s\n", main_task->pid, time_secs, time_nsecs, cooldown_secs, cooldown_nsecs, iterations, state, @@ -542,6 +584,9 @@ static int __init test_lockup_init(void) pr_notice("FINISH in %llu ns\n", local_clock() - test_start); + if (test_file) + fput(test_file); + if (signal_pending(main_task)) return -EINTR; From a44ce5137026493b3bec3768cff92861dbeede5c Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Mon, 6 Apr 2020 20:09:57 -0700 Subject: [PATCH 583/635] lib/bch.c: replace zero-length array with flexible-array member The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertenly introduced[3] to the codebase from now on. This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Signed-off-by: Andrew Morton Link: http://lkml.kernel.org/r/20200211205119.GA21234@embeddedor Signed-off-by: Linus Torvalds --- lib/bch.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bch.c b/lib/bch.c index 5db6d3a4c8a6..052d3fb753a0 100644 --- a/lib/bch.c +++ b/lib/bch.c @@ -102,7 +102,7 @@ */ struct gf_poly { unsigned int deg; /* polynomial degree */ - unsigned int c[0]; /* polynomial terms */ + unsigned int c[]; /* polynomial terms */ }; /* given its degree, compute a polynomial size in bytes */ From c6e2ac3b476b458f9bd2069d85b95ca4634ce680 Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Mon, 6 Apr 2020 20:10:00 -0700 Subject: [PATCH 584/635] lib/ts_bm.c: replace zero-length array with flexible-array member The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertenly introduced[3] to the codebase from now on. This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Signed-off-by: Andrew Morton Link: http://lkml.kernel.org/r/20200211205620.GA24694@embeddedor Signed-off-by: Linus Torvalds --- lib/ts_bm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ts_bm.c b/lib/ts_bm.c index b352903c50e3..277cb4417ac2 100644 --- a/lib/ts_bm.c +++ b/lib/ts_bm.c @@ -52,7 +52,7 @@ struct ts_bm u8 * pattern; unsigned int patlen; unsigned int bad_shift[ASIZE]; - unsigned int good_shift[0]; + unsigned int good_shift[]; }; static unsigned int bm_find(struct ts_config *conf, struct ts_state *state) From 842ae1f52b44a4fa3e4e7e5dd61e8b53e8a09ece Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Mon, 6 Apr 2020 20:10:03 -0700 Subject: [PATCH 585/635] lib/ts_fsm.c: replace zero-length array with flexible-array member The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertenly introduced[3] to the codebase from now on. This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Signed-off-by: Andrew Morton Link: http://lkml.kernel.org/r/20200211205813.GA25602@embeddedor Signed-off-by: Linus Torvalds --- lib/ts_fsm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ts_fsm.c b/lib/ts_fsm.c index 9c873cadab7c..ab749ec10ab5 100644 --- a/lib/ts_fsm.c +++ b/lib/ts_fsm.c @@ -32,7 +32,7 @@ struct ts_fsm { unsigned int ntokens; - struct ts_fsm_token tokens[0]; + struct ts_fsm_token tokens[]; }; /* other values derived from ctype.h */ From 51022f8715bbb365de8521a5b6a2fcf967ee76ca Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Mon, 6 Apr 2020 20:10:06 -0700 Subject: [PATCH 586/635] lib/ts_kmp.c: replace zero-length array with flexible-array member The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertenly introduced[3] to the codebase from now on. This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Signed-off-by: Andrew Morton Link: http://lkml.kernel.org/r/20200211205948.GA26459@embeddedor Signed-off-by: Linus Torvalds --- lib/ts_kmp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ts_kmp.c b/lib/ts_kmp.c index 94617e014b3a..c77a3d537f24 100644 --- a/lib/ts_kmp.c +++ b/lib/ts_kmp.c @@ -36,7 +36,7 @@ struct ts_kmp { u8 * pattern; unsigned int pattern_len; - unsigned int prefix_tbl[0]; + unsigned int prefix_tbl[]; }; static unsigned int kmp_find(struct ts_config *conf, struct ts_state *state) From 6e85318521c001b6af82830774a8a7d05eec6adf Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Mon, 6 Apr 2020 20:10:09 -0700 Subject: [PATCH 587/635] lib/scatterlist: fix sg_copy_buffer() kerneldoc Add the missing closing parenthesis to the description for the to_buffer parameter of sg_copy_buffer(). Signed-off-by: Geert Uytterhoeven Signed-off-by: Andrew Morton Cc: Akinobu Mita --- lib/scatterlist.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/scatterlist.c b/lib/scatterlist.c index 5813072bc589..5d63a8857f36 100644 --- a/lib/scatterlist.c +++ b/lib/scatterlist.c @@ -832,7 +832,7 @@ EXPORT_SYMBOL(sg_miter_stop); * @buflen: The number of bytes to copy * @skip: Number of bytes to skip before copying * @to_buffer: transfer direction (true == from an sg list to a - * buffer, false == from a buffer to an sg list + * buffer, false == from a buffer to an sg list) * * Returns the number of copied bytes. * From 9cf016e6b49b53d6c15d4137c034178148149ef4 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Mon, 6 Apr 2020 20:10:12 -0700 Subject: [PATCH 588/635] lib: test_stackinit.c: XFAIL switch variable init tests The tests for initializing a variable defined between a switch statement's test and its first "case" statement are currently not initialized in Clang[1] nor the proposed auto-initialization feature in GCC. We should retain the test (so that we can evaluate compiler fixes), but mark it as an "expected fail". The rest of the kernel source will be adjusted to avoid this corner case. Also disable -Wswitch-unreachable for the test so that the intentionally broken code won't trigger warnings for GCC (nor future Clang) when initialization happens this unhandled place. [1] https://bugs.llvm.org/show_bug.cgi?id=44916 Suggested-by: Alexander Potapenko Signed-off-by: Kees Cook Signed-off-by: Andrew Morton Cc: Jann Horn Cc: Ard Biesheuvel Link: http://lkml.kernel.org/r/202002191358.2897A07C6@keescook Signed-off-by: Linus Torvalds --- lib/Makefile | 1 + lib/test_stackinit.c | 28 ++++++++++++++++++---------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index 0fd125c4ad07..93d05ff4f501 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -87,6 +87,7 @@ obj-$(CONFIG_TEST_KMOD) += test_kmod.o obj-$(CONFIG_TEST_DEBUG_VIRTUAL) += test_debug_virtual.o obj-$(CONFIG_TEST_MEMCAT_P) += test_memcat_p.o obj-$(CONFIG_TEST_OBJAGG) += test_objagg.o +CFLAGS_test_stackinit.o += $(call cc-disable-warning, switch-unreachable) obj-$(CONFIG_TEST_STACKINIT) += test_stackinit.o obj-$(CONFIG_TEST_BLACKHOLE_DEV) += test_blackhole_dev.o obj-$(CONFIG_TEST_MEMINIT) += test_meminit.o diff --git a/lib/test_stackinit.c b/lib/test_stackinit.c index 2d7d257a430e..f93b1e145ada 100644 --- a/lib/test_stackinit.c +++ b/lib/test_stackinit.c @@ -92,8 +92,9 @@ static bool range_contains(char *haystack_start, size_t haystack_size, * @var_type: type to be tested for zeroing initialization * @which: is this a SCALAR, STRING, or STRUCT type? * @init_level: what kind of initialization is performed + * @xfail: is this test expected to fail? */ -#define DEFINE_TEST_DRIVER(name, var_type, which) \ +#define DEFINE_TEST_DRIVER(name, var_type, which, xfail) \ /* Returns 0 on success, 1 on failure. */ \ static noinline __init int test_ ## name (void) \ { \ @@ -139,13 +140,14 @@ static noinline __init int test_ ## name (void) \ for (sum = 0, i = 0; i < target_size; i++) \ sum += (check_buf[i] == 0xFF); \ \ - if (sum == 0) \ + if (sum == 0) { \ pr_info(#name " ok\n"); \ - else \ - pr_warn(#name " FAIL (uninit bytes: %d)\n", \ - sum); \ - \ - return (sum != 0); \ + return 0; \ + } else { \ + pr_warn(#name " %sFAIL (uninit bytes: %d)\n", \ + (xfail) ? "X" : "", sum); \ + return (xfail) ? 0 : 1; \ + } \ } #define DEFINE_TEST(name, var_type, which, init_level) \ /* no-op to force compiler into ignoring "uninitialized" vars */\ @@ -189,7 +191,7 @@ static noinline __init int leaf_ ## name(unsigned long sp, \ \ return (int)buf[0] | (int)buf[sizeof(buf) - 1]; \ } \ -DEFINE_TEST_DRIVER(name, var_type, which) +DEFINE_TEST_DRIVER(name, var_type, which, 0) /* Structure with no padding. */ struct test_packed { @@ -326,8 +328,14 @@ static noinline __init int leaf_switch_2_none(unsigned long sp, bool fill, return __leaf_switch_none(2, fill); } -DEFINE_TEST_DRIVER(switch_1_none, uint64_t, SCALAR); -DEFINE_TEST_DRIVER(switch_2_none, uint64_t, SCALAR); +/* + * These are expected to fail for most configurations because neither + * GCC nor Clang have a way to perform initialization of variables in + * non-code areas (i.e. in a switch statement before the first "case"). + * https://bugs.llvm.org/show_bug.cgi?id=44916 + */ +DEFINE_TEST_DRIVER(switch_1_none, uint64_t, SCALAR, 1); +DEFINE_TEST_DRIVER(switch_2_none, uint64_t, SCALAR, 1); static int __init test_stackinit_init(void) { From 69866e156ce28c47dc0085b5ba5e734f5511812e Mon Sep 17 00:00:00 2001 From: Alexander Potapenko Date: Mon, 6 Apr 2020 20:10:15 -0700 Subject: [PATCH 589/635] lib/stackdepot.c: check depot_index before accessing the stack slab Avoid crashes on corrupted stack ids. Despite stack ID corruption may indicate other bugs in the program, we'd better fail gracefully on such IDs instead of crashing the kernel. This patch has been previously mailed as part of KMSAN RFC patch series. Link: http://lkml.kernel.org/r/20200220141916.55455-1-glider@google.com Signed-off-by: Alexander Potapenko Cc: Vegard Nossum Cc: Dmitry Vyukov Cc: Marco Elver Cc: Andrey Konovalov Cc: Andrey Ryabinin Cc: Arnd Bergmann Cc: Sergey Senozhatsky From: Dan Carpenter Subject: lib/stackdepot.c: fix a condition in stack_depot_fetch() We should check for a NULL pointer first before adding the offset. Otherwise if the pointer is NULL and the offset is non-zero, it will lead to an Oops. Fixes: d45048e65a59 ("lib/stackdepot.c: check depot_index before accessing the stack slab") Signed-off-by: Dan Carpenter Signed-off-by: Andrew Morton Acked-by: Alexander Potapenko Link: http://lkml.kernel.org/r/20200312113006.GA20562@mwanda Signed-off-by: Linus Torvalds --- lib/stackdepot.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 81c69c08d1d1..1ec36ee344e0 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -202,9 +202,20 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle, unsigned long **entries) { union handle_parts parts = { .handle = handle }; - void *slab = stack_slabs[parts.slabindex]; + void *slab; size_t offset = parts.offset << STACK_ALLOC_ALIGN; - struct stack_record *stack = slab + offset; + struct stack_record *stack; + + *entries = NULL; + if (parts.slabindex > depot_index) { + WARN(1, "slab index %d out of bounds (%d) for stack id %08x\n", + parts.slabindex, depot_index, handle); + return 0; + } + slab = stack_slabs[parts.slabindex]; + if (!slab) + return 0; + stack = slab + offset; *entries = stack->entries; return stack->size; From 7b65942fb2f0ac939be9c659bb889e78b399f84e Mon Sep 17 00:00:00 2001 From: Alexander Potapenko Date: Mon, 6 Apr 2020 20:10:19 -0700 Subject: [PATCH 590/635] lib/stackdepot.c: build with -fno-builtin Clang may replace stackdepot_memcmp() with a call to instrumented bcmp(), which is exactly what we wanted to avoid creating stackdepot_memcmp(). Building the file with -fno-builtin prevents such optimizations. This patch has been previously mailed as part of KMSAN RFC patch series. Signed-off-by: Alexander Potapenko Signed-off-by: Andrew Morton Cc: Vegard Nossum Cc: Dmitry Vyukov Cc: Marco Elver Cc: Andrey Konovalov Cc: Sergey Senozhatsky Cc: Arnd Bergmann Cc: Andrey Ryabinin Link: http://lkml.kernel.org/r/20200220141916.55455-2-glider@google.com Signed-off-by: Linus Torvalds --- lib/Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/Makefile b/lib/Makefile index 93d05ff4f501..3fc06399295d 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -223,6 +223,10 @@ obj-$(CONFIG_MEMREGION) += memregion.o obj-$(CONFIG_STMP_DEVICE) += stmp_device.o obj-$(CONFIG_IRQ_POLL) += irq_poll.o +# stackdepot.c should not be instrumented or call instrumented functions. +# Prevent the compiler from calling builtins like memcmp() or bcmp() from this +# file. +CFLAGS_stackdepot.o += -fno-builtin obj-$(CONFIG_STACKDEPOT) += stackdepot.o KASAN_SANITIZE_stackdepot.o := n KCOV_INSTRUMENT_stackdepot.o := n From 505a0ef15f96c6c43ec719c9fc1833d98957bb39 Mon Sep 17 00:00:00 2001 From: Alexander Potapenko Date: Mon, 6 Apr 2020 20:10:22 -0700 Subject: [PATCH 591/635] kasan: stackdepot: move filter_irq_stacks() to stackdepot.c filter_irq_stacks() can be used by other tools (e.g. KMSAN), so it needs to be moved to a common location. lib/stackdepot.c seems a good place, as filter_irq_stacks() is usually applied to the output of stack_trace_save(). This patch has been previously mailed as part of KMSAN RFC patch series. [glider@google.co: nds32: linker script: add SOFTIRQENTRY_TEXT\ Link: http://lkml.kernel.org/r/20200311121002.241430-1-glider@google.com [glider@google.com: add IRQENTRY_TEXT and SOFTIRQENTRY_TEXT to linker script] Link: http://lkml.kernel.org/r/20200311121124.243352-1-glider@google.com Signed-off-by: Alexander Potapenko Signed-off-by: Andrew Morton Cc: Vegard Nossum Cc: Dmitry Vyukov Cc: Marco Elver Cc: Andrey Konovalov Cc: Andrey Ryabinin Cc: Arnd Bergmann Cc: Sergey Senozhatsky Link: http://lkml.kernel.org/r/20200220141916.55455-3-glider@google.com Signed-off-by: Linus Torvalds --- arch/ia64/kernel/vmlinux.lds.S | 2 ++ arch/nds32/kernel/vmlinux.lds.S | 1 + include/linux/stackdepot.h | 2 ++ lib/stackdepot.c | 24 ++++++++++++++++++++++++ mm/kasan/common.c | 23 ----------------------- 5 files changed, 29 insertions(+), 23 deletions(-) diff --git a/arch/ia64/kernel/vmlinux.lds.S b/arch/ia64/kernel/vmlinux.lds.S index 1ec6b703c5b4..6b5652ee76f9 100644 --- a/arch/ia64/kernel/vmlinux.lds.S +++ b/arch/ia64/kernel/vmlinux.lds.S @@ -54,6 +54,8 @@ SECTIONS { CPUIDLE_TEXT LOCK_TEXT KPROBES_TEXT + IRQENTRY_TEXT + SOFTIRQENTRY_TEXT *(.gnu.linkonce.t*) } diff --git a/arch/nds32/kernel/vmlinux.lds.S b/arch/nds32/kernel/vmlinux.lds.S index f679d3397436..7a6c1cefe3fe 100644 --- a/arch/nds32/kernel/vmlinux.lds.S +++ b/arch/nds32/kernel/vmlinux.lds.S @@ -47,6 +47,7 @@ SECTIONS LOCK_TEXT KPROBES_TEXT IRQENTRY_TEXT + SOFTIRQENTRY_TEXT *(.fixup) } diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index 3efa97d482cb..24d49c732341 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -19,4 +19,6 @@ depot_stack_handle_t stack_depot_save(unsigned long *entries, unsigned int stack_depot_fetch(depot_stack_handle_t handle, unsigned long **entries); +unsigned int filter_irq_stacks(unsigned long *entries, unsigned int nr_entries); + #endif diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 1ec36ee344e0..2caffc64e4c8 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -20,6 +20,7 @@ */ #include +#include #include #include #include @@ -316,3 +317,26 @@ fast_exit: return retval; } EXPORT_SYMBOL_GPL(stack_depot_save); + +static inline int in_irqentry_text(unsigned long ptr) +{ + return (ptr >= (unsigned long)&__irqentry_text_start && + ptr < (unsigned long)&__irqentry_text_end) || + (ptr >= (unsigned long)&__softirqentry_text_start && + ptr < (unsigned long)&__softirqentry_text_end); +} + +unsigned int filter_irq_stacks(unsigned long *entries, + unsigned int nr_entries) +{ + unsigned int i; + + for (i = 0; i < nr_entries; i++) { + if (in_irqentry_text(entries[i])) { + /* Include the irqentry function into the stack. */ + return i + 1; + } + } + return nr_entries; +} +EXPORT_SYMBOL_GPL(filter_irq_stacks); diff --git a/mm/kasan/common.c b/mm/kasan/common.c index e61b4a492218..2906358e42f0 100644 --- a/mm/kasan/common.c +++ b/mm/kasan/common.c @@ -15,7 +15,6 @@ */ #include -#include #include #include #include @@ -42,28 +41,6 @@ #include "kasan.h" #include "../slab.h" -static inline int in_irqentry_text(unsigned long ptr) -{ - return (ptr >= (unsigned long)&__irqentry_text_start && - ptr < (unsigned long)&__irqentry_text_end) || - (ptr >= (unsigned long)&__softirqentry_text_start && - ptr < (unsigned long)&__softirqentry_text_end); -} - -static inline unsigned int filter_irq_stacks(unsigned long *entries, - unsigned int nr_entries) -{ - unsigned int i; - - for (i = 0; i < nr_entries; i++) { - if (in_irqentry_text(entries[i])) { - /* Include the irqentry function into the stack. */ - return i + 1; - } - } - return nr_entries; -} - static inline depot_stack_handle_t save_stack(gfp_t flags) { unsigned long entries[KASAN_STACK_DEPTH]; From 7e2345200262e4a6056580f0231cccdaffc825f3 Mon Sep 17 00:00:00 2001 From: Qian Cai Date: Mon, 6 Apr 2020 20:10:25 -0700 Subject: [PATCH 592/635] percpu_counter: fix a data race at vm_committed_as "vm_committed_as.count" could be accessed concurrently as reported by KCSAN, BUG: KCSAN: data-race in __vm_enough_memory / percpu_counter_add_batch write to 0xffffffff9451c538 of 8 bytes by task 65879 on cpu 35: percpu_counter_add_batch+0x83/0xd0 percpu_counter_add_batch at lib/percpu_counter.c:91 __vm_enough_memory+0xb9/0x260 dup_mm+0x3a4/0x8f0 copy_process+0x2458/0x3240 _do_fork+0xaa/0x9f0 __do_sys_clone+0x125/0x160 __x64_sys_clone+0x70/0x90 do_syscall_64+0x91/0xb05 entry_SYSCALL_64_after_hwframe+0x49/0xbe read to 0xffffffff9451c538 of 8 bytes by task 66773 on cpu 19: __vm_enough_memory+0x199/0x260 percpu_counter_read_positive at include/linux/percpu_counter.h:81 (inlined by) __vm_enough_memory at mm/util.c:839 mmap_region+0x1b2/0xa10 do_mmap+0x45c/0x700 vm_mmap_pgoff+0xc0/0x130 ksys_mmap_pgoff+0x6e/0x300 __x64_sys_mmap+0x33/0x40 do_syscall_64+0x91/0xb05 entry_SYSCALL_64_after_hwframe+0x49/0xbe The read is outside percpu_counter::lock critical section which results in a data race. Fix it by adding a READ_ONCE() in percpu_counter_read_positive() which could also service as the existing compiler memory barrier. Signed-off-by: Qian Cai Signed-off-by: Andrew Morton Acked-by: Marco Elver Link: http://lkml.kernel.org/r/1582302724-2804-1-git-send-email-cai@lca.pw Signed-off-by: Linus Torvalds --- include/linux/percpu_counter.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/percpu_counter.h b/include/linux/percpu_counter.h index 4f052496cdfd..0a4f54dd4737 100644 --- a/include/linux/percpu_counter.h +++ b/include/linux/percpu_counter.h @@ -78,9 +78,9 @@ static inline s64 percpu_counter_read(struct percpu_counter *fbc) */ static inline s64 percpu_counter_read_positive(struct percpu_counter *fbc) { - s64 ret = fbc->count; + /* Prevent reloads of fbc->count */ + s64 ret = READ_ONCE(fbc->count); - barrier(); /* Prevent reloads of fbc->count */ if (ret >= 0) return ret; return 0; From caa7f776ccd3067ddea1f36d92662cee7608f141 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 6 Apr 2020 20:10:28 -0700 Subject: [PATCH 593/635] lib/test_bitmap.c: make use of EXP2_IN_BITS Commit 30544ed5de43 ("lib/bitmap: introduce bitmap_replace() helper") introduced some new test cases to the test_bitmap.c module. Among these it also introduced an (unused) definition. Let's make use of EXP2_IN_BITS. Reported-by: Alex Shi Signed-off-by: Andy Shevchenko Signed-off-by: Andrew Morton Reviewed-by: Alex Shi Link: http://lkml.kernel.org/r/20200121151847.75223-1-andriy.shevchenko@linux.intel.com Signed-off-by: Linus Torvalds --- lib/test_bitmap.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/test_bitmap.c b/lib/test_bitmap.c index 61ed71c1daba..6b13150667f5 100644 --- a/lib/test_bitmap.c +++ b/lib/test_bitmap.c @@ -278,6 +278,8 @@ static void __init test_replace(void) unsigned int nlongs = DIV_ROUND_UP(nbits, BITS_PER_LONG); DECLARE_BITMAP(bmap, 1024); + BUILD_BUG_ON(EXP2_IN_BITS < nbits * 2); + bitmap_zero(bmap, 1024); bitmap_replace(bmap, &exp2[0 * nlongs], &exp2[1 * nlongs], exp2_to_exp3_mask, nbits); expect_eq_bitmap(bmap, exp3_0_1, nbits); From 8d994cada27c565adfaa5ac524cc1bc099668dfd Mon Sep 17 00:00:00 2001 From: chenqiwu Date: Mon, 6 Apr 2020 20:10:31 -0700 Subject: [PATCH 594/635] lib/rbtree: fix coding style of assignments Leave blank space between the right-hand and left-hand side of the assignment to meet the kernel coding style better. Signed-off-by: chenqiwu Signed-off-by: Andrew Morton Reviewed-by: Michel Lespinasse Link: http://lkml.kernel.org/r/1582621140-25850-1-git-send-email-qiwuchen55@gmail.com Signed-off-by: Linus Torvalds --- lib/rbtree.c | 4 ++-- tools/lib/rbtree.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/rbtree.c b/lib/rbtree.c index abc86c6a3177..8545872e61db 100644 --- a/lib/rbtree.c +++ b/lib/rbtree.c @@ -503,7 +503,7 @@ struct rb_node *rb_next(const struct rb_node *node) if (node->rb_right) { node = node->rb_right; while (node->rb_left) - node=node->rb_left; + node = node->rb_left; return (struct rb_node *)node; } @@ -535,7 +535,7 @@ struct rb_node *rb_prev(const struct rb_node *node) if (node->rb_left) { node = node->rb_left; while (node->rb_right) - node=node->rb_right; + node = node->rb_right; return (struct rb_node *)node; } diff --git a/tools/lib/rbtree.c b/tools/lib/rbtree.c index 2548ff8c4d9c..06ac7bd2144b 100644 --- a/tools/lib/rbtree.c +++ b/tools/lib/rbtree.c @@ -497,7 +497,7 @@ struct rb_node *rb_next(const struct rb_node *node) if (node->rb_right) { node = node->rb_right; while (node->rb_left) - node=node->rb_left; + node = node->rb_left; return (struct rb_node *)node; } @@ -528,7 +528,7 @@ struct rb_node *rb_prev(const struct rb_node *node) if (node->rb_left) { node = node->rb_left; while (node->rb_right) - node=node->rb_right; + node = node->rb_right; return (struct rb_node *)node; } From 8f0259c27c850403c6a2076ef07180dd802da559 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Mon, 6 Apr 2020 20:10:35 -0700 Subject: [PATCH 595/635] lib/test_kmod.c: remove a NULL test The "info" pointer has already been dereferenced so checking here is too late. Fortunately, we never pass NULL pointers to the test_kmod_put_module() function so the test can simply be removed. Signed-off-by: Dan Carpenter Signed-off-by: Andrew Morton Acked-by: Luis Chamberlain Link: http://lkml.kernel.org/r/20200228092452.vwkhthsn77nrxdy6@kili.mountain Signed-off-by: Linus Torvalds --- lib/test_kmod.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/test_kmod.c b/lib/test_kmod.c index 9cf77628fc91..e651c37d56db 100644 --- a/lib/test_kmod.c +++ b/lib/test_kmod.c @@ -204,7 +204,7 @@ static void test_kmod_put_module(struct kmod_test_device_info *info) case TEST_KMOD_DRIVER: break; case TEST_KMOD_FS_TYPE: - if (info && info->fs_sync && info->fs_sync->owner) + if (info->fs_sync && info->fs_sync->owner) module_put(info->fs_sync->owner); break; default: From 295bcca84916cb5079140a89fccb472bb8d1f6e2 Mon Sep 17 00:00:00 2001 From: Rikard Falkeborn Date: Mon, 6 Apr 2020 20:10:38 -0700 Subject: [PATCH 596/635] linux/bits.h: add compile time sanity check of GENMASK inputs GENMASK() and GENMASK_ULL() are supposed to be called with the high bit as the first argument and the low bit as the second argument. Mixing them will return a mask with zero bits set. Recent commits show getting this wrong is not uncommon, see e.g. commit aa4c0c9091b0 ("net: stmmac: Fix misuses of GENMASK macro") and commit 9bdd7bb3a844 ("clocksource/drivers/npcm: Fix misuse of GENMASK macro"). To prevent such mistakes from appearing again, add compile time sanity checking to the arguments of GENMASK() and GENMASK_ULL(). If both arguments are known at compile time, and the low bit is higher than the high bit, break the build to detect the mistake immediately. Since GENMASK() is used in declarations, BUILD_BUG_ON_ZERO() must be used instead of BUILD_BUG_ON(). __builtin_constant_p does not evaluate is argument, it only checks if it is a constant or not at compile time, and __builtin_choose_expr does not evaluate the expression that is not chosen. Therefore, GENMASK(x++, 0) does only evaluate x++ once. Commit 95b980d62d52 ("linux/bits.h: make BIT(), GENMASK(), and friends available in assembly") made the macros in linux/bits.h available in assembly. Since BUILD_BUG_OR_ZERO() is not asm compatible, disable the checks if the file is included in an asm file. Due to bugs in GCC versions before 4.9 [0], disable the check if building with a too old GCC compiler. [0]: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=19449 Signed-off-by: Rikard Falkeborn Signed-off-by: Andrew Morton Reviewed-by: Masahiro Yamada Reviewed-by: Kees Cook Cc: Borislav Petkov Cc: Geert Uytterhoeven Cc: Haren Myneni Cc: Joe Perches Cc: Johannes Berg Cc: lkml Cc: Ingo Molnar Cc: Thomas Gleixner Link: http://lkml.kernel.org/r/20200308193954.2372399-1-rikard.falkeborn@gmail.com Signed-off-by: Linus Torvalds --- include/linux/bits.h | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/include/linux/bits.h b/include/linux/bits.h index a740bbcf3cd2..4671fbf28842 100644 --- a/include/linux/bits.h +++ b/include/linux/bits.h @@ -18,12 +18,30 @@ * position @h. For example * GENMASK_ULL(39, 21) gives us the 64bit vector 0x000000ffffe00000. */ -#define GENMASK(h, l) \ +#if !defined(__ASSEMBLY__) && \ + (!defined(CONFIG_CC_IS_GCC) || CONFIG_GCC_VERSION >= 49000) +#include +#define GENMASK_INPUT_CHECK(h, l) \ + (BUILD_BUG_ON_ZERO(__builtin_choose_expr( \ + __builtin_constant_p((l) > (h)), (l) > (h), 0))) +#else +/* + * BUILD_BUG_ON_ZERO is not available in h files included from asm files, + * disable the input check if that is the case. + */ +#define GENMASK_INPUT_CHECK(h, l) 0 +#endif + +#define __GENMASK(h, l) \ (((~UL(0)) - (UL(1) << (l)) + 1) & \ (~UL(0) >> (BITS_PER_LONG - 1 - (h)))) +#define GENMASK(h, l) \ + (GENMASK_INPUT_CHECK(h, l) + __GENMASK(h, l)) -#define GENMASK_ULL(h, l) \ +#define __GENMASK_ULL(h, l) \ (((~ULL(0)) - (ULL(1) << (l)) + 1) & \ (~ULL(0) >> (BITS_PER_LONG_LONG - 1 - (h)))) +#define GENMASK_ULL(h, l) \ + (GENMASK_INPUT_CHECK(h, l) + __GENMASK_ULL(h, l)) #endif /* __LINUX_BITS_H */ From 8306b057a85ec07482da5d4b99d5c0b47af69be1 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Mon, 6 Apr 2020 20:10:45 -0700 Subject: [PATCH 597/635] lib/dynamic_debug.c: use address-of operator on section symbols Clang warns: ../lib/dynamic_debug.c:1034:24: warning: array comparison always evaluates to false [-Wtautological-compare] if (__start___verbose == __stop___verbose) { ^ 1 warning generated. These are not true arrays, they are linker defined symbols, which are just addresses. Using the address of operator silences the warning and does not change the resulting assembly with either clang/ld.lld or gcc/ld (tested with diff + objdump -Dr). Suggested-by: Nick Desaulniers Signed-off-by: Nathan Chancellor Signed-off-by: Andrew Morton Acked-by: Jason Baron Link: https://github.com/ClangBuiltLinux/linux/issues/894 Link: http://lkml.kernel.org/r/20200220051320.10739-1-natechancellor@gmail.com Signed-off-by: Linus Torvalds --- lib/dynamic_debug.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/dynamic_debug.c b/lib/dynamic_debug.c index aae17d9522e5..8f199f403ab5 100644 --- a/lib/dynamic_debug.c +++ b/lib/dynamic_debug.c @@ -1031,7 +1031,7 @@ static int __init dynamic_debug_init(void) int n = 0, entries = 0, modct = 0; int verbose_bytes = 0; - if (__start___verbose == __stop___verbose) { + if (&__start___verbose == &__stop___verbose) { pr_warn("_ddebug table is empty in a CONFIG_DYNAMIC_DEBUG build\n"); return 1; } From dfa05c28ca7ffc0a94e7a35b91014d088c1a1ff5 Mon Sep 17 00:00:00 2001 From: Joe Perches Date: Mon, 6 Apr 2020 20:10:48 -0700 Subject: [PATCH 598/635] checkpatch: remove email address comment from email address comparisons About 2% of the last 100K commits have email addresses that include an RFC2822 compliant comment like: Peter Zijlstra (Intel) checkpatch currently does a comparison of the complete name and address to the submitted author to determine if the author has signed-off and emits a warning if the exact email names and addresses do not match. Unfortunately, the author email address can be written without the comment like: Peter Zijlstra Add logic to compare the comment stripped email addresses to avoid this warning. Reported-by: Peter Zijlstra Signed-off-by: Joe Perches Signed-off-by: Andrew Morton Acked-by: Peter Zijlstra (Intel) Link: http://lkml.kernel.org/r/ebaa2f7c8f94e25520981945cddcc1982e70e072.camel@perches.com Signed-off-by: Linus Torvalds --- scripts/checkpatch.pl | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index a63380c6b0d2..3f62971c7c98 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -1118,6 +1118,7 @@ sub parse_email { my ($formatted_email) = @_; my $name = ""; + my $name_comment = ""; my $address = ""; my $comment = ""; @@ -1150,6 +1151,10 @@ sub parse_email { $name = trim($name); $name =~ s/^\"|\"$//g; + $name =~ s/(\s*\([^\)]+\))\s*//; + if (defined($1)) { + $name_comment = trim($1); + } $address = trim($address); $address =~ s/^\<|\>$//g; @@ -1158,7 +1163,7 @@ sub parse_email { $name = "\"$name\""; } - return ($name, $address, $comment); + return ($name, $name_comment, $address, $comment); } sub format_email { @@ -1184,6 +1189,23 @@ sub format_email { return $formatted_email; } +sub reformat_email { + my ($email) = @_; + + my ($email_name, $name_comment, $email_address, $comment) = parse_email($email); + return format_email($email_name, $email_address); +} + +sub same_email_addresses { + my ($email1, $email2) = @_; + + my ($email1_name, $name1_comment, $email1_address, $comment1) = parse_email($email1); + my ($email2_name, $name2_comment, $email2_address, $comment2) = parse_email($email2); + + return $email1_name eq $email2_name && + $email1_address eq $email2_address; +} + sub which { my ($bin) = @_; @@ -2604,17 +2626,16 @@ sub process { $author = $1; $author = encode("utf8", $author) if ($line =~ /=\?utf-8\?/i); $author =~ s/"//g; + $author = reformat_email($author); } # Check the patch for a signoff: - if ($line =~ /^\s*signed-off-by:/i) { + if ($line =~ /^\s*signed-off-by:\s*(.*)/i) { $signoff++; $in_commit_log = 0; if ($author ne '') { - my $l = $line; - $l =~ s/"//g; - if ($l =~ /^\s*signed-off-by:\s*\Q$author\E/i) { - $authorsignoff = 1; + if (same_email_addresses($1, $author)) { + $authorsignoff = 1; } } } @@ -2664,7 +2685,7 @@ sub process { } } - my ($email_name, $email_address, $comment) = parse_email($email); + my ($email_name, $name_comment, $email_address, $comment) = parse_email($email); my $suggested_email = format_email(($email_name, $email_address)); if ($suggested_email eq "") { ERROR("BAD_SIGN_OFF", @@ -2675,9 +2696,7 @@ sub process { $dequoted =~ s/" $comment" ne $email && - "$suggested_email$comment" ne $email) { + if (!same_email_addresses($email, $suggested_email)) { WARN("BAD_SIGN_OFF", "email address '$email' might be better as '$suggested_email$comment'\n" . $herecurr); } From c8df0ab61454aa1c84e97af3673ee8db44e39f05 Mon Sep 17 00:00:00 2001 From: Lubomir Rintel Date: Mon, 6 Apr 2020 20:10:51 -0700 Subject: [PATCH 599/635] checkpatch: check SPDX tags in YAML files This adds a warning when a YAML file is lacking a SPDX header on first line, or it uses incorrect commenting style. Currently the only YAML files in the tree are Devicetree binding documents. Signed-off-by: Lubomir Rintel Signed-off-by: Andrew Morton Acked-by: Joe Perches Cc: Rob Herring Link: http://lkml.kernel.org/r/20200129123356.388669-1-lkundrak@v3.sk Signed-off-by: Linus Torvalds --- scripts/checkpatch.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index 3f62971c7c98..977d4c0c2dc9 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -3106,7 +3106,7 @@ sub process { $comment = '/*'; } elsif ($realfile =~ /\.(c|dts|dtsi)$/) { $comment = '//'; - } elsif (($checklicenseline == 2) || $realfile =~ /\.(sh|pl|py|awk|tc)$/) { + } elsif (($checklicenseline == 2) || $realfile =~ /\.(sh|pl|py|awk|tc|yaml)$/) { $comment = '#'; } elsif ($realfile =~ /\.rst$/) { $comment = '..'; From a8972573eb5c784a9c199fedbfdfb694e0ca4233 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Mon, 6 Apr 2020 20:10:55 -0700 Subject: [PATCH 600/635] checkpatch: support "base-commit:" format In order to support the get-lore-mbox.py tool described in [1], I ran: git format-patch --base= --cover-letter ... which generated a "base-commit: " tag at the end of the cover letter. However, checkpatch.pl generated an error upon encounting "base-commit:" in the cover letter: "ERROR: Please use git commit description style..." ... because it found the "commit" keyword, and failed to recognize that it was part of the "base-commit" phrase, and as such, should not be subjected to the same commit description style rules. Update checkpatch.pl to include a special case for "base-commit:" (at the start of the line, possibly with some leading whitespace) so that that tag no longer generates a checkpatch error. [1] https://lwn.net/Articles/811528/ "Better tools for kernel developers" Suggested-by: Joe Perches Signed-off-by: John Hubbard Signed-off-by: Andrew Morton Acked-by: Joe Perches Cc: Andy Whitcroft Cc: Konstantin Ryabitsev Cc: Jonathan Corbet Link: http://lkml.kernel.org/r/20200213055004.69235-2-jhubbard@nvidia.com Signed-off-by: Linus Torvalds --- scripts/checkpatch.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index 977d4c0c2dc9..ede55afaadf0 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -2780,7 +2780,7 @@ sub process { # Check for git id commit length and improperly formed commit descriptions if ($in_commit_log && !$commit_log_possible_stack_dump && - $line !~ /^\s*(?:Link|Patchwork|http|https|BugLink):/i && + $line !~ /^\s*(?:Link|Patchwork|http|https|BugLink|base-commit):/i && $line !~ /^This reverts commit [0-9a-f]{7,40}/ && ($line =~ /\bcommit\s+[0-9a-f]{5,}\b/i || ($line =~ /(?:\s|^)[0-9a-f]{12,40}(?:[\s"'\(\[]|$)/i && From f36d3eb89a43047d3eba222a8132585da25cebfd Mon Sep 17 00:00:00 2001 From: Joe Perches Date: Mon, 6 Apr 2020 20:10:58 -0700 Subject: [PATCH 601/635] checkpatch: prefer fallthrough; over fallthrough comments commit 294f69e662d1 ("compiler_attributes.h: Add 'fallthrough' pseudo keyword for switch/case use") added the pseudo keyword so add a test for it to checkpatch. Warn on a patch or use --strict for files. Signed-off-by: Joe Perches Signed-off-by: Andrew Morton Link: http://lkml.kernel.org/r/8b6c1b9031ab9f3cdebada06b8d46467f1492d68.camel@perches.com Signed-off-by: Linus Torvalds --- scripts/checkpatch.pl | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index ede55afaadf0..2b32816f5c2e 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -2294,6 +2294,19 @@ sub pos_last_openparen { return length(expand_tabs(substr($line, 0, $last_openparen))) + 1; } +sub get_raw_comment { + my ($line, $rawline) = @_; + my $comment = ''; + + for my $i (0 .. (length($line) - 1)) { + if (substr($line, $i, 1) eq "$;") { + $comment .= substr($rawline, $i, 1); + } + } + + return $comment; +} + sub process { my $filename = shift; @@ -2455,6 +2468,7 @@ sub process { $sline =~ s/$;/ /g; #with comments as spaces my $rawline = $rawlines[$linenr - 1]; + my $raw_comment = get_raw_comment($line, $rawline); # check if it's a mode change, rename or start of a patch if (!$in_commit_log && @@ -6408,6 +6422,28 @@ sub process { } } +# check for /* fallthrough */ like comment, prefer fallthrough; + my @fallthroughs = ( + 'fallthrough', + '@fallthrough@', + 'lint -fallthrough[ \t]*', + 'intentional(?:ly)?[ \t]*fall(?:(?:s | |-)[Tt]|t)hr(?:ough|u|ew)', + '(?:else,?\s*)?FALL(?:S | |-)?THR(?:OUGH|U|EW)[ \t.!]*(?:-[^\n\r]*)?', + 'Fall(?:(?:s | |-)[Tt]|t)hr(?:ough|u|ew)[ \t.!]*(?:-[^\n\r]*)?', + 'fall(?:s | |-)?thr(?:ough|u|ew)[ \t.!]*(?:-[^\n\r]*)?', + ); + if ($raw_comment ne '') { + foreach my $ft (@fallthroughs) { + if ($raw_comment =~ /$ft/) { + my $msg_level = \&WARN; + $msg_level = \&CHK if ($file); + &{$msg_level}("PREFER_FALLTHROUGH", + "Prefer 'fallthrough;' over fallthrough comment\n" . $herecurr); + last; + } + } + } + # check for switch/default statements without a break; if ($perl_version_ok && defined $stat && From 342d3d2f136837cbe9aaee368dbea39865b9c0f4 Mon Sep 17 00:00:00 2001 From: Antonio Borneo Date: Mon, 6 Apr 2020 20:11:01 -0700 Subject: [PATCH 602/635] checkpatch: fix minor typo and mixed space+tab in indentation Fix spelling of "concatenation". Don't use tab after space in indentation. Signed-off-by: Antonio Borneo Signed-off-by: Andrew Morton Cc: Joe Perches Link: http://lkml.kernel.org/r/20200122163852.124417-2-borneo.antonio@gmail.com Signed-off-by: Linus Torvalds --- scripts/checkpatch.pl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index 2b32816f5c2e..879ccf33b441 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -4615,7 +4615,7 @@ sub process { ($op eq '>' && $ca =~ /<\S+\@\S+$/)) { - $ok = 1; + $ok = 1; } # for asm volatile statements @@ -4950,7 +4950,7 @@ sub process { # conditional. substr($s, 0, length($c), ''); $s =~ s/\n.*//g; - $s =~ s/$;//g; # Remove any comments + $s =~ s/$;//g; # Remove any comments if (length($c) && $s !~ /^\s*{?\s*\\*\s*$/ && $c !~ /}\s*while\s*/) { @@ -4989,7 +4989,7 @@ sub process { # if and else should not have general statements after it if ($line =~ /^.\s*(?:}\s*)?else\b(.*)/) { my $s = $1; - $s =~ s/$;//g; # Remove any comments + $s =~ s/$;//g; # Remove any comments if ($s !~ /^\s*(?:\sif|(?:{|)\s*\\?\s*$)/) { ERROR("TRAILING_STATEMENTS", "trailing statements should be on next line\n" . $herecurr); @@ -5165,7 +5165,7 @@ sub process { { } - # Flatten any obvious string concatentation. + # Flatten any obvious string concatenation. while ($dstat =~ s/($String)\s*$Ident/$1/ || $dstat =~ s/$Ident\s*($String)/$1/) { From 7b18496cbc9af07f5dfbb1ce56ea839344061b54 Mon Sep 17 00:00:00 2001 From: Antonio Borneo Date: Mon, 6 Apr 2020 20:11:04 -0700 Subject: [PATCH 603/635] checkpatch: fix multiple const * types Commit 1574a29f8e76 ("checkpatch: allow multiple const * types") claims to support repetition of pattern "const *", but it actually allows only one extra instance. Check the following lines int a(char const * const x[]); int b(char const * const *x); int c(char const * const * const x[]); int d(char const * const * const *x); with command ./scripts/checkpatch.pl --show-types -f filename to find that only the first line passes the test, while a warning is triggered by the other 3 lines: WARNING:FUNCTION_ARGUMENTS: function definition argument 'char const * const' should also have an identifier name The reason is that the pattern match halts at the second asterisk in the line, thus the remaining text starting with asterisk fails to match a valid name for a variable. Fixed by replacing "?" (Match 1 or 0 times) with "{0,4}" (Match no more than 4 times) in the regular expression. Fix also the similar test for types in unusual order. Fixes: 1574a29f8e76 ("checkpatch: allow multiple const * types") Signed-off-by: Antonio Borneo Signed-off-by: Andrew Morton Cc: Joe Perches Link: http://lkml.kernel.org/r/20200122163852.124417-1-borneo.antonio@gmail.com Signed-off-by: Linus Torvalds --- scripts/checkpatch.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index 879ccf33b441..0d2b95036ea5 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -804,12 +804,12 @@ sub build_types { }x; $Type = qr{ $NonptrType - (?:(?:\s|\*|\[\])+\s*const|(?:\s|\*\s*(?:const\s*)?|\[\])+|(?:\s*\[\s*\])+)? + (?:(?:\s|\*|\[\])+\s*const|(?:\s|\*\s*(?:const\s*)?|\[\])+|(?:\s*\[\s*\])+){0,4} (?:\s+$Inline|\s+$Modifier)* }x; $TypeMisordered = qr{ $NonptrTypeMisordered - (?:(?:\s|\*|\[\])+\s*const|(?:\s|\*\s*(?:const\s*)?|\[\])+|(?:\s*\[\s*\])+)? + (?:(?:\s|\*|\[\])+\s*const|(?:\s|\*\s*(?:const\s*)?|\[\])+|(?:\s*\[\s*\])+){0,4} (?:\s+$Inline|\s+$Modifier)* }x; $Declare = qr{(?:$Storage\s+(?:$Inline\s+)?)?$Type}; From 713a09de9ca9ac6277cb43d79967e7852238f998 Mon Sep 17 00:00:00 2001 From: Antonio Borneo Date: Mon, 6 Apr 2020 20:11:07 -0700 Subject: [PATCH 604/635] checkpatch: add command-line option for TAB size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Linux kernel coding style requires a size of 8 characters for both TAB and indentation, and such value is embedded as magic value allover the checkpatch script. This makes hard to reuse the script by other projects with different requirements in their coding style (e.g. OpenOCD [1] requires TAB size of 4 characters [2]). Replace the magic value 8 with a variable. Add a command-line option "--tab-size" to let the user select a TAB size value other than 8. [1] http://openocd.org/ [2] http://openocd.org/doc/doxygen/html/stylec.html#styleformat Signed-off-by: Antonio Borneo Signed-off-by: Erik Ahlén Signed-off-by: Spencer Oliver Signed-off-by: Andrew Morton Cc: Joe Perches Link: http://lkml.kernel.org/r/20200122163852.124417-3-borneo.antonio@gmail.com Signed-off-by: Linus Torvalds --- scripts/checkpatch.pl | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index 0d2b95036ea5..fd16f2239a00 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -64,6 +64,7 @@ my $color = "auto"; my $allow_c99_comments = 1; # Can be overridden by --ignore C99_COMMENT_TOLERANCE # git output parsing needs US English output, so first set backtick child process LANGUAGE my $git_command ='export LANGUAGE=en_US.UTF-8; git'; +my $tabsize = 8; sub help { my ($exitcode) = @_; @@ -98,6 +99,7 @@ Options: --show-types show the specific message type in the output --max-line-length=n set the maximum line length, if exceeded, warn --min-conf-desc-length=n set the min description length, if shorter, warn + --tab-size=n set the number of spaces for tab (default 8) --root=PATH PATH to the kernel tree root --no-summary suppress the per-file summary --mailback only produce a report in case of warnings/errors @@ -215,6 +217,7 @@ GetOptions( 'list-types!' => \$list_types, 'max-line-length=i' => \$max_line_length, 'min-conf-desc-length=i' => \$min_conf_desc_length, + 'tab-size=i' => \$tabsize, 'root=s' => \$root, 'summary!' => \$summary, 'mailback!' => \$mailback, @@ -267,6 +270,9 @@ if ($color =~ /^[01]$/) { die "Invalid color mode: $color\n"; } +# skip TAB size 1 to avoid additional checks on $tabsize - 1 +die "Invalid TAB size: $tabsize\n" if ($tabsize < 2); + sub hash_save_array_words { my ($hashRef, $arrayRef) = @_; @@ -1239,7 +1245,7 @@ sub expand_tabs { if ($c eq "\t") { $res .= ' '; $n++; - for (; ($n % 8) != 0; $n++) { + for (; ($n % $tabsize) != 0; $n++) { $res .= ' '; } next; @@ -2252,7 +2258,7 @@ sub string_find_replace { sub tabify { my ($leading) = @_; - my $source_indent = 8; + my $source_indent = $tabsize; my $max_spaces_before_tab = $source_indent - 1; my $spaces_to_tab = " " x $source_indent; @@ -3231,7 +3237,7 @@ sub process { next if ($realfile !~ /\.(h|c|pl|dtsi|dts)$/); # at the beginning of a line any tabs must come first and anything -# more than 8 must use tabs. +# more than $tabsize must use tabs. if ($rawline =~ /^\+\s* \t\s*\S/ || $rawline =~ /^\+\s* \s*/) { my $herevet = "$here\n" . cat_vet($rawline) . "\n"; @@ -3250,7 +3256,7 @@ sub process { "please, no space before tabs\n" . $herevet) && $fix) { while ($fixed[$fixlinenr] =~ - s/(^\+.*) {8,8}\t/$1\t\t/) {} + s/(^\+.*) {$tabsize,$tabsize}\t/$1\t\t/) {} while ($fixed[$fixlinenr] =~ s/(^\+.*) +\t/$1\t/) {} } @@ -3272,11 +3278,11 @@ sub process { if ($perl_version_ok && $sline =~ /^\+\t+( +)(?:$c90_Keywords\b|\{\s*$|\}\s*(?:else\b|while\b|\s*$)|$Declare\s*$Ident\s*[;=])/) { my $indent = length($1); - if ($indent % 8) { + if ($indent % $tabsize) { if (WARN("TABSTOP", "Statements should start on a tabstop\n" . $herecurr) && $fix) { - $fixed[$fixlinenr] =~ s@(^\+\t+) +@$1 . "\t" x ($indent/8)@e; + $fixed[$fixlinenr] =~ s@(^\+\t+) +@$1 . "\t" x ($indent/$tabsize)@e; } } } @@ -3294,8 +3300,8 @@ sub process { my $newindent = $2; my $goodtabindent = $oldindent . - "\t" x ($pos / 8) . - " " x ($pos % 8); + "\t" x ($pos / $tabsize) . + " " x ($pos % $tabsize); my $goodspaceindent = $oldindent . " " x $pos; if ($newindent ne $goodtabindent && @@ -3766,11 +3772,11 @@ sub process { #print "line<$line> prevline<$prevline> indent<$indent> sindent<$sindent> check<$check> continuation<$continuation> s<$s> cond_lines<$cond_lines> stat_real<$stat_real> stat<$stat>\n"; if ($check && $s ne '' && - (($sindent % 8) != 0 || + (($sindent % $tabsize) != 0 || ($sindent < $indent) || ($sindent == $indent && ($s !~ /^\s*(?:\}|\{|else\b)/)) || - ($sindent > $indent + 8))) { + ($sindent > $indent + $tabsize))) { WARN("SUSPECT_CODE_INDENT", "suspect code indent for conditional statements ($indent, $sindent)\n" . $herecurr . "$stat_real\n"); } From 44d303eb05ef1a24fec96580aa8a8d6f555e9b7e Mon Sep 17 00:00:00 2001 From: Joe Perches Date: Mon, 6 Apr 2020 20:11:10 -0700 Subject: [PATCH 605/635] checkpatch: improve Gerrit Change-Id: test The Gerrit Change-Id: entry is sometimes placed after a Signed-off-by: line. When this occurs, the Gerrit warning is not currently emitted as the first Signed-off-by: signature sets a flag to stop looking. Change the test to add a test for the --- patch separator and emit the warning before any before the --- and also before any diff file name. Signed-off-by: Joe Perches Signed-off-by: Andrew Morton Tested-by: John Stultz Link: http://lkml.kernel.org/r/2f6d5f8766fe7439a116c77ea8cc721a3f2d77a2.camel@perches.com Signed-off-by: Linus Torvalds --- scripts/checkpatch.pl | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index fd16f2239a00..64cfd617eefc 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -2335,6 +2335,7 @@ sub process { my $is_binding_patch = -1; my $in_header_lines = $file ? 0 : 1; my $in_commit_log = 0; #Scanning lines before patch + my $has_patch_separator = 0; #Found a --- line my $has_commit_log = 0; #Encountered lines before patch my $commit_log_lines = 0; #Number of commit log lines my $commit_log_possible_stack_dump = 0; @@ -2660,6 +2661,12 @@ sub process { } } +# Check for patch separator + if ($line =~ /^---$/) { + $has_patch_separator = 1; + $in_commit_log = 0; + } + # Check if MAINTAINERS is being updated. If so, there's probably no need to # emit the "does MAINTAINERS need updating?" message on file add/move/delete if ($line =~ /^\s*MAINTAINERS\s*\|/) { @@ -2759,10 +2766,10 @@ sub process { "A patch subject line should describe the change not the tool that found it\n" . $herecurr); } -# Check for unwanted Gerrit info - if ($in_commit_log && $line =~ /^\s*change-id:/i) { +# Check for Gerrit Change-Ids not in any patch context + if ($realfile eq '' && !$has_patch_separator && $line =~ /^\s*change-id:/i) { ERROR("GERRIT_CHANGE_ID", - "Remove Gerrit Change-Id's before submitting upstream.\n" . $herecurr); + "Remove Gerrit Change-Id's before submitting upstream\n" . $herecurr); } # Check if the commit log is in a possible stack dump From 50c92900214dd9a55bcecc3c53e90d072aff6560 Mon Sep 17 00:00:00 2001 From: Lubomir Rintel Date: Mon, 6 Apr 2020 20:11:13 -0700 Subject: [PATCH 606/635] checkpatch: check proper licensing of Devicetree bindings According to Devicetree maintainers (see Link: below), the Devicetree binding documents are preferrably licensed (GPL-2.0-only OR BSD-2-Clause). Let's check that. The actual check is a bit more relaxed, to allow more liberal but compatible licensing (e.g. GPL-2.0-or-later OR BSD-2-Clause). Signed-off-by: Lubomir Rintel Signed-off-by: Andrew Morton Acked-by: Joe Perches Acked-by: Laurent Pinchart Cc: Rob Herring Cc: Neil Armstrong Cc: Laurent Pinchart , Cc: Jonas Karlman , Cc: Jernej Skrabec , Cc: Mark Rutland , Cc: David Airlie Cc: Daniel Vetter , Link: https://lore.kernel.org/lkml/20200108142132.GA4830@bogus/ Link: http://lkml.kernel.org/r/20200309215153.38824-1-lkundrak@v3.sk Signed-off-by: Linus Torvalds --- scripts/checkpatch.pl | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index 64cfd617eefc..460a6b5b9d9a 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -3157,6 +3157,17 @@ sub process { WARN("SPDX_LICENSE_TAG", "'$spdx_license' is not supported in LICENSES/...\n" . $herecurr); } + if ($realfile =~ m@^Documentation/devicetree/bindings/@ && + not $spdx_license =~ /GPL-2\.0.*BSD-2-Clause/) { + my $msg_level = \&WARN; + $msg_level = \&CHK if ($file); + if (&{$msg_level}("SPDX_LICENSE_TAG", + + "DT binding documents should be licensed (GPL-2.0-only OR BSD-2-Clause)\n" . $herecurr) && + $fix) { + $fixed[$fixlinenr] =~ s/SPDX-License-Identifier: .*/SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)/; + } + } } } } From 16b7f3c89907acf947500f0c20dcff44f651d795 Mon Sep 17 00:00:00 2001 From: Joe Perches Date: Mon, 6 Apr 2020 20:11:17 -0700 Subject: [PATCH 607/635] checkpatch: avoid warning about uninitialized_var() WARNING: function definition argument 'flags' should also have an identifier name #26: FILE: drivers/tty/serial/sh-sci.c:1348: + unsigned long uninitialized_var(flags); Special-case uninitialized_var() to prevent this. Signed-off-by: Joe Perches Signed-off-by: Andrew Morton Tested-by: Andrew Morton Link: http://lkml.kernel.org/r/7db7944761b0bd88c70eb17d4b7f40fe589e14ed.camel@perches.com Signed-off-by: Linus Torvalds --- scripts/checkpatch.pl | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index 460a6b5b9d9a..d64c67b67e3c 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -4071,7 +4071,7 @@ sub process { } # check for function declarations without arguments like "int foo()" - if ($line =~ /(\b$Type\s+$Ident)\s*\(\s*\)/) { + if ($line =~ /(\b$Type\s*$Ident)\s*\(\s*\)/) { if (ERROR("FUNCTION_WITHOUT_ARGS", "Bad function definition - $1() should probably be $1(void)\n" . $herecurr) && $fix) { @@ -6287,13 +6287,17 @@ sub process { } # check for function declarations that have arguments without identifier names +# while avoiding uninitialized_var(x) if (defined $stat && - $stat =~ /^.\s*(?:extern\s+)?$Type\s*(?:$Ident|\(\s*\*\s*$Ident\s*\))\s*\(\s*([^{]+)\s*\)\s*;/s && - $1 ne "void") { - my $args = trim($1); + $stat =~ /^.\s*(?:extern\s+)?$Type\s*(?:($Ident)|\(\s*\*\s*$Ident\s*\))\s*\(\s*([^{]+)\s*\)\s*;/s && + (!defined($1) || + (defined($1) && $1 ne "uninitialized_var")) && + $2 ne "void") { + my $args = trim($2); while ($args =~ m/\s*($Type\s*(?:$Ident|\(\s*\*\s*$Ident?\s*\)\s*$balanced_parens)?)/g) { my $arg = trim($1); - if ($arg =~ /^$Type$/ && $arg !~ /enum\s+$Ident$/) { + if ($arg =~ /^$Type$/ && + $arg !~ /enum\s+$Ident$/) { WARN("FUNCTION_ARGUMENTS", "function definition argument '$arg' should also have an identifier name\n" . $herecurr); } From 282144e04b9a88b53dc16dc46e47f7d810e0b63b Mon Sep 17 00:00:00 2001 From: Roman Penyaev Date: Mon, 6 Apr 2020 20:11:20 -0700 Subject: [PATCH 608/635] kselftest: introduce new epoll test case This testcase repeats epollbug.c from the bug: https://bugzilla.kernel.org/show_bug.cgi?id=205933 What it tests? It tests the race between epoll_ctl() and epoll_wait(). New event mask passed to epoll_ctl() triggers wake up, which can be missed because of the bug described in the link. Reproduction is 100%, so easy to fix. Kudos, Max, for wonderful test case. Signed-off-by: Roman Penyaev Signed-off-by: Andrew Morton Cc: Max Neunhoeffer Cc: Jakub Kicinski Cc: Christopher Kohlhoff Cc: Davidlohr Bueso Cc: Jason Baron Cc: Jes Sorensen Link: http://lkml.kernel.org/r/20200214170211.561524-2-rpenyaev@suse.de Signed-off-by: Linus Torvalds --- .../filesystems/epoll/epoll_wakeup_test.c | 67 ++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/filesystems/epoll/epoll_wakeup_test.c b/tools/testing/selftests/filesystems/epoll/epoll_wakeup_test.c index 37a04dab56f0..11eee0b60040 100644 --- a/tools/testing/selftests/filesystems/epoll/epoll_wakeup_test.c +++ b/tools/testing/selftests/filesystems/epoll/epoll_wakeup_test.c @@ -7,13 +7,14 @@ #include #include #include +#include #include "../../kselftest_harness.h" struct epoll_mtcontext { int efd[3]; int sfd[4]; - int count; + volatile int count; pthread_t main; pthread_t waiter; @@ -3071,4 +3072,68 @@ TEST(epoll58) close(ctx.sfd[3]); } +static void *epoll59_thread(void *ctx_) +{ + struct epoll_mtcontext *ctx = ctx_; + struct epoll_event e; + int i; + + for (i = 0; i < 100000; i++) { + while (ctx->count == 0) + ; + + e.events = EPOLLIN | EPOLLERR | EPOLLET; + epoll_ctl(ctx->efd[0], EPOLL_CTL_MOD, ctx->sfd[0], &e); + ctx->count = 0; + } + + return NULL; +} + +/* + * t0 + * (p) \ + * e0 + * (et) / + * e0 + * + * Based on https://bugzilla.kernel.org/show_bug.cgi?id=205933 + */ +TEST(epoll59) +{ + pthread_t emitter; + struct pollfd pfd; + struct epoll_event e; + struct epoll_mtcontext ctx = { 0 }; + int i, ret; + + signal(SIGUSR1, signal_handler); + + ctx.efd[0] = epoll_create1(0); + ASSERT_GE(ctx.efd[0], 0); + + ctx.sfd[0] = eventfd(1, 0); + ASSERT_GE(ctx.sfd[0], 0); + + e.events = EPOLLIN | EPOLLERR | EPOLLET; + ASSERT_EQ(epoll_ctl(ctx.efd[0], EPOLL_CTL_ADD, ctx.sfd[0], &e), 0); + + ASSERT_EQ(pthread_create(&emitter, NULL, epoll59_thread, &ctx), 0); + + for (i = 0; i < 100000; i++) { + ret = epoll_wait(ctx.efd[0], &e, 1, 1000); + ASSERT_GT(ret, 0); + + while (ctx.count != 0) + ; + ctx.count = 1; + } + if (pthread_tryjoin_np(emitter, NULL) < 0) { + pthread_kill(emitter, SIGUSR1); + pthread_join(emitter, NULL); + } + close(ctx.efd[0]); + close(ctx.sfd[0]); +} + TEST_HARNESS_MAIN From efcdd350d1f8a98fa9fb5280e5af0a22e2059a26 Mon Sep 17 00:00:00 2001 From: Jason Baron Date: Mon, 6 Apr 2020 20:11:23 -0700 Subject: [PATCH 609/635] fs/epoll: make nesting accounting safe for -rt kernel Davidlohr Bueso pointed out that when CONFIG_DEBUG_LOCK_ALLOC is set ep_poll_safewake() can take several non-raw spinlocks after disabling interrupts. Since a spinlock can block in the -rt kernel, we can't take a spinlock after disabling interrupts. So let's re-work how we determine the nesting level such that it plays nicely with the -rt kernel. Let's introduce a 'nests' field in struct eventpoll that records the current nesting level during ep_poll_callback(). Then, if we nest again we can find the previous struct eventpoll that we were called from and increase our count by 1. The 'nests' field is protected by ep->poll_wait.lock. I've also moved the visited field to reduce the size of struct eventpoll from 184 bytes to 176 bytes on x86_64 for !CONFIG_DEBUG_LOCK_ALLOC, which is typical for a production config. Reported-by: Davidlohr Bueso Signed-off-by: Jason Baron Signed-off-by: Andrew Morton Reviewed-by: Davidlohr Bueso Cc: Roman Penyaev Cc: Eric Wong Cc: Al Viro Link: http://lkml.kernel.org/r/1582739816-13167-1-git-send-email-jbaron@akamai.com Signed-off-by: Linus Torvalds --- fs/eventpoll.c | 64 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/fs/eventpoll.c b/fs/eventpoll.c index eee3c92a9ebf..8c596641a72b 100644 --- a/fs/eventpoll.c +++ b/fs/eventpoll.c @@ -218,13 +218,18 @@ struct eventpoll { struct file *file; /* used to optimize loop detection check */ - int visited; struct list_head visited_list_link; + int visited; #ifdef CONFIG_NET_RX_BUSY_POLL /* used to track busy poll napi_id */ unsigned int napi_id; #endif + +#ifdef CONFIG_DEBUG_LOCK_ALLOC + /* tracks wakeup nests for lockdep validation */ + u8 nests; +#endif }; /* Wait structure used by the poll hooks */ @@ -545,30 +550,47 @@ out_unlock: */ #ifdef CONFIG_DEBUG_LOCK_ALLOC -static DEFINE_PER_CPU(int, wakeup_nest); - -static void ep_poll_safewake(wait_queue_head_t *wq) +static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi) { + struct eventpoll *ep_src; unsigned long flags; - int subclass; + u8 nests = 0; - local_irq_save(flags); - preempt_disable(); - subclass = __this_cpu_read(wakeup_nest); - spin_lock_nested(&wq->lock, subclass + 1); - __this_cpu_inc(wakeup_nest); - wake_up_locked_poll(wq, POLLIN); - __this_cpu_dec(wakeup_nest); - spin_unlock(&wq->lock); - local_irq_restore(flags); - preempt_enable(); + /* + * To set the subclass or nesting level for spin_lock_irqsave_nested() + * it might be natural to create a per-cpu nest count. However, since + * we can recurse on ep->poll_wait.lock, and a non-raw spinlock can + * schedule() in the -rt kernel, the per-cpu variable are no longer + * protected. Thus, we are introducing a per eventpoll nest field. + * If we are not being call from ep_poll_callback(), epi is NULL and + * we are at the first level of nesting, 0. Otherwise, we are being + * called from ep_poll_callback() and if a previous wakeup source is + * not an epoll file itself, we are at depth 1 since the wakeup source + * is depth 0. If the wakeup source is a previous epoll file in the + * wakeup chain then we use its nests value and record ours as + * nests + 1. The previous epoll file nests value is stable since its + * already holding its own poll_wait.lock. + */ + if (epi) { + if ((is_file_epoll(epi->ffd.file))) { + ep_src = epi->ffd.file->private_data; + nests = ep_src->nests; + } else { + nests = 1; + } + } + spin_lock_irqsave_nested(&ep->poll_wait.lock, flags, nests); + ep->nests = nests + 1; + wake_up_locked_poll(&ep->poll_wait, EPOLLIN); + ep->nests = 0; + spin_unlock_irqrestore(&ep->poll_wait.lock, flags); } #else -static void ep_poll_safewake(wait_queue_head_t *wq) +static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi) { - wake_up_poll(wq, EPOLLIN); + wake_up_poll(&ep->poll_wait, EPOLLIN); } #endif @@ -789,7 +811,7 @@ static void ep_free(struct eventpoll *ep) /* We need to release all tasks waiting for these file */ if (waitqueue_active(&ep->poll_wait)) - ep_poll_safewake(&ep->poll_wait); + ep_poll_safewake(ep, NULL); /* * We need to lock this because we could be hit by @@ -1258,7 +1280,7 @@ out_unlock: /* We have to call this outside the lock */ if (pwake) - ep_poll_safewake(&ep->poll_wait); + ep_poll_safewake(ep, epi); if (!(epi->event.events & EPOLLEXCLUSIVE)) ewake = 1; @@ -1562,7 +1584,7 @@ static int ep_insert(struct eventpoll *ep, const struct epoll_event *event, /* We have to call this outside the lock */ if (pwake) - ep_poll_safewake(&ep->poll_wait); + ep_poll_safewake(ep, NULL); return 0; @@ -1666,7 +1688,7 @@ static int ep_modify(struct eventpoll *ep, struct epitem *epi, /* We have to call this outside the lock */ if (pwake) - ep_poll_safewake(&ep->poll_wait); + ep_poll_safewake(ep, NULL); return 0; } From c69bcc932ef3568a13cf6d67398cf5e9da88e812 Mon Sep 17 00:00:00 2001 From: Alexey Dobriyan Date: Mon, 6 Apr 2020 20:11:26 -0700 Subject: [PATCH 610/635] fs/binfmt_elf.c: delete "loc" variable "loc" variable became just a wrapper for PT_INTERP ELF header after main ELF header was moved to "bprm->buf". Delete it. Signed-off-by: Alexey Dobriyan Signed-off-by: Andrew Morton Link: http://lkml.kernel.org/r/20200219184847.GA4871@avx2 Signed-off-by: Linus Torvalds --- fs/binfmt_elf.c | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index 1eb63867e266..3cb888cb2b2b 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -699,15 +699,13 @@ static int load_elf_binary(struct linux_binprm *bprm) unsigned long reloc_func_desc __maybe_unused = 0; int executable_stack = EXSTACK_DEFAULT; struct elfhdr *elf_ex = (struct elfhdr *)bprm->buf; - struct { - struct elfhdr interp_elf_ex; - } *loc; + struct elfhdr *interp_elf_ex; struct arch_elf_state arch_state = INIT_ARCH_ELF_STATE; struct mm_struct *mm; struct pt_regs *regs; - loc = kmalloc(sizeof(*loc), GFP_KERNEL); - if (!loc) { + interp_elf_ex = kmalloc(sizeof(*interp_elf_ex), GFP_KERNEL); + if (!interp_elf_ex) { retval = -ENOMEM; goto out_ret; } @@ -772,8 +770,8 @@ static int load_elf_binary(struct linux_binprm *bprm) would_dump(bprm, interpreter); /* Get the exec headers */ - retval = elf_read(interpreter, &loc->interp_elf_ex, - sizeof(loc->interp_elf_ex), 0); + retval = elf_read(interpreter, interp_elf_ex, + sizeof(*interp_elf_ex), 0); if (retval < 0) goto out_free_dentry; @@ -807,25 +805,25 @@ out_free_interp: if (interpreter) { retval = -ELIBBAD; /* Not an ELF interpreter */ - if (memcmp(loc->interp_elf_ex.e_ident, ELFMAG, SELFMAG) != 0) + if (memcmp(interp_elf_ex->e_ident, ELFMAG, SELFMAG) != 0) goto out_free_dentry; /* Verify the interpreter has a valid arch */ - if (!elf_check_arch(&loc->interp_elf_ex) || - elf_check_fdpic(&loc->interp_elf_ex)) + if (!elf_check_arch(interp_elf_ex) || + elf_check_fdpic(interp_elf_ex)) goto out_free_dentry; /* Load the interpreter program headers */ - interp_elf_phdata = load_elf_phdrs(&loc->interp_elf_ex, + interp_elf_phdata = load_elf_phdrs(interp_elf_ex, interpreter); if (!interp_elf_phdata) goto out_free_dentry; /* Pass PT_LOPROC..PT_HIPROC headers to arch code */ elf_ppnt = interp_elf_phdata; - for (i = 0; i < loc->interp_elf_ex.e_phnum; i++, elf_ppnt++) + for (i = 0; i < interp_elf_ex->e_phnum; i++, elf_ppnt++) switch (elf_ppnt->p_type) { case PT_LOPROC ... PT_HIPROC: - retval = arch_elf_pt_proc(&loc->interp_elf_ex, + retval = arch_elf_pt_proc(interp_elf_ex, elf_ppnt, interpreter, true, &arch_state); if (retval) @@ -840,7 +838,7 @@ out_free_interp: * the exec syscall. */ retval = arch_check_elf(elf_ex, - !!interpreter, &loc->interp_elf_ex, + !!interpreter, interp_elf_ex, &arch_state); if (retval) goto out_free_dentry; @@ -1056,7 +1054,7 @@ out_free_interp: } if (interpreter) { - elf_entry = load_elf_interp(&loc->interp_elf_ex, + elf_entry = load_elf_interp(interp_elf_ex, interpreter, load_bias, interp_elf_phdata); if (!IS_ERR((void *)elf_entry)) { @@ -1065,7 +1063,7 @@ out_free_interp: * adjustment */ interp_load_addr = elf_entry; - elf_entry += loc->interp_elf_ex.e_entry; + elf_entry += interp_elf_ex->e_entry; } if (BAD_ADDR(elf_entry)) { retval = IS_ERR((void *)elf_entry) ? @@ -1154,7 +1152,7 @@ out_free_interp: start_thread(regs, elf_entry, bprm->p); retval = 0; out: - kfree(loc); + kfree(interp_elf_ex); out_ret: return retval; From 0693ffebcfe5ac7b31f63ad54587007f7d96fb7b Mon Sep 17 00:00:00 2001 From: Alexey Dobriyan Date: Mon, 6 Apr 2020 20:11:29 -0700 Subject: [PATCH 611/635] fs/binfmt_elf.c: allocate less for static executable PT_INTERP ELF header can be spared if executable is static. Signed-off-by: Alexey Dobriyan Signed-off-by: Andrew Morton Link: http://lkml.kernel.org/r/20200219185012.GB4871@avx2 Signed-off-by: Linus Torvalds --- fs/binfmt_elf.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index 3cb888cb2b2b..a3a9429ef1d2 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -699,17 +699,11 @@ static int load_elf_binary(struct linux_binprm *bprm) unsigned long reloc_func_desc __maybe_unused = 0; int executable_stack = EXSTACK_DEFAULT; struct elfhdr *elf_ex = (struct elfhdr *)bprm->buf; - struct elfhdr *interp_elf_ex; + struct elfhdr *interp_elf_ex = NULL; struct arch_elf_state arch_state = INIT_ARCH_ELF_STATE; struct mm_struct *mm; struct pt_regs *regs; - interp_elf_ex = kmalloc(sizeof(*interp_elf_ex), GFP_KERNEL); - if (!interp_elf_ex) { - retval = -ENOMEM; - goto out_ret; - } - retval = -ENOEXEC; /* First of all, some simple consistency checks */ if (memcmp(elf_ex->e_ident, ELFMAG, SELFMAG) != 0) @@ -769,6 +763,12 @@ static int load_elf_binary(struct linux_binprm *bprm) */ would_dump(bprm, interpreter); + interp_elf_ex = kmalloc(sizeof(*interp_elf_ex), GFP_KERNEL); + if (!interp_elf_ex) { + retval = -ENOMEM; + goto out_free_ph; + } + /* Get the exec headers */ retval = elf_read(interpreter, interp_elf_ex, sizeof(*interp_elf_ex), 0); @@ -1074,6 +1074,8 @@ out_free_interp: allow_write_access(interpreter); fput(interpreter); + + kfree(interp_elf_ex); } else { elf_entry = e_entry; if (BAD_ADDR(elf_entry)) { @@ -1152,12 +1154,11 @@ out_free_interp: start_thread(regs, elf_entry, bprm->p); retval = 0; out: - kfree(interp_elf_ex); -out_ret: return retval; /* error cleanup */ out_free_dentry: + kfree(interp_elf_ex); kfree(interp_elf_phdata); allow_write_access(interpreter); if (interpreter) From aa0d1564b10f9165f913229d4428bdeea4e0d945 Mon Sep 17 00:00:00 2001 From: Alexey Dobriyan Date: Mon, 6 Apr 2020 20:11:32 -0700 Subject: [PATCH 612/635] fs/binfmt_elf.c: don't free interpreter's ELF pheaders on common path Static executables don't need to free NULL pointer. It doesn't matter really because static executable is not common scenario but do it anyway out of pedantry. Signed-off-by: Alexey Dobriyan Signed-off-by: Andrew Morton Link: http://lkml.kernel.org/r/20200219185330.GA4933@avx2 Signed-off-by: Linus Torvalds --- fs/binfmt_elf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index a3a9429ef1d2..13f25e241ac4 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -1076,6 +1076,7 @@ out_free_interp: fput(interpreter); kfree(interp_elf_ex); + kfree(interp_elf_phdata); } else { elf_entry = e_entry; if (BAD_ADDR(elf_entry)) { @@ -1084,7 +1085,6 @@ out_free_interp: } } - kfree(interp_elf_phdata); kfree(elf_phdata); set_binfmt(&elf_format); From 4800314e19d976d88c87e99bcd37d7635eb57f78 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Mon, 6 Apr 2020 20:11:36 -0700 Subject: [PATCH 613/635] samples/hw_breakpoint: drop HW_BREAKPOINT_R when reporting writes Patch series "Unexport kallsyms_lookup_name() and kallsyms_on_each_symbol()". Despite having just a single modular in-tree user that I could spot, kallsyms_lookup_name() is exported to modules and provides a mechanism for out-of-tree modules to access and invoke arbitrary, non-exported kernel symbols when kallsyms is enabled. This patch series fixes up that one user and unexports the symbol along with kallsyms_on_each_symbol(), since that could also be abused in a similar manner. I would like to avoid out-of-tree modules being easily able to call functions that are not exported. kallsyms_lookup_name() makes this trivial to the point that there is very little incentive to rework these modules to either use upstream interfaces correctly or propose functionality which may be otherwise missing upstream. Both of these latter solutions would be pre-requisites to upstreaming these modules, and the current state of things actively discourages that approach. The background here is that we are aiming for Android devices to be able to use a generic binary kernel image closely following upstream, with any vendor extensions coming in as kernel modules. In this case, we (Google) end up maintaining the binary module ABI within the scope of a single LTS kernel. Monitoring and managing the ABI surface is not feasible if it effectively includes all data and functions via kallsyms_lookup_name(). Of course, we could just carry this patch in the Android kernel tree, but we're aiming to carry as little as possible (ideally nothing) and I think it's a sensible change in its own right. I'm surprised you object to it, in all honesty. Now, you could turn around and say "that's not upstream's problem", but it still seems highly undesirable to me to have an upstream bypass for exported symbols that isn't even used by upstream modules. It's ripe for abuse and encourages people to work outside of the upstream tree. The usual rule is that we don't export symbols without a user in the tree and that seems especially relevant in this case. Joe Lawrence said: : FWIW, kallsyms was historically used by the out-of-tree kpatch support : module to resolve external symbols as well as call set_memory_r{w,o}() : API. All of that support code has been merged upstream, so modern kpatch : modules* no longer leverage kallsyms by default. : : That said, there are still some users who still use the deprecated support : module with newer kernels, but that is not officially supported by the : project. This patch (of 3): Given the name of a kernel symbol, the 'data_breakpoint' test claims to "report any write operations on the kernel symbol". However, it creates the breakpoint using both HW_BREAKPOINT_W and HW_BREAKPOINT_R, which menas it also fires for read access. Drop HW_BREAKPOINT_R from the breakpoint attributes. Signed-off-by: Will Deacon Signed-off-by: Andrew Morton Reviewed-by: Christoph Hellwig Reviewed-by: Masami Hiramatsu Reviewed-by: Quentin Perret Cc: K.Prasad Cc: Thomas Gleixner Cc: Greg Kroah-Hartman Cc: Frederic Weisbecker Cc: Alexei Starovoitov Cc: Miroslav Benes Cc: Petr Mladek Cc: Joe Lawrence Link: http://lkml.kernel.org/r/20200221114404.14641-2-will@kernel.org Signed-off-by: Linus Torvalds --- samples/hw_breakpoint/data_breakpoint.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/hw_breakpoint/data_breakpoint.c b/samples/hw_breakpoint/data_breakpoint.c index c58504774118..469b36f93696 100644 --- a/samples/hw_breakpoint/data_breakpoint.c +++ b/samples/hw_breakpoint/data_breakpoint.c @@ -45,7 +45,7 @@ static int __init hw_break_module_init(void) hw_breakpoint_init(&attr); attr.bp_addr = kallsyms_lookup_name(ksym_name); attr.bp_len = HW_BREAKPOINT_LEN_4; - attr.bp_type = HW_BREAKPOINT_W | HW_BREAKPOINT_R; + attr.bp_type = HW_BREAKPOINT_W; sample_hbp = register_wide_hw_breakpoint(&attr, sample_hbp_handler, NULL); if (IS_ERR((void __force *)sample_hbp)) { From d8a84d33a4954b85c53faf77be48b5c2b559692e Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Mon, 6 Apr 2020 20:11:39 -0700 Subject: [PATCH 614/635] samples/hw_breakpoint: drop use of kallsyms_lookup_name() The 'data_breakpoint' test code is the only modular user of kallsyms_lookup_name(), which was exported as part of fixing the test in f60d24d2ad04 ("hw-breakpoints: Fix broken hw-breakpoint sample module"). In preparation for un-exporting this symbol, switch the test over to using __symbol_get(), which can be used to place breakpoints on exported symbols. Signed-off-by: Will Deacon Signed-off-by: Andrew Morton Reviewed-by: Christoph Hellwig Reviewed-by: Masami Hiramatsu Reviewed-by: Quentin Perret Cc: K.Prasad Cc: Thomas Gleixner Cc: Greg Kroah-Hartman Cc: Frederic Weisbecker Cc: Alexei Starovoitov Cc: Miroslav Benes Cc: Petr Mladek Cc: Joe Lawrence Link: http://lkml.kernel.org/r/20200221114404.14641-3-will@kernel.org Signed-off-by: Linus Torvalds --- samples/hw_breakpoint/data_breakpoint.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/samples/hw_breakpoint/data_breakpoint.c b/samples/hw_breakpoint/data_breakpoint.c index 469b36f93696..418c46fe5ffc 100644 --- a/samples/hw_breakpoint/data_breakpoint.c +++ b/samples/hw_breakpoint/data_breakpoint.c @@ -23,7 +23,7 @@ struct perf_event * __percpu *sample_hbp; -static char ksym_name[KSYM_NAME_LEN] = "pid_max"; +static char ksym_name[KSYM_NAME_LEN] = "jiffies"; module_param_string(ksym, ksym_name, KSYM_NAME_LEN, S_IRUGO); MODULE_PARM_DESC(ksym, "Kernel symbol to monitor; this module will report any" " write operations on the kernel symbol"); @@ -41,9 +41,13 @@ static int __init hw_break_module_init(void) { int ret; struct perf_event_attr attr; + void *addr = __symbol_get(ksym_name); + + if (!addr) + return -ENXIO; hw_breakpoint_init(&attr); - attr.bp_addr = kallsyms_lookup_name(ksym_name); + attr.bp_addr = (unsigned long)addr; attr.bp_len = HW_BREAKPOINT_LEN_4; attr.bp_type = HW_BREAKPOINT_W; @@ -66,6 +70,7 @@ fail: static void __exit hw_break_module_exit(void) { unregister_wide_hw_breakpoint(sample_hbp); + symbol_put(ksym_name); printk(KERN_INFO "HW Breakpoint for %s write uninstalled\n", ksym_name); } From 0bd476e6c67190b5eb7b6e105c8db8ff61103281 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Mon, 6 Apr 2020 20:11:43 -0700 Subject: [PATCH 615/635] kallsyms: unexport kallsyms_lookup_name() and kallsyms_on_each_symbol() kallsyms_lookup_name() and kallsyms_on_each_symbol() are exported to modules despite having no in-tree users and being wide open to abuse by out-of-tree modules that can use them as a method to invoke arbitrary non-exported kernel functions. Unexport kallsyms_lookup_name() and kallsyms_on_each_symbol(). Signed-off-by: Will Deacon Signed-off-by: Andrew Morton Reviewed-by: Greg Kroah-Hartman Reviewed-by: Christoph Hellwig Reviewed-by: Masami Hiramatsu Reviewed-by: Quentin Perret Acked-by: Alexei Starovoitov Cc: Thomas Gleixner Cc: Frederic Weisbecker Cc: K.Prasad Cc: Miroslav Benes Cc: Petr Mladek Cc: Joe Lawrence Cc: Mathieu Desnoyers Link: http://lkml.kernel.org/r/20200221114404.14641-4-will@kernel.org Signed-off-by: Linus Torvalds --- kernel/kallsyms.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c index a9b3f660dee7..16c8c605f4b0 100644 --- a/kernel/kallsyms.c +++ b/kernel/kallsyms.c @@ -175,7 +175,6 @@ unsigned long kallsyms_lookup_name(const char *name) } return module_kallsyms_lookup_name(name); } -EXPORT_SYMBOL_GPL(kallsyms_lookup_name); int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, unsigned long), @@ -194,7 +193,6 @@ int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, } return module_kallsyms_on_each_symbol(fn, data); } -EXPORT_SYMBOL_GPL(kallsyms_on_each_symbol); static unsigned long get_symbol_pos(unsigned long addr, unsigned long *symbolsize, From 5404e7e0ac0cd03b83dd2476155254ee1e50529f Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Mon, 6 Apr 2020 20:11:46 -0700 Subject: [PATCH 616/635] reiserfs: clean up several indentation issues There are several places where code is indented incorrectly. Fix these. Signed-off-by: Colin Ian King Signed-off-by: Andrew Morton Link: http://lkml.kernel.org/r/20200325135018.113431-1-colin.king@canonical.com Signed-off-by: Linus Torvalds --- fs/reiserfs/do_balan.c | 2 +- fs/reiserfs/ioctl.c | 11 ++++++----- fs/reiserfs/namei.c | 10 +++++----- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/fs/reiserfs/do_balan.c b/fs/reiserfs/do_balan.c index 4075e41408b4..5129efc6f2e6 100644 --- a/fs/reiserfs/do_balan.c +++ b/fs/reiserfs/do_balan.c @@ -842,7 +842,7 @@ static void balance_leaf_paste_right_whole(struct tree_balance *tb, struct item_head *pasted; struct buffer_info bi; - buffer_info_init_right(tb, &bi); + buffer_info_init_right(tb, &bi); leaf_shift_right(tb, tb->rnum[0], tb->rbytes); /* append item in R[0] */ diff --git a/fs/reiserfs/ioctl.c b/fs/reiserfs/ioctl.c index 45e1a5d11af3..adb21bea3d60 100644 --- a/fs/reiserfs/ioctl.c +++ b/fs/reiserfs/ioctl.c @@ -184,11 +184,12 @@ int reiserfs_unpack(struct inode *inode, struct file *filp) } /* we need to make sure nobody is changing the file size beneath us */ -{ - int depth = reiserfs_write_unlock_nested(inode->i_sb); - inode_lock(inode); - reiserfs_write_lock_nested(inode->i_sb, depth); -} + { + int depth = reiserfs_write_unlock_nested(inode->i_sb); + + inode_lock(inode); + reiserfs_write_lock_nested(inode->i_sb, depth); + } reiserfs_write_lock(inode->i_sb); diff --git a/fs/reiserfs/namei.c b/fs/reiserfs/namei.c index 959a066b7bb0..1594687582f0 100644 --- a/fs/reiserfs/namei.c +++ b/fs/reiserfs/namei.c @@ -838,10 +838,10 @@ static int reiserfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode */ INC_DIR_INODE_NLINK(dir) - retval = reiserfs_new_inode(&th, dir, mode, NULL /*symlink */ , - old_format_only(dir->i_sb) ? - EMPTY_DIR_SIZE_V1 : EMPTY_DIR_SIZE, - dentry, inode, &security); + retval = reiserfs_new_inode(&th, dir, mode, NULL /*symlink */, + old_format_only(dir->i_sb) ? + EMPTY_DIR_SIZE_V1 : EMPTY_DIR_SIZE, + dentry, inode, &security); if (retval) { DEC_DIR_INODE_NLINK(dir) goto out_failed; @@ -967,7 +967,7 @@ static int reiserfs_rmdir(struct inode *dir, struct dentry *dentry) reiserfs_update_sd(&th, inode); DEC_DIR_INODE_NLINK(dir) - dir->i_size -= (DEH_SIZE + de.de_entrylen); + dir->i_size -= (DEH_SIZE + de.de_entrylen); reiserfs_update_sd(&th, dir); /* prevent empty directory from getting lost */ From 06d4f8152a01034b36591885321da54eb77398c1 Mon Sep 17 00:00:00 2001 From: Qiujun Huang Date: Mon, 6 Apr 2020 20:11:49 -0700 Subject: [PATCH 617/635] kernel/kmod.c: fix a typo "assuems" -> "assumes" There is a typo in comment. Fix it. s/assuems/assumes/ Signed-off-by: Qiujun Huang Signed-off-by: Andrew Morton Acked-by: Luis Chamberlain Link: http://lkml.kernel.org/r/1585891029-6450-1-git-send-email-hqjagain@gmail.com Signed-off-by: Linus Torvalds --- kernel/kmod.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/kmod.c b/kernel/kmod.c index bc6addd9152b..8b2b311afa95 100644 --- a/kernel/kmod.c +++ b/kernel/kmod.c @@ -35,7 +35,7 @@ * (u64) THREAD_SIZE * 8UL); * * If you need less than 50 threads would mean we're dealing with systems - * smaller than 3200 pages. This assuems you are capable of having ~13M memory, + * smaller than 3200 pages. This assumes you are capable of having ~13M memory, * and this would only be an be an upper limit, after which the OOM killer * would take effect. Systems like these are very unlikely if modules are * enabled. From fba4168edecdd2781bcd83cb131977ec1157f87c Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Mon, 6 Apr 2020 20:11:52 -0700 Subject: [PATCH 618/635] gcov: gcc_4_7: replace zero-length array with flexible-array member The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertently introduced[3] to the codebase from now on. Also, notice that, dynamic memory allocations won't be affected by this change: "Flexible array members have incomplete type, and so the sizeof operator may not be applied. As a quirk of the original implementation of zero-length arrays, sizeof evaluates to zero."[1] This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Signed-off-by: Andrew Morton Acked-by: Peter Oberparleiter Link: http://lkml.kernel.org/r/20200213152241.GA877@embeddedor Signed-off-by: Linus Torvalds --- kernel/gcov/gcc_4_7.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/gcov/gcc_4_7.c b/kernel/gcov/gcc_4_7.c index ec37563674d6..908fdf5098c3 100644 --- a/kernel/gcov/gcc_4_7.c +++ b/kernel/gcov/gcc_4_7.c @@ -68,7 +68,7 @@ struct gcov_fn_info { unsigned int ident; unsigned int lineno_checksum; unsigned int cfg_checksum; - struct gcov_ctr_info ctrs[0]; + struct gcov_ctr_info ctrs[]; }; /** From 7ff87182d156e90b59455aac8503da512a1c5dba Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Mon, 6 Apr 2020 20:11:55 -0700 Subject: [PATCH 619/635] gcov: gcc_3_4: replace zero-length array with flexible-array member The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertently introduced[3] to the codebase from now on. Also, notice that, dynamic memory allocations won't be affected by this change: "Flexible array members have incomplete type, and so the sizeof operator may not be applied. As a quirk of the original implementation of zero-length arrays, sizeof evaluates to zero."[1] This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Signed-off-by: Andrew Morton Cc: Peter Oberparleiter Link: http://lkml.kernel.org/r/20200302224501.GA14175@embeddedor Signed-off-by: Linus Torvalds --- kernel/gcov/gcc_3_4.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/gcov/gcc_3_4.c b/kernel/gcov/gcc_3_4.c index 801ee4b0b969..acb83558e5df 100644 --- a/kernel/gcov/gcc_3_4.c +++ b/kernel/gcov/gcc_3_4.c @@ -38,7 +38,7 @@ static struct gcov_info *gcov_info_head; struct gcov_fn_info { unsigned int ident; unsigned int checksum; - unsigned int n_ctrs[0]; + unsigned int n_ctrs[]; }; /** @@ -78,7 +78,7 @@ struct gcov_info { unsigned int n_functions; const struct gcov_fn_info *functions; unsigned int ctr_mask; - struct gcov_ctr_info counts[0]; + struct gcov_ctr_info counts[]; }; /** @@ -352,7 +352,7 @@ struct gcov_iterator { unsigned int count; int num_types; - struct type_info type_info[0]; + struct type_info type_info[]; }; static struct gcov_fn_info *get_func(struct gcov_iterator *iter) From 6524d79413f4ffd28480f541ef68f9ea199b2e0c Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Mon, 6 Apr 2020 20:11:58 -0700 Subject: [PATCH 620/635] kernel/gcov/fs.c: replace zero-length array with flexible-array member The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertently introduced[3] to the codebase from now on. Also, notice that, dynamic memory allocations won't be affected by this change: "Flexible array members have incomplete type, and so the sizeof operator may not be applied. As a quirk of the original implementation of zero-length arrays, sizeof evaluates to zero."[1] This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Signed-off-by: Andrew Morton Cc: Peter Oberparleiter Link: http://lkml.kernel.org/r/20200302224851.GA26467@embeddedor Signed-off-by: Linus Torvalds --- kernel/gcov/fs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/gcov/fs.c b/kernel/gcov/fs.c index e5eb5ea7ea59..5e891c3c2d93 100644 --- a/kernel/gcov/fs.c +++ b/kernel/gcov/fs.c @@ -58,7 +58,7 @@ struct gcov_node { struct dentry *dentry; struct dentry **links; int num_loaded; - char name[0]; + char name[]; }; static const char objtree[] = OBJTREE; From 7baf219982281034f8c3a806869873bf098fb611 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Mon, 6 Apr 2020 20:12:02 -0700 Subject: [PATCH 621/635] init/Kconfig: clean up ANON_INODES and old IO schedulers options CONFIG_ANON_INODES is gone since commit 5dd50aaeb185 ("Make anon_inodes unconditional"). CONFIG_CFQ_GROUP_IOSCHED was replaced with CONFIG_BFQ_GROUP_IOSCHED in commit f382fb0bcef4 ("block: remove legacy IO schedulers"). Signed-off-by: Krzysztof Kozlowski Signed-off-by: Andrew Morton Cc: Masahiro Yamada Cc: Thomas Gleixner Cc: Greg Kroah-Hartman Link: http://lkml.kernel.org/r/20200130192419.3026-1-krzk@kernel.org Signed-off-by: Linus Torvalds --- init/Kconfig | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/init/Kconfig b/init/Kconfig index 2bc0e47689d8..574b7215aa6a 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -872,7 +872,7 @@ config BLK_CGROUP This option only enables generic Block IO controller infrastructure. One needs to also enable actual IO controlling logic/policy. For enabling proportional weight division of disk bandwidth in CFQ, set - CONFIG_CFQ_GROUP_IOSCHED=y; for enabling throttling policy, set + CONFIG_BFQ_GROUP_IOSCHED=y; for enabling throttling policy, set CONFIG_BLK_DEV_THROTTLING=y. See Documentation/admin-guide/cgroup-v1/blkio-controller.rst for more information. @@ -1538,7 +1538,6 @@ config AIO config IO_URING bool "Enable IO uring support" if EXPERT - select ANON_INODES select IO_WQ default y help From 0887a7ebc97770c7870abf3075a2e8cd502a7f52 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Mon, 6 Apr 2020 20:12:27 -0700 Subject: [PATCH 622/635] ubsan: add trap instrumentation option Patch series "ubsan: Split out bounds checker", v5. This splits out the bounds checker so it can be individually used. This is enabled in Android and hopefully for syzbot. Includes LKDTM tests for behavioral corner-cases (beyond just the bounds checker), and adjusts ubsan and kasan slightly for correct panic handling. This patch (of 6): The Undefined Behavior Sanitizer can operate in two modes: warning reporting mode via lib/ubsan.c handler calls, or trap mode, which uses __builtin_trap() as the handler. Using lib/ubsan.c means the kernel image is about 5% larger (due to all the debugging text and reporting structures to capture details about the warning conditions). Using the trap mode, the image size changes are much smaller, though at the loss of the "warning only" mode. In order to give greater flexibility to system builders that want minimal changes to image size and are prepared to deal with kernel code being aborted and potentially destabilizing the system, this introduces CONFIG_UBSAN_TRAP. The resulting image sizes comparison: text data bss dec hex filename 19533663 6183037 18554956 44271656 2a38828 vmlinux.stock 19991849 7618513 18874448 46484810 2c54d4a vmlinux.ubsan 19712181 6284181 18366540 44362902 2a4ec96 vmlinux.ubsan-trap CONFIG_UBSAN=y: image +4.8% (text +2.3%, data +18.9%) CONFIG_UBSAN_TRAP=y: image +0.2% (text +0.9%, data +1.6%) Additionally adjusts the CONFIG_UBSAN Kconfig help for clarity and removes the mention of non-existing boot param "ubsan_handle". Suggested-by: Elena Petrova Signed-off-by: Kees Cook Signed-off-by: Andrew Morton Acked-by: Dmitry Vyukov Cc: Andrey Ryabinin Cc: Andrey Konovalov Cc: Alexander Potapenko Cc: Dan Carpenter Cc: "Gustavo A. R. Silva" Cc: Arnd Bergmann Cc: Ard Biesheuvel Link: http://lkml.kernel.org/r/20200227193516.32566-2-keescook@chromium.org Signed-off-by: Linus Torvalds --- lib/Kconfig.ubsan | 22 ++++++++++++++++++---- lib/Makefile | 2 ++ scripts/Makefile.ubsan | 9 +++++++-- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/lib/Kconfig.ubsan b/lib/Kconfig.ubsan index 0e04fcb3ab3d..9deb655838b0 100644 --- a/lib/Kconfig.ubsan +++ b/lib/Kconfig.ubsan @@ -5,11 +5,25 @@ config ARCH_HAS_UBSAN_SANITIZE_ALL config UBSAN bool "Undefined behaviour sanity checker" help - This option enables undefined behaviour sanity checker + This option enables the Undefined Behaviour sanity checker. Compile-time instrumentation is used to detect various undefined - behaviours in runtime. Various types of checks may be enabled - via boot parameter ubsan_handle - (see: Documentation/dev-tools/ubsan.rst). + behaviours at runtime. For more details, see: + Documentation/dev-tools/ubsan.rst + +config UBSAN_TRAP + bool "On Sanitizer warnings, abort the running kernel code" + depends on UBSAN + depends on $(cc-option, -fsanitize-undefined-trap-on-error) + help + Building kernels with Sanitizer features enabled tends to grow + the kernel size by around 5%, due to adding all the debugging + text on failure paths. To avoid this, Sanitizer instrumentation + can just issue a trap. This reduces the kernel size overhead but + turns all warnings (including potentially harmless conditions) + into full exceptions that abort the running kernel code + (regardless of context, locks held, etc), which may destabilize + the system. For some system builders this is an acceptable + trade-off. config UBSAN_SANITIZE_ALL bool "Enable instrumentation for the entire kernel" diff --git a/lib/Makefile b/lib/Makefile index 3fc06399295d..685aee60de1d 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -286,7 +286,9 @@ quiet_cmd_build_OID_registry = GEN $@ clean-files += oid_registry_data.c obj-$(CONFIG_UCS2_STRING) += ucs2_string.o +ifneq ($(CONFIG_UBSAN_TRAP),y) obj-$(CONFIG_UBSAN) += ubsan.o +endif UBSAN_SANITIZE_ubsan.o := n KASAN_SANITIZE_ubsan.o := n diff --git a/scripts/Makefile.ubsan b/scripts/Makefile.ubsan index 019771b845c5..668a91510bfe 100644 --- a/scripts/Makefile.ubsan +++ b/scripts/Makefile.ubsan @@ -1,5 +1,10 @@ # SPDX-License-Identifier: GPL-2.0 ifdef CONFIG_UBSAN + +ifdef CONFIG_UBSAN_ALIGNMENT + CFLAGS_UBSAN += $(call cc-option, -fsanitize=alignment) +endif + CFLAGS_UBSAN += $(call cc-option, -fsanitize=shift) CFLAGS_UBSAN += $(call cc-option, -fsanitize=integer-divide-by-zero) CFLAGS_UBSAN += $(call cc-option, -fsanitize=unreachable) @@ -9,8 +14,8 @@ ifdef CONFIG_UBSAN CFLAGS_UBSAN += $(call cc-option, -fsanitize=bool) CFLAGS_UBSAN += $(call cc-option, -fsanitize=enum) -ifdef CONFIG_UBSAN_ALIGNMENT - CFLAGS_UBSAN += $(call cc-option, -fsanitize=alignment) +ifdef CONFIG_UBSAN_TRAP + CFLAGS_UBSAN += $(call cc-option, -fsanitize-undefined-trap-on-error) endif # -fsanitize=* options makes GCC less smart than usual and From 277a10850f9f4cb3429faf59293e2c89b1a320be Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Mon, 6 Apr 2020 20:12:31 -0700 Subject: [PATCH 623/635] ubsan: split "bounds" checker from other options In order to do kernel builds with the bounds checker individually available, introduce CONFIG_UBSAN_BOUNDS, with the remaining options under CONFIG_UBSAN_MISC. For example, using this, we can start to expand the coverage syzkaller is providing. Right now, all of UBSan is disabled for syzbot builds because taken as a whole, it is too noisy. This will let us focus on one feature at a time. For the bounds checker specifically, this provides a mechanism to eliminate an entire class of array overflows with close to zero performance overhead (I cannot measure a difference). In my (mostly) defconfig, enabling bounds checking adds ~4200 checks to the kernel. Performance changes are in the noise, likely due to the branch predictors optimizing for the non-fail path. Some notes on the bounds checker: - it does not instrument {mem,str}*()-family functions, it only instruments direct indexed accesses (e.g. "foo[i]"). Dealing with the {mem,str}*()-family functions is a work-in-progress around CONFIG_FORTIFY_SOURCE[1]. - it ignores flexible array members, including the very old single byte (e.g. "int foo[1];") declarations. (Note that GCC's implementation appears to ignore _all_ trailing arrays, but Clang only ignores empty, 0, and 1 byte arrays[2].) [1] https://github.com/KSPP/linux/issues/6 [2] https://gcc.gnu.org/bugzilla/show_bug.cgi?id=92589 Suggested-by: Elena Petrova Signed-off-by: Kees Cook Signed-off-by: Andrew Morton Reviewed-by: Andrey Ryabinin Acked-by: Dmitry Vyukov Cc: Alexander Potapenko Cc: Andrey Konovalov Cc: Ard Biesheuvel Cc: Arnd Bergmann Cc: Dan Carpenter Cc: "Gustavo A. R. Silva" Link: http://lkml.kernel.org/r/20200227193516.32566-3-keescook@chromium.org Signed-off-by: Linus Torvalds --- lib/Kconfig.ubsan | 29 ++++++++++++++++++++++++----- scripts/Makefile.ubsan | 7 ++++++- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/lib/Kconfig.ubsan b/lib/Kconfig.ubsan index 9deb655838b0..48469c95d78e 100644 --- a/lib/Kconfig.ubsan +++ b/lib/Kconfig.ubsan @@ -2,7 +2,7 @@ config ARCH_HAS_UBSAN_SANITIZE_ALL bool -config UBSAN +menuconfig UBSAN bool "Undefined behaviour sanity checker" help This option enables the Undefined Behaviour sanity checker. @@ -10,9 +10,10 @@ config UBSAN behaviours at runtime. For more details, see: Documentation/dev-tools/ubsan.rst +if UBSAN + config UBSAN_TRAP bool "On Sanitizer warnings, abort the running kernel code" - depends on UBSAN depends on $(cc-option, -fsanitize-undefined-trap-on-error) help Building kernels with Sanitizer features enabled tends to grow @@ -25,9 +26,26 @@ config UBSAN_TRAP the system. For some system builders this is an acceptable trade-off. +config UBSAN_BOUNDS + bool "Perform array index bounds checking" + default UBSAN + help + This option enables detection of directly indexed out of bounds + array accesses, where the array size is known at compile time. + Note that this does not protect array overflows via bad calls + to the {str,mem}*cpy() family of functions (that is addressed + by CONFIG_FORTIFY_SOURCE). + +config UBSAN_MISC + bool "Enable all other Undefined Behavior sanity checks" + default UBSAN + help + This option enables all sanity checks that don't have their + own Kconfig options. Disable this if you only want to have + individually selected checks. + config UBSAN_SANITIZE_ALL bool "Enable instrumentation for the entire kernel" - depends on UBSAN depends on ARCH_HAS_UBSAN_SANITIZE_ALL # We build with -Wno-maybe-uninitilzed, but we still want to @@ -44,7 +62,6 @@ config UBSAN_SANITIZE_ALL config UBSAN_NO_ALIGNMENT bool "Disable checking of pointers alignment" - depends on UBSAN default y if HAVE_EFFICIENT_UNALIGNED_ACCESS help This option disables the check of unaligned memory accesses. @@ -57,7 +74,9 @@ config UBSAN_ALIGNMENT config TEST_UBSAN tristate "Module for testing for undefined behavior detection" - depends on m && UBSAN + depends on m help This is a test module for UBSAN. It triggers various undefined behavior, and detect it. + +endif # if UBSAN diff --git a/scripts/Makefile.ubsan b/scripts/Makefile.ubsan index 668a91510bfe..5b15bc425ec9 100644 --- a/scripts/Makefile.ubsan +++ b/scripts/Makefile.ubsan @@ -5,14 +5,19 @@ ifdef CONFIG_UBSAN_ALIGNMENT CFLAGS_UBSAN += $(call cc-option, -fsanitize=alignment) endif +ifdef CONFIG_UBSAN_BOUNDS + CFLAGS_UBSAN += $(call cc-option, -fsanitize=bounds) +endif + +ifdef CONFIG_UBSAN_MISC CFLAGS_UBSAN += $(call cc-option, -fsanitize=shift) CFLAGS_UBSAN += $(call cc-option, -fsanitize=integer-divide-by-zero) CFLAGS_UBSAN += $(call cc-option, -fsanitize=unreachable) CFLAGS_UBSAN += $(call cc-option, -fsanitize=signed-integer-overflow) - CFLAGS_UBSAN += $(call cc-option, -fsanitize=bounds) CFLAGS_UBSAN += $(call cc-option, -fsanitize=object-size) CFLAGS_UBSAN += $(call cc-option, -fsanitize=bool) CFLAGS_UBSAN += $(call cc-option, -fsanitize=enum) +endif ifdef CONFIG_UBSAN_TRAP CFLAGS_UBSAN += $(call cc-option, -fsanitize-undefined-trap-on-error) From ae2e1aad3e48e495878d9f149e437a308bfdaefa Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Mon, 6 Apr 2020 20:12:34 -0700 Subject: [PATCH 624/635] drivers/misc/lkdtm/bugs.c: add arithmetic overflow and array bounds checks Adds LKDTM tests for arithmetic overflow (both signed and unsigned), as well as array bounds checking. Signed-off-by: Kees Cook Signed-off-by: Andrew Morton Acked-by: Dmitry Vyukov Cc: Alexander Potapenko Cc: Andrey Konovalov Cc: Andrey Ryabinin Cc: Ard Biesheuvel Cc: Arnd Bergmann Cc: Dan Carpenter Cc: Elena Petrova Cc: "Gustavo A. R. Silva" Link: http://lkml.kernel.org/r/20200227193516.32566-4-keescook@chromium.org Signed-off-by: Linus Torvalds --- drivers/misc/lkdtm/bugs.c | 75 ++++++++++++++++++++++++++++++++++++++ drivers/misc/lkdtm/core.c | 3 ++ drivers/misc/lkdtm/lkdtm.h | 3 ++ 3 files changed, 81 insertions(+) diff --git a/drivers/misc/lkdtm/bugs.c b/drivers/misc/lkdtm/bugs.c index cc92bc3ed820..886459e0ddd9 100644 --- a/drivers/misc/lkdtm/bugs.c +++ b/drivers/misc/lkdtm/bugs.c @@ -11,6 +11,7 @@ #include #include #include +#include #ifdef CONFIG_X86_32 #include @@ -175,6 +176,80 @@ void lkdtm_HUNG_TASK(void) schedule(); } +volatile unsigned int huge = INT_MAX - 2; +volatile unsigned int ignored; + +void lkdtm_OVERFLOW_SIGNED(void) +{ + int value; + + value = huge; + pr_info("Normal signed addition ...\n"); + value += 1; + ignored = value; + + pr_info("Overflowing signed addition ...\n"); + value += 4; + ignored = value; +} + + +void lkdtm_OVERFLOW_UNSIGNED(void) +{ + unsigned int value; + + value = huge; + pr_info("Normal unsigned addition ...\n"); + value += 1; + ignored = value; + + pr_info("Overflowing unsigned addition ...\n"); + value += 4; + ignored = value; +} + +/* Intentially using old-style flex array definition of 1 byte. */ +struct array_bounds_flex_array { + int one; + int two; + char data[1]; +}; + +struct array_bounds { + int one; + int two; + char data[8]; + int three; +}; + +void lkdtm_ARRAY_BOUNDS(void) +{ + struct array_bounds_flex_array *not_checked; + struct array_bounds *checked; + volatile int i; + + not_checked = kmalloc(sizeof(*not_checked) * 2, GFP_KERNEL); + checked = kmalloc(sizeof(*checked) * 2, GFP_KERNEL); + + pr_info("Array access within bounds ...\n"); + /* For both, touch all bytes in the actual member size. */ + for (i = 0; i < sizeof(checked->data); i++) + checked->data[i] = 'A'; + /* + * For the uninstrumented flex array member, also touch 1 byte + * beyond to verify it is correctly uninstrumented. + */ + for (i = 0; i < sizeof(not_checked->data) + 1; i++) + not_checked->data[i] = 'A'; + + pr_info("Array access beyond bounds ...\n"); + for (i = 0; i < sizeof(checked->data) + 1; i++) + checked->data[i] = 'B'; + + kfree(not_checked); + kfree(checked); +} + void lkdtm_CORRUPT_LIST_ADD(void) { /* diff --git a/drivers/misc/lkdtm/core.c b/drivers/misc/lkdtm/core.c index 5ce4ac8c06fc..a5e344df9166 100644 --- a/drivers/misc/lkdtm/core.c +++ b/drivers/misc/lkdtm/core.c @@ -130,6 +130,9 @@ static const struct crashtype crashtypes[] = { CRASHTYPE(HARDLOCKUP), CRASHTYPE(SPINLOCKUP), CRASHTYPE(HUNG_TASK), + CRASHTYPE(OVERFLOW_SIGNED), + CRASHTYPE(OVERFLOW_UNSIGNED), + CRASHTYPE(ARRAY_BOUNDS), CRASHTYPE(EXEC_DATA), CRASHTYPE(EXEC_STACK), CRASHTYPE(EXEC_KMALLOC), diff --git a/drivers/misc/lkdtm/lkdtm.h b/drivers/misc/lkdtm/lkdtm.h index 8d13d0176624..601a2156a0d4 100644 --- a/drivers/misc/lkdtm/lkdtm.h +++ b/drivers/misc/lkdtm/lkdtm.h @@ -22,6 +22,9 @@ void lkdtm_SOFTLOCKUP(void); void lkdtm_HARDLOCKUP(void); void lkdtm_SPINLOCKUP(void); void lkdtm_HUNG_TASK(void); +void lkdtm_OVERFLOW_SIGNED(void); +void lkdtm_OVERFLOW_UNSIGNED(void); +void lkdtm_ARRAY_BOUNDS(void); void lkdtm_CORRUPT_LIST_ADD(void); void lkdtm_CORRUPT_LIST_DEL(void); void lkdtm_CORRUPT_USER_DS(void); From 1d28c8d6d076efdec762026f7192c7cf251c66be Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Mon, 6 Apr 2020 20:12:38 -0700 Subject: [PATCH 625/635] ubsan: check panic_on_warn Syzkaller expects kernel warnings to panic when the panic_on_warn sysctl is set. More work is needed here to have UBSan reuse the WARN infrastructure, but for now, just check the flag manually. Signed-off-by: Kees Cook Signed-off-by: Andrew Morton Cc: Alexander Potapenko Cc: Andrey Konovalov Cc: Andrey Ryabinin Cc: Ard Biesheuvel Cc: Arnd Bergmann Cc: Dan Carpenter Cc: Dmitry Vyukov Cc: Elena Petrova Cc: "Gustavo A. R. Silva" Link: https://lore.kernel.org/lkml/CACT4Y+bsLJ-wFx_TaXqax3JByUOWB3uk787LsyMVcfW6JzzGvg@mail.gmail.com Link: http://lkml.kernel.org/r/20200227193516.32566-5-keescook@chromium.org Signed-off-by: Linus Torvalds --- lib/ubsan.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/ubsan.c b/lib/ubsan.c index 7b9b58aee72c..429663eef6a7 100644 --- a/lib/ubsan.c +++ b/lib/ubsan.c @@ -156,6 +156,17 @@ static void ubsan_epilogue(void) "========================================\n"); current->in_ubsan--; + + if (panic_on_warn) { + /* + * This thread may hit another WARN() in the panic path. + * Resetting this prevents additional WARN() from panicking the + * system on this thread. Other threads are blocked by the + * panic_mutex in panic(). + */ + panic_on_warn = 0; + panic("panic_on_warn set ...\n"); + } } static void handle_overflow(struct overflow_data *data, void *lhs, From 1d2252fab9a1c9206d1c981c0c15d5e01e0f5729 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Mon, 6 Apr 2020 20:12:42 -0700 Subject: [PATCH 626/635] kasan: unset panic_on_warn before calling panic() As done in the full WARN() handler, panic_on_warn needs to be cleared before calling panic() to avoid recursive panics. Signed-off-by: Kees Cook Signed-off-by: Andrew Morton Acked-by: Dmitry Vyukov Cc: Alexander Potapenko Cc: Andrey Konovalov Cc: Andrey Ryabinin Cc: Ard Biesheuvel Cc: Arnd Bergmann Cc: Dan Carpenter Cc: Elena Petrova Cc: "Gustavo A. R. Silva" Link: http://lkml.kernel.org/r/20200227193516.32566-6-keescook@chromium.org Signed-off-by: Linus Torvalds --- mm/kasan/report.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mm/kasan/report.c b/mm/kasan/report.c index cf5c17d5e361..80f23c9da6b0 100644 --- a/mm/kasan/report.c +++ b/mm/kasan/report.c @@ -92,8 +92,16 @@ static void end_report(unsigned long *flags) pr_err("==================================================================\n"); add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); spin_unlock_irqrestore(&report_lock, *flags); - if (panic_on_warn) + if (panic_on_warn) { + /* + * This thread may hit another WARN() in the panic path. + * Resetting this prevents additional WARN() from panicking the + * system on this thread. Other threads are blocked by the + * panic_mutex in panic(). + */ + panic_on_warn = 0; panic("panic_on_warn set ...\n"); + } kasan_enable_current(); } From ef065653e526a020d2a71549f413f14a830db799 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Mon, 6 Apr 2020 20:12:45 -0700 Subject: [PATCH 627/635] ubsan: include bug type in report header When syzbot tries to figure out how to deduplicate bug reports, it prefers seeing a hint about a specific bug type (we can do better than just "UBSAN"). This lifts the handler reason into the UBSAN report line that includes the file path that tripped a check. Unfortunately, UBSAN does not provide function names. Suggested-by: Dmitry Vyukov Signed-off-by: Kees Cook Signed-off-by: Andrew Morton Cc: Alexander Potapenko Cc: Andrey Konovalov Cc: Andrey Ryabinin Cc: Ard Biesheuvel Cc: Arnd Bergmann Cc: Dan Carpenter Cc: Elena Petrova Cc: "Gustavo A. R. Silva" Link: http://lkml.kernel.org/r/20200227193516.32566-7-keescook@chromium.org Link: https://lore.kernel.org/lkml/CACT4Y+bsLJ-wFx_TaXqax3JByUOWB3uk787LsyMVcfW6JzzGvg@mail.gmail.com Signed-off-by: Linus Torvalds --- lib/ubsan.c | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/lib/ubsan.c b/lib/ubsan.c index 429663eef6a7..f8c0ccf35f29 100644 --- a/lib/ubsan.c +++ b/lib/ubsan.c @@ -45,13 +45,6 @@ static bool was_reported(struct source_location *location) return test_and_set_bit(REPORTED_BIT, &location->reported); } -static void print_source_location(const char *prefix, - struct source_location *loc) -{ - pr_err("%s %s:%d:%d\n", prefix, loc->file_name, - loc->line & LINE_MASK, loc->column & COLUMN_MASK); -} - static bool suppress_report(struct source_location *loc) { return current->in_ubsan || was_reported(loc); @@ -140,13 +133,14 @@ static void val_to_string(char *str, size_t size, struct type_descriptor *type, } } -static void ubsan_prologue(struct source_location *location) +static void ubsan_prologue(struct source_location *loc, const char *reason) { current->in_ubsan++; pr_err("========================================" "========================================\n"); - print_source_location("UBSAN: Undefined behaviour in", location); + pr_err("UBSAN: %s in %s:%d:%d\n", reason, loc->file_name, + loc->line & LINE_MASK, loc->column & COLUMN_MASK); } static void ubsan_epilogue(void) @@ -180,12 +174,12 @@ static void handle_overflow(struct overflow_data *data, void *lhs, if (suppress_report(&data->location)) return; - ubsan_prologue(&data->location); + ubsan_prologue(&data->location, type_is_signed(type) ? + "signed-integer-overflow" : + "unsigned-integer-overflow"); val_to_string(lhs_val_str, sizeof(lhs_val_str), type, lhs); val_to_string(rhs_val_str, sizeof(rhs_val_str), type, rhs); - pr_err("%s integer overflow:\n", - type_is_signed(type) ? "signed" : "unsigned"); pr_err("%s %c %s cannot be represented in type %s\n", lhs_val_str, op, @@ -225,7 +219,7 @@ void __ubsan_handle_negate_overflow(struct overflow_data *data, if (suppress_report(&data->location)) return; - ubsan_prologue(&data->location); + ubsan_prologue(&data->location, "negation-overflow"); val_to_string(old_val_str, sizeof(old_val_str), data->type, old_val); @@ -245,7 +239,7 @@ void __ubsan_handle_divrem_overflow(struct overflow_data *data, if (suppress_report(&data->location)) return; - ubsan_prologue(&data->location); + ubsan_prologue(&data->location, "division-overflow"); val_to_string(rhs_val_str, sizeof(rhs_val_str), data->type, rhs); @@ -264,7 +258,7 @@ static void handle_null_ptr_deref(struct type_mismatch_data_common *data) if (suppress_report(data->location)) return; - ubsan_prologue(data->location); + ubsan_prologue(data->location, "null-ptr-deref"); pr_err("%s null pointer of type %s\n", type_check_kinds[data->type_check_kind], @@ -279,7 +273,7 @@ static void handle_misaligned_access(struct type_mismatch_data_common *data, if (suppress_report(data->location)) return; - ubsan_prologue(data->location); + ubsan_prologue(data->location, "misaligned-access"); pr_err("%s misaligned address %p for type %s\n", type_check_kinds[data->type_check_kind], @@ -295,7 +289,7 @@ static void handle_object_size_mismatch(struct type_mismatch_data_common *data, if (suppress_report(data->location)) return; - ubsan_prologue(data->location); + ubsan_prologue(data->location, "object-size-mismatch"); pr_err("%s address %p with insufficient space\n", type_check_kinds[data->type_check_kind], (void *) ptr); @@ -354,7 +348,7 @@ void __ubsan_handle_out_of_bounds(struct out_of_bounds_data *data, void *index) if (suppress_report(&data->location)) return; - ubsan_prologue(&data->location); + ubsan_prologue(&data->location, "array-index-out-of-bounds"); val_to_string(index_str, sizeof(index_str), data->index_type, index); pr_err("index %s is out of range for type %s\n", index_str, @@ -375,7 +369,7 @@ void __ubsan_handle_shift_out_of_bounds(struct shift_out_of_bounds_data *data, if (suppress_report(&data->location)) goto out; - ubsan_prologue(&data->location); + ubsan_prologue(&data->location, "shift-out-of-bounds"); val_to_string(rhs_str, sizeof(rhs_str), rhs_type, rhs); val_to_string(lhs_str, sizeof(lhs_str), lhs_type, lhs); @@ -407,7 +401,7 @@ EXPORT_SYMBOL(__ubsan_handle_shift_out_of_bounds); void __ubsan_handle_builtin_unreachable(struct unreachable_data *data) { - ubsan_prologue(&data->location); + ubsan_prologue(&data->location, "unreachable"); pr_err("calling __builtin_unreachable()\n"); ubsan_epilogue(); panic("can't return from __builtin_unreachable()"); @@ -422,7 +416,7 @@ void __ubsan_handle_load_invalid_value(struct invalid_value_data *data, if (suppress_report(&data->location)) return; - ubsan_prologue(&data->location); + ubsan_prologue(&data->location, "invalid-load"); val_to_string(val_str, sizeof(val_str), data->type, val); From 29b46fa3dc57def8af3cec1d4e2984b5150be9d5 Mon Sep 17 00:00:00 2001 From: Qiujun Huang Date: Mon, 6 Apr 2020 20:12:49 -0700 Subject: [PATCH 628/635] lib/Kconfig.debug: fix a typo "capabilitiy" -> "capability" s/capabilitiy/capability Signed-off-by: Qiujun Huang Signed-off-by: Andrew Morton Link: http://lkml.kernel.org/r/1585818594-27373-1-git-send-email-hqjagain@gmail.com Signed-off-by: Linus Torvalds --- lib/Kconfig.debug | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index ddcf000022ae..50c1f5f08e6f 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1655,7 +1655,7 @@ config FAILSLAB Provide fault-injection capability for kmalloc. config FAIL_PAGE_ALLOC - bool "Fault-injection capabilitiy for alloc_pages()" + bool "Fault-injection capability for alloc_pages()" depends on FAULT_INJECTION help Provide fault-injection capability for alloc_pages(). From 43afe4d3661b04915aa357c5ab0f2a08718af9fd Mon Sep 17 00:00:00 2001 From: Somala Swaraj Date: Mon, 6 Apr 2020 20:12:53 -0700 Subject: [PATCH 629/635] ipc/mqueue.c: fix a brace coding style issue Signed-off-by: somala swaraj Signed-off-by: Andrew Morton Link: http://lkml.kernel.org/r/20200301135530.18340-1-somalaswaraj@gmail.com Signed-off-by: Linus Torvalds --- ipc/mqueue.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ipc/mqueue.c b/ipc/mqueue.c index 49a05ba3000d..dc8307bf2d74 100644 --- a/ipc/mqueue.c +++ b/ipc/mqueue.c @@ -239,11 +239,10 @@ static inline void msg_tree_erase(struct posix_msg_tree_node *leaf, info->msg_tree_rightmost = rb_prev(node); rb_erase(node, &info->msg_tree); - if (info->node_cache) { + if (info->node_cache) kfree(leaf); - } else { + else info->node_cache = leaf; - } } static inline struct msg_msg *msg_get(struct mqueue_inode_info *info) From 1cd377baa91844b9f87a2b72eabf7ff783946b5e Mon Sep 17 00:00:00 2001 From: Jason Yan Date: Mon, 6 Apr 2020 20:12:56 -0700 Subject: [PATCH 630/635] ipc/shm.c: make compat_ksys_shmctl() static Fix the following sparse warning: ipc/shm.c:1335:6: warning: symbol 'compat_ksys_shmctl' was not declared. Should it be static? Reported-by: Hulk Robot Signed-off-by: Jason Yan Signed-off-by: Andrew Morton Link: http://lkml.kernel.org/r/20200403063933.24785-1-yanaijie@huawei.com Signed-off-by: Linus Torvalds --- ipc/shm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipc/shm.c b/ipc/shm.c index ce1ca9f7c6e9..0ba6add05b35 100644 --- a/ipc/shm.c +++ b/ipc/shm.c @@ -1332,7 +1332,7 @@ static int copy_compat_shmid_from_user(struct shmid64_ds *out, void __user *buf, } } -long compat_ksys_shmctl(int shmid, int cmd, void __user *uptr, int version) +static long compat_ksys_shmctl(int shmid, int cmd, void __user *uptr, int version) { struct ipc_namespace *ns; struct shmid64_ds sem64; From a46387712da12b61bf1ce1a3f63b60a17b098960 Mon Sep 17 00:00:00 2001 From: Benson Leung Date: Tue, 7 Apr 2020 14:36:06 -0700 Subject: [PATCH 631/635] platform/chrome: cros_ec_spi: Wait for USECS, not NSECS The use of `delay_usecs` in terminate_request() was replaced with the new `delay` struct used by the SPI subsystem, however the unit was set to SPI_DELAY_UNIT_NSECS instead of SPI_DELAY_UNIT_USECS. This fixes that. Fixes: 7d3ca507fda9 ("platform/chrome: cros_ec_spi: Use new structure for SPI transfer delays") Signed-off-by: Benson Leung --- drivers/platform/chrome/cros_ec_spi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/platform/chrome/cros_ec_spi.c b/drivers/platform/chrome/cros_ec_spi.c index 665ab154bb4f..debea5c4c829 100644 --- a/drivers/platform/chrome/cros_ec_spi.c +++ b/drivers/platform/chrome/cros_ec_spi.c @@ -128,7 +128,7 @@ static int terminate_request(struct cros_ec_device *ec_dev) spi_message_init(&msg); memset(&trans, 0, sizeof(trans)); trans.delay.value = ec_spi->end_of_msg_delay; - trans.delay.unit = SPI_DELAY_UNIT_NSECS; + trans.delay.unit = SPI_DELAY_UNIT_USECS; spi_message_add_tail(&trans, &msg); ret = spi_sync_locked(ec_spi->spi, &msg); From ba841078cd0557b43b59c63f5c048b12168f0db2 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Tue, 7 Apr 2020 21:40:09 -0400 Subject: [PATCH 632/635] mm/mempolicy: Allow lookup_node() to handle fatal signal lookup_node() uses gup to pin the page and get node information. It checks against ret>=0 assuming the page will be filled in. However it's also possible that gup will return zero, for example, when the thread is quickly killed with a fatal signal. Teach lookup_node() to gracefully return an error -EFAULT if it happens. Meanwhile, initialize "page" to NULL to avoid potential risk of exploiting the pointer. Fixes: 4426e945df58 ("mm/gup: allow VM_FAULT_RETRY for multiple times") Reported-by: syzbot+693dc11fcb53120b5559@syzkaller.appspotmail.com Signed-off-by: Peter Xu Signed-off-by: Linus Torvalds --- mm/mempolicy.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mm/mempolicy.c b/mm/mempolicy.c index b4a039d779b0..fb678a7254f0 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -896,12 +896,15 @@ static void get_policy_nodemask(struct mempolicy *p, nodemask_t *nodes) static int lookup_node(struct mm_struct *mm, unsigned long addr) { - struct page *p; + struct page *p = NULL; int err; int locked = 1; err = get_user_pages_locked(addr & PAGE_MASK, 1, 0, &p, &locked); - if (err >= 0) { + if (err == 0) { + /* E.g. GUP interrupted by fatal signal */ + err = -EFAULT; + } else if (err > 0) { err = page_to_nid(p); put_page(p); } From c7b6a566b98524baea6a244186e665d22b633545 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Tue, 7 Apr 2020 21:40:10 -0400 Subject: [PATCH 633/635] mm/gup: Mark lock taken only after a successful retake It's definitely incorrect to mark the lock as taken even if down_read_killable() failed. This wass overlooked when we switched from down_read() to down_read_killable() because down_read() won't fail while down_read_killable() could. Fixes: 71335f37c5e8 ("mm/gup: allow to react to fatal signals") Reported-by: syzbot+a8c70b7f3579fc0587dc@syzkaller.appspotmail.com Signed-off-by: Peter Xu Signed-off-by: Linus Torvalds --- mm/gup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/gup.c b/mm/gup.c index a21230569520..afce0bc47e70 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -1329,7 +1329,6 @@ retry: if (fatal_signal_pending(current)) break; - *locked = 1; ret = down_read_killable(&mm->mmap_sem); if (ret) { BUG_ON(ret > 0); @@ -1338,6 +1337,7 @@ retry: break; } + *locked = 1; ret = __get_user_pages(tsk, mm, start, 1, flags | FOLL_TRIED, pages, NULL, locked); if (!*locked) { From c6f141412d24c8d8a9d98ef45303c0235829044b Mon Sep 17 00:00:00 2001 From: Dominique Martinet Date: Mon, 6 Apr 2020 19:19:32 +0200 Subject: [PATCH 634/635] 9p: document short read behaviour with O_NONBLOCK Regular files opened with O_NONBLOCK allow read to return after a single round-trip with the server instead of trying to fill buffer. Add a few lines in 9p documentation to describe that. Link: http://lkml.kernel.org/r/1586193572-1375-1-git-send-email-asmadeus@codewreck.org Signed-off-by: Dominique Martinet --- Documentation/filesystems/9p.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Documentation/filesystems/9p.txt b/Documentation/filesystems/9p.txt index fec7144e817c..3fb780ffdf23 100644 --- a/Documentation/filesystems/9p.txt +++ b/Documentation/filesystems/9p.txt @@ -133,6 +133,16 @@ OPTIONS cache tags for existing cache sessions can be listed at /sys/fs/9p/caches. (applies only to cache=fscache) +BEHAVIOR +======== + +This section aims at describing 9p 'quirks' that can be different +from a local filesystem behaviors. + + - Setting O_NONBLOCK on a file will make client reads return as early + as the server returns some data instead of trying to fill the read + buffer with the requested amount of bytes or end of file is reached. + RESOURCES ========= From ae46d2aa6a7fbe8ca0946f24b061b6ccdc6c3f25 Mon Sep 17 00:00:00 2001 From: Hillf Danton Date: Wed, 8 Apr 2020 11:59:24 -0400 Subject: [PATCH 635/635] mm/gup: Let __get_user_pages_locked() return -EINTR for fatal signal __get_user_pages_locked() will return 0 instead of -EINTR after commit 4426e945df588 ("mm/gup: allow VM_FAULT_RETRY for multiple times") which added extra code to allow gup detect fatal signal faster. Restore the original -EINTR behavior. Cc: Andrew Morton Cc: Thomas Gleixner Cc: Peter Zijlstra Fixes: 4426e945df58 ("mm/gup: allow VM_FAULT_RETRY for multiple times") Reported-by: syzbot+3be1a33f04dc782e9fd5@syzkaller.appspotmail.com Signed-off-by: Hillf Danton Acked-by: Michal Hocko Signed-off-by: Peter Xu Signed-off-by: Linus Torvalds --- mm/gup.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mm/gup.c b/mm/gup.c index afce0bc47e70..6076df8e04a4 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -1326,8 +1326,11 @@ retry: * start trying again otherwise it can loop forever. */ - if (fatal_signal_pending(current)) + if (fatal_signal_pending(current)) { + if (!pages_done) + pages_done = -EINTR; break; + } ret = down_read_killable(&mm->mmap_sem); if (ret) {