From 4cedc115fd567a653848612afa6559c36246a12a Mon Sep 17 00:00:00 2001 From: Zhen Chen Date: Mon, 12 Aug 2024 15:27:25 +0800 Subject: [PATCH 1/2] MALI: rockchip: upgrade bifrost DDK to g24p0-00eac0, from g22p0-01eac0 mali_csffw.bin from Valhall DDK g24(r49) is included. Change-Id: Ic48b63e744457163fbae3f41b477fc2827a1380e Signed-off-by: Zhen Chen --- Documentation/ABI/testing/sysfs-device-mali | 3 +- .../sysfs-device-mali-coresight-source | 2 +- .../devicetree/bindings/arm/mali-bifrost.txt | 7 +- drivers/base/arm/Makefile | 2 + .../memory_group_manager.c | 8 +- drivers/gpu/arm/bifrost/Kbuild | 2 +- drivers/gpu/arm/bifrost/Kconfig | 15 +- drivers/gpu/arm/bifrost/Makefile | 12 +- drivers/gpu/arm/bifrost/arbiter/Kbuild | 3 +- .../arm/bifrost/arbiter/mali_kbase_arbif.c | 137 ++-- .../arm/bifrost/arbiter/mali_kbase_arbif.h | 3 +- .../bifrost/arbiter/mali_kbase_arbiter_pm.c | 97 ++- .../bifrost/arbiter/mali_kbase_arbiter_pm.h | 4 +- drivers/gpu/arm/bifrost/backend/gpu/Kbuild | 9 +- .../bifrost/backend/gpu/mali_kbase_devfreq.c | 11 +- .../backend/gpu/mali_kbase_instr_backend.c | 23 +- .../backend/gpu/mali_kbase_irq_internal.h | 2 +- .../backend/gpu/mali_kbase_irq_linux.c | 28 +- .../bifrost/backend/gpu/mali_kbase_jm_hw.c | 6 +- .../bifrost/backend/gpu/mali_kbase_jm_rb.c | 2 +- .../backend/gpu/mali_kbase_model_dummy.c | 56 +- .../gpu/mali_kbase_model_error_generator.c | 172 ---- .../backend/gpu/mali_kbase_model_linux.h | 14 - .../backend/gpu/mali_kbase_pm_backend.c | 88 +- .../bifrost/backend/gpu/mali_kbase_pm_defs.h | 9 +- .../backend/gpu/mali_kbase_pm_driver.c | 99 ++- .../backend/gpu/mali_kbase_pm_policy.c | 4 +- .../arm/bifrost/backend/gpu/mali_kbase_time.c | 111 ++- drivers/gpu/arm/bifrost/build.bp | 16 +- .../context/backend/mali_kbase_context_csf.c | 15 +- .../context/backend/mali_kbase_context_jm.c | 7 +- .../arm/bifrost/context/mali_kbase_context.c | 10 +- .../arm/bifrost/context/mali_kbase_context.h | 11 +- drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c | 424 ++++++---- drivers/gpu/arm/bifrost/csf/mali_kbase_csf.h | 34 +- .../gpu/arm/bifrost/csf/mali_kbase_csf_defs.h | 127 ++- .../arm/bifrost/csf/mali_kbase_csf_firmware.c | 80 +- .../arm/bifrost/csf/mali_kbase_csf_firmware.h | 3 +- .../bifrost/csf/mali_kbase_csf_firmware_cfg.c | 4 +- .../csf/mali_kbase_csf_firmware_no_mali.c | 85 +- .../gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c | 218 ++--- .../gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.h | 30 +- .../bifrost/csf/mali_kbase_csf_registers.h | 24 +- .../bifrost/csf/mali_kbase_csf_reset_gpu.c | 15 +- .../bifrost/csf/mali_kbase_csf_scheduler.c | 594 ++++++++------ .../bifrost/csf/mali_kbase_csf_scheduler.h | 52 +- .../gpu/arm/bifrost/csf/mali_kbase_csf_sync.c | 90 +- .../bifrost/csf/mali_kbase_csf_tiler_heap.c | 5 +- .../csf/mali_kbase_csf_tiler_heap_reclaim.c | 26 +- .../csf/mali_kbase_csf_tiler_heap_reclaim.h | 4 +- .../bifrost/csf/mali_kbase_csf_tl_reader.c | 37 +- .../bifrost/csf/mali_kbase_csf_trace_buffer.c | 8 + .../bifrost/csf/mali_kbase_csf_trace_buffer.h | 9 + .../gpu/arm/bifrost/csf/mali_kbase_csf_util.c | 6 +- .../backend/mali_kbase_debug_ktrace_csf.c | 31 +- .../backend/mali_kbase_debug_ktrace_jm.c | 32 +- .../bifrost/debug/mali_kbase_debug_ktrace.c | 32 +- .../device/backend/mali_kbase_device_csf.c | 14 +- .../device/backend/mali_kbase_device_hw_csf.c | 15 +- .../device/backend/mali_kbase_device_hw_jm.c | 10 +- .../device/backend/mali_kbase_device_jm.c | 8 +- .../arm/bifrost/device/mali_kbase_device.c | 49 +- .../arm/bifrost/device/mali_kbase_device.h | 5 +- .../arm/bifrost/device/mali_kbase_device_hw.c | 10 +- .../mali_kbase_hw_access_model_linux.c | 12 +- .../backend/mali_kbase_hw_access_real_hw.c | 36 +- .../bifrost/hw_access/mali_kbase_hw_access.c | 54 +- .../bifrost/hw_access/mali_kbase_hw_access.h | 21 +- .../hw_access/mali_kbase_hw_access_regmap.h | 10 + .../regmap/mali_kbase_regmap_csf_macros.h | 2 +- .../hw_access/regmap/mali_kbase_regmap_jm.c | 65 +- .../regmap/mali_kbase_regmap_jm_enums.h | 17 +- .../regmap/mali_kbase_regmap_jm_macros.h | 11 +- .../backend/mali_kbase_hwcnt_backend_csf.c | 138 ++-- .../backend/mali_kbase_hwcnt_backend_csf.h | 28 +- .../backend/mali_kbase_hwcnt_backend_csf_if.h | 6 +- .../mali_kbase_hwcnt_backend_csf_if_fw.c | 14 +- .../backend/mali_kbase_hwcnt_backend_jm.c | 24 +- .../arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.c | 38 +- .../arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.h | 50 +- .../bifrost/hwcnt/mali_kbase_hwcnt_types.c | 5 +- .../arm/bifrost/mali_base_hwconfig_features.h | 38 +- .../arm/bifrost/mali_base_hwconfig_issues.h | 205 ++--- drivers/gpu/arm/bifrost/mali_csffw.bin | Bin 274432 -> 278528 bytes drivers/gpu/arm/bifrost/mali_kbase.h | 120 +-- drivers/gpu/arm/bifrost/mali_kbase_caps.h | 33 +- drivers/gpu/arm/bifrost/mali_kbase_config.h | 35 +- .../arm/bifrost/mali_kbase_config_defaults.h | 20 +- .../gpu/arm/bifrost/mali_kbase_core_linux.c | 774 +++++++----------- .../arm/bifrost/mali_kbase_debug_mem_view.c | 12 +- drivers/gpu/arm/bifrost/mali_kbase_defs.h | 86 +- drivers/gpu/arm/bifrost/mali_kbase_fence.h | 31 +- .../gpu/arm/bifrost/mali_kbase_gpu_metrics.c | 113 +-- .../gpu/arm/bifrost/mali_kbase_gpu_metrics.h | 12 +- drivers/gpu/arm/bifrost/mali_kbase_gpuprops.c | 18 +- drivers/gpu/arm/bifrost/mali_kbase_gwt.c | 19 +- drivers/gpu/arm/bifrost/mali_kbase_hw.c | 6 +- .../gpu/arm/bifrost/mali_kbase_hwaccess_pm.h | 10 +- .../arm/bifrost/mali_kbase_hwaccess_time.h | 41 +- drivers/gpu/arm/bifrost/mali_kbase_js.c | 8 +- drivers/gpu/arm/bifrost/mali_kbase_linux.h | 7 +- drivers/gpu/arm/bifrost/mali_kbase_mem.c | 254 ++++-- drivers/gpu/arm/bifrost/mali_kbase_mem.h | 52 +- .../gpu/arm/bifrost/mali_kbase_mem_linux.c | 174 ++-- .../gpu/arm/bifrost/mali_kbase_mem_migrate.c | 40 +- .../gpu/arm/bifrost/mali_kbase_mem_migrate.h | 2 +- drivers/gpu/arm/bifrost/mali_kbase_mem_pool.c | 31 +- .../gpu/arm/bifrost/mali_kbase_native_mgm.c | 43 +- drivers/gpu/arm/bifrost/mali_kbase_pbha.c | 6 +- .../gpu/arm/bifrost/mali_kbase_pbha_debugfs.c | 6 +- drivers/gpu/arm/bifrost/mali_kbase_pm.c | 69 +- drivers/gpu/arm/bifrost/mali_kbase_pm.h | 27 +- drivers/gpu/arm/bifrost/mali_kbase_softjobs.c | 4 +- drivers/gpu/arm/bifrost/mmu/Kbuild | 11 +- .../bifrost/mmu/backend/mali_kbase_mmu_csf.c | 144 +++- .../mali_kbase_mmu_faults_decoder_luts_csf.c | 139 ++++ .../mali_kbase_mmu_faults_decoder_luts_csf.h | 50 ++ .../mali_kbase_mmu_faults_decoder_luts_jm.c | 74 ++ .../mali_kbase_mmu_faults_decoder_luts_jm.h | 37 + .../bifrost/mmu/backend/mali_kbase_mmu_jm.c | 41 +- drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.c | 266 ++++-- .../mmu/mali_kbase_mmu_faults_decoder.c | 124 +++ .../mmu/mali_kbase_mmu_faults_decoder.h | 143 ++++ .../mmu/mali_kbase_mmu_faults_decoder_luts.c | 660 +++++++++++++++ .../mmu/mali_kbase_mmu_faults_decoder_luts.h | 119 +++ .../bifrost/mmu/mali_kbase_mmu_mode_aarch64.c | 2 +- .../devicetree/mali_kbase_runtime_pm.c | 2 +- drivers/gpu/arm/bifrost/tests/Kbuild | 2 +- .../bifrost/tests/include/kutf/kutf_kprobe.h | 6 +- .../gpu/arm/bifrost/tests/kutf/kutf_kprobe.c | 28 +- .../arm/bifrost/thirdparty/mali_kbase_mmap.c | 193 ++++- .../arm/bifrost/tl/mali_kbase_tracepoints.c | 2 +- .../arm/bifrost/tl/mali_kbase_tracepoints.h | 2 +- drivers/hwtracing/coresight/mali/Makefile | 4 +- .../mali/sources/coresight_mali_sources.c | 6 +- drivers/xen/arm/Makefile | 2 + include/linux/mali_arbiter_interface.h | 4 +- include/linux/mali_hw_access.h | 61 ++ include/linux/memory_group_manager.h | 4 + include/linux/version_compat_defs.h | 48 +- .../backend/gpu/mali_kbase_model_dummy.h | 2 +- .../arm/bifrost/csf/mali_base_csf_kernel.h | 49 +- .../arm/bifrost/csf/mali_kbase_csf_ioctl.h | 43 +- .../gpu/arm/bifrost/gpu/mali_kbase_gpu_id.h | 2 +- .../gpu/arm/bifrost/jm/mali_base_jm_kernel.h | 23 +- .../gpu/arm/bifrost/jm/mali_kbase_jm_ioctl.h | 14 +- .../uapi/gpu/arm/bifrost/mali_base_kernel.h | 25 +- 147 files changed, 5342 insertions(+), 2886 deletions(-) delete mode 100644 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_error_generator.c create mode 100644 drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_faults_decoder_luts_csf.c create mode 100644 drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_faults_decoder_luts_csf.h create mode 100644 drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_faults_decoder_luts_jm.c create mode 100644 drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_faults_decoder_luts_jm.h create mode 100644 drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_faults_decoder.c create mode 100644 drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_faults_decoder.h create mode 100644 drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_faults_decoder_luts.c create mode 100644 drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_faults_decoder_luts.h create mode 100644 include/linux/mali_hw_access.h diff --git a/Documentation/ABI/testing/sysfs-device-mali b/Documentation/ABI/testing/sysfs-device-mali index 1ec265c5add4..12a1667feeb2 100644 --- a/Documentation/ABI/testing/sysfs-device-mali +++ b/Documentation/ABI/testing/sysfs-device-mali @@ -341,8 +341,7 @@ Description: device-driver that supports a CSF GPU. Used to enable firmware logs, logging levels valid values - are indicated using 'min and 'max' attribute values - values that are read-only. + are indicated using 'min' and 'max' attributes, which are read-only. Log level can be set using the 'cur' read, write attribute, we can use a valid log level value from min and max range values diff --git a/Documentation/ABI/testing/sysfs-device-mali-coresight-source b/Documentation/ABI/testing/sysfs-device-mali-coresight-source index 0f31a6acaa87..58d9085b8bb6 100644 --- a/Documentation/ABI/testing/sysfs-device-mali-coresight-source +++ b/Documentation/ABI/testing/sysfs-device-mali-coresight-source @@ -19,7 +19,7 @@ Description: What: /sys/bus/coresight/devices/mali-source-etm/is_enabled Description: - Attribute used to check if Coresight Source ITM is enabled. + Attribute used to check if Coresight Source ETM is enabled. What: /sys/bus/coresight/devices/mali-source-etm/trcconfigr Description: diff --git a/Documentation/devicetree/bindings/arm/mali-bifrost.txt b/Documentation/devicetree/bindings/arm/mali-bifrost.txt index 85672c6c6258..3f80d97b0064 100644 --- a/Documentation/devicetree/bindings/arm/mali-bifrost.txt +++ b/Documentation/devicetree/bindings/arm/mali-bifrost.txt @@ -111,7 +111,10 @@ for details. - idvs-group-size : Override the IDVS group size value. Tasks are sent to cores in groups of N + 1, so i.e. 0xF means 16 tasks. Valid values are between 0 to 0x3F (including). -- l2-size : Override L2 cache size on GPU that supports it +- l2-size : Override L2 cache size on GPU that supports it. Value should be larger than the minimum + size 1KiB and smaller than the maximum size. Maximum size is Hardware integration dependent. + The value passed should be of log2(Cache Size in Bytes). + For example for a 1KiB of cache size, 0xa should be passed. - l2-hash : Override L2 hash function on GPU that supports it - l2-hash-values : Override L2 hash function using provided hash values, on GPUs that supports it. It is mutually exclusive with 'l2-hash'. Only one or the other must be @@ -237,7 +240,7 @@ gpu@0xfc010000 { ... pbha { int-id-override = <2 0x32>, <9 0x05>, <16 0x32>; - propagate-bits = /bits/ 4 <0x03>; + propagate-bits = /bits/ 8 <0x03>; }; ... }; diff --git a/drivers/base/arm/Makefile b/drivers/base/arm/Makefile index 4aa68f89d3d9..42071f769729 100644 --- a/drivers/base/arm/Makefile +++ b/drivers/base/arm/Makefile @@ -125,6 +125,8 @@ CFLAGS_MODULE += -Wno-sign-compare CFLAGS_MODULE += -Wno-shift-negative-value # This flag is needed to avoid build errors on older kernels CFLAGS_MODULE += $(call cc-option, -Wno-cast-function-type) +# The following ensures the stack frame does not get larger than a page +CFLAGS_MODULE += -Wframe-larger-than=4096 KBUILD_CPPFLAGS += -DKBUILD_EXTRA_WARN1 diff --git a/drivers/base/arm/memory_group_manager/memory_group_manager.c b/drivers/base/arm/memory_group_manager/memory_group_manager.c index 389b0f051f3a..c5fba5f1b522 100644 --- a/drivers/base/arm/memory_group_manager/memory_group_manager.c +++ b/drivers/base/arm/memory_group_manager/memory_group_manager.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -51,10 +51,6 @@ static inline vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigne } #endif -#define PTE_PBHA_SHIFT (59) -#define PTE_PBHA_MASK ((uint64_t)0xf << PTE_PBHA_SHIFT) -#define PTE_RES_BIT_MULTI_AS_SHIFT (63) - #define IMPORTED_MEMORY_ID (MEMORY_GROUP_MANAGER_NR_GROUPS - 1) /** @@ -263,7 +259,7 @@ static struct page *example_mgm_alloc_page(struct memory_group_manager_device *m } else { struct mgm_groups *data = mgm_dev->data; - dev_err(data->dev, "alloc_pages failed\n"); + dev_dbg(data->dev, "alloc_pages failed\n"); } return p; diff --git a/drivers/gpu/arm/bifrost/Kbuild b/drivers/gpu/arm/bifrost/Kbuild index b35fcee88baa..b19e4abf3e2b 100644 --- a/drivers/gpu/arm/bifrost/Kbuild +++ b/drivers/gpu/arm/bifrost/Kbuild @@ -69,7 +69,7 @@ endif # # Driver version string which is returned to userspace via an ioctl -MALI_RELEASE_NAME ?= '"g22p0-01eac0"' +MALI_RELEASE_NAME ?= '"g24p0-00eac0"' # Set up defaults if not defined by build system ifeq ($(CONFIG_MALI_BIFROST_DEBUG), y) MALI_UNIT_TEST = 1 diff --git a/drivers/gpu/arm/bifrost/Kconfig b/drivers/gpu/arm/bifrost/Kconfig index 22fdfe80405a..685ce4f423ad 100644 --- a/drivers/gpu/arm/bifrost/Kconfig +++ b/drivers/gpu/arm/bifrost/Kconfig @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2012-2023 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2012-2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -70,7 +70,6 @@ config MALI_NO_MALI_DEFAULT_GPU help This option sets the default GPU to identify as for No Mali builds. - endchoice menu "Platform specific options" @@ -214,16 +213,6 @@ config MALI_CORESTACK If unsure, say N. -comment "Platform options" - depends on MALI_BIFROST && MALI_BIFROST_EXPERT - -config MALI_BIFROST_ERROR_INJECT - bool "Enable No Mali error injection" - depends on MALI_BIFROST && MALI_BIFROST_EXPERT && MALI_BIFROST_NO_MALI - default n - help - Enables insertion of errors to test module failure and recovery mechanisms. - comment "Debug options" depends on MALI_BIFROST && MALI_BIFROST_EXPERT @@ -304,7 +293,7 @@ endchoice config MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS bool "Enable runtime selection of performance counters set via debugfs" - depends on MALI_BIFROST && MALI_BIFROST_EXPERT && DEBUG_FS + depends on MALI_BIFROST && MALI_BIFROST_EXPERT && DEBUG_FS && !MALI_CSF_SUPPORT default n help Select this option to make the secondary set of performance counters diff --git a/drivers/gpu/arm/bifrost/Makefile b/drivers/gpu/arm/bifrost/Makefile index 69dbe3750a10..9b636f58c6bc 100644 --- a/drivers/gpu/arm/bifrost/Makefile +++ b/drivers/gpu/arm/bifrost/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -41,11 +41,12 @@ ifeq ($(MALI_KCONFIG_EXT_PREFIX),) CONFIG_MALI_BIFROST_GATOR_SUPPORT ?= y CONFIG_MALI_ARBITRATION ?= n CONFIG_MALI_PARTITION_MANAGER ?= n - CONFIG_MALI_64BIT_HW_ACCESS ?= n + ifneq ($(CONFIG_MALI_BIFROST_NO_MALI),y) - # Prevent misuse when CONFIG_MALI_BIFROST_NO_MALI=y + # Prevent misuse when CONFIG_MALI_BIFROST_NO_MALI!=y CONFIG_MALI_REAL_HW ?= y + else CONFIG_MALI_CORESIGHT = n endif @@ -76,7 +77,6 @@ ifeq ($(MALI_KCONFIG_EXT_PREFIX),) else # Prevent misuse when CONFIG_MALI_BIFROST_NO_MALI=n CONFIG_MALI_REAL_HW = y - CONFIG_MALI_BIFROST_ERROR_INJECT = n endif @@ -108,7 +108,6 @@ ifeq ($(MALI_KCONFIG_EXT_PREFIX),) CONFIG_MALI_JOB_DUMP = n CONFIG_MALI_BIFROST_NO_MALI = n CONFIG_MALI_REAL_HW = y - CONFIG_MALI_BIFROST_ERROR_INJECT = n CONFIG_MALI_HW_ERRATA_1485982_NOT_AFFECTED = n CONFIG_MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE = n CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS = n @@ -171,7 +170,6 @@ ifeq ($(MALI_KCONFIG_EXT_PREFIX),) CONFIG_MALI_PWRSOFT_765 \ CONFIG_MALI_JOB_DUMP \ CONFIG_MALI_BIFROST_NO_MALI \ - CONFIG_MALI_BIFROST_ERROR_INJECT \ CONFIG_MALI_HW_ERRATA_1485982_NOT_AFFECTED \ CONFIG_MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE \ CONFIG_MALI_PRFCNT_SET_PRIMARY \ @@ -272,6 +270,8 @@ CFLAGS_MODULE += -Wmissing-field-initializers CFLAGS_MODULE += -Wno-type-limits CFLAGS_MODULE += $(call cc-option, -Wmaybe-uninitialized) CFLAGS_MODULE += $(call cc-option, -Wunused-macros) +# The following ensures the stack frame does not get larger than a page +CFLAGS_MODULE += -Wframe-larger-than=4096 KBUILD_CPPFLAGS += -DKBUILD_EXTRA_WARN2 diff --git a/drivers/gpu/arm/bifrost/arbiter/Kbuild b/drivers/gpu/arm/bifrost/arbiter/Kbuild index 2e6b111441ca..de339ccae394 100644 --- a/drivers/gpu/arm/bifrost/arbiter/Kbuild +++ b/drivers/gpu/arm/bifrost/arbiter/Kbuild @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -21,3 +21,4 @@ bifrost_kbase-y += \ arbiter/mali_kbase_arbif.o \ arbiter/mali_kbase_arbiter_pm.o + diff --git a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.c b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.c index c290dd6b086f..8cdae33cf919 100644 --- a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.c +++ b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -160,28 +160,19 @@ static void on_gpu_lost(struct device *dev) kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_LOST_EVT); } -/** - * kbase_arbif_init() - Kbase Arbiter interface initialisation. - * @kbdev: The kbase device structure for the device (must be a valid pointer) - * - * Initialise Kbase Arbiter interface and assign callback functions. - * - * Return: - * * 0 - the interface was initialized or was not specified - * * in the device tree. - * * -EFAULT - the interface was specified but failed to initialize. - * * -EPROBE_DEFER - module dependencies are not yet available. - */ -int kbase_arbif_init(struct kbase_device *kbdev) +static int kbase_arbif_of_init(struct kbase_device *kbdev) { -#if IS_ENABLED(CONFIG_OF) - struct arbiter_if_arb_vm_ops ops; struct arbiter_if_dev *arb_if; struct device_node *arbiter_if_node; struct platform_device *pdev; - int err; - dev_dbg(kbdev->dev, "%s\n", __func__); + if (!IS_ENABLED(CONFIG_OF)) { + /* + * Return -ENODEV in the event CONFIG_OF is not available and let the + * internal AW check for suitability for arbitration. + */ + return -ENODEV; + } arbiter_if_node = of_parse_phandle(kbdev->dev->of_node, "arbiter-if", 0); if (!arbiter_if_node) @@ -191,7 +182,7 @@ int kbase_arbif_init(struct kbase_device *kbdev) /* no arbiter interface defined in device tree */ kbdev->arb.arb_dev = NULL; kbdev->arb.arb_if = NULL; - return 0; + return -ENODEV; } pdev = of_find_device_by_node(arbiter_if_node); @@ -215,6 +206,47 @@ int kbase_arbif_init(struct kbase_device *kbdev) } kbdev->arb.arb_if = arb_if; + return 0; +} + +static void kbase_arbif_of_term(struct kbase_device *kbdev) +{ + if (!IS_ENABLED(CONFIG_OF)) + return; + + if (kbdev->arb.arb_dev) { + module_put(kbdev->arb.arb_dev->driver->owner); + put_device(kbdev->arb.arb_dev); + } + kbdev->arb.arb_dev = NULL; +} + + +/** + * kbase_arbif_init() - Kbase Arbiter interface initialisation. + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Initialise Kbase Arbiter interface and assign callback functions. + * + * Return: + * * 0 - the interface was initialized or was not specified + * * in the device tree. + * * -EFAULT - the interface was specified but failed to initialize. + * * -EPROBE_DEFER - module dependencies are not yet available. + */ +int kbase_arbif_init(struct kbase_device *kbdev) +{ + struct arbiter_if_arb_vm_ops ops; + struct arbiter_if_dev *arb_if; + int err = 0; + + /* Tries to init with 'arbiter-if' if present in devicetree */ + err = kbase_arbif_of_init(kbdev); + + + if (err) + return err; + ops.arb_vm_gpu_stop = on_gpu_stop; ops.arb_vm_gpu_granted = on_gpu_granted; ops.arb_vm_gpu_lost = on_gpu_lost; @@ -225,25 +257,35 @@ int kbase_arbif_init(struct kbase_device *kbdev) kbdev->arb.arb_freq.freq_updated = false; mutex_init(&kbdev->arb.arb_freq.arb_freq_lock); - /* register kbase arbiter_if callbacks */ - if (arb_if->vm_ops.vm_arb_register_dev) { - err = arb_if->vm_ops.vm_arb_register_dev(arb_if, kbdev->dev, &ops); - if (err) { - dev_err(&pdev->dev, "Failed to register with arbiter. (err = %d)\n", err); - module_put(pdev->dev.driver->owner); - put_device(&pdev->dev); - if (err != -EPROBE_DEFER) - err = -EFAULT; - return err; - } + arb_if = kbdev->arb.arb_if; + + if (arb_if == NULL) { + dev_err(kbdev->dev, "No arbiter interface present\n"); + goto failure_term; + } + + if (!arb_if->vm_ops.vm_arb_register_dev) { + dev_err(kbdev->dev, "arbiter_if registration callback not present\n"); + goto failure_term; + } + + /* register kbase arbiter_if callbacks */ + err = arb_if->vm_ops.vm_arb_register_dev(arb_if, kbdev->dev, &ops); + if (err) { + dev_err(kbdev->dev, "Failed to register with arbiter. (err = %d)\n", err); + goto failure_term; } -#else /* CONFIG_OF */ - dev_dbg(kbdev->dev, "No arbiter without Device Tree support\n"); - kbdev->arb.arb_dev = NULL; - kbdev->arb.arb_if = NULL; -#endif return 0; + +failure_term: + { + kbase_arbif_of_term(kbdev); + } + + if (err != -EPROBE_DEFER) + err = -EFAULT; + return err; } /** @@ -256,16 +298,13 @@ void kbase_arbif_destroy(struct kbase_device *kbdev) { struct arbiter_if_dev *arb_if = kbdev->arb.arb_if; - if (arb_if && arb_if->vm_ops.vm_arb_unregister_dev) { - dev_dbg(kbdev->dev, "%s\n", __func__); + if (arb_if && arb_if->vm_ops.vm_arb_unregister_dev) arb_if->vm_ops.vm_arb_unregister_dev(kbdev->arb.arb_if); + + { + kbase_arbif_of_term(kbdev); } kbdev->arb.arb_if = NULL; - if (kbdev->arb.arb_dev) { - module_put(kbdev->arb.arb_dev->driver->owner); - put_device(kbdev->arb.arb_dev); - } - kbdev->arb.arb_dev = NULL; } /** @@ -278,10 +317,8 @@ void kbase_arbif_get_max_config(struct kbase_device *kbdev) { struct arbiter_if_dev *arb_if = kbdev->arb.arb_if; - if (arb_if && arb_if->vm_ops.vm_arb_get_max_config) { - dev_dbg(kbdev->dev, "%s\n", __func__); + if (arb_if && arb_if->vm_ops.vm_arb_get_max_config) arb_if->vm_ops.vm_arb_get_max_config(arb_if); - } } /** @@ -295,7 +332,6 @@ void kbase_arbif_gpu_request(struct kbase_device *kbdev) struct arbiter_if_dev *arb_if = kbdev->arb.arb_if; if (arb_if && arb_if->vm_ops.vm_arb_gpu_request) { - dev_dbg(kbdev->dev, "%s\n", __func__); KBASE_TLSTREAM_TL_ARBITER_REQUESTED(kbdev, kbdev); arb_if->vm_ops.vm_arb_gpu_request(arb_if); } @@ -312,7 +348,6 @@ void kbase_arbif_gpu_stopped(struct kbase_device *kbdev, u8 gpu_required) struct arbiter_if_dev *arb_if = kbdev->arb.arb_if; if (arb_if && arb_if->vm_ops.vm_arb_gpu_stopped) { - dev_dbg(kbdev->dev, "%s\n", __func__); KBASE_TLSTREAM_TL_ARBITER_STOPPED(kbdev, kbdev); if (gpu_required) KBASE_TLSTREAM_TL_ARBITER_REQUESTED(kbdev, kbdev); @@ -330,10 +365,8 @@ void kbase_arbif_gpu_active(struct kbase_device *kbdev) { struct arbiter_if_dev *arb_if = kbdev->arb.arb_if; - if (arb_if && arb_if->vm_ops.vm_arb_gpu_active) { - dev_dbg(kbdev->dev, "%s\n", __func__); + if (arb_if && arb_if->vm_ops.vm_arb_gpu_active) arb_if->vm_ops.vm_arb_gpu_active(arb_if); - } } /** @@ -346,8 +379,6 @@ void kbase_arbif_gpu_idle(struct kbase_device *kbdev) { struct arbiter_if_dev *arb_if = kbdev->arb.arb_if; - if (arb_if && arb_if->vm_ops.vm_arb_gpu_idle) { - dev_dbg(kbdev->dev, "vm_arb_gpu_idle\n"); + if (arb_if && arb_if->vm_ops.vm_arb_gpu_idle) arb_if->vm_ops.vm_arb_gpu_idle(arb_if); - } } diff --git a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.h b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.h index 701ffd42f6f7..c77792115e4d 100644 --- a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.h +++ b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -50,6 +50,7 @@ enum kbase_arbif_evt { KBASE_VM_OS_RESUME_EVENT, }; + /** * kbase_arbif_init() - Initialize the arbiter interface functionality. * @kbdev: The kbase device structure for the device (must be a valid pointer) diff --git a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.c b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.c index 616b0a78cbe5..a27085d0f4f4 100644 --- a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.c +++ b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -48,7 +48,7 @@ MODULE_PARM_DESC( "On a virtualized platform, if the GPU is not granted within this time(ms) kbase will defer the probe"); static void kbase_arbiter_pm_vm_wait_gpu_assignment(struct kbase_device *kbdev); -static inline bool kbase_arbiter_pm_vm_gpu_assigned_lockheld(struct kbase_device *kbdev); +static inline bool kbase_arbiter_pm_vm_gpu_assigned_locked(struct kbase_device *kbdev); /** * kbase_arbiter_pm_vm_state_str() - Helper function to get string @@ -85,7 +85,6 @@ static inline const char *kbase_arbiter_pm_vm_state_str(enum kbase_vm_state stat case KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT: return "KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT"; default: - KBASE_DEBUG_ASSERT(false); return "[UnknownState]"; } } @@ -117,14 +116,13 @@ static inline const char *kbase_arbiter_pm_vm_event_str(enum kbase_arbif_evt evt case KBASE_VM_REF_EVENT: return "KBASE_VM_REF_EVENT"; default: - KBASE_DEBUG_ASSERT(false); return "[UnknownEvent]"; } } /** * kbase_arbiter_pm_vm_set_state() - Sets new kbase_arbiter_vm_state - * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @kbdev: The kbase device structure for the device * @new_state: kbase VM new state * * This function sets the new state for the VM @@ -229,7 +227,7 @@ static enum hrtimer_restart request_timer_callback(struct hrtimer *timer) /** * start_request_timer() - Start a timer after requesting GPU - * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @kbdev: The kbase device structure for the device * * Start a timer to track when kbase is waiting for the GPU from the * Arbiter. If the timer expires before GPU is granted, a warning in @@ -245,7 +243,7 @@ static void start_request_timer(struct kbase_device *kbdev) /** * cancel_request_timer() - Stop the request timer - * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @kbdev: The kbase device structure for the device * * Stops the request timer once GPU has been granted. Safe to call * even if timer is no longer running. @@ -260,7 +258,7 @@ static void cancel_request_timer(struct kbase_device *kbdev) /** * kbase_arbiter_pm_early_init() - Initialize arbiter for VM * Paravirtualized use. - * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @kbdev: The kbase device structure for the device * * Initialize the arbiter and other required resources during the runtime * and request the GPU for the VM for the first time. @@ -272,7 +270,7 @@ int kbase_arbiter_pm_early_init(struct kbase_device *kbdev) int err; struct kbase_arbiter_vm_state *arb_vm_state = NULL; - arb_vm_state = kmalloc(sizeof(struct kbase_arbiter_vm_state), GFP_KERNEL); + arb_vm_state = kzalloc(sizeof(struct kbase_arbiter_vm_state), GFP_KERNEL); if (arb_vm_state == NULL) return -ENOMEM; @@ -311,7 +309,7 @@ int kbase_arbiter_pm_early_init(struct kbase_device *kbdev) msecs_to_jiffies((unsigned int)gpu_req_timeout)); if (!err) { - dev_dbg(kbdev->dev, + dev_err(kbdev->dev, "Kbase probe Deferred after waiting %d ms to receive GPU_GRANT\n", gpu_req_timeout); @@ -336,7 +334,7 @@ arbif_init_fail: /** * kbase_arbiter_pm_early_term() - Shutdown arbiter and free resources - * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @kbdev: The kbase device structure for the device * * Clean up all the resources */ @@ -344,6 +342,11 @@ void kbase_arbiter_pm_early_term(struct kbase_device *kbdev) { struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + if (arb_vm_state == NULL) + return; + + kbase_arbiter_pm_release_interrupts(kbdev); + cancel_request_timer(kbdev); mutex_lock(&arb_vm_state->vm_state_lock); if (arb_vm_state->vm_state > KBASE_VM_STATE_STOPPED_GPU_REQUESTED) { @@ -358,12 +361,6 @@ void kbase_arbiter_pm_early_term(struct kbase_device *kbdev) kbdev->pm.arb_vm_state = NULL; } -/** - * kbase_arbiter_pm_release_interrupts() - Release the GPU interrupts - * @kbdev: The kbase device structure for the device (must be a valid pointer) - * - * Releases interrupts and set the interrupt flag to false - */ void kbase_arbiter_pm_release_interrupts(struct kbase_device *kbdev) { struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; @@ -376,29 +373,25 @@ void kbase_arbiter_pm_release_interrupts(struct kbase_device *kbdev) mutex_unlock(&arb_vm_state->vm_state_lock); } -/** - * kbase_arbiter_pm_install_interrupts() - Install the GPU interrupts - * @kbdev: The kbase device structure for the device (must be a valid pointer) - * - * Install interrupts and set the interrupt_install flag to true. - * - * Return: 0 if success, or a Linux error code - */ int kbase_arbiter_pm_install_interrupts(struct kbase_device *kbdev) { struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; - int err; + int err = 0; mutex_lock(&arb_vm_state->vm_state_lock); - arb_vm_state->interrupts_installed = true; - err = kbase_install_interrupts(kbdev); + if (arb_vm_state->interrupts_installed == false) { + arb_vm_state->interrupts_installed = true; + err = kbase_install_interrupts(kbdev); + } else { + dev_dbg(kbdev->dev, "%s: interrupts installed already", __func__); + } mutex_unlock(&arb_vm_state->vm_state_lock); return err; } /** * kbase_arbiter_pm_vm_stopped() - Handle stop state for the VM - * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @kbdev: The kbase device structure for the device * * Handles a stop state for the VM */ @@ -416,7 +409,13 @@ void kbase_arbiter_pm_vm_stopped(struct kbase_device *kbdev) dev_dbg(kbdev->dev, "%s %s\n", __func__, kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state)); - if (arb_vm_state->interrupts_installed) { + /* + * Release the interrupts on external arb_if to address Xen requirements. + * Interrupts are not released with internal arb_if as the IRQs are required + * to handle messaging to/from Arbiter/Resource Group. + */ + if (arb_vm_state->interrupts_installed + ) { arb_vm_state->interrupts_installed = false; kbase_release_interrupts(kbdev); } @@ -507,7 +506,7 @@ int kbase_arbiter_pm_gpu_assigned(struct kbase_device *kbdev) /** * kbase_arbiter_pm_vm_gpu_start() - Handles the start state of the VM - * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @kbdev: The kbase device structure for the device * * Handles the start state of the VM */ @@ -532,7 +531,15 @@ static void kbase_arbiter_pm_vm_gpu_start(struct kbase_device *kbdev) case KBASE_VM_STATE_STOPPED_GPU_REQUESTED: kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_STARTING); arb_vm_state->interrupts_installed = true; - kbase_install_interrupts(kbdev); + /* + * Re-install interrupts that were released for external arb_if to + * address Xen requirements. Interrupts are not released with internal + * arb_if as the IRQs are required to handle messaging to/from + * Arbiter/Resource Group. + */ + { + kbase_install_interrupts(kbdev); + } /* * GPU GRANTED received while in stop can be a result of a * repartitioning. @@ -561,7 +568,7 @@ static void kbase_arbiter_pm_vm_gpu_start(struct kbase_device *kbdev) /** * kbase_arbiter_pm_vm_gpu_stop() - Handles the stop state of the VM - * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @kbdev: The kbase device structure for the device * * Handles the start state of the VM */ @@ -603,7 +610,7 @@ static void kbase_arbiter_pm_vm_gpu_stop(struct kbase_device *kbdev) /** * kbase_gpu_lost() - Kbase signals GPU is lost on a lost event signal - * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @kbdev: The kbase device structure for the device * * On GPU lost event signals GPU_LOST to the aribiter */ @@ -658,7 +665,7 @@ static void kbase_gpu_lost(struct kbase_device *kbdev) /** * kbase_arbiter_pm_vm_os_suspend_ready_state() - checks if VM is ready * to be moved to suspended state. - * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @kbdev: The kbase device structure for the device * * Return: True if its ready to be suspended else False. */ @@ -678,7 +685,7 @@ static inline bool kbase_arbiter_pm_vm_os_suspend_ready_state(struct kbase_devic /** * kbase_arbiter_pm_vm_os_prepare_suspend() - Prepare OS to be in suspend state * until it receives the grant message from arbiter - * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @kbdev: The kbase device structure for the device * * Prepares OS to be in suspend state until it receives GRANT message * from Arbiter asynchronously. @@ -745,7 +752,7 @@ static void kbase_arbiter_pm_vm_os_prepare_suspend(struct kbase_device *kbdev) /** * kbase_arbiter_pm_vm_os_resume() - Resume OS function once it receives * a grant message from arbiter - * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @kbdev: The kbase device structure for the device * * Resume OS function once it receives GRANT message * from Arbiter asynchronously. @@ -774,7 +781,7 @@ static void kbase_arbiter_pm_vm_os_resume(struct kbase_device *kbdev) /** * kbase_arbiter_pm_vm_event() - Dispatch VM event to the state machine. - * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @kbdev: The kbase device structure for the device * @evt: VM event * * The state machine function. Receives events and transitions states @@ -853,7 +860,7 @@ void kbase_arbiter_pm_vm_event(struct kbase_device *kbdev, enum kbase_arbif_evt break; default: - dev_alert(kbdev->dev, "Got Unknown Event!"); + dev_err(kbdev->dev, "Got Unknown Event!"); break; } mutex_unlock(&arb_vm_state->vm_state_lock); @@ -863,7 +870,7 @@ KBASE_EXPORT_TEST_API(kbase_arbiter_pm_vm_event); /** * kbase_arbiter_pm_vm_wait_gpu_assignment() - VM wait for a GPU assignment. - * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @kbdev: The kbase device structure for the device * * VM waits for a GPU assignment. */ @@ -879,14 +886,14 @@ static void kbase_arbiter_pm_vm_wait_gpu_assignment(struct kbase_device *kbdev) } /** - * kbase_arbiter_pm_vm_gpu_assigned_lockheld() - Check if VM holds VM state lock - * @kbdev: The kbase device structure for the device (must be a valid pointer) + * kbase_arbiter_pm_vm_gpu_assigned_locked() - Check if VM holds VM state lock + * @kbdev: The kbase device structure for the device * * Checks if the virtual machine holds VM state lock. * * Return: true if GPU is assigned, else false. */ -static inline bool kbase_arbiter_pm_vm_gpu_assigned_lockheld(struct kbase_device *kbdev) +static inline bool kbase_arbiter_pm_vm_gpu_assigned_locked(struct kbase_device *kbdev) { struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; @@ -898,7 +905,7 @@ static inline bool kbase_arbiter_pm_vm_gpu_assigned_lockheld(struct kbase_device /** * kbase_arbiter_pm_ctx_active_handle_suspend() - Handle suspend operation for * arbitration mode - * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @kbdev: The kbase device structure for the device * @suspend_handler: The handler code for how to handle a suspend * that might occur * @@ -916,7 +923,7 @@ int kbase_arbiter_pm_ctx_active_handle_suspend(struct kbase_device *kbdev, if (kbdev->arb.arb_if) { mutex_lock(&arb_vm_state->vm_state_lock); - while (!kbase_arbiter_pm_vm_gpu_assigned_lockheld(kbdev)) { + while (!kbase_arbiter_pm_vm_gpu_assigned_locked(kbdev)) { /* Update VM state since we have GPU work to do */ if (arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_IDLE) kbase_arbiter_pm_vm_set_state(kbdev, diff --git a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.h b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.h index 3734d32b6e2b..649f488d4f67 100644 --- a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.h +++ b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -102,7 +102,7 @@ void kbase_arbiter_pm_release_interrupts(struct kbase_device *kbdev); * * Install interrupts and set the interrupt_install flag to true. * - * Return: 0 if success, or a Linux error code + * Return: 0 if success or already installed. Otherwise a Linux error code */ int kbase_arbiter_pm_install_interrupts(struct kbase_device *kbdev); diff --git a/drivers/gpu/arm/bifrost/backend/gpu/Kbuild b/drivers/gpu/arm/bifrost/backend/gpu/Kbuild index c3db14217c6d..ffec0417aa5c 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/Kbuild +++ b/drivers/gpu/arm/bifrost/backend/gpu/Kbuild @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2014-2022 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -47,12 +47,7 @@ endif bifrost_kbase-$(CONFIG_MALI_BIFROST_DEVFREQ) += \ backend/gpu/mali_kbase_devfreq.o -ifneq ($(CONFIG_MALI_REAL_HW),y) - bifrost_kbase-y += backend/gpu/mali_kbase_model_linux.o -endif +bifrost_kbase-$(CONFIG_MALI_BIFROST_NO_MALI) += backend/gpu/mali_kbase_model_linux.o # NO_MALI Dummy model interface bifrost_kbase-$(CONFIG_MALI_BIFROST_NO_MALI) += backend/gpu/mali_kbase_model_dummy.o -# HW error simulation -bifrost_kbase-$(CONFIG_MALI_BIFROST_NO_MALI) += backend/gpu/mali_kbase_model_error_generator.o - diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.c index 2649f1815e9f..e223535d01f7 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -366,7 +366,7 @@ static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev) err = of_property_read_u64(node, "opp-hz-real", real_freqs); #endif if (err < 0) { - dev_warn(kbdev->dev, "Failed to read opp-hz-real property with error %d\n", + dev_warn(kbdev->dev, "Failed to read opp-hz-real property with error %d", err); continue; } @@ -374,8 +374,8 @@ static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev) err = of_property_read_u32_array(node, "opp-microvolt", opp_volts, kbdev->nr_regulators); if (err < 0) { - dev_warn(kbdev->dev, - "Failed to read opp-microvolt property with error %d\n", err); + dev_warn(kbdev->dev, "Failed to read opp-microvolt property with error %d", + err); continue; } #endif @@ -386,11 +386,12 @@ static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev) dev_warn( kbdev->dev, - "Ignoring OPP %llu - Dynamic Core Scaling not supported on this GPU\n", + "Ignoring OPP %llu - Dynamic Core Scaling not supported on this GPU", opp_freq); continue; } + core_count_p = of_get_property(node, "opp-core-count", NULL); if (core_count_p) { u64 remaining_core_mask = kbdev->gpu_props.shader_present; diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_backend.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_backend.c index 131cfe32df9f..07960713f75a 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_backend.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_backend.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -29,6 +29,8 @@ #include #include +#define WAIT_FOR_DUMP_TIMEOUT_MS 5000 + static int wait_prfcnt_ready(struct kbase_device *kbdev) { u32 val; @@ -163,6 +165,7 @@ int kbase_instr_hwcnt_disable_internal(struct kbase_context *kctx) { unsigned long flags, pm_flags; struct kbase_device *kbdev = kctx->kbdev; + const unsigned long timeout = msecs_to_jiffies(WAIT_FOR_DUMP_TIMEOUT_MS); while (1) { spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); @@ -199,7 +202,8 @@ int kbase_instr_hwcnt_disable_internal(struct kbase_context *kctx) spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); /* Ongoing dump/setup - wait for its completion */ - wait_event(kbdev->hwcnt.backend.wait, kbdev->hwcnt.backend.triggered != 0); + wait_event_timeout(kbdev->hwcnt.backend.wait, kbdev->hwcnt.backend.triggered != 0, + timeout); } kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED; @@ -319,8 +323,19 @@ int kbase_instr_hwcnt_wait_for_dump(struct kbase_context *kctx) unsigned long flags; int err; + unsigned long remaining; + const unsigned long timeout = msecs_to_jiffies(WAIT_FOR_DUMP_TIMEOUT_MS); + /* Wait for dump & cache clean to complete */ - wait_event(kbdev->hwcnt.backend.wait, kbdev->hwcnt.backend.triggered != 0); + remaining = wait_event_timeout(kbdev->hwcnt.backend.wait, + kbdev->hwcnt.backend.triggered != 0, timeout); + if (remaining == 0) { + err = -ETIME; + /* Set the backend state so it's clear things have gone bad (could be a HW issue) + */ + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_UNRECOVERABLE_ERROR; + goto timed_out; + } spin_lock_irqsave(&kbdev->hwcnt.lock, flags); @@ -336,7 +351,7 @@ int kbase_instr_hwcnt_wait_for_dump(struct kbase_context *kctx) } spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - +timed_out: return err; } diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_internal.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_internal.h index 34e8178d1d76..feb76757f955 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_internal.h +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_internal.h @@ -74,7 +74,7 @@ void kbase_synchronize_irqs(struct kbase_device *kbdev); * Return: 0 on success. Error code (negative) on failure. */ int kbase_validate_interrupts(struct kbase_device *const kbdev); -#endif /* CONFIG_MALI_REAL_HW */ +#endif /* IS_ENABLED(CONFIG_MALI_REAL_HW) */ #endif /* CONFIG_MALI_BIFROST_DEBUG */ /** diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_linux.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_linux.c index 9cb367508dde..152b140b5381 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_linux.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_linux.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -23,6 +23,7 @@ #include #include + #include #if IS_ENABLED(CONFIG_MALI_REAL_HW) @@ -163,13 +164,9 @@ static irqreturn_t kbase_gpu_irq_handler(int irq, void *data) static irqreturn_t kbase_combined_irq_handler(int irq, void *data) { irqreturn_t irq_state = IRQ_NONE; - - if (kbase_job_irq_handler(irq, data) == IRQ_HANDLED) - irq_state = IRQ_HANDLED; - if (kbase_mmu_irq_handler(irq, data) == IRQ_HANDLED) - irq_state = IRQ_HANDLED; - if (kbase_gpu_irq_handler(irq, data) == IRQ_HANDLED) - irq_state = IRQ_HANDLED; + irq_state |= kbase_job_irq_handler(irq, data); + irq_state |= kbase_mmu_irq_handler(irq, data); + irq_state |= kbase_gpu_irq_handler(irq, data); return irq_state; } @@ -212,8 +209,7 @@ int kbase_set_custom_irq_handler(struct kbase_device *kbdev, irq_handler_t custo if (!handler) handler = kbase_get_interrupt_handler(kbdev, irq_tag); - if (request_irq(kbdev->irqs[irq].irq, handler, - kbdev->irqs[irq].flags | ((kbdev->nr_irqs == 1) ? 0 : IRQF_SHARED), + if (request_irq(kbdev->irqs[irq].irq, handler, kbdev->irqs[irq].flags | IRQF_SHARED, dev_name(kbdev->dev), kbase_tag(kbdev, irq)) != 0) { result = -EINVAL; dev_err(kbdev->dev, "Can't request interrupt %u (index %u)\n", kbdev->irqs[irq].irq, @@ -396,8 +392,8 @@ static int validate_interrupt(struct kbase_device *const kbdev, u32 tag) /* restore original interrupt */ if (request_irq(kbdev->irqs[irq].irq, kbase_get_interrupt_handler(kbdev, tag), - kbdev->irqs[irq].flags | ((kbdev->nr_irqs == 1) ? 0 : IRQF_SHARED), - dev_name(kbdev->dev), kbase_tag(kbdev, irq))) { + kbdev->irqs[irq].flags | IRQF_SHARED, dev_name(kbdev->dev), + kbase_tag(kbdev, irq))) { dev_err(kbdev->dev, "Can't restore original interrupt %u (index %u)\n", kbdev->irqs[irq].irq, tag); err = -EINVAL; @@ -449,10 +445,10 @@ int kbase_install_interrupts(struct kbase_device *kbdev) u32 i; for (i = 0; i < kbdev->nr_irqs; i++) { - const int result = request_irq( - kbdev->irqs[i].irq, kbase_get_interrupt_handler(kbdev, i), - kbdev->irqs[i].flags | ((kbdev->nr_irqs == 1) ? 0 : IRQF_SHARED), - dev_name(kbdev->dev), kbase_tag(kbdev, i)); + const int result = request_irq(kbdev->irqs[i].irq, + kbase_get_interrupt_handler(kbdev, i), + kbdev->irqs[i].flags | IRQF_SHARED, + dev_name(kbdev->dev), kbase_tag(kbdev, i)); if (result) { dev_err(kbdev->dev, "Can't request interrupt %u (index %u)\n", kbdev->irqs[i].irq, i); diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c index e822dc59977b..cc8a0ff7fa42 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -1328,7 +1328,7 @@ void kbase_reset_gpu(struct kbase_device *kbdev) if (!kbase_is_quick_reset_enabled(kbdev)) dev_err(kbdev->dev, - "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n", + "Preparing to soft-reset GPU: Waiting (up to %d ms) for all jobs to complete soft-stop\n", kbdev->reset_timeout_ms); hrtimer_start(&kbdev->hwaccess.backend.reset_timer, @@ -1350,7 +1350,7 @@ void kbase_reset_gpu_locked(struct kbase_device *kbdev) if (!kbase_is_quick_reset_enabled(kbdev)) dev_err(kbdev->dev, - "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n", + "Preparing to soft-reset GPU: Waiting (up to %d ms) for all jobs to complete soft-stop\n", kbdev->reset_timeout_ms); hrtimer_start(&kbdev->hwaccess.backend.reset_timer, HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms), HRTIMER_MODE_REL); diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c index 842209f9c049..e1105bf90899 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c @@ -1437,7 +1437,7 @@ void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp) * then leave it in the RB and next time we're kicked * it will be processed again from the starting state. */ - if (keep_in_jm_rb) { + if (!kbase_is_gpu_removed(kbdev) && keep_in_jm_rb) { katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; /* As the atom was not removed, increment the * index so that we read the correct atom in the diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_dummy.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_dummy.c index 41b9b37797d3..b034ffef0ceb 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_dummy.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_dummy.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -25,42 +25,8 @@ * insmod'ing mali_kbase.ko with no arguments after a build with "scons * gpu=tXYZ" will yield the expected GPU ID for tXYZ. This can always be * overridden by passing the 'no_mali_gpu' argument to insmod. - * - * - if CONFIG_MALI_BIFROST_ERROR_INJECT is defined the error injection system is - * activated. */ -/* Implementation of failure injection system: - * - * Error conditions are generated by gpu_generate_error(). - * According to CONFIG_MALI_BIFROST_ERROR_INJECT definition gpu_generate_error() either - * generates an error HW condition randomly (CONFIG_MALI_ERROR_INJECT_RANDOM) or - * checks if there is (in error_track_list) an error configuration to be set for - * the current job chain (CONFIG_MALI_ERROR_INJECT_RANDOM not defined). - * Each error condition will trigger a specific "state" for a certain set of - * registers as per Midgard Architecture Specifications doc. - * - * According to Midgard Architecture Specifications doc the following registers - * are always affected by error conditions: - * - * JOB Exception: - * JOB_IRQ_RAWSTAT - * JOB STATUS AREA - * - * MMU Exception: - * MMU_IRQ_RAWSTAT - * AS_FAULTSTATUS - * AS_FAULTADDRESS - * - * GPU Exception: - * GPU_IRQ_RAWSTAT - * GPU_FAULTSTATUS - * GPU_FAULTADDRESS - * - * For further clarification on the model behaviour upon specific error - * conditions the user may refer to the Midgard Architecture Specification - * document - */ #include #include #include @@ -126,7 +92,7 @@ struct error_status_t hw_error_status; */ struct control_reg_values_t { const char *name; - u32 gpu_id; + u64 gpu_id; u32 as_present; u32 thread_max_threads; u32 thread_max_workgroup_size; @@ -524,7 +490,7 @@ MODULE_PARM_DESC(no_mali_gpu, "GPU to identify as"); static u32 gpu_model_get_prfcnt_value(enum kbase_ipa_core_type core_type, u32 cnt_idx, bool is_low_word) { - u64 *counters_data; + u64 *counters_data = NULL; u32 core_count = 0; u32 event_index; u64 value = 0; @@ -580,6 +546,9 @@ static u32 gpu_model_get_prfcnt_value(enum kbase_ipa_core_type core_type, u32 cn break; } + if (unlikely(counters_data == NULL)) + return 0; + for (core = 0; core < core_count; core++) { value += counters_data[event_index]; event_index += KBASE_DUMMY_MODEL_COUNTER_PER_CORE; @@ -1172,9 +1141,6 @@ static void midgard_model_update(void *h) /*this job is done assert IRQ lines */ signal_int(dummy, i); -#ifdef CONFIG_MALI_BIFROST_ERROR_INJECT - midgard_set_error(i); -#endif /* CONFIG_MALI_BIFROST_ERROR_INJECT */ update_register_statuses(dummy, i); /*if this job slot returned failures we cannot use it */ if (hw_error_status.job_irq_rawstat & (1u << (i + 16))) { @@ -1564,6 +1530,7 @@ void midgard_model_write_reg(void *h, u32 addr, u32 value) case L2_PWROFF_HI: case PWR_KEY: case PWR_OVERRIDE0: + case PWR_OVERRIDE1: #if MALI_USE_CSF case SHADER_PWRFEATURES: case CSF_CONFIG: @@ -1607,8 +1574,7 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value) #else /* !MALI_USE_CSF */ if (addr == GPU_CONTROL_REG(GPU_ID)) { #endif /* !MALI_USE_CSF */ - - *value = dummy->control_reg_values->gpu_id; + *value = dummy->control_reg_values->gpu_id & U32_MAX; } else if (addr == JOB_CONTROL_REG(JOB_IRQ_RAWSTAT)) { *value = hw_error_status.job_irq_rawstat; pr_debug("%s", "JS_IRQ_RAWSTAT being read"); @@ -2166,9 +2132,3 @@ int gpu_model_control(void *model, struct kbase_model_control_params *params) return 0; } - -u64 midgard_model_arch_timer_get_cntfrq(void *h) -{ - CSTD_UNUSED(h); - return arch_timer_get_cntfrq(); -} diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_error_generator.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_error_generator.c deleted file mode 100644 index 86d4e26bd6b4..000000000000 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_error_generator.c +++ /dev/null @@ -1,172 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2014-2015, 2018-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include -#include -#include "backend/gpu/mali_kbase_model_linux.h" - -static struct kbase_error_atom *error_track_list; - -#ifdef CONFIG_MALI_ERROR_INJECT_RANDOM - -/** Kernel 6.1.0 has dropped prandom_u32(), use get_random_u32() */ -#if (KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE) -#define prandom_u32 get_random_u32 -#endif - -/*following error probability are set quite high in order to stress the driver*/ -static unsigned int error_probability = 50; /* to be set between 0 and 100 */ -/* probability to have multiple error give that there is an error */ -static unsigned int multiple_error_probability = 50; - -/* all the error conditions supported by the model */ -#define TOTAL_FAULTS 27 -/* maximum number of levels in the MMU translation table tree */ -#define MAX_MMU_TABLE_LEVEL 4 -/* worst case scenario is <1 MMU fault + 1 job fault + 2 GPU faults> */ -#define MAX_CONCURRENT_FAULTS 3 - -/** - * gpu_generate_error - Generate GPU error - */ -static void gpu_generate_error(void) -{ - unsigned int errors_num = 0; - - /*is there at least one error? */ - if ((prandom_u32() % 100) < error_probability) { - /* pick up a faulty mmu address space */ - hw_error_status.faulty_mmu_as = prandom_u32() % NUM_MMU_AS; - /* pick up an mmu table level */ - hw_error_status.mmu_table_level = 1 + (prandom_u32() % MAX_MMU_TABLE_LEVEL); - hw_error_status.errors_mask = (u32)(1 << (prandom_u32() % TOTAL_FAULTS)); - - /*is there also one or more errors? */ - if ((prandom_u32() % 100) < multiple_error_probability) { - errors_num = 1 + (prandom_u32() % (MAX_CONCURRENT_FAULTS - 1)); - while (errors_num-- > 0) { - u32 temp_mask; - - temp_mask = (u32)(1 << (prandom_u32() % TOTAL_FAULTS)); - /* below we check that no bit of the same error - * type is set again in the error mask - */ - if ((temp_mask & IS_A_JOB_ERROR) && - (hw_error_status.errors_mask & IS_A_JOB_ERROR)) { - errors_num++; - continue; - } - if ((temp_mask & IS_A_MMU_ERROR) && - (hw_error_status.errors_mask & IS_A_MMU_ERROR)) { - errors_num++; - continue; - } - if ((temp_mask & IS_A_GPU_ERROR) && - (hw_error_status.errors_mask & IS_A_GPU_ERROR)) { - errors_num++; - continue; - } - /* this error mask is already set */ - if ((hw_error_status.errors_mask | temp_mask) == - hw_error_status.errors_mask) { - errors_num++; - continue; - } - hw_error_status.errors_mask |= temp_mask; - } - } - } -} -#endif - -int job_atom_inject_error(struct kbase_error_params *params) -{ - struct kbase_error_atom *new_elem; - - KBASE_DEBUG_ASSERT(params); - - new_elem = kzalloc(sizeof(*new_elem), GFP_KERNEL); - - if (!new_elem) { - model_error_log(KBASE_CORE, - "\njob_atom_inject_error: kzalloc failed for new_elem\n"); - return -ENOMEM; - } - new_elem->params.jc = params->jc; - new_elem->params.errors_mask = params->errors_mask; - new_elem->params.mmu_table_level = params->mmu_table_level; - new_elem->params.faulty_mmu_as = params->faulty_mmu_as; - - /*circular list below */ - if (error_track_list == NULL) { /*no elements */ - error_track_list = new_elem; - new_elem->next = error_track_list; - } else { - struct kbase_error_atom *walker = error_track_list; - - while (walker->next != error_track_list) - walker = walker->next; - - new_elem->next = error_track_list; - walker->next = new_elem; - } - return 0; -} - -void midgard_set_error(u32 job_slot) -{ -#ifdef CONFIG_MALI_ERROR_INJECT_RANDOM - gpu_generate_error(); -#else - struct kbase_error_atom *walker, *auxiliar; - - if (error_track_list != NULL) { - walker = error_track_list->next; - auxiliar = error_track_list; - do { - if (walker->params.jc == hw_error_status.current_jc) { - /* found a faulty atom matching with the - * current one - */ - hw_error_status.errors_mask = walker->params.errors_mask; - hw_error_status.mmu_table_level = walker->params.mmu_table_level; - hw_error_status.faulty_mmu_as = walker->params.faulty_mmu_as; - hw_error_status.current_job_slot = job_slot; - - if (walker->next == walker) { - /* only one element */ - kfree(error_track_list); - error_track_list = NULL; - } else { - auxiliar->next = walker->next; - if (walker == error_track_list) - error_track_list = walker->next; - - kfree(walker); - } - break; - } - auxiliar = walker; - walker = walker->next; - } while (auxiliar->next != error_track_list); - } -#endif /* CONFIG_MALI_ERROR_INJECT_RANDOM */ -} diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_linux.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_linux.h index 77e089ef45c8..d38bb8891be1 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_linux.h +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_linux.h @@ -48,12 +48,8 @@ /* * Include Model definitions */ - -#if IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) #include -#endif /* IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) */ -#if !IS_ENABLED(CONFIG_MALI_REAL_HW) /** * kbase_gpu_device_create() - Generic create function. * @@ -116,15 +112,6 @@ void midgard_model_write_reg(void *h, u32 addr, u32 value); */ void midgard_model_read_reg(void *h, u32 addr, u32 *const value); -/** - * midgard_model_arch_timer_get_cntfrq - Get Model specific System Timer Frequency - * - * @h: Model handle. - * - * Return: Frequency in Hz - */ -u64 midgard_model_arch_timer_get_cntfrq(void *h); - /** * gpu_device_raise_irq() - Private IRQ raise function. * @@ -155,6 +142,5 @@ void gpu_device_set_data(void *model, void *data); * Return: Pointer to the data carried by model. */ void *gpu_device_get_data(void *model); -#endif /* !IS_ENABLED(CONFIG_MALI_REAL_HW) */ #endif /* _KBASE_MODEL_LINUX_H_ */ diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c index ca4e73d3fbb7..6db242af0578 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -36,6 +36,9 @@ #include #include #include +#ifdef CONFIG_MALI_ARBITER_SUPPORT +#include +#endif /* !CONFIG_MALI_ARBITER_SUPPORT */ #endif /* !MALI_USE_CSF */ #include #include @@ -393,7 +396,7 @@ static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data) backend->poweron_required = false; kbdev->pm.backend.l2_desired = true; #if MALI_USE_CSF - kbdev->pm.backend.mcu_desired = true; + kbdev->pm.backend.mcu_desired = kbdev->pm.backend.mcu_poweron_required; #endif kbase_pm_update_state(kbdev); kbase_pm_update_cores_state_nolock(kbdev); @@ -860,9 +863,11 @@ void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, u64 new_core_mask) } KBASE_EXPORT_TEST_API(kbase_pm_set_debug_core_mask); #else -void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, u64 new_core_mask_js0, - u64 new_core_mask_js1, u64 new_core_mask_js2) +void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, u64 *new_core_mask, + size_t new_core_mask_size) { + size_t i; + lockdep_assert_held(&kbdev->hwaccess_lock); lockdep_assert_held(&kbdev->pm.lock); @@ -870,13 +875,14 @@ void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, u64 new_core_mask_ dev_warn_once( kbdev->dev, "Change of core mask not supported for slot 0 as dummy job WA is enabled"); - new_core_mask_js0 = kbdev->pm.debug_core_mask[0]; + new_core_mask[0] = kbdev->pm.debug_core_mask[0]; } - kbdev->pm.debug_core_mask[0] = new_core_mask_js0; - kbdev->pm.debug_core_mask[1] = new_core_mask_js1; - kbdev->pm.debug_core_mask[2] = new_core_mask_js2; - kbdev->pm.debug_core_mask_all = new_core_mask_js0 | new_core_mask_js1 | new_core_mask_js2; + kbdev->pm.debug_core_mask_all = 0; + for (i = 0; i < new_core_mask_size; i++) { + kbdev->pm.debug_core_mask[i] = new_core_mask[i]; + kbdev->pm.debug_core_mask_all |= new_core_mask[i]; + } kbase_pm_update_dynamic_cores_onoff(kbdev); } @@ -962,7 +968,9 @@ void kbase_hwaccess_pm_resume(struct kbase_device *kbdev) void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev) { unsigned long flags; -#if !MALI_USE_CSF +#if MALI_USE_CSF + unsigned long flags_sched; +#else ktime_t end_timestamp = ktime_get_raw(); #endif struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; @@ -981,24 +989,44 @@ void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev) */ WARN(!kbase_is_gpu_removed(kbdev), "GPU is still available after GPU lost event\n"); - /* Full GPU reset will have been done by hypervisor, so - * cancel - */ +#if MALI_USE_CSF + /* Full GPU reset will have been done by hypervisor, so cancel */ + kbase_reset_gpu_prevent_and_wait(kbdev); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_csf_scheduler_spin_lock(kbdev, &flags_sched); + atomic_set(&kbdev->hwaccess.backend.reset_gpu, KBASE_RESET_GPU_NOT_PENDING); + kbase_csf_scheduler_spin_unlock(kbdev, flags_sched); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + kbase_synchronize_irqs(kbdev); + + /* Scheduler reset happens outside of spinlock due to the mutex it acquires */ + kbase_csf_scheduler_reset(kbdev); + + /* Update kbase status */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbdev->protected_mode = false; + kbase_pm_update_state(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + /* Cancel any pending HWC dumps */ + kbase_hwcnt_backend_csf_on_unrecoverable_error(&kbdev->hwcnt_gpu_iface); +#else + /* Full GPU reset will have been done by hypervisor, so cancel */ atomic_set(&kbdev->hwaccess.backend.reset_gpu, KBASE_RESET_GPU_NOT_PENDING); hrtimer_cancel(&kbdev->hwaccess.backend.reset_timer); + kbase_synchronize_irqs(kbdev); /* Clear all jobs running on the GPU */ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbdev->protected_mode = false; -#if !MALI_USE_CSF kbase_backend_reset(kbdev, &end_timestamp); kbase_pm_metrics_update(kbdev, NULL); -#endif kbase_pm_update_state(kbdev); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#if !MALI_USE_CSF /* Cancel any pending HWC dumps */ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DUMPING || @@ -1008,12 +1036,11 @@ void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev) wake_up(&kbdev->hwcnt.backend.wait); } spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -#endif +#endif /* MALI_USE_CSF */ } mutex_unlock(&arb_vm_state->vm_state_lock); mutex_unlock(&kbdev->pm.lock); } - #endif /* CONFIG_MALI_ARBITER_SUPPORT */ #if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) @@ -1063,26 +1090,15 @@ static int pm_handle_mcu_sleep_on_runtime_suspend(struct kbase_device *kbdev) } /* Check if a Doorbell mirror interrupt occurred meanwhile. - * Also check if GPU idle work item is pending. If FW had sent the GPU idle notification - * after the wake up of MCU then it can be assumed that Userspace submission didn't make - * GPU non-idle, so runtime suspend doesn't need to be aborted. */ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - if (kbdev->pm.backend.gpu_sleep_mode_active && kbdev->pm.backend.exit_gpu_sleep_mode && - !work_pending(&kbdev->csf.scheduler.gpu_idle_work)) { - u32 glb_req = - kbase_csf_firmware_global_input_read(&kbdev->csf.global_iface, GLB_REQ); - u32 glb_ack = kbase_csf_firmware_global_output(&kbdev->csf.global_iface, GLB_ACK); - - /* Only abort the runtime suspend if GPU idle event is not pending */ - if (!((glb_req ^ glb_ack) & GLB_REQ_IDLE_EVENT_MASK)) { - dev_dbg(kbdev->dev, - "DB mirror interrupt occurred during runtime suspend after L2 power up"); - kbdev->pm.backend.gpu_wakeup_override = false; - kbdev->pm.backend.runtime_suspend_abort_reason = ABORT_REASON_DB_MIRROR_IRQ; - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - return -EBUSY; - } + if (kbdev->pm.backend.gpu_sleep_mode_active && kbdev->pm.backend.exit_gpu_sleep_mode) { + dev_dbg(kbdev->dev, + "DB mirror interrupt occurred during runtime suspend after L2 power up"); + kbdev->pm.backend.gpu_wakeup_override = false; + kbdev->pm.backend.runtime_suspend_abort_reason = ABORT_REASON_DB_MIRROR_IRQ; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + return -EBUSY; } spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); /* Need to release the kbdev->pm.lock to avoid lock ordering issue diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h index a0b8b9500077..34c34df7f82f 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -332,7 +332,11 @@ union kbase_pm_policy_data { * cores may be different, but there should be transitions in * progress that will eventually achieve this state (assuming * that the policy doesn't change its mind in the mean time). - * @mcu_desired: True if the micro-control unit should be powered on + * @mcu_desired: True if the micro-control unit should be powered on by the MCU state + * machine. Updated as per the value of @mcu_poweron_required. + * @mcu_poweron_required: Boolean flag updated mainly by the CSF Scheduler code, + * before updating the PM active count, to indicate to the + * PM code that micro-control unit needs to be powered up/down. * @policy_change_clamp_state_to_off: Signaling the backend is in PM policy * change transition, needs the mcu/L2 to be brought back to the * off state and remain in that state until the flag is cleared. @@ -485,6 +489,7 @@ struct kbase_pm_backend_data { u64 shaders_desired_mask; #if MALI_USE_CSF bool mcu_desired; + bool mcu_poweron_required; bool policy_change_clamp_state_to_off; unsigned int csf_pm_sched_flags; struct mutex policy_change_lock; diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c index 506e168f86d2..9e85cf5589f4 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -70,6 +70,19 @@ MODULE_PARM_DESC(corestack_driver_control, "to the Mali GPU is known to be problematic."); KBASE_EXPORT_TEST_API(corestack_driver_control); +/** + * enum kbase_gpu_state - The state of data in the GPU. + * + * @GPU_STATE_INTACT: The GPU state is intact + * @GPU_STATE_LOST: The GPU state is lost + * @GPU_STATE_IN_RESET: The GPU is in reset state + * + * This enumeration is private to the file. It is used as + * the return values of platform specific PM + * callback (*power_on_callback). + */ +enum kbase_gpu_state { GPU_STATE_INTACT = 0, GPU_STATE_LOST, GPU_STATE_IN_RESET }; + /** * enum kbasep_pm_action - Actions that can be performed on a core. * @@ -110,7 +123,15 @@ bool kbase_pm_is_mcu_desired(struct kbase_device *kbdev) if (kbdev->pm.backend.l2_force_off_after_mcu_halt) return false; - if (kbdev->csf.scheduler.pm_active_count && kbdev->pm.backend.mcu_desired) + /* Check if policy changing transition needs MCU to be off. */ + if (unlikely(kbdev->pm.backend.policy_change_clamp_state_to_off)) + return false; + + if (kbdev->pm.backend.mcu_desired) + return true; + + /* For always_on policy, the MCU needs to be kept on */ + if (kbase_pm_no_mcu_core_pwroff(kbdev)) return true; #ifdef KBASE_PM_RUNTIME @@ -119,13 +140,7 @@ bool kbase_pm_is_mcu_desired(struct kbase_device *kbdev) return true; #endif - /* MCU is supposed to be ON, only when scheduler.pm_active_count is - * non zero. But for always_on policy, the MCU needs to be kept on, - * unless policy changing transition needs it off. - */ - - return (kbdev->pm.backend.mcu_desired && kbase_pm_no_mcu_core_pwroff(kbdev) && - !kbdev->pm.backend.policy_change_clamp_state_to_off); + return false; } #endif @@ -979,8 +994,8 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) kbase_hwcnt_backend_csf_set_hw_availability( &kbdev->hwcnt_gpu_iface, kbdev->gpu_props.curr_config.l2_slices, - kbdev->gpu_props.curr_config.shader_present & - kbdev->pm.debug_core_mask); + kbdev->gpu_props.curr_config.shader_present, + kbdev->pm.debug_core_mask); kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); kbase_csf_scheduler_spin_unlock(kbdev, flags); backend->hwcnt_disabled = false; @@ -1342,6 +1357,8 @@ static void kbase_pm_l2_clear_backend_slot_submit_kctx(struct kbase_device *kbde static bool can_power_down_l2(struct kbase_device *kbdev) { + lockdep_assert_held(&kbdev->hwaccess_lock); + /* Defer the power-down if MMU is in process of page migration. */ return !kbdev->mmu_page_migrate_in_progress; } @@ -2797,7 +2814,7 @@ static void update_user_reg_page_mapping(struct kbase_device *kbdev) void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) { struct kbase_pm_backend_data *backend = &kbdev->pm.backend; - bool reset_required = is_resume; + int ret = is_resume; unsigned long flags; KBASE_DEBUG_ASSERT(kbdev != NULL); @@ -2836,7 +2853,7 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) backend->callback_power_resume(kbdev); return; } else if (backend->callback_power_on) { - reset_required = backend->callback_power_on(kbdev); + ret = backend->callback_power_on(kbdev); } spin_lock_irqsave(&kbdev->hwaccess_lock, flags); @@ -2849,7 +2866,12 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) #endif - if (reset_required) { + if (ret == GPU_STATE_IN_RESET) { + /* GPU is already in reset state after power on and no + * soft-reset needed. Just reconfiguration is needed. + */ + kbase_pm_init_hw(kbdev, PM_ENABLE_IRQS | PM_NO_RESET); + } else if (ret == GPU_STATE_LOST) { /* GPU state was lost, reset GPU to ensure it is in a * consistent state */ @@ -2898,7 +2920,7 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) backend->l2_desired = true; #if MALI_USE_CSF { - if (reset_required) { + if (ret != GPU_STATE_INTACT) { /* GPU reset was done after the power on, so send the post * reset event instead. This is okay as GPU power off event * is same as pre GPU reset event. @@ -3139,6 +3161,7 @@ static int kbase_set_tiler_quirks(struct kbase_device *kbdev) return 0; } + static int kbase_pm_hw_issues_detect(struct kbase_device *kbdev) { struct device_node *np = kbdev->dev->of_node; @@ -3191,6 +3214,7 @@ static int kbase_pm_hw_issues_detect(struct kbase_device *kbdev) error = kbase_set_mmu_quirks(kbdev); } + return error; } @@ -3210,6 +3234,7 @@ static void kbase_pm_hw_issues_apply(struct kbase_device *kbdev) #else kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(JM_CONFIG), kbdev->hw_quirks_gpu); #endif + } void kbase_pm_cache_snoop_enable(struct kbase_device *kbdev) @@ -3257,16 +3282,10 @@ static void reenable_protected_mode_hwcnt(struct kbase_device *kbdev) } #endif -static int kbase_pm_do_reset(struct kbase_device *kbdev) +static int kbase_pm_do_reset_soft(struct kbase_device *kbdev) { - struct kbasep_reset_timeout_data rtdata; - u32 reg_offset, reg_val; int ret; - KBASE_KTRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, 0); - - KBASE_TLSTREAM_JD_GPU_SOFT_RESET(kbdev, kbdev); - if (kbdev->pm.backend.callback_soft_reset) { ret = kbdev->pm.backend.callback_soft_reset(kbdev); if (ret < 0) @@ -3279,12 +3298,30 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev) GPU_COMMAND_SOFT_RESET); } } + return 0; +} - reg_offset = GPU_CONTROL_ENUM(GPU_IRQ_MASK); - reg_val = RESET_COMPLETED; +static int kbase_pm_do_reset(struct kbase_device *kbdev) +{ + struct kbasep_reset_timeout_data rtdata; + u32 reg_offset, reg_val; + int ret; - /* Unmask the reset complete interrupt only */ - kbase_reg_write32(kbdev, reg_offset, reg_val); + KBASE_KTRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, 0); + + KBASE_TLSTREAM_JD_GPU_SOFT_RESET(kbdev, kbdev); + + { + ret = kbase_pm_do_reset_soft(kbdev); + if (ret) + return ret; + + reg_offset = GPU_CONTROL_ENUM(GPU_IRQ_MASK); + reg_val = RESET_COMPLETED; + + /* Unmask the reset complete interrupt only */ + kbase_reg_write32(kbdev, reg_offset, reg_val); + } /* Initialize a structure for tracking the status of the reset */ rtdata.kbdev = kbdev; @@ -3335,7 +3372,7 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev) */ #ifdef CONFIG_MALI_ARBITER_SUPPORT if (!kbdev->arb.arb_if) { -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ +#endif dev_err(kbdev->dev, "Failed to soft-reset GPU (timed out after %d ms), now attempting a hard reset\n", RESET_TIMEOUT); @@ -3367,7 +3404,7 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev) RESET_TIMEOUT); #ifdef CONFIG_MALI_ARBITER_SUPPORT } -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ +#endif return -EINVAL; } @@ -3418,9 +3455,7 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); /* Soft reset the GPU */ -#ifdef CONFIG_MALI_ARBITER_SUPPORT if (!(flags & PM_NO_RESET)) -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ err = kbdev->protected_ops->protected_mode_disable(kbdev->protected_dev); spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); @@ -3441,7 +3476,6 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) if (err) goto exit; - if (flags & PM_HW_ISSUES_DETECT) { err = kbase_pm_hw_issues_detect(kbdev); if (err) @@ -3451,6 +3485,9 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) kbase_pm_hw_issues_apply(kbdev); kbase_cache_set_coherency_mode(kbdev, kbdev->system_coherency); kbase_amba_set_shareable_cache_support(kbdev); +#if MALI_USE_CSF + kbase_backend_update_gpu_timestamp_offset(kbdev); +#endif /* Sanity check protected mode was left after reset */ WARN_ON(kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_STATUS)) & diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c index 23e447b15767..c8e3f40335d6 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -127,7 +127,7 @@ void kbase_pm_update_active(struct kbase_device *kbdev) pm->backend.poweroff_wait_in_progress = false; pm->backend.l2_desired = true; #if MALI_USE_CSF - pm->backend.mcu_desired = true; + pm->backend.mcu_desired = pm->backend.mcu_poweron_required; #endif spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c index 0bf0f5a062d3..331c26c6a310 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -30,10 +30,7 @@ #include #include #include - -#if !IS_ENABLED(CONFIG_MALI_REAL_HW) -#include -#endif +#include struct kbase_timeout_info { char *selector_str; @@ -41,12 +38,16 @@ struct kbase_timeout_info { }; #if MALI_USE_CSF + +#define GPU_TIMESTAMP_OFFSET_INVALID S64_MAX + static struct kbase_timeout_info timeout_info[KBASE_TIMEOUT_SELECTOR_COUNT] = { [CSF_FIRMWARE_TIMEOUT] = { "CSF_FIRMWARE_TIMEOUT", MIN(CSF_FIRMWARE_TIMEOUT_CYCLES, CSF_FIRMWARE_PING_TIMEOUT_CYCLES) }, [CSF_PM_TIMEOUT] = { "CSF_PM_TIMEOUT", CSF_PM_TIMEOUT_CYCLES }, [CSF_GPU_RESET_TIMEOUT] = { "CSF_GPU_RESET_TIMEOUT", CSF_GPU_RESET_TIMEOUT_CYCLES }, [CSF_CSG_SUSPEND_TIMEOUT] = { "CSF_CSG_SUSPEND_TIMEOUT", CSF_CSG_SUSPEND_TIMEOUT_CYCLES }, + [CSF_CSG_TERM_TIMEOUT] = { "CSF_CSG_TERM_TIMEOUT", CSF_CSG_TERM_TIMEOUT_CYCLES }, [CSF_FIRMWARE_BOOT_TIMEOUT] = { "CSF_FIRMWARE_BOOT_TIMEOUT", CSF_FIRMWARE_BOOT_TIMEOUT_CYCLES }, [CSF_FIRMWARE_PING_TIMEOUT] = { "CSF_FIRMWARE_PING_TIMEOUT", @@ -82,6 +83,68 @@ static struct kbase_timeout_info timeout_info[KBASE_TIMEOUT_SELECTOR_COUNT] = { }; #endif +#if MALI_USE_CSF +void kbase_backend_invalidate_gpu_timestamp_offset(struct kbase_device *kbdev) +{ + kbdev->backend_time.gpu_timestamp_offset = GPU_TIMESTAMP_OFFSET_INVALID; +} +KBASE_EXPORT_TEST_API(kbase_backend_invalidate_gpu_timestamp_offset); + +/** + * kbase_backend_compute_gpu_ts_offset() - Compute GPU TS offset. + * + * @kbdev: Kbase device. + * + * This function compute the value of GPU and CPU TS offset: + * - set to zero current TIMESTAMP_OFFSET register + * - read CPU TS and convert it to ticks + * - read GPU TS + * - calculate diff between CPU and GPU ticks + * - cache the diff as the GPU TS offset + * + * To reduce delays, preemption must be disabled during reads of both CPU and GPU TS + * this function require access to GPU register to be enabled + */ +static inline void kbase_backend_compute_gpu_ts_offset(struct kbase_device *kbdev) +{ + s64 cpu_ts_ticks = 0; + s64 gpu_ts_ticks = 0; + + if (kbdev->backend_time.gpu_timestamp_offset != GPU_TIMESTAMP_OFFSET_INVALID) + return; + + kbase_reg_write64(kbdev, GPU_CONTROL_ENUM(TIMESTAMP_OFFSET), 0); + + gpu_ts_ticks = kbase_reg_read64_coherent(kbdev, GPU_CONTROL_ENUM(TIMESTAMP)); + cpu_ts_ticks = ktime_get_raw_ns(); + cpu_ts_ticks = div64_u64(cpu_ts_ticks * kbdev->backend_time.divisor, + kbdev->backend_time.multiplier); + kbdev->backend_time.gpu_timestamp_offset = cpu_ts_ticks - gpu_ts_ticks; +} + +void kbase_backend_update_gpu_timestamp_offset(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->pm.lock); + + kbase_backend_compute_gpu_ts_offset(kbdev); + + dev_dbg(kbdev->dev, "Setting GPU timestamp offset register to %lld (%lld ns)", + kbdev->backend_time.gpu_timestamp_offset, + div64_s64(kbdev->backend_time.gpu_timestamp_offset * + (s64)kbdev->backend_time.multiplier, + (s64)kbdev->backend_time.divisor)); + kbase_reg_write64(kbdev, GPU_CONTROL_ENUM(TIMESTAMP_OFFSET), + kbdev->backend_time.gpu_timestamp_offset); +} +#if MALI_UNIT_TEST +u64 kbase_backend_read_gpu_timestamp_offset_reg(struct kbase_device *kbdev) +{ + return kbase_reg_read64_coherent(kbdev, GPU_CONTROL_ENUM(TIMESTAMP_OFFSET)); +} +KBASE_EXPORT_TEST_API(kbase_backend_read_gpu_timestamp_offset_reg); +#endif +#endif + void kbase_backend_get_gpu_time_norequest(struct kbase_device *kbdev, u64 *cycle_counter, u64 *system_time, struct timespec64 *ts) { @@ -100,6 +163,7 @@ void kbase_backend_get_gpu_time_norequest(struct kbase_device *kbdev, u64 *cycle ktime_get_raw_ts64(ts); #endif } +KBASE_EXPORT_TEST_API(kbase_backend_get_gpu_time_norequest); #if !MALI_USE_CSF /** @@ -143,6 +207,7 @@ void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, kbase_pm_release_gpu_cycle_counter(kbdev); #endif } +KBASE_EXPORT_TEST_API(kbase_backend_get_gpu_time); static u64 kbase_device_get_scaling_frequency(struct kbase_device *kbdev) { @@ -282,36 +347,14 @@ u64 __maybe_unused kbase_backend_time_convert_gpu_to_cpu(struct kbase_device *kb if (WARN_ON(!kbdev)) return 0; - return div64_u64(gpu_ts * kbdev->backend_time.multiplier, kbdev->backend_time.divisor) + - kbdev->backend_time.offset; -} - -/** - * get_cpu_gpu_time() - Get current CPU and GPU timestamps. - * - * @kbdev: Kbase device. - * @cpu_ts: Output CPU timestamp. - * @gpu_ts: Output GPU timestamp. - * @gpu_cycle: Output GPU cycle counts. - */ -static void get_cpu_gpu_time(struct kbase_device *kbdev, u64 *cpu_ts, u64 *gpu_ts, u64 *gpu_cycle) -{ - struct timespec64 ts; - - kbase_backend_get_gpu_time(kbdev, gpu_cycle, gpu_ts, &ts); - - if (cpu_ts) - *cpu_ts = (u64)(ts.tv_sec * NSEC_PER_SEC + ts.tv_nsec); + return div64_u64(gpu_ts * kbdev->backend_time.multiplier, kbdev->backend_time.divisor); } +KBASE_EXPORT_TEST_API(kbase_backend_time_convert_gpu_to_cpu); #endif u64 kbase_arch_timer_get_cntfrq(struct kbase_device *kbdev) { - u64 freq = arch_timer_get_cntfrq(); - -#if !IS_ENABLED(CONFIG_MALI_REAL_HW) - freq = midgard_model_arch_timer_get_cntfrq(kbdev->model); -#endif + u64 freq = mali_arch_timer_get_cntfrq(); dev_dbg(kbdev->dev, "System Timer Freq = %lluHz", freq); @@ -322,13 +365,10 @@ int kbase_backend_time_init(struct kbase_device *kbdev) { int err = 0; #if MALI_USE_CSF - u64 cpu_ts = 0; - u64 gpu_ts = 0; u64 freq; u64 common_factor; kbase_pm_register_access_enable(kbdev); - get_cpu_gpu_time(kbdev, &cpu_ts, &gpu_ts, NULL); freq = kbase_arch_timer_get_cntfrq(kbdev); if (!freq) { @@ -348,9 +388,8 @@ int kbase_backend_time_init(struct kbase_device *kbdev) goto disable_registers; } - kbdev->backend_time.offset = - (s64)(cpu_ts - div64_u64(gpu_ts * kbdev->backend_time.multiplier, - kbdev->backend_time.divisor)); + kbase_backend_invalidate_gpu_timestamp_offset( + kbdev); /* force computation of GPU Timestamp offset */ #endif if (kbase_timeout_scaling_init(kbdev)) { diff --git a/drivers/gpu/arm/bifrost/build.bp b/drivers/gpu/arm/bifrost/build.bp index 9ee968af8de5..861282bd4fab 100644 --- a/drivers/gpu/arm/bifrost/build.bp +++ b/drivers/gpu/arm/bifrost/build.bp @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2017-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2017-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -71,18 +71,6 @@ bob_defaults { mali_real_hw: { kbuild_options: ["CONFIG_MALI_REAL_HW=y"], }, - mali_error_inject_none: { - kbuild_options: ["CONFIG_MALI_ERROR_INJECT_NONE=y"], - }, - mali_error_inject_track_list: { - kbuild_options: ["CONFIG_MALI_ERROR_INJECT_TRACK_LIST=y"], - }, - mali_error_inject_random: { - kbuild_options: ["CONFIG_MALI_ERROR_INJECT_RANDOM=y"], - }, - mali_error_inject: { - kbuild_options: ["CONFIG_MALI_BIFROST_ERROR_INJECT=y"], - }, mali_debug: { kbuild_options: [ "CONFIG_MALI_BIFROST_DEBUG=y", @@ -239,6 +227,7 @@ bob_kernel_module { "jm/*.h", "tl/backend/*_jm.c", "mmu/backend/*_jm.c", + "mmu/backend/*_jm.h", "ipa/backend/*_jm.c", "ipa/backend/*_jm.h", ], @@ -263,6 +252,7 @@ bob_kernel_module { "hwcnt/backend/*_csf_*.h", "tl/backend/*_csf.c", "mmu/backend/*_csf.c", + "mmu/backend/*_csf.h", "ipa/backend/*_csf.c", "ipa/backend/*_csf.h", ], diff --git a/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_csf.c b/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_csf.c index 8b1410886b05..f973d39ebb22 100644 --- a/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_csf.c +++ b/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_csf.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -116,8 +116,7 @@ static void kbase_context_term_partial(struct kbase_context *kctx, unsigned int struct kbase_context *kbase_create_context(struct kbase_device *kbdev, bool is_compat, base_context_create_flags const flags, - unsigned long const api_version, - struct kbase_file *const kfile) + unsigned long const api_version, struct file *const filp) { struct kbase_context *kctx; unsigned int i = 0; @@ -136,7 +135,7 @@ struct kbase_context *kbase_create_context(struct kbase_device *kbdev, bool is_c kctx->kbdev = kbdev; kctx->api_version = api_version; - kctx->kfile = kfile; + kctx->filp = filp; kctx->create_flags = flags; memcpy(kctx->comm, current->comm, sizeof(current->comm)); @@ -187,11 +186,17 @@ void kbase_destroy_context(struct kbase_context *kctx) * Customer side that a hang could occur if context termination is * not blocked until the resume of GPU device. */ +#ifdef CONFIG_MALI_ARBITER_SUPPORT + atomic_inc(&kbdev->pm.gpu_users_waiting); +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ while (kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE)) { - dev_info(kbdev->dev, "Suspend in progress when destroying context"); + dev_dbg(kbdev->dev, "Suspend in progress when destroying context"); wait_event(kbdev->pm.resume_wait, !kbase_pm_is_suspending(kbdev)); } +#ifdef CONFIG_MALI_ARBITER_SUPPORT + atomic_dec(&kbdev->pm.gpu_users_waiting); +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ /* Have synchronized against the System suspend and incremented the * pm.active_count. So any subsequent invocation of System suspend diff --git a/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_jm.c b/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_jm.c index f2eefe9ddcd0..06c2ed813de3 100644 --- a/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_jm.c +++ b/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_jm.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -168,8 +168,7 @@ static void kbase_context_term_partial(struct kbase_context *kctx, unsigned int struct kbase_context *kbase_create_context(struct kbase_device *kbdev, bool is_compat, base_context_create_flags const flags, - unsigned long const api_version, - struct kbase_file *const kfile) + unsigned long const api_version, struct file *const filp) { struct kbase_context *kctx; unsigned int i = 0; @@ -188,7 +187,7 @@ struct kbase_context *kbase_create_context(struct kbase_device *kbdev, bool is_c kctx->kbdev = kbdev; kctx->api_version = api_version; - kctx->kfile = kfile; + kctx->filp = filp; kctx->create_flags = flags; if (is_compat) diff --git a/drivers/gpu/arm/bifrost/context/mali_kbase_context.c b/drivers/gpu/arm/bifrost/context/mali_kbase_context.c index 36cfde3cdab1..2c7417bd6506 100644 --- a/drivers/gpu/arm/bifrost/context/mali_kbase_context.c +++ b/drivers/gpu/arm/bifrost/context/mali_kbase_context.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -141,7 +141,7 @@ int kbase_context_common_init(struct kbase_context *kctx) kctx->pid = task_pid_vnr(current); /* Check if this is a Userspace created context */ - if (likely(kctx->kfile)) { + if (likely(kctx->filp)) { struct pid *pid_struct; rcu_read_lock(); @@ -184,6 +184,8 @@ int kbase_context_common_init(struct kbase_context *kctx) spin_lock_init(&kctx->waiting_soft_jobs_lock); INIT_LIST_HEAD(&kctx->waiting_soft_jobs); + init_waitqueue_head(&kctx->event_queue); + kbase_gpu_vm_lock(kctx); bitmap_copy(kctx->cookies, &cookies_mask, BITS_PER_LONG); kbase_gpu_vm_unlock(kctx); @@ -195,7 +197,7 @@ int kbase_context_common_init(struct kbase_context *kctx) mutex_unlock(&kctx->kbdev->kctx_list_lock); if (err) { dev_err(kctx->kbdev->dev, "(err:%d) failed to insert kctx to kbase_process", err); - if (likely(kctx->kfile)) { + if (likely(kctx->filp)) { mmdrop(kctx->process_mm); put_task_struct(kctx->task); } @@ -284,7 +286,7 @@ void kbase_context_common_term(struct kbase_context *kctx) kbase_remove_kctx_from_process(kctx); mutex_unlock(&kctx->kbdev->kctx_list_lock); - if (likely(kctx->kfile)) { + if (likely(kctx->filp)) { mmdrop(kctx->process_mm); put_task_struct(kctx->task); } diff --git a/drivers/gpu/arm/bifrost/context/mali_kbase_context.h b/drivers/gpu/arm/bifrost/context/mali_kbase_context.h index e2295d020292..07c235fab11e 100644 --- a/drivers/gpu/arm/bifrost/context/mali_kbase_context.h +++ b/drivers/gpu/arm/bifrost/context/mali_kbase_context.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2011-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -56,9 +56,9 @@ void kbase_context_debugfs_term(struct kbase_context *const kctx); * BASEP_CONTEXT_CREATE_KERNEL_FLAGS. * @api_version: Application program interface version, as encoded in * a single integer by the KBASE_API_VERSION macro. - * @kfile: Pointer to the object representing the /dev/malixx device - * file instance. Shall be passed as NULL for internally created - * contexts. + * @filp: Pointer to the struct file corresponding to device file + * /dev/malixx instance, passed to the file's open method. + * Shall be passed as NULL for internally created contexts. * * Up to one context can be created for each client that opens the device file * /dev/malixx. Context creation is deferred until a special ioctl() system call @@ -68,8 +68,7 @@ void kbase_context_debugfs_term(struct kbase_context *const kctx); */ struct kbase_context *kbase_create_context(struct kbase_device *kbdev, bool is_compat, base_context_create_flags const flags, - unsigned long api_version, - struct kbase_file *const kfile); + unsigned long api_version, struct file *filp); /** * kbase_destroy_context - Destroy a kernel base context. diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c index 9dffe34f095b..9a33169ec554 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -539,6 +539,8 @@ static int csf_queue_register_internal(struct kbase_context *kctx, queue->blocked_reason = CS_STATUS_BLOCKED_REASON_REASON_UNBLOCKED; + queue->clear_faults = true; + INIT_LIST_HEAD(&queue->link); atomic_set(&queue->pending_kick, 0); INIT_LIST_HEAD(&queue->pending_kick_link); @@ -589,11 +591,19 @@ int kbase_csf_queue_register_ex(struct kbase_context *kctx, u32 const glb_version = iface->version; u32 instr = iface->instr_features; u8 max_size = GLB_INSTR_FEATURES_EVENT_SIZE_MAX_GET(instr); - u32 min_buf_size = - (1u << reg->ex_event_size) * GLB_INSTR_FEATURES_OFFSET_UPDATE_RATE_GET(instr); + const u8 event_size = reg->ex_event_size; + u64 min_buf_size; /* If cs_trace_command not supported, the call fails */ if (glb_version < kbase_csf_interface_version(1, 1, 0)) + return -EPERM; + + /* Sanity check to avoid shift-out-of-bounds */ + if (event_size >= 32) + return -EINVAL; + + min_buf_size = ((u64)1 << event_size) * GLB_INSTR_FEATURES_OFFSET_UPDATE_RATE_GET(instr); + if (min_buf_size > UINT32_MAX) return -EINVAL; /* Validate the ring buffer configuration parameters */ @@ -605,8 +615,8 @@ int kbase_csf_queue_register_ex(struct kbase_context *kctx, /* Validate the cs_trace configuration parameters */ if (reg->ex_buffer_size && - ((reg->ex_event_size > max_size) || (reg->ex_buffer_size & (reg->ex_buffer_size - 1)) || - (reg->ex_buffer_size < min_buf_size))) + ((event_size > max_size) || (reg->ex_buffer_size & (reg->ex_buffer_size - 1)) || + (reg->ex_buffer_size < (u32)min_buf_size))) return -EINVAL; return csf_queue_register_internal(kctx, NULL, reg); @@ -734,7 +744,7 @@ out: } /** - * get_bound_queue_group - Get the group to which a queue was bound + * get_bound_queue_group() - Get the group to which a queue was bound * * @queue: Pointer to the queue for this group * @@ -847,6 +857,47 @@ void kbase_csf_ring_cs_kernel_doorbell(struct kbase_device *kbdev, int csi_index kbase_csf_ring_csg_doorbell(kbdev, csg_nr); } +int kbase_csf_queue_group_clear_faults(struct kbase_context *kctx, + struct kbase_ioctl_queue_group_clear_faults *faults) +{ + void __user *user_bufs = u64_to_user_ptr(faults->addr); + u32 i; + struct kbase_device *kbdev = kctx->kbdev; + const u32 nr_queues = faults->nr_queues; + + if (unlikely(nr_queues > kbdev->csf.global_iface.groups[0].stream_num)) { + dev_warn(kbdev->dev, "Invalid nr_queues %u", nr_queues); + return -EINVAL; + } + + for (i = 0; i < nr_queues; ++i) { + u64 buf_gpu_addr; + struct kbase_va_region *region; + + if (copy_from_user(&buf_gpu_addr, user_bufs, sizeof(buf_gpu_addr))) + return -EFAULT; + mutex_lock(&kctx->csf.lock); + kbase_gpu_vm_lock(kctx); + region = kbase_region_tracker_find_region_enclosing_address(kctx, buf_gpu_addr); + if (likely(!kbase_is_region_invalid_or_free(region))) { + struct kbase_queue *queue = region->user_data; + + queue->clear_faults = true; + } else { + dev_warn(kbdev->dev, "GPU queue %u without a valid command buffer region", + i); + kbase_gpu_vm_unlock(kctx); + mutex_unlock(&kctx->csf.lock); + return -EFAULT; + } + kbase_gpu_vm_unlock(kctx); + mutex_unlock(&kctx->csf.lock); + user_bufs = (void __user *)((uintptr_t)user_bufs + sizeof(buf_gpu_addr)); + } + + return 0; +} + int kbase_csf_queue_kick(struct kbase_context *kctx, struct kbase_ioctl_cs_queue_kick *kick) { struct kbase_device *kbdev = kctx->kbdev; @@ -868,7 +919,7 @@ int kbase_csf_queue_kick(struct kbase_context *kctx, struct kbase_ioctl_cs_queue struct kbase_queue *queue = region->user_data; if (queue && (queue->bind_state == KBASE_CSF_QUEUE_BOUND)) { - spin_lock(&kbdev->csf.pending_gpuq_kicks_lock); + spin_lock(&kbdev->csf.pending_gpuq_kick_queues_lock); if (list_empty(&queue->pending_kick_link)) { /* Queue termination shall block until this * kick has been handled. @@ -876,10 +927,12 @@ int kbase_csf_queue_kick(struct kbase_context *kctx, struct kbase_ioctl_cs_queue atomic_inc(&queue->pending_kick); list_add_tail( &queue->pending_kick_link, - &kbdev->csf.pending_gpuq_kicks[queue->group_priority]); - complete(&kbdev->csf.scheduler.kthread_signal); + &kbdev->csf.pending_gpuq_kick_queues[queue->group_priority]); + if (atomic_cmpxchg(&kbdev->csf.pending_gpuq_kicks, false, true) == + false) + complete(&kbdev->csf.scheduler.kthread_signal); } - spin_unlock(&kbdev->csf.pending_gpuq_kicks_lock); + spin_unlock(&kbdev->csf.pending_gpuq_kick_queues_lock); } } else { dev_dbg(kbdev->dev, @@ -1095,12 +1148,11 @@ static int create_normal_suspend_buffer(struct kbase_context *const kctx, } static void timer_event_worker(struct work_struct *data); -static void protm_event_worker(struct work_struct *data); static void term_normal_suspend_buffer(struct kbase_context *const kctx, struct kbase_normal_suspend_buffer *s_buf); /** - * create_suspend_buffers - Setup normal and protected mode + * create_suspend_buffers() - Setup normal and protected mode * suspend buffers. * * @kctx: Address of the kbase context within which the queue group @@ -1199,6 +1251,8 @@ static int create_queue_group(struct kbase_context *const kctx, group->deschedule_deferred_cnt = 0; #endif + group->cs_fault_report_enable = create->in.cs_fault_report_enable; + group->group_uid = generate_group_uid(); create->out.group_uid = group->group_uid; @@ -1206,7 +1260,8 @@ static int create_queue_group(struct kbase_context *const kctx, INIT_LIST_HEAD(&group->link_to_schedule); INIT_LIST_HEAD(&group->error_fatal.link); INIT_WORK(&group->timer_event_work, timer_event_worker); - INIT_WORK(&group->protm_event_work, protm_event_worker); + INIT_LIST_HEAD(&group->protm_event_work); + atomic_set(&group->pending_protm_event_work, 0); bitmap_zero(group->protm_pending_bitmap, MAX_SUPPORTED_STREAMS_PER_GROUP); group->run_state = KBASE_CSF_GROUP_INACTIVE; @@ -1254,10 +1309,8 @@ int kbase_csf_queue_group_create(struct kbase_context *const kctx, size_t i; for (i = 0; i < ARRAY_SIZE(create->in.padding); i++) { - if (create->in.padding[i] != 0) { - dev_warn(kctx->kbdev->dev, "Invalid padding not 0 in queue group create\n"); + if (create->in.padding[i] != 0) return -EINVAL; - } } mutex_lock(&kctx->csf.lock); @@ -1379,7 +1432,7 @@ void kbase_csf_term_descheduled_queue_group(struct kbase_queue_group *group) } /** - * term_queue_group - Terminate a GPU command queue group. + * term_queue_group() - Terminate a GPU command queue group. * * @group: Pointer to GPU command queue group data. * @@ -1407,8 +1460,8 @@ static void term_queue_group(struct kbase_queue_group *group) } /** - * wait_group_deferred_deschedule_completion - Wait for refcount of the group to - * become 0 that was taken when the group deschedule had to be deferred. + * wait_group_deferred_deschedule_completion() - Wait for refcount of the group + * to become 0 that was taken when the group deschedule had to be deferred. * * @group: Pointer to GPU command queue group that is being deleted. * @@ -1437,7 +1490,10 @@ static void wait_group_deferred_deschedule_completion(struct kbase_queue_group * static void cancel_queue_group_events(struct kbase_queue_group *group) { cancel_work_sync(&group->timer_event_work); - cancel_work_sync(&group->protm_event_work); + + /* Drain a pending protected mode request if any */ + kbase_csf_scheduler_wait_for_kthread_pending_work(group->kctx->kbdev, + &group->pending_protm_event_work); } static void remove_pending_group_fatal_error(struct kbase_queue_group *group) @@ -1592,6 +1648,7 @@ int kbase_csf_ctx_init(struct kbase_context *kctx) INIT_LIST_HEAD(&kctx->csf.queue_list); INIT_LIST_HEAD(&kctx->csf.link); + atomic_set(&kctx->csf.pending_sync_update, 0); kbase_csf_event_init(kctx); @@ -1827,7 +1884,7 @@ void kbase_csf_ctx_term(struct kbase_context *kctx) } /** - * handle_oom_event - Handle the OoM event generated by the firmware for the + * handle_oom_event() - Handle the OoM event generated by the firmware for the * CSI. * * @group: Pointer to the CSG group the oom-event belongs to. @@ -1902,7 +1959,7 @@ static int handle_oom_event(struct kbase_queue_group *const group, } /** - * report_tiler_oom_error - Report a CSG error due to a tiler heap OOM event + * report_tiler_oom_error() - Report a CSG error due to a tiler heap OOM event * * @group: Pointer to the GPU command queue group that encountered the error */ @@ -1945,7 +2002,7 @@ static void flush_gpu_cache_on_fatal_error(struct kbase_device *kbdev) } /** - * kbase_queue_oom_event - Handle tiler out-of-memory for a GPU command queue. + * kbase_queue_oom_event() - Handle tiler out-of-memory for a GPU command queue. * * @queue: Pointer to queue for which out-of-memory event was received. * @@ -2033,7 +2090,7 @@ unlock: } /** - * oom_event_worker - Tiler out-of-memory handler called from a workqueue. + * oom_event_worker() - Tiler out-of-memory handler called from a workqueue. * * @data: Pointer to a work_struct embedded in GPU command queue data. * @@ -2061,7 +2118,8 @@ static void oom_event_worker(struct work_struct *data) } /** - * report_group_timeout_error - Report the timeout error for the group to userspace. + * report_group_timeout_error() - Report the timeout error for the group to + * userspace. * * @group: Pointer to the group for which timeout error occurred */ @@ -2085,7 +2143,7 @@ static void report_group_timeout_error(struct kbase_queue_group *const group) } /** - * timer_event_worker - Handle the progress timeout error for the group + * timer_event_worker() - Handle the progress timeout error for the group * * @data: Pointer to a work_struct embedded in GPU command queue group data. * @@ -2120,7 +2178,7 @@ static void timer_event_worker(struct work_struct *data) } /** - * handle_progress_timer_event - Progress timer timeout event handler. + * handle_progress_timer_event() - Progress timer timeout event handler. * * @group: Pointer to GPU queue group for which the timeout event is received. * @@ -2211,41 +2269,7 @@ static void report_group_fatal_error(struct kbase_queue_group *const group) } /** - * protm_event_worker - Protected mode switch request event handler - * called from a workqueue. - * - * @data: Pointer to a work_struct embedded in GPU command queue group data. - * - * Request to switch to protected mode. - */ -static void protm_event_worker(struct work_struct *data) -{ - struct kbase_queue_group *const group = - container_of(data, struct kbase_queue_group, protm_event_work); - struct kbase_protected_suspend_buffer *sbuf = &group->protected_suspend_buf; - int err = 0; - - KBASE_KTRACE_ADD_CSF_GRP(group->kctx->kbdev, PROTM_EVENT_WORKER_START, group, 0u); - - err = alloc_grp_protected_suspend_buffer_pages(group); - if (!err) { - kbase_csf_scheduler_group_protm_enter(group); - } else if (err == -ENOMEM && sbuf->alloc_retries <= PROTM_ALLOC_MAX_RETRIES) { - sbuf->alloc_retries++; - /* try again to allocate pages */ - queue_work(group->kctx->csf.wq, &group->protm_event_work); - } else if (sbuf->alloc_retries >= PROTM_ALLOC_MAX_RETRIES || err != -ENOMEM) { - dev_err(group->kctx->kbdev->dev, - "Failed to allocate physical pages for Protected mode suspend buffer for the group %d of context %d_%d", - group->handle, group->kctx->tgid, group->kctx->id); - report_group_fatal_error(group); - } - - KBASE_KTRACE_ADD_CSF_GRP(group->kctx->kbdev, PROTM_EVENT_WORKER_END, group, 0u); -} - -/** - * handle_fault_event - Handler for CS fault. + * handle_fault_event() - Handler for CS fault. * * @queue: Pointer to queue for which fault event was received. * @cs_ack: Value of the CS_ACK register in the CS kernel input page used for @@ -2286,47 +2310,32 @@ static void handle_fault_event(struct kbase_queue *const queue, const u32 cs_ack cs_fault_info_exception_data); -#if IS_ENABLED(CONFIG_DEBUG_FS) - /* CS_RESOURCE_TERMINATED type fault event can be ignored from the - * standpoint of dump on error. It is used to report fault for the CSIs - * that are associated with the same CSG as the CSI for which the actual - * fault was reported by the Iterator. - * Dumping would be triggered when the actual fault is reported. + /* If dump-on-fault daemon is waiting for a fault, wake up the daemon. + * Acknowledging the fault is deferred to the bottom-half until the wait + * of the dump completion is done. * - * CS_INHERIT_FAULT can also be ignored. It could happen due to the error - * in other types of queues (cpu/kcpu). If a fault had occurred in some - * other GPU queue then the dump would have been performed anyways when - * that fault was reported. + * Otherwise acknowledge the fault and ring the doorbell for the faulty queue + * to enter into recoverable state. */ - if ((cs_fault_exception_type != CS_FAULT_EXCEPTION_TYPE_CS_INHERIT_FAULT) && - (cs_fault_exception_type != CS_FAULT_EXCEPTION_TYPE_CS_RESOURCE_TERMINATED)) { - if (unlikely(kbase_debug_csf_fault_notify(kbdev, queue->kctx, DF_CS_FAULT))) { - queue->cs_error = cs_fault; - queue->cs_error_info = cs_fault_info; - queue->cs_error_fatal = false; - queue_work(queue->kctx->csf.wq, &queue->cs_error_work); - return; - } - } -#endif + if (likely(!kbase_debug_csf_fault_notify(kbdev, queue->kctx, DF_CS_FAULT))) { + kbase_csf_firmware_cs_input_mask(stream, CS_REQ, cs_ack, CS_REQ_FAULT_MASK); + kbase_csf_ring_cs_kernel_doorbell(kbdev, queue->csi_index, queue->group->csg_nr, + true); + queue->cs_error_acked = true; + } else + queue->cs_error_acked = false; - kbase_csf_firmware_cs_input_mask(stream, CS_REQ, cs_ack, CS_REQ_FAULT_MASK); - kbase_csf_ring_cs_kernel_doorbell(kbdev, queue->csi_index, queue->group->csg_nr, true); + queue->cs_error = cs_fault; + queue->cs_error_info = cs_fault_info; + queue->cs_error_fatal = false; + if (!queue_work(queue->kctx->csf.wq, &queue->cs_error_work)) + dev_warn(kbdev->dev, "%s: failed to enqueue a work", __func__); } -static void report_queue_fatal_error(struct kbase_queue *const queue, u32 cs_fatal, - u64 cs_fatal_info, struct kbase_queue_group *group) +static void report_queue_error(struct kbase_queue *const queue, u32 cs_error, u64 cs_error_info, + struct kbase_queue_group *group, bool fatal) { - struct base_csf_notification - error = { .type = BASE_CSF_NOTIFICATION_GPU_QUEUE_GROUP_ERROR, - .payload = { - .csg_error = { - .error = { .error_type = - BASE_GPU_QUEUE_GROUP_QUEUE_ERROR_FATAL, - .payload = { .fatal_queue = { - .sideband = cs_fatal_info, - .status = cs_fatal, - } } } } } }; + struct base_csf_notification error = { .type = BASE_CSF_NOTIFICATION_GPU_QUEUE_GROUP_ERROR }; if (!queue) return; @@ -2335,17 +2344,30 @@ static void report_queue_fatal_error(struct kbase_queue *const queue, u32 cs_fat return; error.payload.csg_error.handle = group->handle; - error.payload.csg_error.error.payload.fatal_queue.csi_index = (__u8)queue->csi_index; + if (fatal) { + error.payload.csg_error.error.error_type = BASE_GPU_QUEUE_GROUP_QUEUE_ERROR_FATAL; + error.payload.csg_error.error.payload.fatal_queue.sideband = cs_error_info; + error.payload.csg_error.error.payload.fatal_queue.status = cs_error; + error.payload.csg_error.error.payload.fatal_queue.csi_index = queue->csi_index; + } else { + error.payload.csg_error.error.error_type = BASE_GPU_QUEUE_GROUP_QUEUE_ERROR_FAULT; + error.payload.csg_error.error.payload.fault_queue.sideband = cs_error_info; + error.payload.csg_error.error.payload.fault_queue.status = cs_error; + error.payload.csg_error.error.payload.fault_queue.csi_index = queue->csi_index; + } kbase_csf_event_add_error(queue->kctx, &group->error_fatal, &error); kbase_event_wakeup(queue->kctx); + + if (!fatal) + queue->clear_faults = false; } /** - * cs_error_worker - Handle the CS_FATAL/CS_FAULT error for the GPU queue + * cs_error_worker() - Handle the CS_FATAL/CS_FAULT error for the GPU queue * * @data: Pointer to a work_struct embedded in GPU command queue. * - * Terminate the CSG and report the error to userspace. + * Terminate the CSG for CS_FATAL and report the error to userspace. */ static void cs_error_worker(struct work_struct *const data) { @@ -2356,6 +2378,7 @@ static void cs_error_worker(struct work_struct *const data) struct kbase_queue_group *group; bool reset_prevented = false; int err; + const bool cs_fatal = queue->cs_error_fatal; kbase_debug_csf_fault_wait_completion(kbdev); err = kbase_reset_gpu_prevent_and_wait(kbdev); @@ -2371,45 +2394,57 @@ static void cs_error_worker(struct work_struct *const data) group = get_bound_queue_group(queue); if (!group) { - dev_warn(kbdev->dev, "queue not bound when handling fatal event"); + dev_warn(kbdev->dev, "queue not bound when handling an error event"); goto unlock; } -#if IS_ENABLED(CONFIG_DEBUG_FS) - if (!queue->cs_error_fatal) { - unsigned long flags; - int slot_num; + if (!cs_fatal) { + if (group->cs_fault_report_enable && queue->clear_faults) + report_queue_error(queue, queue->cs_error, queue->cs_error_info, group, + false); + if (unlikely(!queue->cs_error_acked)) { + unsigned long flags; + int slot_num; - kbase_csf_scheduler_spin_lock(kbdev, &flags); - slot_num = kbase_csf_scheduler_group_get_slot_locked(group); - if (slot_num >= 0) { - struct kbase_csf_cmd_stream_group_info const *ginfo = - &kbdev->csf.global_iface.groups[slot_num]; - struct kbase_csf_cmd_stream_info const *stream = - &ginfo->streams[queue->csi_index]; - u32 const cs_ack = kbase_csf_firmware_cs_output(stream, CS_ACK); + kbase_csf_scheduler_spin_lock(kbdev, &flags); + slot_num = kbase_csf_scheduler_group_get_slot_locked(group); + if (likely(slot_num >= 0)) { + struct kbase_csf_cmd_stream_group_info const *ginfo = + &kbdev->csf.global_iface.groups[slot_num]; + struct kbase_csf_cmd_stream_info const *stream = + &ginfo->streams[queue->csi_index]; + u32 const cs_ack = kbase_csf_firmware_cs_output(stream, CS_ACK); + u32 const cs_req = kbase_csf_firmware_cs_input_read(stream, CS_REQ); - kbase_csf_firmware_cs_input_mask(stream, CS_REQ, cs_ack, CS_REQ_FAULT_MASK); - kbase_csf_ring_cs_kernel_doorbell(kbdev, queue->csi_index, slot_num, true); + /* Acknowledge the fault and ring the doorbell for the queue + * if it hasn't yet done. + */ + if ((cs_ack & CS_ACK_FAULT_MASK) != (cs_req & CS_REQ_FAULT_MASK)) { + kbase_csf_firmware_cs_input_mask(stream, CS_REQ, cs_ack, + CS_REQ_FAULT_MASK); + kbase_csf_ring_cs_kernel_doorbell(kbdev, queue->csi_index, + slot_num, true); + } + } + kbase_csf_scheduler_spin_unlock(kbdev, flags); } - kbase_csf_scheduler_spin_unlock(kbdev, flags); - goto unlock; - } -#endif - - term_queue_group(group); - flush_gpu_cache_on_fatal_error(kbdev); - /* For an invalid GPU page fault, CS_BUS_FAULT fatal error is expected after the - * page fault handler disables the AS of faulty context. Need to skip reporting the - * CS_BUS_FAULT fatal error to the Userspace as it doesn't have the full fault info. - * Page fault handler will report the fatal error with full page fault info. - */ - if ((cs_fatal_exception_type == CS_FATAL_EXCEPTION_TYPE_CS_BUS_FAULT) && group->faulted) { - dev_dbg(kbdev->dev, - "Skipped reporting CS_BUS_FAULT for queue %d of group %d of ctx %d_%d", - queue->csi_index, group->handle, kctx->tgid, kctx->id); } else { - report_queue_fatal_error(queue, queue->cs_error, queue->cs_error_info, group); + term_queue_group(group); + flush_gpu_cache_on_fatal_error(kbdev); + /* For an invalid GPU page fault, CS_BUS_FAULT fatal error is expected after the + * page fault handler disables the AS of faulty context. Need to skip reporting the + * CS_BUS_FAULT fatal error to the Userspace as it doesn't have the full fault info. + * Page fault handler will report the fatal error with full page fault info. + */ + if ((cs_fatal_exception_type == CS_FATAL_EXCEPTION_TYPE_CS_BUS_FAULT) && + group->faulted) { + dev_dbg(kbdev->dev, + "Skipped reporting CS_BUS_FAULT for queue %d of group %d of ctx %d_%d", + queue->csi_index, group->handle, kctx->tgid, kctx->id); + } else { + report_queue_error(queue, queue->cs_error, queue->cs_error_info, group, + true); + } } unlock: @@ -2419,7 +2454,7 @@ unlock: } /** - * handle_fatal_event - Handler for CS fatal. + * handle_fatal_event() - Handler for CS fatal. * * @queue: Pointer to queue for which fatal event was received. * @stream: Pointer to the structure containing info provided by the @@ -2481,7 +2516,7 @@ static void handle_fatal_event(struct kbase_queue *const queue, } /** - * process_cs_interrupts - Process interrupts for a CS. + * process_cs_interrupts() - Process interrupts for a CS. * * @group: Pointer to GPU command queue group data. * @ginfo: The CSG interface provided by the firmware. @@ -2595,7 +2630,7 @@ static void process_cs_interrupts(struct kbase_queue_group *const group, } if (!group->protected_suspend_buf.pma) - queue_work(group->kctx->csf.wq, &group->protm_event_work); + kbase_csf_scheduler_enqueue_protm_event_work(group); if (test_bit(group->csg_nr, scheduler->csg_slots_idle_mask)) { clear_bit(group->csg_nr, scheduler->csg_slots_idle_mask); @@ -2608,7 +2643,7 @@ static void process_cs_interrupts(struct kbase_queue_group *const group, } /** - * process_csg_interrupts - Process interrupts for a CSG. + * process_csg_interrupts() - Process interrupts for a CSG. * * @kbdev: Instance of a GPU platform device that implements a CSF interface. * @csg_nr: CSG number. @@ -2728,7 +2763,7 @@ static void process_csg_interrupts(struct kbase_device *const kbdev, u32 const c } /** - * process_prfcnt_interrupts - Process performance counter interrupts. + * process_prfcnt_interrupts() - Process performance counter interrupts. * * @kbdev: Instance of a GPU platform device that implements a CSF interface. * @glb_req: Global request register value. @@ -2800,7 +2835,7 @@ static void process_prfcnt_interrupts(struct kbase_device *kbdev, u32 glb_req, u } /** - * check_protm_enter_req_complete - Check if PROTM_ENTER request completed + * check_protm_enter_req_complete() - Check if PROTM_ENTER request completed * * @kbdev: Instance of a GPU platform device that implements a CSF interface. * @glb_req: Global request register value. @@ -2834,7 +2869,7 @@ static inline void check_protm_enter_req_complete(struct kbase_device *kbdev, u3 } /** - * process_protm_exit - Handle the protected mode exit interrupt + * process_protm_exit() - Handle the protected mode exit interrupt * * @kbdev: Instance of a GPU platform device that implements a CSF interface. * @glb_ack: Global acknowledge register value. @@ -2923,7 +2958,7 @@ static inline void process_tracked_info_for_protm(struct kbase_device *kbdev, if (!tock_triggered) { dev_dbg(kbdev->dev, "Group-%d on slot-%d start protm work\n", group->handle, group->csg_nr); - queue_work(group->kctx->csf.wq, &group->protm_event_work); + kbase_csf_scheduler_enqueue_protm_event_work(group); } } } @@ -2952,6 +2987,46 @@ static void order_job_irq_clear_with_iface_mem_read(void) dmb(osh); } +static const char *const glb_fatal_status_errors[GLB_FATAL_STATUS_VALUE_COUNT] = { + [GLB_FATAL_STATUS_VALUE_OK] = "OK", + [GLB_FATAL_STATUS_VALUE_ASSERT] = "Firmware assert triggered", + [GLB_FATAL_STATUS_VALUE_UNEXPECTED_EXCEPTION] = + "Hardware raised an exception firmware did not expect", + [GLB_FATAL_STATUS_VALUE_HANG] = "Firmware hangs and watchdog timer expired", +}; + +/** + * handle_glb_fatal_event() - Handle the GLB fatal event + * + * @kbdev: Instance of GPU device. + * @global_iface: CSF global interface + */ +static void handle_glb_fatal_event(struct kbase_device *kbdev, + const struct kbase_csf_global_iface *const global_iface) +{ + const char *error_string = NULL; + const u32 fatal_status = kbase_csf_firmware_global_output(global_iface, GLB_FATAL_STATUS); + + lockdep_assert_held(&kbdev->hwaccess_lock); + kbase_csf_scheduler_spin_lock_assert_held(kbdev); + dev_warn(kbdev->dev, "MCU encountered unrecoverable error"); + + if (fatal_status < GLB_FATAL_STATUS_VALUE_COUNT) + error_string = glb_fatal_status_errors[fatal_status]; + else { + dev_err(kbdev->dev, "Invalid GLB_FATAL_STATUS (%u)", fatal_status); + return; + } + + if (fatal_status == GLB_FATAL_STATUS_VALUE_OK) + dev_err(kbdev->dev, "GLB_FATAL_STATUS(OK) must be set with proper reason"); + else { + dev_warn(kbdev->dev, "GLB_FATAL_STATUS: %s", error_string); + if (kbase_prepare_to_reset_gpu_locked(kbdev, RESET_FLAGS_NONE)) + kbase_reset_gpu_locked(kbdev); + } +} + void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val) { bool deferred_handling_glb_idle_irq = false; @@ -3026,6 +3101,9 @@ void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val) deferred_handling_glb_idle_irq = true; } + if (glb_ack & GLB_ACK_FATAL_MASK) + handle_glb_fatal_event(kbdev, global_iface); + process_prfcnt_interrupts(kbdev, glb_req, glb_ack); kbase_csf_scheduler_spin_unlock(kbdev, flags); @@ -3050,13 +3128,10 @@ void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val) if (deferred_handling_glb_idle_irq) { unsigned long flags; - bool invoke_pm_state_machine; kbase_csf_scheduler_spin_lock(kbdev, &flags); - invoke_pm_state_machine = kbase_csf_scheduler_process_gpu_idle_event(kbdev); + kbase_csf_scheduler_process_gpu_idle_event(kbdev); kbase_csf_scheduler_spin_unlock(kbdev, flags); - if (unlikely(invoke_pm_state_machine)) - kbase_pm_update_state(kbdev); } wake_up_all(&kbdev->csf.event_wait); @@ -3087,6 +3162,11 @@ void kbase_csf_doorbell_mapping_term(struct kbase_device *kbdev) if (kbdev->csf.db_filp) { struct page *page = as_page(kbdev->csf.dummy_db_page); + /* This is a shared dummy sink page for avoiding potential segmentation fault + * to user-side library when a csi is off slot. Additionally, the call is on + * module unload path, so the page can be left uncleared before returning it + * back to kbdev memory pool. + */ kbase_mem_pool_free(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], page, false); fput(kbdev->csf.db_filp); @@ -3118,26 +3198,27 @@ int kbase_csf_doorbell_mapping_init(struct kbase_device *kbdev) return 0; } -void kbase_csf_pending_gpuq_kicks_init(struct kbase_device *kbdev) +void kbase_csf_pending_gpuq_kick_queues_init(struct kbase_device *kbdev) { size_t i; - for (i = 0; i != ARRAY_SIZE(kbdev->csf.pending_gpuq_kicks); ++i) - INIT_LIST_HEAD(&kbdev->csf.pending_gpuq_kicks[i]); - spin_lock_init(&kbdev->csf.pending_gpuq_kicks_lock); + atomic_set(&kbdev->csf.pending_gpuq_kicks, false); + for (i = 0; i != ARRAY_SIZE(kbdev->csf.pending_gpuq_kick_queues); ++i) + INIT_LIST_HEAD(&kbdev->csf.pending_gpuq_kick_queues[i]); + spin_lock_init(&kbdev->csf.pending_gpuq_kick_queues_lock); } -void kbase_csf_pending_gpuq_kicks_term(struct kbase_device *kbdev) +void kbase_csf_pending_gpuq_kick_queues_term(struct kbase_device *kbdev) { size_t i; - spin_lock(&kbdev->csf.pending_gpuq_kicks_lock); - for (i = 0; i != ARRAY_SIZE(kbdev->csf.pending_gpuq_kicks); ++i) { - if (!list_empty(&kbdev->csf.pending_gpuq_kicks[i])) + spin_lock(&kbdev->csf.pending_gpuq_kick_queues_lock); + for (i = 0; i != ARRAY_SIZE(kbdev->csf.pending_gpuq_kick_queues); ++i) { + if (!list_empty(&kbdev->csf.pending_gpuq_kick_queues[i])) dev_warn(kbdev->dev, "Some GPU queue kicks for priority %zu were not handled", i); } - spin_unlock(&kbdev->csf.pending_gpuq_kicks_lock); + spin_unlock(&kbdev->csf.pending_gpuq_kick_queues_lock); } void kbase_csf_free_dummy_user_reg_page(struct kbase_device *kbdev) @@ -3145,6 +3226,11 @@ void kbase_csf_free_dummy_user_reg_page(struct kbase_device *kbdev) if (kbdev->csf.user_reg.filp) { struct page *page = as_page(kbdev->csf.user_reg.dummy_page); + /* This is a shared dummy page in place of the real USER Register page just + * before the GPU is powered down. Additionally, the call is on module unload + * path, so the page can be left uncleared before returning it back to kbdev + * memory pool. + */ kbase_mem_pool_free(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], page, false); fput(kbdev->csf.user_reg.filp); } @@ -3227,17 +3313,17 @@ void kbase_csf_process_queue_kick(struct kbase_queue *queue) if (err == -EBUSY) { retry_kick = true; - spin_lock(&kbdev->csf.pending_gpuq_kicks_lock); + spin_lock(&kbdev->csf.pending_gpuq_kick_queues_lock); if (list_empty(&queue->pending_kick_link)) { /* A failed queue kick shall be pushed to the * back of the queue to avoid potential abuse. */ list_add_tail( &queue->pending_kick_link, - &kbdev->csf.pending_gpuq_kicks[queue->group_priority]); - spin_unlock(&kbdev->csf.pending_gpuq_kicks_lock); + &kbdev->csf.pending_gpuq_kick_queues[queue->group_priority]); + spin_unlock(&kbdev->csf.pending_gpuq_kick_queues_lock); } else { - spin_unlock(&kbdev->csf.pending_gpuq_kicks_lock); + spin_unlock(&kbdev->csf.pending_gpuq_kick_queues_lock); WARN_ON(atomic_read(&queue->pending_kick) == 0); } @@ -3260,3 +3346,27 @@ out_release_queue: WARN_ON(atomic_read(&queue->pending_kick) == 0); atomic_dec(&queue->pending_kick); } + +void kbase_csf_process_protm_event_request(struct kbase_queue_group *group) +{ + struct kbase_protected_suspend_buffer *sbuf = &group->protected_suspend_buf; + int err = 0; + + KBASE_KTRACE_ADD_CSF_GRP(group->kctx->kbdev, PROTM_EVENT_WORKER_START, group, 0u); + + err = alloc_grp_protected_suspend_buffer_pages(group); + if (!err) { + kbase_csf_scheduler_group_protm_enter(group); + } else if (err == -ENOMEM && sbuf->alloc_retries <= PROTM_ALLOC_MAX_RETRIES) { + sbuf->alloc_retries++; + /* try again to allocate pages */ + kbase_csf_scheduler_enqueue_protm_event_work(group); + } else if (sbuf->alloc_retries >= PROTM_ALLOC_MAX_RETRIES || err != -ENOMEM) { + dev_err(group->kctx->kbdev->dev, + "Failed to allocate physical pages for Protected mode suspend buffer for the group %d of context %d_%d", + group->handle, group->kctx->tgid, group->kctx->id); + report_group_fatal_error(group); + } + + KBASE_KTRACE_ADD_CSF_GRP(group->kctx->kbdev, PROTM_EVENT_WORKER_END, group, 0u); +} diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.h index b2f6ab2c4a27..566136342a06 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.h +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.h @@ -243,6 +243,19 @@ struct kbase_queue_group *kbase_csf_find_queue_group(struct kbase_context *kctx, */ int kbase_csf_queue_group_handle_is_valid(struct kbase_context *kctx, u8 group_handle); +/** + * kbase_csf_queue_group_clear_faults - Re-enable CS Fault reporting. + * + * @kctx: Pointer to the kbase context within which the + * CS Faults for the queues has to be re-enabled. + * @clear_faults: Pointer to the structure which contains details of the + * queues for which the CS Fault reporting has to be re-enabled. + * + * Return: 0 on success, or negative on failure. + */ +int kbase_csf_queue_group_clear_faults(struct kbase_context *kctx, + struct kbase_ioctl_queue_group_clear_faults *clear_faults); + /** * kbase_csf_queue_group_create - Create a GPU command queue group. * @@ -379,20 +392,20 @@ int kbase_csf_setup_dummy_user_reg_page(struct kbase_device *kbdev); void kbase_csf_free_dummy_user_reg_page(struct kbase_device *kbdev); /** - * kbase_csf_pending_gpuq_kicks_init - Initialize the data used for handling - * GPU queue kicks. + * kbase_csf_pending_gpuq_kick_queues_init - Initialize the data used for handling + * GPU queue kicks. * * @kbdev: Instance of a GPU platform device that implements a CSF interface. */ -void kbase_csf_pending_gpuq_kicks_init(struct kbase_device *kbdev); +void kbase_csf_pending_gpuq_kick_queues_init(struct kbase_device *kbdev); /** - * kbase_csf_pending_gpuq_kicks_term - De-initialize the data used for handling - * GPU queue kicks. + * kbase_csf_pending_gpuq_kick_queues_term - De-initialize the data used for handling + * GPU queue kicks. * * @kbdev: Instance of a GPU platform device that implements a CSF interface. */ -void kbase_csf_pending_gpuq_kicks_term(struct kbase_device *kbdev); +void kbase_csf_pending_gpuq_kick_queues_term(struct kbase_device *kbdev); /** * kbase_csf_ring_csg_doorbell - ring the doorbell for a CSG interface. @@ -546,4 +559,13 @@ static inline u64 kbase_csf_ktrace_gpu_cycle_cnt(struct kbase_device *kbdev) */ void kbase_csf_process_queue_kick(struct kbase_queue *queue); +/** + * kbase_csf_process_protm_event_request - Handle protected mode switch request + * + * @group: The group to handle protected mode request + * + * Request to switch to protected mode. + */ +void kbase_csf_process_protm_event_request(struct kbase_queue_group *group); + #endif /* _KBASE_CSF_H_ */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h index 8d7c896e1051..155c20aaa356 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -268,6 +268,7 @@ enum kbase_queue_group_priority { * Shader, L2 and MCU state. * @CSF_GPU_RESET_TIMEOUT: Waiting timeout for GPU reset to complete. * @CSF_CSG_SUSPEND_TIMEOUT: Timeout given for a CSG to be suspended. + * @CSF_CSG_TERM_TIMEOUT: Timeout given for a CSG to be terminated. * @CSF_FIRMWARE_BOOT_TIMEOUT: Maximum time to wait for firmware to boot. * @CSF_FIRMWARE_PING_TIMEOUT: Maximum time to wait for firmware to respond * to a ping from KBase. @@ -290,6 +291,7 @@ enum kbase_timeout_selector { CSF_PM_TIMEOUT, CSF_GPU_RESET_TIMEOUT, CSF_CSG_SUSPEND_TIMEOUT, + CSF_CSG_TERM_TIMEOUT, CSF_FIRMWARE_BOOT_TIMEOUT, CSF_FIRMWARE_PING_TIMEOUT, CSF_SCHED_PROTM_PROGRESS_TIMEOUT, @@ -398,6 +400,10 @@ struct kbase_csf_notification { * @cs_error: Records information about the CS fatal event or * about CS fault event if dump on fault is enabled. * @cs_error_fatal: Flag to track if the CS fault or CS fatal event occurred. + * @cs_error_acked: Flag to indicate that acknowledging the fault has been done + * at top-half of fault handler. + * @clear_faults: Flag to track if the CS fault reporting is enabled for this queue. + * It's protected by &kbase_context.csf.lock. * @extract_ofs: The current EXTRACT offset, this is only updated when handling * the GLB IDLE IRQ if the idle timeout value is non-0 in order * to help detect a queue's true idle status. @@ -441,6 +447,8 @@ struct kbase_queue { u64 cs_error_info; u32 cs_error; bool cs_error_fatal; + bool cs_error_acked; + bool clear_faults; u64 extract_ofs; u64 saved_cmd_ptr; }; @@ -501,6 +509,8 @@ struct kbase_protected_suspend_buffer { * @compute_max: Maximum number of compute endpoints the group is * allowed to use. * @csi_handlers: Requested CSI exception handler flags for the group. + * @cs_fault_report_enable: Indicated if reporting of CS_FAULTs to + * userspace is enabled. * @tiler_mask: Mask of tiler endpoints the group is allowed to use. * @fragment_mask: Mask of fragment endpoints the group is allowed to use. * @compute_mask: Mask of compute endpoints the group is allowed to use. @@ -531,8 +541,13 @@ struct kbase_protected_suspend_buffer { * @bound_queues: Array of registered queues bound to this queue group. * @doorbell_nr: Index of the hardware doorbell page assigned to the * group. - * @protm_event_work: Work item corresponding to the protected mode entry - * event for this queue. + * @protm_event_work: List item corresponding to the protected mode entry + * event for this queue. This would be handled by + * kbase_csf_scheduler_kthread(). + * @pending_protm_event_work: Indicates that kbase_csf_scheduler_kthread() should + * handle PROTM request for this group. This would + * be set to false when the work is done. This is used + * mainly for synchronisation with group termination. * @protm_pending_bitmap: Bit array to keep a track of CSs that * have pending protected mode entry requests. * @error_fatal: An error of type BASE_GPU_QUEUE_GROUP_ERROR_FATAL to be @@ -569,7 +584,7 @@ struct kbase_queue_group { u8 compute_max; u8 csi_handlers; - + __u8 cs_fault_report_enable; u64 tiler_mask; u64 fragment_mask; u64 compute_mask; @@ -588,7 +603,8 @@ struct kbase_queue_group { struct kbase_queue *bound_queues[MAX_SUPPORTED_STREAMS_PER_GROUP]; int doorbell_nr; - struct work_struct protm_event_work; + struct list_head protm_event_work; + atomic_t pending_protm_event_work; DECLARE_BITMAP(protm_pending_bitmap, MAX_SUPPORTED_STREAMS_PER_GROUP); struct kbase_csf_notification error_fatal; @@ -625,6 +641,9 @@ struct kbase_queue_group { * @cmd_seq_num: The sequence number assigned to an enqueued command, * in incrementing order (older commands shall have a * smaller number). + * @kcpu_wq: Work queue to process KCPU commands for all queues in this + * context. This would be used if the context is not prioritised, + * otherwise it would be handled by kbase_csf_scheduler_kthread(). * @jit_lock: Lock to serialise JIT operations. * @jit_cmds_head: A list of the just-in-time memory commands, both * allocate & free, in submission order, protected @@ -640,6 +659,8 @@ struct kbase_csf_kcpu_queue_context { DECLARE_BITMAP(in_use, KBASEP_MAX_KCPU_QUEUES); atomic64_t cmd_seq_num; + struct workqueue_struct *kcpu_wq; + struct mutex jit_lock; struct list_head jit_cmds_head; struct list_head jit_blocked_queues; @@ -747,15 +768,7 @@ struct kbase_csf_ctx_heap_reclaim_info { * GPU command queues are idle and at least one of them * is blocked on a sync wait operation. * @num_idle_wait_grps: Length of the @idle_wait_groups list. - * @sync_update_wq_high_prio: high-priority work queue to process the - * SYNC_UPDATE events by sync_set / sync_add - * instruction execution on command streams bound to - * groups of @idle_wait_groups list. This WQ would - * be used if the context is prioritised. - * @sync_update_wq_normal_prio: similar to sync_update_wq_high_prio, but this - * WQ would be used if the context is not - * prioritised. - * @sync_update_work: Work item to process the SYNC_UPDATE events. + * @sync_update_work: List item to process the SYNC_UPDATE event. * @ngrp_to_schedule: Number of groups added for the context to the * 'groups_to_schedule' list of scheduler instance. * @heap_info: Heap reclaim information data of the kctx. As the @@ -768,9 +781,7 @@ struct kbase_csf_scheduler_context { u32 num_runnable_grps; struct list_head idle_wait_groups; u32 num_idle_wait_grps; - struct workqueue_struct *sync_update_wq_high_prio; - struct workqueue_struct *sync_update_wq_normal_prio; - struct work_struct sync_update_work; + struct list_head sync_update_work; u32 ngrp_to_schedule; struct kbase_csf_ctx_heap_reclaim_info heap_info; }; @@ -865,17 +876,16 @@ struct kbase_csf_user_reg_context { * @wq: Dedicated workqueue to process work items corresponding * to the OoM events raised for chunked tiler heaps being * used by GPU command queues, and progress timeout events. - * @kcpu_wq_high_prio: High-priority work queue to process KCPU commands for - * all queues in this context. This WQ would be used if - * the context is prioritised. - * @kcpu_wq_normal_prio: Similar to kcpu_wq_high_prio, but this WQ would be - * used if the context is not prioritised. * @link: Link to this csf context in the 'runnable_kctxs' list of * the scheduler instance * @sched: Object representing the scheduler's context * @cpu_queue: CPU queue information. Only be available when DEBUG_FS * is enabled. * @user_reg: Collective information to support mapping to USER Register page. + * @pending_sync_update: Indicates that kbase_csf_scheduler_kthread() should + * handle SYNC_UPDATE event for this context. This would + * be set to false when the work is done. This is used + * mainly for synchronisation with context termination. */ struct kbase_csf_context { struct list_head event_pages_head; @@ -888,12 +898,11 @@ struct kbase_csf_context { struct kbase_csf_event event; struct kbase_csf_tiler_heap_context tiler_heaps; struct workqueue_struct *wq; - struct workqueue_struct *kcpu_wq_high_prio; - struct workqueue_struct *kcpu_wq_normal_prio; struct list_head link; struct kbase_csf_scheduler_context sched; struct kbase_csf_cpu_queue_context cpu_queue; struct kbase_csf_user_reg_context user_reg; + atomic_t pending_sync_update; }; /** @@ -936,14 +945,15 @@ struct kbase_csf_csg_slot { * struct kbase_csf_sched_heap_reclaim_mgr - Object for managing tiler heap reclaim * kctx lists inside the CSF device's scheduler. * - * @heap_reclaim: Tiler heap reclaim shrinker object. + * @heap_reclaim: Defines Tiler heap reclaim shrinker object. * @ctx_lists: Array of kctx lists, size matching CSG defined priorities. The * lists track the kctxs attached to the reclaim manager. * @unused_pages: Estimated number of unused pages from the @ctxlist array. The * number is indicative for use with reclaim shrinker's count method. */ struct kbase_csf_sched_heap_reclaim_mgr { - struct shrinker heap_reclaim; + DEFINE_KBASE_SHRINKER heap_reclaim; + struct list_head ctx_lists[KBASE_QUEUE_GROUP_PRIORITY_COUNT]; atomic_t unused_pages; }; @@ -1042,10 +1052,29 @@ struct kbase_csf_mcu_shared_regions { * workqueue items (kernel-provided delayed_work * items do not use hrtimer and for some reason do * not provide sufficiently reliable periodicity). - * @pending_tick_work: Indicates that kbase_csf_scheduler_kthread() should perform - * a scheduling tick. - * @pending_tock_work: Indicates that kbase_csf_scheduler_kthread() should perform - * a scheduling tock. + * @pending_sync_update_works: Indicates that kbase_csf_scheduler_kthread() + * should handle SYNC_UPDATE events. + * @sync_update_work_ctxs_lock: Lock protecting the list of contexts that + * require handling SYNC_UPDATE events. + * @sync_update_work_ctxs: The list of contexts that require handling + * SYNC_UPDATE events. + * @pending_protm_event_works: Indicates that kbase_csf_scheduler_kthread() + * should handle PROTM requests. + * @protm_event_work_grps_lock: Lock protecting the list of groups that + * have requested protected mode. + * @protm_event_work_grps: The list of groups that have requested + * protected mode. + * @pending_kcpuq_works: Indicates that kbase_csf_scheduler_kthread() + * should process pending KCPU queue works. + * @kcpuq_work_queues_lock: Lock protecting the list of KCPU queues that + * need to be processed. + * @kcpuq_work_queues: The list of KCPU queue that need to be processed + * @pending_tick_work: Indicates that kbase_csf_scheduler_kthread() should + * perform a scheduling tick. + * @pending_tock_work: Indicates that kbase_csf_scheduler_kthread() should + * perform a scheduling tock. + * @pending_gpu_idle_work: Indicates that kbase_csf_scheduler_kthread() should + * handle the GPU IDLE event. * @ping_work: Work item that would ping the firmware at regular * intervals, only if there is a single active CSG * slot, to check if firmware is alive and would @@ -1063,10 +1092,6 @@ struct kbase_csf_mcu_shared_regions { * This pointer being set doesn't necessarily indicates * that GPU is in protected mode, kbdev->protected_mode * needs to be checked for that. - * @idle_wq: Workqueue for executing GPU idle notification - * handler. - * @gpu_idle_work: Work item for facilitating the scheduler to bring - * the GPU to a low-power mode on becoming idle. * @fast_gpu_idle_handling: Indicates whether to relax many of the checks * normally done in the GPU idle worker. This is * set to true when handling the GLB IDLE IRQ if the @@ -1109,7 +1134,8 @@ struct kbase_csf_mcu_shared_regions { * thread when a queue needs attention. * @kthread_running: Whether the GPU queue submission thread should keep * executing. - * @gpuq_kthread: High-priority thread used to handle GPU queue + * @gpuq_kthread: Dedicated thread primarily used to handle + * latency-sensitive tasks such as GPU queue * submissions. */ struct kbase_csf_scheduler { @@ -1134,14 +1160,22 @@ struct kbase_csf_scheduler { unsigned long last_schedule; atomic_t timer_enabled; struct hrtimer tick_timer; + atomic_t pending_sync_update_works; + spinlock_t sync_update_work_ctxs_lock; + struct list_head sync_update_work_ctxs; + atomic_t pending_protm_event_works; + spinlock_t protm_event_work_grps_lock; + struct list_head protm_event_work_grps; + atomic_t pending_kcpuq_works; + spinlock_t kcpuq_work_queues_lock; + struct list_head kcpuq_work_queues; atomic_t pending_tick_work; atomic_t pending_tock_work; + atomic_t pending_gpu_idle_work; struct delayed_work ping_work; struct kbase_context *top_kctx; struct kbase_queue_group *top_grp; struct kbase_queue_group *active_protm_grp; - struct workqueue_struct *idle_wq; - struct work_struct gpu_idle_work; bool fast_gpu_idle_handling; atomic_t gpu_no_longer_idle; atomic_t non_idle_offslot_grps; @@ -1653,12 +1687,16 @@ struct kbase_csf_user_reg { * @dof: Structure for dump on fault. * @user_reg: Collective information to support the mapping to * USER Register page for user processes. - * @pending_gpuq_kicks: Lists of GPU queue that have been kicked but not - * yet processed, categorised by queue group's priority. - * @pending_gpuq_kicks_lock: Protect @pending_gpu_kicks and - * kbase_queue.pending_kick_link. + * @pending_gpuq_kicks: Indicates that kbase_csf_scheduler_kthread() + * should handle GPU queue kicks. + * @pending_gpuq_kick_queues: Lists of GPU queued that have been kicked but not + * yet processed, categorised by queue group's priority. + * @pending_gpuq_kick_queues_lock: Protect @pending_gpuq_kick_queues and + * kbase_queue.pending_kick_link. * @quirks_ext: Pointer to an allocated buffer containing the firmware * workarounds configuration. + * @pmode_sync_sem: RW Semaphore to prevent MMU operations during P.Mode entrance. + * @gpu_idle_timer_enabled: Tracks whether the GPU idle timer is enabled or disabled. */ struct kbase_csf_device { struct kbase_mmu_table mcu_mmu; @@ -1710,9 +1748,12 @@ struct kbase_csf_device { struct kbase_debug_coresight_device coresight; #endif /* IS_ENABLED(CONFIG_MALI_CORESIGHT) */ struct kbase_csf_user_reg user_reg; - struct list_head pending_gpuq_kicks[KBASE_QUEUE_GROUP_PRIORITY_COUNT]; - spinlock_t pending_gpuq_kicks_lock; + atomic_t pending_gpuq_kicks; + struct list_head pending_gpuq_kick_queues[KBASE_QUEUE_GROUP_PRIORITY_COUNT]; + spinlock_t pending_gpuq_kick_queues_lock; u32 *quirks_ext; + struct rw_semaphore pmode_sync_sem; + bool gpu_idle_timer_enabled; }; /** diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c index 952a9b9cdd94..35f09028098c 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -1552,7 +1552,6 @@ static bool global_request_complete(struct kbase_device *const kbdev, u32 const unsigned long flags; kbase_csf_scheduler_spin_lock(kbdev, &flags); - if ((kbase_csf_firmware_global_output(global_iface, GLB_ACK) & req_mask) == (kbase_csf_firmware_global_input_read(global_iface, GLB_REQ) & req_mask)) complete = true; @@ -1644,6 +1643,23 @@ static void set_timeout_global(const struct kbase_csf_global_iface *const global set_global_request(global_iface, GLB_REQ_CFG_PROGRESS_TIMER_MASK); } +static inline void set_gpu_idle_timer_glb_req(struct kbase_device *const kbdev, bool set) +{ + struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; + + kbase_csf_scheduler_spin_lock_assert_held(kbdev); + + if (set) { + kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, GLB_REQ_REQ_IDLE_ENABLE, + GLB_REQ_IDLE_ENABLE_MASK); + } else { + kbase_csf_firmware_global_input_mask( + global_iface, GLB_REQ, GLB_REQ_REQ_IDLE_DISABLE, GLB_REQ_IDLE_DISABLE_MASK); + } + + kbdev->csf.gpu_idle_timer_enabled = set; +} + static void enable_gpu_idle_timer(struct kbase_device *const kbdev) { struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; @@ -1657,8 +1673,7 @@ static void enable_gpu_idle_timer(struct kbase_device *const kbdev) kbdev->csf.gpu_idle_dur_count_no_modifier, GLB_IDLE_TIMER_CONFIG_NO_MODIFIER_MASK); - kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, GLB_REQ_REQ_IDLE_ENABLE, - GLB_REQ_IDLE_ENABLE_MASK); + set_gpu_idle_timer_glb_req(kbdev, true); dev_dbg(kbdev->dev, "Enabling GPU idle timer with count-value: 0x%.8x", kbdev->csf.gpu_idle_dur_count); } @@ -1890,6 +1905,7 @@ static void kbase_csf_firmware_reload_worker(struct work_struct *work) { struct kbase_device *kbdev = container_of(work, struct kbase_device, csf.firmware_reload_work); + unsigned long flags; int err; dev_info(kbdev->dev, "reloading firmware"); @@ -1908,7 +1924,9 @@ static void kbase_csf_firmware_reload_worker(struct work_struct *work) return; /* Reboot the firmware */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbase_csf_firmware_enable_mcu(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } void kbase_csf_firmware_trigger_reload(struct kbase_device *kbdev) @@ -2045,29 +2063,33 @@ u32 kbase_csf_firmware_set_gpu_idle_hysteresis_time(struct kbase_device *kbdev, return kbdev->csf.gpu_idle_dur_count; } - /* The 'reg_lock' is also taken and is held till the update is not + /* The scheduler lock is also taken and is held till the update is not * complete, to ensure the update of idle timer value by multiple Users * gets serialized. */ - mutex_lock(&kbdev->csf.reg_lock); - /* The firmware only reads the new idle timer value when the timer is - * disabled. - */ - kbase_csf_scheduler_spin_lock(kbdev, &flags); - kbase_csf_firmware_disable_gpu_idle_timer(kbdev); - kbase_csf_scheduler_spin_unlock(kbdev, flags); - /* Ensure that the request has taken effect */ - wait_for_global_request(kbdev, GLB_REQ_IDLE_DISABLE_MASK); - + kbase_csf_scheduler_lock(kbdev); kbase_csf_scheduler_spin_lock(kbdev, &flags); kbdev->csf.gpu_idle_hysteresis_ns = dur_ns; kbdev->csf.gpu_idle_dur_count = hysteresis_val; kbdev->csf.gpu_idle_dur_count_no_modifier = no_modifier; - kbase_csf_firmware_enable_gpu_idle_timer(kbdev); - kbase_csf_scheduler_spin_unlock(kbdev, flags); - wait_for_global_request(kbdev, GLB_REQ_IDLE_ENABLE_MASK); - mutex_unlock(&kbdev->csf.reg_lock); + if (kbdev->csf.gpu_idle_timer_enabled) { + /* Timer is already enabled. Disable the timer as FW only reads + * the new idle timer value when timer is re-enabled. + */ + kbase_csf_firmware_disable_gpu_idle_timer(kbdev); + kbase_csf_scheduler_spin_unlock(kbdev, flags); + /* Ensure that the request has taken effect */ + wait_for_global_request(kbdev, GLB_REQ_IDLE_DISABLE_MASK); + kbase_csf_scheduler_spin_lock(kbdev, &flags); + kbase_csf_firmware_enable_gpu_idle_timer(kbdev); + kbase_csf_scheduler_spin_unlock(kbdev, flags); + wait_for_global_request(kbdev, GLB_REQ_IDLE_ENABLE_MASK); + } else { + kbase_csf_scheduler_spin_unlock(kbdev, flags); + } + + kbase_csf_scheduler_unlock(kbdev); kbase_csf_scheduler_pm_idle(kbdev); kbase_reset_gpu_allow(kbdev); end: @@ -2255,8 +2277,9 @@ int kbase_csf_firmware_early_init(struct kbase_device *kbdev) kbdev->csf.glb_init_request_pending = true; + init_rwsem(&kbdev->csf.pmode_sync_sem); mutex_init(&kbdev->csf.reg_lock); - kbase_csf_pending_gpuq_kicks_init(kbdev); + kbase_csf_pending_gpuq_kick_queues_init(kbdev); kbdev->csf.fw = (struct kbase_csf_mcu_fw){ .data = NULL }; @@ -2265,7 +2288,7 @@ int kbase_csf_firmware_early_init(struct kbase_device *kbdev) void kbase_csf_firmware_early_term(struct kbase_device *kbdev) { - kbase_csf_pending_gpuq_kicks_term(kbdev); + kbase_csf_pending_gpuq_kick_queues_term(kbdev); mutex_destroy(&kbdev->csf.reg_lock); } @@ -2731,7 +2754,7 @@ int kbase_csf_firmware_mcu_register_poll(struct kbase_device *const kbdev, u32 c unsigned long remaining = kbase_csf_timeout_in_jiffies(kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT)) + jiffies; - u32 read_val; + u32 read_val = 0; dev_dbg(kbdev->dev, "p: reg %08x val %08x mask %08x", reg_addr, reg_val, val_mask); @@ -2778,12 +2801,9 @@ void kbase_csf_firmware_enable_gpu_idle_timer(struct kbase_device *kbdev) void kbase_csf_firmware_disable_gpu_idle_timer(struct kbase_device *kbdev) { - struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; - kbase_csf_scheduler_spin_lock_assert_held(kbdev); - kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, GLB_REQ_REQ_IDLE_DISABLE, - GLB_REQ_IDLE_DISABLE_MASK); + set_gpu_idle_timer_glb_req(kbdev, false); dev_dbg(kbdev->dev, "Sending request to disable gpu idle timer"); kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); @@ -2807,6 +2827,7 @@ int kbase_csf_firmware_ping_wait(struct kbase_device *const kbdev, unsigned int return wait_for_global_request_with_timeout(kbdev, GLB_REQ_PING_MASK, wait_timeout_ms); } + int kbase_csf_firmware_set_timeout(struct kbase_device *const kbdev, u64 const timeout) { const struct kbase_csf_global_iface *const global_iface = &kbdev->csf.global_iface; @@ -2845,8 +2866,6 @@ int kbase_csf_wait_protected_mode_enter(struct kbase_device *kbdev) { int err; - lockdep_assert_held(&kbdev->mmu_hw_mutex); - err = wait_for_global_request(kbdev, GLB_REQ_PROTM_ENTER_MASK); if (!err) { @@ -2912,6 +2931,7 @@ void kbase_csf_firmware_enable_mcu(struct kbase_device *kbdev) { struct kbase_csf_global_iface *iface = &kbdev->csf.global_iface; + lockdep_assert_held(&kbdev->hwaccess_lock); /* Clear the HALT bit before triggering the boot of MCU firmware */ kbase_csf_firmware_global_input_mask(iface, GLB_REQ, 0, GLB_REQ_HALT_MASK); @@ -2927,6 +2947,7 @@ void kbase_csf_firmware_trigger_mcu_sleep(struct kbase_device *kbdev) KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_REQUEST_SLEEP(kbdev, kbase_backend_get_cycle_cnt(kbdev)); kbase_csf_scheduler_spin_lock(kbdev, &flags); + set_gpu_idle_timer_glb_req(kbdev, false); set_global_request(global_iface, GLB_REQ_SLEEP_MASK); dev_dbg(kbdev->dev, "Sending sleep request to MCU"); kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); @@ -3191,6 +3212,9 @@ void kbase_csf_firmware_mcu_shared_mapping_term(struct kbase_device *kbdev, } if (csf_mapping->phys) { + /* This is on module unload path, so the pages can be left uncleared before + * returning them back to kbdev memory pool. + */ kbase_mem_pool_free_pages(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], csf_mapping->num_pages, csf_mapping->phys, false, false); } diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.h index a2948a98e9a7..f7a9c07dd6a9 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.h +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -618,6 +618,7 @@ void kbase_csf_firmware_trigger_mcu_sleep(struct kbase_device *kbdev); bool kbase_csf_firmware_is_mcu_in_sleep(struct kbase_device *kbdev); #endif + /** * kbase_csf_firmware_trigger_reload() - Trigger the reboot of MCU firmware, for * the cold boot case firmware image would diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_cfg.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_cfg.c index d08686f5829b..030a1ebf0ac6 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_cfg.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_cfg.c @@ -367,10 +367,10 @@ int kbase_csf_firmware_cfg_fw_wa_init(struct kbase_device *kbdev) */ entry_count = of_property_count_u32_elems(kbdev->dev->of_node, "quirks-ext"); - if (entry_count == -EINVAL) + if (entry_count < 0) entry_count = of_property_count_u32_elems(kbdev->dev->of_node, "quirks_ext"); - if (entry_count == -EINVAL || entry_count == -ENODATA) + if (entry_count < 0) return 0; entry_bytes = (size_t)entry_count * sizeof(u32); diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_no_mali.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_no_mali.c index 90568f6fa09f..0af560fd4260 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_no_mali.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_no_mali.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -670,6 +670,23 @@ static void set_timeout_global(const struct kbase_csf_global_iface *const global set_global_request(global_iface, GLB_REQ_CFG_PROGRESS_TIMER_MASK); } +static inline void set_gpu_idle_timer_glb_req(struct kbase_device *const kbdev, bool set) +{ + struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; + + kbase_csf_scheduler_spin_lock_assert_held(kbdev); + + if (set) { + kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, GLB_REQ_REQ_IDLE_ENABLE, + GLB_REQ_IDLE_ENABLE_MASK); + } else { + kbase_csf_firmware_global_input_mask( + global_iface, GLB_REQ, GLB_REQ_REQ_IDLE_DISABLE, GLB_REQ_IDLE_DISABLE_MASK); + } + + kbdev->csf.gpu_idle_timer_enabled = set; +} + static void enable_gpu_idle_timer(struct kbase_device *const kbdev) { struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; @@ -678,8 +695,11 @@ static void enable_gpu_idle_timer(struct kbase_device *const kbdev) kbase_csf_firmware_global_input(global_iface, GLB_IDLE_TIMER, kbdev->csf.gpu_idle_dur_count); - kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, GLB_REQ_REQ_IDLE_ENABLE, - GLB_REQ_IDLE_ENABLE_MASK); + kbase_csf_firmware_global_input_mask(global_iface, GLB_IDLE_TIMER_CONFIG, + kbdev->csf.gpu_idle_dur_count_no_modifier, + GLB_IDLE_TIMER_CONFIG_NO_MODIFIER_MASK); + + set_gpu_idle_timer_glb_req(kbdev, true); dev_dbg(kbdev->dev, "Enabling GPU idle timer with count-value: 0x%.8x", kbdev->csf.gpu_idle_dur_count); } @@ -857,11 +877,11 @@ static void kbase_csf_firmware_reload_worker(struct work_struct *work) container_of(work, struct kbase_device, csf.firmware_reload_work); unsigned long flags; + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); /* Reboot the firmware */ kbase_csf_firmware_enable_mcu(kbdev); /* Tell MCU state machine to transit to next state */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbdev->csf.firmware_reloaded = true; kbase_pm_update_state(kbdev); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); @@ -900,7 +920,7 @@ static u32 convert_dur_to_idle_count(struct kbase_device *kbdev, const u32 dur_n /* Get the cntfreq_el0 value, which drives the SYSTEM_TIMESTAMP */ u64 freq = kbase_arch_timer_get_cntfrq(kbdev); u64 dur_val = dur_ns; - u32 cnt_val_u32, reg_val_u32; + u32 cnt_val_u32, reg_val_u32, timer_src; bool src_system_timestamp = freq > 0; if (!src_system_timestamp) { @@ -932,9 +952,9 @@ static u32 convert_dur_to_idle_count(struct kbase_device *kbdev, const u32 dur_n reg_val_u32 = GLB_IDLE_TIMER_TIMEOUT_SET(0, cnt_val_u32); /* add the source flag */ - reg_val_u32 = GLB_IDLE_TIMER_TIMER_SOURCE_SET( - reg_val_u32, (src_system_timestamp ? GLB_IDLE_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP : - GLB_IDLE_TIMER_TIMER_SOURCE_GPU_COUNTER)); + timer_src = src_system_timestamp ? GLB_IDLE_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP : + GLB_IDLE_TIMER_TIMER_SOURCE_GPU_COUNTER; + reg_val_u32 = GLB_IDLE_TIMER_TIMER_SOURCE_SET(reg_val_u32, timer_src); return reg_val_u32; } @@ -989,29 +1009,33 @@ u32 kbase_csf_firmware_set_gpu_idle_hysteresis_time(struct kbase_device *kbdev, return kbdev->csf.gpu_idle_dur_count; } - /* The 'reg_lock' is also taken and is held till the update is not + /* The scheduler lock is also taken and is held till the update is not * complete, to ensure the update of idle timer value by multiple Users * gets serialized. */ - mutex_lock(&kbdev->csf.reg_lock); - /* The firmware only reads the new idle timer value when the timer is - * disabled. - */ - kbase_csf_scheduler_spin_lock(kbdev, &flags); - kbase_csf_firmware_disable_gpu_idle_timer(kbdev); - kbase_csf_scheduler_spin_unlock(kbdev, flags); - /* Ensure that the request has taken effect */ - wait_for_global_request(kbdev, GLB_REQ_IDLE_DISABLE_MASK); - + kbase_csf_scheduler_lock(kbdev); kbase_csf_scheduler_spin_lock(kbdev, &flags); kbdev->csf.gpu_idle_hysteresis_ns = dur_ns; kbdev->csf.gpu_idle_dur_count = hysteresis_val; kbdev->csf.gpu_idle_dur_count_no_modifier = no_modifier; - kbase_csf_firmware_enable_gpu_idle_timer(kbdev); - kbase_csf_scheduler_spin_unlock(kbdev, flags); - wait_for_global_request(kbdev, GLB_REQ_IDLE_ENABLE_MASK); - mutex_unlock(&kbdev->csf.reg_lock); + if (kbdev->csf.gpu_idle_timer_enabled) { + /* Timer is already enabled. Disable the timer as FW only reads + * the new idle timer value when timer is re-enabled. + */ + kbase_csf_firmware_disable_gpu_idle_timer(kbdev); + kbase_csf_scheduler_spin_unlock(kbdev, flags); + /* Ensure that the request has taken effect */ + wait_for_global_request(kbdev, GLB_REQ_IDLE_DISABLE_MASK); + kbase_csf_scheduler_spin_lock(kbdev, &flags); + kbase_csf_firmware_enable_gpu_idle_timer(kbdev); + kbase_csf_scheduler_spin_unlock(kbdev, flags); + wait_for_global_request(kbdev, GLB_REQ_IDLE_ENABLE_MASK); + } else { + kbase_csf_scheduler_spin_unlock(kbdev, flags); + } + + kbase_csf_scheduler_unlock(kbdev); kbase_csf_scheduler_pm_idle(kbdev); kbase_reset_gpu_allow(kbdev); end: @@ -1118,15 +1142,16 @@ int kbase_csf_firmware_early_init(struct kbase_device *kbdev) INIT_WORK(&kbdev->csf.firmware_reload_work, kbase_csf_firmware_reload_worker); INIT_WORK(&kbdev->csf.fw_error_work, firmware_error_worker); + init_rwsem(&kbdev->csf.pmode_sync_sem); mutex_init(&kbdev->csf.reg_lock); - kbase_csf_pending_gpuq_kicks_init(kbdev); + kbase_csf_pending_gpuq_kick_queues_init(kbdev); return 0; } void kbase_csf_firmware_early_term(struct kbase_device *kbdev) { - kbase_csf_pending_gpuq_kicks_term(kbdev); + kbase_csf_pending_gpuq_kick_queues_term(kbdev); mutex_destroy(&kbdev->csf.reg_lock); } @@ -1278,13 +1303,9 @@ void kbase_csf_firmware_enable_gpu_idle_timer(struct kbase_device *kbdev) void kbase_csf_firmware_disable_gpu_idle_timer(struct kbase_device *kbdev) { - struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; - kbase_csf_scheduler_spin_lock_assert_held(kbdev); - kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, GLB_REQ_REQ_IDLE_DISABLE, - GLB_REQ_IDLE_DISABLE_MASK); - + set_gpu_idle_timer_glb_req(kbdev, false); dev_dbg(kbdev->dev, "Sending request to disable gpu idle timer"); kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); @@ -1308,6 +1329,7 @@ int kbase_csf_firmware_ping_wait(struct kbase_device *const kbdev, unsigned int return wait_for_global_request(kbdev, GLB_REQ_PING_MASK); } + int kbase_csf_firmware_set_timeout(struct kbase_device *const kbdev, u64 const timeout) { const struct kbase_csf_global_iface *const global_iface = &kbdev->csf.global_iface; @@ -1370,6 +1392,8 @@ void kbase_csf_firmware_trigger_mcu_halt(struct kbase_device *kbdev) void kbase_csf_firmware_enable_mcu(struct kbase_device *kbdev) { + lockdep_assert_held(&kbdev->hwaccess_lock); + /* Trigger the boot of MCU firmware, Use the AUTO mode as * otherwise on fast reset, to exit protected mode, MCU will * not reboot by itself to enter normal mode. @@ -1384,6 +1408,7 @@ void kbase_csf_firmware_trigger_mcu_sleep(struct kbase_device *kbdev) unsigned long flags; kbase_csf_scheduler_spin_lock(kbdev, &flags); + set_gpu_idle_timer_glb_req(kbdev, false); set_global_request(global_iface, GLB_REQ_SLEEP_MASK); dev_dbg(kbdev->dev, "Sending sleep request to MCU"); kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c index 76e42e847fc3..ba47b7190395 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -39,13 +39,7 @@ static DEFINE_SPINLOCK(kbase_csf_fence_lock); #endif -#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG #define FENCE_WAIT_TIMEOUT_MS 3000 -#endif - -static void kcpu_queue_process(struct kbase_kcpu_command_queue *kcpu_queue, bool drain_queue); - -static void kcpu_queue_process_worker(struct work_struct *data); static int kbase_kcpu_map_import_prepare(struct kbase_kcpu_command_queue *kcpu_queue, struct base_kcpu_command_import_info *import_info, @@ -445,6 +439,16 @@ static void kbase_kcpu_jit_allocate_finish(struct kbase_kcpu_command_queue *queu kfree(cmd->info.jit_alloc.info); } +static void enqueue_kcpuq_work(struct kbase_kcpu_command_queue *queue) +{ + struct kbase_context *const kctx = queue->kctx; + + if (!atomic_read(&kctx->prioritized)) + queue_work(kctx->csf.kcpu_queues.kcpu_wq, &queue->work); + else + kbase_csf_scheduler_enqueue_kcpuq_work(queue); +} + /** * kbase_kcpu_jit_retry_pending_allocs() - Retry blocked JIT_ALLOC commands * @@ -464,9 +468,7 @@ static void kbase_kcpu_jit_retry_pending_allocs(struct kbase_context *kctx) * kbase_csf_kcpu_queue_context.jit_lock . */ list_for_each_entry(blocked_queue, &kctx->csf.kcpu_queues.jit_blocked_queues, jit_blocked) - queue_work(atomic_read(&kctx->prioritized) ? kctx->csf.kcpu_wq_high_prio : - kctx->csf.kcpu_wq_normal_prio, - &blocked_queue->work); + enqueue_kcpuq_work(blocked_queue); } static int kbase_kcpu_jit_free_process(struct kbase_kcpu_command_queue *queue, @@ -717,11 +719,8 @@ static int kbase_csf_queue_group_suspend_process(struct kbase_context *kctx, static enum kbase_csf_event_callback_action event_cqs_callback(void *param) { struct kbase_kcpu_command_queue *kcpu_queue = (struct kbase_kcpu_command_queue *)param; - struct kbase_context *kctx = kcpu_queue->kctx; - queue_work(atomic_read(&kctx->prioritized) ? kctx->csf.kcpu_wq_high_prio : - kctx->csf.kcpu_wq_normal_prio, - &kcpu_queue->work); + enqueue_kcpuq_work(kcpu_queue); return KBASE_CSF_EVENT_CALLBACK_KEEP; } @@ -1322,9 +1321,7 @@ static void kbase_csf_fence_wait_callback(struct dma_fence *fence, struct dma_fe fence->seqno); /* Resume kcpu command queue processing. */ - queue_work(atomic_read(&kctx->prioritized) ? kctx->csf.kcpu_wq_high_prio : - kctx->csf.kcpu_wq_normal_prio, - &kcpu_queue->work); + enqueue_kcpuq_work(kcpu_queue); } static void kbasep_kcpu_fence_wait_cancel(struct kbase_kcpu_command_queue *kcpu_queue, @@ -1360,7 +1357,6 @@ static void kbasep_kcpu_fence_wait_cancel(struct kbase_kcpu_command_queue *kcpu_ fence_info->fence = NULL; } -#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG /** * fence_timeout_callback() - Timeout callback function for fence-wait * @@ -1399,9 +1395,7 @@ static void fence_timeout_callback(struct timer_list *timer) kbase_sync_fence_info_get(fence, &info); if (info.status == 1) { - queue_work(atomic_read(&kctx->prioritized) ? kctx->csf.kcpu_wq_high_prio : - kctx->csf.kcpu_wq_normal_prio, - &kcpu_queue->work); + enqueue_kcpuq_work(kcpu_queue); } else if (info.status == 0) { dev_warn(kctx->kbdev->dev, "fence has not yet signalled in %ums", FENCE_WAIT_TIMEOUT_MS); @@ -1430,7 +1424,6 @@ static void fence_wait_timeout_start(struct kbase_kcpu_command_queue *cmd) { mod_timer(&cmd->fence_timeout, jiffies + msecs_to_jiffies(FENCE_WAIT_TIMEOUT_MS)); } -#endif /** * kbase_kcpu_fence_wait_process() - Process the kcpu fence wait command @@ -1469,9 +1462,8 @@ static int kbase_kcpu_fence_wait_process(struct kbase_kcpu_command_queue *kcpu_q fence_status = cb_err; if (cb_err == 0) { kcpu_queue->fence_wait_processed = true; -#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG - fence_wait_timeout_start(kcpu_queue); -#endif + if (IS_ENABLED(CONFIG_MALI_BIFROST_FENCE_DEBUG)) + fence_wait_timeout_start(kcpu_queue); } else if (cb_err == -ENOENT) { fence_status = dma_fence_get_status(fence); if (!fence_status) { @@ -1692,9 +1684,7 @@ static void fence_signal_timeout_cb(struct timer_list *timer) if (atomic_read(&kcpu_queue->fence_signal_pending_cnt) > 1) fence_signal_timeout_start(kcpu_queue); - queue_work(atomic_read(&kctx->prioritized) ? kctx->csf.kcpu_wq_high_prio : - kctx->csf.kcpu_wq_normal_prio, - &kcpu_queue->timeout_work); + queue_work(kctx->csf.kcpu_queues.kcpu_wq, &kcpu_queue->timeout_work); } } @@ -1973,7 +1963,7 @@ static void kcpu_queue_process_worker(struct work_struct *data) container_of(data, struct kbase_kcpu_command_queue, work); mutex_lock(&queue->lock); - kcpu_queue_process(queue, false); + kbase_csf_kcpu_queue_process(queue, false); mutex_unlock(&queue->lock); } @@ -2006,7 +1996,7 @@ static int delete_queue(struct kbase_context *kctx, u32 id) /* Drain the remaining work for this queue first and go past * all the waits. */ - kcpu_queue_process(queue, true); + kbase_csf_kcpu_queue_process(queue, true); /* All commands should have been processed */ WARN_ON(queue->num_pending_cmds); @@ -2022,11 +2012,20 @@ static int delete_queue(struct kbase_context *kctx, u32 id) mutex_unlock(&queue->lock); cancel_work_sync(&queue->timeout_work); + + /* + * Drain a pending request to process this queue in + * kbase_csf_scheduler_kthread() if any. By this point the + * queue would be empty so this would be a no-op. + */ + kbase_csf_scheduler_wait_for_kthread_pending_work(kctx->kbdev, + &queue->pending_kick); + cancel_work_sync(&queue->work); mutex_destroy(&queue->lock); - kfree(queue); + vfree(queue); } else { dev_dbg(kctx->kbdev->dev, "Attempt to delete a non-existent KCPU queue"); mutex_unlock(&kctx->csf.kcpu_queues.lock); @@ -2079,7 +2078,7 @@ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_END(struct kbase_device *kbde KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END(kbdev, queue); } -static void kcpu_queue_process(struct kbase_kcpu_command_queue *queue, bool drain_queue) +void kbase_csf_kcpu_queue_process(struct kbase_kcpu_command_queue *queue, bool drain_queue) { struct kbase_device *kbdev = queue->kctx->kbdev; bool process_next = true; @@ -2199,10 +2198,10 @@ static void kcpu_queue_process(struct kbase_kcpu_command_queue *queue, bool drai KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START(kbdev, queue); - kbase_gpu_vm_lock(queue->kctx); + kbase_gpu_vm_lock_with_pmode_sync(queue->kctx); meta = kbase_sticky_resource_acquire(queue->kctx, - cmd->info.import.gpu_va); - kbase_gpu_vm_unlock(queue->kctx); + cmd->info.import.gpu_va, NULL); + kbase_gpu_vm_unlock_with_pmode_sync(queue->kctx); if (meta == NULL) { queue->has_error = true; @@ -2219,10 +2218,10 @@ static void kcpu_queue_process(struct kbase_kcpu_command_queue *queue, bool drai KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START(kbdev, queue); - kbase_gpu_vm_lock(queue->kctx); + kbase_gpu_vm_lock_with_pmode_sync(queue->kctx); ret = kbase_sticky_resource_release(queue->kctx, NULL, cmd->info.import.gpu_va); - kbase_gpu_vm_unlock(queue->kctx); + kbase_gpu_vm_unlock_with_pmode_sync(queue->kctx); if (!ret) { queue->has_error = true; @@ -2240,10 +2239,10 @@ static void kcpu_queue_process(struct kbase_kcpu_command_queue *queue, bool drai KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START(kbdev, queue); - kbase_gpu_vm_lock(queue->kctx); + kbase_gpu_vm_lock_with_pmode_sync(queue->kctx); ret = kbase_sticky_resource_release_force(queue->kctx, NULL, cmd->info.import.gpu_va); - kbase_gpu_vm_unlock(queue->kctx); + kbase_gpu_vm_unlock_with_pmode_sync(queue->kctx); if (!ret) { queue->has_error = true; @@ -2642,7 +2641,7 @@ int kbase_csf_kcpu_queue_enqueue(struct kbase_context *kctx, } queue->num_pending_cmds += enq->nr_commands; - kcpu_queue_process(queue, false); + kbase_csf_kcpu_queue_process(queue, false); } out: @@ -2653,23 +2652,14 @@ out: int kbase_csf_kcpu_queue_context_init(struct kbase_context *kctx) { - kctx->csf.kcpu_wq_high_prio = alloc_workqueue("mali_kcpu_wq_%i_high_prio", - WQ_UNBOUND | WQ_HIGHPRI, 0, kctx->tgid); - if (kctx->csf.kcpu_wq_high_prio == NULL) { + kctx->csf.kcpu_queues.kcpu_wq = + alloc_workqueue("mali_kcpu_wq_%i_%i", 0, 0, kctx->tgid, kctx->id); + if (kctx->csf.kcpu_queues.kcpu_wq == NULL) { dev_err(kctx->kbdev->dev, "Failed to initialize KCPU queue high-priority workqueue"); return -ENOMEM; } - kctx->csf.kcpu_wq_normal_prio = - alloc_workqueue("mali_kcpu_wq_%i_normal_prio", 0, 0, kctx->tgid); - if (kctx->csf.kcpu_wq_normal_prio == NULL) { - dev_err(kctx->kbdev->dev, - "Failed to initialize KCPU queue normal-priority workqueue"); - destroy_workqueue(kctx->csf.kcpu_wq_high_prio); - return -ENOMEM; - } - mutex_init(&kctx->csf.kcpu_queues.lock); return 0; @@ -2688,8 +2678,7 @@ void kbase_csf_kcpu_queue_context_term(struct kbase_context *kctx) mutex_destroy(&kctx->csf.kcpu_queues.lock); - destroy_workqueue(kctx->csf.kcpu_wq_normal_prio); - destroy_workqueue(kctx->csf.kcpu_wq_high_prio); + destroy_workqueue(kctx->csf.kcpu_queues.kcpu_wq); } KBASE_EXPORT_TEST_API(kbase_csf_kcpu_queue_context_term); @@ -2699,15 +2688,42 @@ int kbase_csf_kcpu_queue_delete(struct kbase_context *kctx, return delete_queue(kctx, (u32)del->id); } +static struct kbase_kcpu_dma_fence_meta * +kbase_csf_kcpu_queue_metadata_new(struct kbase_context *kctx, u64 fence_context) +{ + int n; + struct kbase_kcpu_dma_fence_meta *metadata = kzalloc(sizeof(*metadata), GFP_KERNEL); + + if (!metadata) + goto early_ret; + + *metadata = (struct kbase_kcpu_dma_fence_meta){ + .kbdev = kctx->kbdev, + .kctx_id = kctx->id, + }; + + /* Please update MAX_TIMELINE_NAME macro when making changes to the string. */ + n = scnprintf(metadata->timeline_name, MAX_TIMELINE_NAME, "%u-%d_%u-%llu-kcpu", + kctx->kbdev->id, kctx->tgid, kctx->id, fence_context); + if (WARN_ON(n >= MAX_TIMELINE_NAME)) { + kfree(metadata); + metadata = NULL; + goto early_ret; + } + + kbase_refcount_set(&metadata->refcount, 1); + +early_ret: + return metadata; +} +KBASE_ALLOW_ERROR_INJECTION_TEST_API(kbase_csf_kcpu_queue_metadata_new, ERRNO_NULL); + int kbase_csf_kcpu_queue_new(struct kbase_context *kctx, struct kbase_ioctl_kcpu_queue_new *newq) { struct kbase_kcpu_command_queue *queue; - int idx; - int n; - int ret = 0; -#if IS_ENABLED(CONFIG_SYNC_FILE) struct kbase_kcpu_dma_fence_meta *metadata; -#endif + int idx; + int ret = 0; /* The queue id is of u8 type and we use the index of the kcpu_queues * array as an id, so the number of elements in the array can't be * more than 256. @@ -2727,54 +2743,48 @@ int kbase_csf_kcpu_queue_new(struct kbase_context *kctx, struct kbase_ioctl_kcpu goto out; } - queue = kzalloc(sizeof(*queue), GFP_KERNEL); - + queue = vzalloc(sizeof(*queue)); if (!queue) { ret = -ENOMEM; goto out; } + *queue = (struct kbase_kcpu_command_queue) + { + .kctx = kctx, .start_offset = 0, .num_pending_cmds = 0, .enqueue_failed = false, + .command_started = false, .has_error = false, .id = idx, +#if IS_ENABLED(CONFIG_SYNC_FILE) + .fence_context = dma_fence_context_alloc(1), .fence_seqno = 0, + .fence_wait_processed = false, +#endif /* IS_ENABLED(CONFIG_SYNC_FILE) */ + }; + + mutex_init(&queue->lock); + INIT_WORK(&queue->work, kcpu_queue_process_worker); + INIT_LIST_HEAD(&queue->high_prio_work); + atomic_set(&queue->pending_kick, 0); + INIT_WORK(&queue->timeout_work, kcpu_queue_timeout_worker); + INIT_LIST_HEAD(&queue->jit_blocked); + + if (IS_ENABLED(CONFIG_SYNC_FILE)) { + metadata = kbase_csf_kcpu_queue_metadata_new(kctx, queue->fence_context); + if (!metadata) { + vfree(queue); + ret = -ENOMEM; + goto out; + } + + queue->metadata = metadata; + atomic_inc(&kctx->kbdev->live_fence_metadata); + atomic_set(&queue->fence_signal_pending_cnt, 0); + kbase_timer_setup(&queue->fence_signal_timeout, fence_signal_timeout_cb); + } + + if (IS_ENABLED(CONFIG_MALI_BIFROST_FENCE_DEBUG)) + kbase_timer_setup(&queue->fence_timeout, fence_timeout_callback); + bitmap_set(kctx->csf.kcpu_queues.in_use, (unsigned int)idx, 1); kctx->csf.kcpu_queues.array[idx] = queue; - mutex_init(&queue->lock); - queue->kctx = kctx; - queue->start_offset = 0; - queue->num_pending_cmds = 0; -#if IS_ENABLED(CONFIG_SYNC_FILE) - queue->fence_context = dma_fence_context_alloc(1); - queue->fence_seqno = 0; - queue->fence_wait_processed = false; - - metadata = kzalloc(sizeof(*metadata), GFP_KERNEL); - if (!metadata) { - kfree(queue); - ret = -ENOMEM; - goto out; - } - - metadata->kbdev = kctx->kbdev; - metadata->kctx_id = kctx->id; - n = snprintf(metadata->timeline_name, MAX_TIMELINE_NAME, "%u-%d_%u-%llu-kcpu", - kctx->kbdev->id, kctx->tgid, kctx->id, queue->fence_context); - if (WARN_ON(n >= MAX_TIMELINE_NAME)) { - kfree(queue); - kfree(metadata); - ret = -EINVAL; - goto out; - } - - kbase_refcount_set(&metadata->refcount, 1); - queue->metadata = metadata; - atomic_inc(&kctx->kbdev->live_fence_metadata); -#endif /* CONFIG_SYNC_FILE */ - queue->enqueue_failed = false; - queue->command_started = false; - INIT_LIST_HEAD(&queue->jit_blocked); - queue->has_error = false; - INIT_WORK(&queue->work, kcpu_queue_process_worker); - INIT_WORK(&queue->timeout_work, kcpu_queue_timeout_worker); - queue->id = idx; - newq->id = idx; /* Fire the tracepoint with the mutex held to enforce correct ordering @@ -2784,14 +2794,6 @@ int kbase_csf_kcpu_queue_new(struct kbase_context *kctx, struct kbase_ioctl_kcpu queue->num_pending_cmds); KBASE_KTRACE_ADD_CSF_KCPU(kctx->kbdev, KCPU_QUEUE_CREATE, queue, queue->fence_context, 0); -#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG - kbase_timer_setup(&queue->fence_timeout, fence_timeout_callback); -#endif - -#if IS_ENABLED(CONFIG_SYNC_FILE) - atomic_set(&queue->fence_signal_pending_cnt, 0); - kbase_timer_setup(&queue->fence_signal_timeout, fence_signal_timeout_cb); -#endif out: mutex_unlock(&kctx->csf.kcpu_queues.lock); diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.h index d1f18ed5caca..291509bef5a6 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.h +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.h @@ -243,7 +243,19 @@ struct kbase_kcpu_command { * @work: struct work_struct which contains a pointer to * the function which handles processing of kcpu * commands enqueued into a kcpu command queue; - * part of kernel API for processing workqueues + * part of kernel API for processing workqueues. + * This would be used if the context is not + * prioritised, otherwise it would be handled by + * kbase_csf_scheduler_kthread(). + * @high_prio_work: A counterpart to @work, this queue would be + * added to a list to be processed by + * kbase_csf_scheduler_kthread() if it is + * prioritised. + * @pending_kick: Indicates that kbase_csf_scheduler_kthread() + * should re-evaluate pending commands for this + * queue. This would be set to false when the work + * is done. This is used mainly for + * synchronisation with queue termination. * @timeout_work: struct work_struct which contains a pointer to the * function which handles post-timeout actions * queue when a fence signal timeout occurs. @@ -287,6 +299,8 @@ struct kbase_kcpu_command_queue { struct kbase_context *kctx; struct kbase_kcpu_command commands[KBASEP_KCPU_QUEUE_SIZE]; struct work_struct work; + struct list_head high_prio_work; + atomic_t pending_kick; struct work_struct timeout_work; u8 start_offset; u8 id; @@ -299,9 +313,7 @@ struct kbase_kcpu_command_queue { bool command_started; struct list_head jit_blocked; bool has_error; -#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG struct timer_list fence_timeout; -#endif /* CONFIG_MALI_BIFROST_FENCE_DEBUG */ #if IS_ENABLED(CONFIG_SYNC_FILE) struct kbase_kcpu_dma_fence_meta *metadata; #endif /* CONFIG_SYNC_FILE */ @@ -334,6 +346,18 @@ int kbase_csf_kcpu_queue_new(struct kbase_context *kctx, struct kbase_ioctl_kcpu int kbase_csf_kcpu_queue_delete(struct kbase_context *kctx, struct kbase_ioctl_kcpu_queue_delete *del); +/** + * kbase_csf_kcpu_queue_process - Proces pending KCPU queue commands + * + * @queue: The queue to process pending commands for + * @drain_queue: Whether to skip all blocking commands in the queue. + * This is expected to be set to true on queue + * termination. + * + * Return: 0 if successful or a negative error code on failure. + */ +void kbase_csf_kcpu_queue_process(struct kbase_kcpu_command_queue *queue, bool drain_queue); + /** * kbase_csf_kcpu_queue_enqueue - Enqueue a KCPU command into a KCPU command * queue. diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_registers.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_registers.h index d01f3070cf5b..9a7c6e451f66 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_registers.h +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_registers.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -250,7 +250,7 @@ #define GLB_ACK 0x0000 /* () Global acknowledge */ #define GLB_DB_ACK 0x0008 /* () Global doorbell acknowledge */ -#define GLB_HALT_STATUS 0x0010 /* () Global halt status */ +#define GLB_FATAL_STATUS 0x0010 /* () Global fatal error status */ #define GLB_PRFCNT_STATUS 0x0014 /* () Performance counter status */ #define GLB_PRFCNT_INSERT 0x0018 /* () Performance counter buffer insert index */ #define GLB_DEBUG_FWUTF_RESULT GLB_DEBUG_ARG_OUT0 /* () Firmware debug test result */ @@ -1422,6 +1422,12 @@ #define GLB_REQ_PRFCNT_OVERFLOW_SET(reg_val, value) \ (((reg_val) & ~GLB_REQ_PRFCNT_OVERFLOW_MASK) | \ (((value) << GLB_REQ_PRFCNT_OVERFLOW_SHIFT) & GLB_REQ_PRFCNT_OVERFLOW_MASK)) +#define GLB_ACK_FATAL_SHIFT GPU_U(27) +#define GLB_ACK_FATAL_MASK (GPU_U(0x1) << GLB_ACK_FATAL_SHIFT) +#define GLB_ACK_FATAL_GET(reg_val) (((reg_val)&GLB_ACK_FATAL_MASK) >> GLB_ACK_FATAL_SHIFT) +#define GLB_ACK_FATAL_SET(reg_val, value) \ + (~(~(reg_val) | GLB_ACK_FATAL_MASK) | \ + (((value) << GLB_ACK_FATAL_SHIFT) & GLB_ACK_FATAL_MASK)) #define GLB_REQ_DEBUG_CSF_REQ_SHIFT 30 #define GLB_REQ_DEBUG_CSF_REQ_MASK (0x1 << GLB_REQ_DEBUG_CSF_REQ_SHIFT) #define GLB_REQ_DEBUG_CSF_REQ_GET(reg_val) \ @@ -1822,6 +1828,20 @@ (((reg_val) & ~GLB_DEBUG_REQ_RUN_MODE_MASK) | \ (((value) << GLB_DEBUG_REQ_RUN_MODE_SHIFT) & GLB_DEBUG_REQ_RUN_MODE_MASK)) +/* GLB_FATAL_STATUS register */ +#define GLB_FATAL_STATUS_VALUE_SHIFT GPU_U(0) +#define GLB_FATAL_STATUS_VALUE_MASK (GPU_U(0xFFFFFFFF) << GLB_FATAL_STATUS_VALUE_SHIFT) +#define GLB_FATAL_STATUS_VALUE_GET(reg_val) \ + (((reg_val)&GLB_FATAL_STATUS_VALUE_MASK) >> GLB_FATAL_STATUS_VALUE_SHIFT) + +enum glb_fatal_status { + GLB_FATAL_STATUS_VALUE_OK, + GLB_FATAL_STATUS_VALUE_ASSERT, + GLB_FATAL_STATUS_VALUE_UNEXPECTED_EXCEPTION, + GLB_FATAL_STATUS_VALUE_HANG, + GLB_FATAL_STATUS_VALUE_COUNT +}; + /* GLB_DEBUG_ACK register */ #define GLB_DEBUG_ACK_DEBUG_RUN_SHIFT GPU_U(23) #define GLB_DEBUG_ACK_DEBUG_RUN_MASK (GPU_U(0x1) << GLB_DEBUG_ACK_DEBUG_RUN_SHIFT) diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_reset_gpu.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_reset_gpu.c index 240397ebc16d..b07cc9600a04 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_reset_gpu.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_reset_gpu.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -224,8 +224,11 @@ static void kbase_csf_reset_end_hw_access(struct kbase_device *kbdev, int err_du static void kbase_csf_debug_dump_registers(struct kbase_device *kbdev) { + unsigned long flags; + kbase_io_history_dump(kbdev); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); dev_err(kbdev->dev, "Register state:"); dev_err(kbdev->dev, " GPU_IRQ_RAWSTAT=0x%08x GPU_STATUS=0x%08x MCU_STATUS=0x%08x", kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_RAWSTAT)), @@ -251,6 +254,7 @@ static void kbase_csf_debug_dump_registers(struct kbase_device *kbdev) kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(TILER_CONFIG))); } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } /** @@ -396,6 +400,7 @@ static int kbase_csf_reset_gpu_now(struct kbase_device *kbdev, bool firmware_ini */ if (likely(firmware_inited)) kbase_csf_scheduler_reset(kbdev); + cancel_work_sync(&kbdev->csf.firmware_reload_work); dev_dbg(kbdev->dev, "Disable GPU hardware counters.\n"); @@ -403,6 +408,7 @@ static int kbase_csf_reset_gpu_now(struct kbase_device *kbdev, bool firmware_ini kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); ret = kbase_csf_reset_gpu_once(kbdev, firmware_inited, silent); + if (ret == SOFT_RESET_FAILED) { dev_err(kbdev->dev, "Soft-reset failed"); goto err; @@ -490,6 +496,13 @@ static void kbase_csf_reset_gpu_worker(struct work_struct *data) bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev, unsigned int flags) { +#ifdef CONFIG_MALI_ARBITER_SUPPORT + if (kbase_pm_is_gpu_lost(kbdev)) { + /* GPU access has been removed, reset will be done by Arbiter instead */ + return false; + } +#endif + if (flags & RESET_FLAGS_HWC_UNRECOVERABLE_ERROR) kbase_hwcnt_backend_csf_on_unrecoverable_error(&kbdev->hwcnt_gpu_iface); diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.c index 81ddeb667d06..642531c1033c 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -36,6 +36,7 @@ #include "mali_kbase_csf_tiler_heap_reclaim.h" #include "mali_kbase_csf_mcu_shared_reg.h" #include +#include #if IS_ENABLED(CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD) #include #include @@ -84,7 +85,8 @@ scheduler_get_protm_enter_async_group(struct kbase_device *const kbdev, struct kbase_queue_group *const group); static struct kbase_queue_group *get_tock_top_group(struct kbase_csf_scheduler *const scheduler); static void scheduler_enable_tick_timer_nolock(struct kbase_device *kbdev); -static int suspend_active_queue_groups(struct kbase_device *kbdev, unsigned long *slot_mask); +static int suspend_active_queue_groups(struct kbase_device *kbdev, unsigned long *slot_mask, + bool reset); static int suspend_active_groups_on_powerdown(struct kbase_device *kbdev, bool system_suspend); static void schedule_in_cycle(struct kbase_queue_group *group, bool force); static bool queue_group_scheduled_locked(struct kbase_queue_group *group); @@ -119,7 +121,7 @@ static inline int gpu_metrics_ctx_init(struct kbase_context *kctx) put_cred(cred); /* Return early if this is not a Userspace created context */ - if (unlikely(!kctx->kfile)) + if (unlikely(!kctx->filp)) return 0; /* Serialize against the other threads trying to create/destroy Kbase contexts. */ @@ -156,7 +158,7 @@ static inline int gpu_metrics_ctx_init(struct kbase_context *kctx) static inline void gpu_metrics_ctx_term(struct kbase_context *kctx) { /* Return early if this is not a Userspace created context */ - if (unlikely(!kctx->kfile)) + if (unlikely(!kctx->filp)) return; /* Serialize against the other threads trying to create/destroy Kbase contexts. */ @@ -458,11 +460,14 @@ static void wait_for_dump_complete_on_group_deschedule(struct kbase_queue_group * * This function notifies the Userspace client waiting for the faults and wait * for the Client to complete the dumping. - * The function is called only from Scheduling tick/tock when a request sent by - * the Scheduler to FW times out or from the protm event work item of the group - * when the protected mode entry request times out. - * In the latter case there is no wait done as scheduler lock would be released - * immediately. In the former case the function waits and releases the scheduler + * The function is mainly called from Scheduling tick/tock when a request sent by + * the Scheduler to FW times out. It can be called outside the tick/tock when timeout + * happens in the following 3 cases :- + * - Entry to protected mode is initiated from protm event work item. + * - Forced exit from protected mode is triggered when GPU queue of an on-slot group is kicked. + * - CSG termination request is sent when Userspace tries to delete the queue group. + * In the latter 3 cases there is no wait done as scheduler lock would be released + * immediately. In the tick/tock case the function waits and releases the scheduler * lock before the wait. It has been ensured that the Scheduler view of the groups * won't change meanwhile, so no group can enter/exit the Scheduler, become * runnable or go off slot. @@ -478,10 +483,9 @@ static void schedule_actions_trigger_df(struct kbase_device *kbdev, struct kbase if (!kbase_debug_csf_fault_notify(kbdev, kctx, error)) return; - if (unlikely(scheduler->state != SCHED_BUSY)) { - WARN_ON(error != DF_PROTECTED_MODE_ENTRY_FAILURE); + /* Return early if the function was called outside the tick/tock */ + if (unlikely(scheduler->state != SCHED_BUSY)) return; - } mutex_unlock(&scheduler->lock); kbase_debug_csf_fault_wait_completion(kbdev); @@ -788,19 +792,20 @@ static void update_on_slot_queues_offsets(struct kbase_device *kbdev) static void enqueue_gpu_idle_work(struct kbase_csf_scheduler *const scheduler) { atomic_set(&scheduler->gpu_no_longer_idle, false); - queue_work(scheduler->idle_wq, &scheduler->gpu_idle_work); + atomic_inc(&scheduler->pending_gpu_idle_work); + complete(&scheduler->kthread_signal); } -bool kbase_csf_scheduler_process_gpu_idle_event(struct kbase_device *kbdev) +void kbase_csf_scheduler_process_gpu_idle_event(struct kbase_device *kbdev) { struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; bool can_suspend_on_idle; - bool invoke_pm_state_machine = false; lockdep_assert_held(&kbdev->hwaccess_lock); lockdep_assert_held(&scheduler->interrupt_lock); - can_suspend_on_idle = kbase_pm_idle_groups_sched_suspendable(kbdev); + can_suspend_on_idle = kbase_pm_idle_groups_sched_suspendable(kbdev) && + !kbase_pm_is_mcu_inactive(kbdev, kbdev->pm.backend.mcu_state); KBASE_KTRACE_ADD(kbdev, SCHEDULER_GPU_IDLE_EVENT_CAN_SUSPEND, NULL, (((u64)can_suspend_on_idle) << 32)); @@ -812,23 +817,6 @@ bool kbase_csf_scheduler_process_gpu_idle_event(struct kbase_device *kbdev) scheduler->fast_gpu_idle_handling = (kbdev->csf.gpu_idle_hysteresis_ns == 0) || !kbase_csf_scheduler_all_csgs_idle(kbdev); - /* If GPU idle event occurred after the runtime suspend was aborted due to - * DB_MIRROR irq then it suggests that Userspace submission didn't make GPU - * non-idle. So the planned resumption of scheduling can be cancelled and - * MCU can be put back to sleep state to re-trigger the runtime suspend. - */ - if (unlikely(kbdev->pm.backend.exit_gpu_sleep_mode && - kbdev->pm.backend.runtime_suspend_abort_reason == - ABORT_REASON_DB_MIRROR_IRQ)) { - /* Cancel the planned resumption of scheduling */ - kbdev->pm.backend.exit_gpu_sleep_mode = false; - kbdev->pm.backend.runtime_suspend_abort_reason = ABORT_REASON_NONE; - /* PM state machine can be invoked to put MCU back to the sleep - * state right away and thereby re-trigger the runtime suspend. - */ - invoke_pm_state_machine = true; - } - /* The GPU idle worker relies on update_on_slot_queues_offsets() to have * finished. It's queued before to reduce the time it takes till execution * but it'll eventually be blocked by the scheduler->interrupt_lock. @@ -839,8 +827,6 @@ bool kbase_csf_scheduler_process_gpu_idle_event(struct kbase_device *kbdev) /* The extract offsets are unused in fast GPU idle handling */ if (!scheduler->fast_gpu_idle_handling) update_on_slot_queues_offsets(kbdev); - - return invoke_pm_state_machine; } u32 kbase_csf_scheduler_get_nr_active_csgs_locked(struct kbase_device *kbdev) @@ -1027,6 +1013,8 @@ static void scheduler_force_protm_exit(struct kbase_device *kbdev) * * @kbdev: Pointer to the device * @suspend_handler: Handler code for how to handle a suspend that might occur. + * @active_after_sleep: Flag to indicate that Scheduler is being activated from + * the sleeping state. * * This function is usually called when Scheduler needs to be activated. * The PM reference count is acquired for the Scheduler and the power on @@ -1035,7 +1023,8 @@ static void scheduler_force_protm_exit(struct kbase_device *kbdev) * Return: 0 if successful or a negative error code on failure. */ static int scheduler_pm_active_handle_suspend(struct kbase_device *kbdev, - enum kbase_pm_suspend_handler suspend_handler) + enum kbase_pm_suspend_handler suspend_handler, + bool active_after_sleep) { unsigned long flags; u32 prev_count; @@ -1043,24 +1032,35 @@ static int scheduler_pm_active_handle_suspend(struct kbase_device *kbdev, lockdep_assert_held(&kbdev->csf.scheduler.lock); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); prev_count = kbdev->csf.scheduler.pm_active_count; if (!WARN_ON(prev_count == U32_MAX)) kbdev->csf.scheduler.pm_active_count++; - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); /* On 0 => 1, make a pm_ctx_active request */ if (!prev_count) { - ret = kbase_pm_context_active_handle_suspend(kbdev, suspend_handler); - /* Invoke the PM state machines again as the change in MCU - * desired status, due to the update of scheduler.pm_active_count, - * may be missed by the thread that called pm_wait_for_desired_state() - */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - if (ret) + kbase_pm_lock(kbdev); + kbdev->pm.backend.mcu_poweron_required = true; + ret = kbase_pm_context_active_handle_suspend_locked(kbdev, suspend_handler); + if (ret) { kbdev->csf.scheduler.pm_active_count--; - kbase_pm_update_state(kbdev); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + kbdev->pm.backend.mcu_poweron_required = false; + } else { + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + if (active_after_sleep) { + kbdev->pm.backend.runtime_suspend_abort_reason = ABORT_REASON_NONE; + kbdev->pm.backend.gpu_sleep_mode_active = false; + } + /* Check if the GPU is already active */ + if (kbdev->pm.active_count > 1) { + /* GPU is already active, so need to invoke the PM state machines + * explicitly to turn on the MCU. + */ + kbdev->pm.backend.mcu_desired = true; + kbase_pm_update_state(kbdev); + } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } + kbase_pm_unlock(kbdev); } return ret; @@ -1072,8 +1072,6 @@ static int scheduler_pm_active_handle_suspend(struct kbase_device *kbdev, * Scheduler * * @kbdev: Pointer to the device - * @flags: Pointer to the flags variable containing the interrupt state - * when hwaccess lock was acquired. * * This function is called when Scheduler needs to be activated from the * sleeping state. @@ -1081,42 +1079,15 @@ static int scheduler_pm_active_handle_suspend(struct kbase_device *kbdev, * MCU is initiated. It resets the flag that indicates to the MCU state * machine that MCU needs to be put in sleep state. * - * Note: This function shall be called with hwaccess lock held and it may - * release that lock and reacquire it. - * * Return: zero when the PM reference was taken and non-zero when the * system is being suspending/suspended. */ -static int scheduler_pm_active_after_sleep(struct kbase_device *kbdev, unsigned long *flags) +static int scheduler_pm_active_after_sleep(struct kbase_device *kbdev) { - u32 prev_count; - int ret = 0; - lockdep_assert_held(&kbdev->csf.scheduler.lock); - lockdep_assert_held(&kbdev->hwaccess_lock); - prev_count = kbdev->csf.scheduler.pm_active_count; - if (!WARN_ON(prev_count == U32_MAX)) - kbdev->csf.scheduler.pm_active_count++; - - kbdev->pm.backend.runtime_suspend_abort_reason = ABORT_REASON_NONE; - - /* On 0 => 1, make a pm_ctx_active request */ - if (!prev_count) { - spin_unlock_irqrestore(&kbdev->hwaccess_lock, *flags); - - ret = kbase_pm_context_active_handle_suspend( - kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE); - - spin_lock_irqsave(&kbdev->hwaccess_lock, *flags); - if (ret) - kbdev->csf.scheduler.pm_active_count--; - else - kbdev->pm.backend.gpu_sleep_mode_active = false; - kbase_pm_update_state(kbdev); - } - - return ret; + return scheduler_pm_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE, + true); } #endif @@ -1136,28 +1107,32 @@ static void scheduler_pm_idle(struct kbase_device *kbdev) lockdep_assert_held(&kbdev->csf.scheduler.lock); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); prev_count = kbdev->csf.scheduler.pm_active_count; if (!WARN_ON(prev_count == 0)) kbdev->csf.scheduler.pm_active_count--; - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); if (prev_count == 1) { - kbase_pm_context_idle(kbdev); - /* Invoke the PM state machines again as the change in MCU - * desired status, due to the update of scheduler.pm_active_count, - * may be missed by the thread that called pm_wait_for_desired_state() - */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_pm_update_state(kbdev); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + kbase_pm_lock(kbdev); + kbdev->pm.backend.mcu_poweron_required = false; + kbase_pm_context_idle_locked(kbdev); + /* Check if GPU is still active */ + if (kbdev->pm.active_count) { + /* GPU is still active, so need to invoke the PM state machines + * explicitly to turn off the MCU. + */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbdev->pm.backend.mcu_desired = false; + kbase_pm_update_state(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } + kbase_pm_unlock(kbdev); } } #ifdef KBASE_PM_RUNTIME /** * scheduler_pm_idle_before_sleep() - Release the PM reference count and - * trigger the tranistion to sleep state. + * trigger the transition to sleep state. * * @kbdev: Pointer to the device * @@ -1168,28 +1143,15 @@ static void scheduler_pm_idle(struct kbase_device *kbdev) static void scheduler_pm_idle_before_sleep(struct kbase_device *kbdev) { unsigned long flags; - u32 prev_count; lockdep_assert_held(&kbdev->csf.scheduler.lock); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - prev_count = kbdev->csf.scheduler.pm_active_count; - if (!WARN_ON(prev_count == 0)) - kbdev->csf.scheduler.pm_active_count--; kbdev->pm.backend.gpu_sleep_mode_active = true; kbdev->pm.backend.exit_gpu_sleep_mode = false; spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - if (prev_count == 1) { - kbase_pm_context_idle(kbdev); - /* Invoke the PM state machines again as the change in MCU - * desired status, due to the update of scheduler.pm_active_count, - * may be missed by the thread that called pm_wait_for_desired_state() - */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_pm_update_state(kbdev); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - } + scheduler_pm_idle(kbdev); } #endif @@ -1205,8 +1167,8 @@ static void scheduler_wakeup(struct kbase_device *kbdev, bool kick) if (scheduler->state == SCHED_SUSPENDED) { dev_dbg(kbdev->dev, "Re-activating the Scheduler after suspend"); - ret = scheduler_pm_active_handle_suspend(kbdev, - KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE); + ret = scheduler_pm_active_handle_suspend( + kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE, false); #if IS_ENABLED(CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD) hrtimer_start(&scheduler->gpu_metrics_timer, HR_TIMER_DELAY_NSEC(kbase_gpu_metrics_get_tp_emit_interval()), @@ -1214,13 +1176,8 @@ static void scheduler_wakeup(struct kbase_device *kbdev, bool kick) #endif } else { #ifdef KBASE_PM_RUNTIME - unsigned long flags; - dev_dbg(kbdev->dev, "Re-activating the Scheduler out of sleep"); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - ret = scheduler_pm_active_after_sleep(kbdev, &flags); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + ret = scheduler_pm_active_after_sleep(kbdev); #endif } @@ -1228,7 +1185,7 @@ static void scheduler_wakeup(struct kbase_device *kbdev, bool kick) /* GPUCORE-29850 would add the handling for the case where * Scheduler could not be activated due to system suspend. */ - dev_info(kbdev->dev, "Couldn't wakeup Scheduler due to system suspend"); + dev_dbg(kbdev->dev, "Couldn't wakeup Scheduler due to system suspend"); return; } @@ -2410,6 +2367,11 @@ static void cancel_tock_work(struct kbase_csf_scheduler *const scheduler) atomic_set(&scheduler->pending_tock_work, false); } +static void cancel_gpu_idle_work(struct kbase_csf_scheduler *const scheduler) +{ + atomic_set(&scheduler->pending_gpu_idle_work, false); +} + static void remove_group_from_runnable(struct kbase_csf_scheduler *const scheduler, struct kbase_queue_group *group, enum kbase_csf_group_state run_state) @@ -3131,8 +3093,9 @@ static void sched_evict_group(struct kbase_queue_group *group, bool fault, static int term_group_sync(struct kbase_queue_group *group) { struct kbase_device *kbdev = group->kctx->kbdev; - const unsigned int fw_timeout_ms = kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT); - long remaining = kbase_csf_timeout_in_jiffies(fw_timeout_ms); + const unsigned int group_term_timeout_ms = + kbase_get_timeout_ms(kbdev, CSF_CSG_TERM_TIMEOUT); + long remaining = kbase_csf_timeout_in_jiffies(group_term_timeout_ms); int err = 0; term_csg_slot(group); @@ -3148,11 +3111,11 @@ static int term_group_sync(struct kbase_queue_group *group) dev_warn( kbdev->dev, "[%llu] term request timeout (%d ms) for group %d of context %d_%d on slot %d", - kbase_backend_get_cycle_cnt(kbdev), fw_timeout_ms, group->handle, + kbase_backend_get_cycle_cnt(kbdev), group_term_timeout_ms, group->handle, group->kctx->tgid, group->kctx->id, group->csg_nr); if (kbase_csf_firmware_ping_wait(kbdev, FW_PING_AFTER_ERROR_TIMEOUT_MS)) error_type = DF_PING_REQUEST_TIMEOUT; - kbase_debug_csf_fault_notify(kbdev, group->kctx, error_type); + schedule_actions_trigger_df(kbdev, group->kctx, error_type); if (kbase_prepare_to_reset_gpu(kbdev, RESET_FLAGS_NONE)) kbase_reset_gpu(kbdev); @@ -4138,7 +4101,7 @@ static void scheduler_group_check_protm_enter(struct kbase_device *const kbdev, * entry to protected mode happens with a memory region being locked and * the same region is then accessed by the GPU in protected mode. */ - mutex_lock(&kbdev->mmu_hw_mutex); + down_write(&kbdev->csf.pmode_sync_sem); spin_lock_irqsave(&scheduler->interrupt_lock, flags); /* Check if the previous transition to enter & exit the protected @@ -4204,7 +4167,7 @@ static void scheduler_group_check_protm_enter(struct kbase_device *const kbdev, spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); err = kbase_csf_wait_protected_mode_enter(kbdev); - mutex_unlock(&kbdev->mmu_hw_mutex); + up_write(&kbdev->csf.pmode_sync_sem); if (err) schedule_actions_trigger_df( @@ -4219,7 +4182,7 @@ static void scheduler_group_check_protm_enter(struct kbase_device *const kbdev, } spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); - mutex_unlock(&kbdev->mmu_hw_mutex); + up_write(&kbdev->csf.pmode_sync_sem); } /** @@ -4797,8 +4760,9 @@ static int suspend_active_groups_on_powerdown(struct kbase_device *kbdev, bool s { struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; DECLARE_BITMAP(slot_mask, MAX_SUPPORTED_CSGS) = { 0 }; + int ret; - int ret = suspend_active_queue_groups(kbdev, slot_mask); + ret = suspend_active_queue_groups(kbdev, slot_mask, false); if (unlikely(ret)) { const int csg_nr = ffs(slot_mask[0]) - 1; @@ -4841,7 +4805,7 @@ static int suspend_active_groups_on_powerdown(struct kbase_device *kbdev, bool s * Returns false if any of the queues inside any of the groups that have been * assigned a physical CSG slot have work to execute, or have executed work * since having received a GPU idle notification. This function is used to - * handle a rance condition between firmware reporting GPU idle and userspace + * handle a race condition between firmware reporting GPU idle and userspace * submitting more work by directly ringing a doorbell. * * Return: false if any queue inside any resident group has work to be processed @@ -4988,14 +4952,14 @@ static bool scheduler_suspend_on_idle(struct kbase_device *kbdev) return true; } -static void gpu_idle_worker(struct work_struct *work) +static void gpu_idle_worker(struct kbase_device *kbdev) { - struct kbase_device *kbdev = - container_of(work, struct kbase_device, csf.scheduler.gpu_idle_work); struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; bool scheduler_is_idle_suspendable = false; bool all_groups_suspended = false; + WARN_ON_ONCE(atomic_read(&scheduler->pending_gpu_idle_work) == 0); + KBASE_KTRACE_ADD(kbdev, SCHEDULER_GPU_IDLE_WORKER_START, NULL, 0u); #define __ENCODE_KTRACE_INFO(reset, idle, all_suspend) \ @@ -5005,7 +4969,7 @@ static void gpu_idle_worker(struct work_struct *work) dev_warn(kbdev->dev, "Quit idle for failing to prevent gpu reset.\n"); KBASE_KTRACE_ADD(kbdev, SCHEDULER_GPU_IDLE_WORKER_END, NULL, __ENCODE_KTRACE_INFO(true, false, false)); - return; + goto exit; } kbase_debug_csf_fault_wait_completion(kbdev); mutex_lock(&scheduler->lock); @@ -5014,7 +4978,7 @@ static void gpu_idle_worker(struct work_struct *work) if (unlikely(scheduler->state == SCHED_BUSY)) { mutex_unlock(&scheduler->lock); kbase_reset_gpu_allow(kbdev); - return; + goto exit; } #endif @@ -5039,6 +5003,9 @@ static void gpu_idle_worker(struct work_struct *work) __ENCODE_KTRACE_INFO(false, scheduler_is_idle_suspendable, all_groups_suspended)); #undef __ENCODE_KTRACE_INFO + +exit: + atomic_dec(&scheduler->pending_gpu_idle_work); } static int scheduler_prepare(struct kbase_device *kbdev) @@ -5376,6 +5343,20 @@ static void evict_lru_or_blocked_csg(struct kbase_device *kbdev) } } +static void scheduler_enable_gpu_idle_timer(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + unsigned long flags; + + lockdep_assert_held(&scheduler->lock); + + if (!kbdev->csf.gpu_idle_timer_enabled) { + spin_lock_irqsave(&scheduler->interrupt_lock, flags); + kbase_csf_firmware_enable_gpu_idle_timer(kbdev); + spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); + } +} + static void schedule_actions(struct kbase_device *kbdev, bool is_tick) { struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; @@ -5417,8 +5398,10 @@ static void schedule_actions(struct kbase_device *kbdev, bool is_tick) * steps and thus extending the previous tick's arrangement, * in particular, no alterations to on-slot CSGs. */ - if (keep_lru_on_slots(kbdev)) + if (keep_lru_on_slots(kbdev)) { + scheduler_enable_gpu_idle_timer(kbdev); return; + } } if (is_tick) @@ -5489,6 +5472,7 @@ redo_local_tock: wait_csg_slots_start(kbdev); wait_csg_slots_finish_prio_update(kbdev); + scheduler_enable_gpu_idle_timer(kbdev); if (new_protm_top_grp) { scheduler_group_check_protm_enter(kbdev, scheduler->top_grp); @@ -5544,9 +5528,10 @@ static bool can_skip_scheduling(struct kbase_device *kbdev) spin_lock_irqsave(&kbdev->hwaccess_lock, flags); if (kbdev->pm.backend.exit_gpu_sleep_mode) { - int ret = scheduler_pm_active_after_sleep(kbdev, &flags); + int ret; spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + ret = scheduler_pm_active_after_sleep(kbdev); if (!ret) { scheduler->state = SCHED_INACTIVE; KBASE_KTRACE_ADD(kbdev, SCHED_INACTIVE, NULL, scheduler->state); @@ -5662,7 +5647,9 @@ exit_no_schedule_unlock: kbase_reset_gpu_allow(kbdev); } -static int suspend_active_queue_groups(struct kbase_device *kbdev, unsigned long *slot_mask) + +static int suspend_active_queue_groups(struct kbase_device *kbdev, unsigned long *slot_mask, + bool reset) { struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; u32 num_groups = kbdev->csf.global_iface.group_num; @@ -5675,12 +5662,12 @@ static int suspend_active_queue_groups(struct kbase_device *kbdev, unsigned long struct kbase_queue_group *group = scheduler->csg_slots[slot_num].resident_group; if (group) { - suspend_queue_group(group); + suspend_queue_group(group); set_bit(slot_num, slot_mask); } } - ret = wait_csg_slots_suspend(kbdev, slot_mask); + ret = wait_csg_slots_suspend(kbdev, slot_mask); return ret; } @@ -5693,7 +5680,7 @@ static int suspend_active_queue_groups_on_reset(struct kbase_device *kbdev) mutex_lock(&scheduler->lock); - ret = suspend_active_queue_groups(kbdev, slot_mask); + ret = suspend_active_queue_groups(kbdev, slot_mask, true); if (ret) { dev_warn( @@ -5830,9 +5817,9 @@ static void scheduler_inner_reset(struct kbase_device *kbdev) WARN_ON(kbase_csf_scheduler_get_nr_active_csgs(kbdev)); /* Cancel any potential queued delayed work(s) */ - cancel_work_sync(&kbdev->csf.scheduler.gpu_idle_work); cancel_tick_work(scheduler); cancel_tock_work(scheduler); + cancel_gpu_idle_work(scheduler); cancel_delayed_work_sync(&scheduler->ping_work); mutex_lock(&scheduler->lock); @@ -5860,12 +5847,13 @@ static void scheduler_inner_reset(struct kbase_device *kbdev) void kbase_csf_scheduler_reset(struct kbase_device *kbdev) { struct kbase_context *kctx; - WARN_ON(!kbase_reset_gpu_is_active(kbdev)); KBASE_KTRACE_ADD(kbdev, SCHEDULER_RESET_START, NULL, 0u); - kbase_debug_csf_fault_wait_completion(kbdev); + if (kbase_reset_gpu_is_active(kbdev)) + kbase_debug_csf_fault_wait_completion(kbdev); + if (scheduler_handle_reset_in_protected_mode(kbdev) && !suspend_active_queue_groups_on_reset(kbdev)) { @@ -6453,8 +6441,8 @@ static void check_sync_update_in_sleep_mode(struct kbase_device *kbdev) * check_group_sync_update_worker() - Check the sync wait condition for all the * blocked queue groups * - * @work: Pointer to the context-specific work item for evaluating the wait - * condition for all the queue groups in idle_wait_groups list. + * @kctx: The context to evaluate the wait condition for all the queue groups + * in idle_wait_groups list. * * This function checks the gpu queues of all the groups present in both * idle_wait_groups list of a context and all on slot idle groups (if GPU @@ -6464,27 +6452,14 @@ static void check_sync_update_in_sleep_mode(struct kbase_device *kbdev) * runnable groups so that Scheduler can consider scheduling the group * in next tick or exit protected mode. */ -static void check_group_sync_update_worker(struct work_struct *work) +static void check_group_sync_update_worker(struct kbase_context *kctx) { - struct kbase_context *const kctx = - container_of(work, struct kbase_context, csf.sched.sync_update_work); struct kbase_device *const kbdev = kctx->kbdev; struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; bool sync_updated = false; mutex_lock(&scheduler->lock); -#if IS_ENABLED(CONFIG_DEBUG_FS) - if (unlikely(scheduler->state == SCHED_BUSY)) { - queue_work(atomic_read(&kctx->prioritized) ? - kctx->csf.sched.sync_update_wq_high_prio : - kctx->csf.sched.sync_update_wq_normal_prio, - &kctx->csf.sched.sync_update_work); - mutex_unlock(&scheduler->lock); - return; - } -#endif - KBASE_KTRACE_ADD(kbdev, SCHEDULER_GROUP_SYNC_UPDATE_WORKER_START, kctx, 0u); if (kctx->csf.sched.num_idle_wait_grps != 0) { struct kbase_queue_group *group, *temp; @@ -6522,13 +6497,10 @@ static void check_group_sync_update_worker(struct work_struct *work) static enum kbase_csf_event_callback_action check_group_sync_update_cb(void *param) { struct kbase_context *const kctx = param; - struct workqueue_struct *wq = atomic_read(&kctx->prioritized) ? - kctx->csf.sched.sync_update_wq_high_prio : - kctx->csf.sched.sync_update_wq_normal_prio; KBASE_KTRACE_ADD(kctx->kbdev, SCHEDULER_GROUP_SYNC_UPDATE_EVENT, kctx, 0u); - queue_work(wq, &kctx->csf.sched.sync_update_work); + kbase_csf_scheduler_enqueue_sync_update_work(kctx); return KBASE_CSF_EVENT_CALLBACK_KEEP; } @@ -6539,6 +6511,8 @@ int kbase_csf_scheduler_context_init(struct kbase_context *kctx) int err; struct kbase_device *kbdev = kctx->kbdev; + WARN_ON_ONCE(!kbdev->csf.scheduler.kthread_running); + #if IS_ENABLED(CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD) err = gpu_metrics_ctx_init(kctx); if (err) @@ -6551,25 +6525,7 @@ int kbase_csf_scheduler_context_init(struct kbase_context *kctx) INIT_LIST_HEAD(&kctx->csf.sched.idle_wait_groups); - kctx->csf.sched.sync_update_wq_high_prio = alloc_ordered_workqueue( - "mali_sync_wq_%i_high_prio", WQ_UNBOUND | WQ_HIGHPRI, kctx->tgid); - if (kctx->csf.sched.sync_update_wq_high_prio == NULL) { - dev_err(kbdev->dev, - "Failed to initialize scheduler context high-priority workqueue"); - err = -ENOMEM; - goto alloc_high_prio_wq_failed; - } - - kctx->csf.sched.sync_update_wq_normal_prio = - alloc_ordered_workqueue("mali_sync_wq_%i_normal_prio", 0, kctx->tgid); - if (kctx->csf.sched.sync_update_wq_normal_prio == NULL) { - dev_err(kbdev->dev, - "Failed to initialize scheduler context normal-priority workqueue"); - err = -ENOMEM; - goto alloc_normal_prio_wq_failed; - } - - INIT_WORK(&kctx->csf.sched.sync_update_work, check_group_sync_update_worker); + INIT_LIST_HEAD(&kctx->csf.sched.sync_update_work); kbase_csf_tiler_heap_reclaim_ctx_init(kctx); @@ -6583,10 +6539,6 @@ int kbase_csf_scheduler_context_init(struct kbase_context *kctx) return err; event_wait_add_failed: - destroy_workqueue(kctx->csf.sched.sync_update_wq_normal_prio); -alloc_normal_prio_wq_failed: - destroy_workqueue(kctx->csf.sched.sync_update_wq_high_prio); -alloc_high_prio_wq_failed: kbase_ctx_sched_remove_ctx(kctx); #if IS_ENABLED(CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD) gpu_metrics_ctx_term(kctx); @@ -6597,9 +6549,10 @@ alloc_high_prio_wq_failed: void kbase_csf_scheduler_context_term(struct kbase_context *kctx) { kbase_csf_event_wait_remove(kctx, check_group_sync_update_cb, kctx); - cancel_work_sync(&kctx->csf.sched.sync_update_work); - destroy_workqueue(kctx->csf.sched.sync_update_wq_normal_prio); - destroy_workqueue(kctx->csf.sched.sync_update_wq_high_prio); + + /* Drain a pending SYNC_UPDATE work if any */ + kbase_csf_scheduler_wait_for_kthread_pending_work(kctx->kbdev, + &kctx->csf.pending_sync_update); kbase_ctx_sched_remove_ctx(kctx); #if IS_ENABLED(CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD) @@ -6607,53 +6560,157 @@ void kbase_csf_scheduler_context_term(struct kbase_context *kctx) #endif /* CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD */ } +static void handle_pending_sync_update_works(struct kbase_csf_scheduler *scheduler) +{ + struct kbase_context *sync_update_ctx; + + if (atomic_cmpxchg(&scheduler->pending_sync_update_works, true, false) == false) + return; + + do { + unsigned long flags; + + spin_lock_irqsave(&scheduler->sync_update_work_ctxs_lock, flags); + sync_update_ctx = NULL; + if (!list_empty(&scheduler->sync_update_work_ctxs)) { + sync_update_ctx = list_first_entry(&scheduler->sync_update_work_ctxs, + struct kbase_context, + csf.sched.sync_update_work); + list_del_init(&sync_update_ctx->csf.sched.sync_update_work); + } + spin_unlock_irqrestore(&scheduler->sync_update_work_ctxs_lock, flags); + + if (sync_update_ctx != NULL) { + WARN_ON_ONCE(atomic_read(&sync_update_ctx->csf.pending_sync_update) == 0); + check_group_sync_update_worker(sync_update_ctx); + atomic_dec(&sync_update_ctx->csf.pending_sync_update); + } + } while (sync_update_ctx != NULL); +} + +static void handle_pending_protm_requests(struct kbase_csf_scheduler *scheduler) +{ + struct kbase_queue_group *protm_grp; + + if (atomic_cmpxchg(&scheduler->pending_protm_event_works, true, false) == false) + return; + + do { + unsigned long flags; + + spin_lock_irqsave(&scheduler->protm_event_work_grps_lock, flags); + protm_grp = NULL; + if (!list_empty(&scheduler->protm_event_work_grps)) { + protm_grp = list_first_entry(&scheduler->protm_event_work_grps, + struct kbase_queue_group, protm_event_work); + list_del_init(&protm_grp->protm_event_work); + } + spin_unlock_irqrestore(&scheduler->protm_event_work_grps_lock, flags); + + if (protm_grp != NULL) { + WARN_ON_ONCE(atomic_read(&protm_grp->pending_protm_event_work) == 0); + kbase_csf_process_protm_event_request(protm_grp); + atomic_dec(&protm_grp->pending_protm_event_work); + } + } while (protm_grp != NULL); +} + +static void handle_pending_kcpuq_commands(struct kbase_csf_scheduler *scheduler) +{ + struct kbase_kcpu_command_queue *kcpuq; + + if (atomic_cmpxchg(&scheduler->pending_kcpuq_works, true, false) == false) + return; + + do { + unsigned long flags; + + spin_lock_irqsave(&scheduler->kcpuq_work_queues_lock, flags); + kcpuq = NULL; + if (!list_empty(&scheduler->kcpuq_work_queues)) { + kcpuq = list_first_entry(&scheduler->kcpuq_work_queues, + struct kbase_kcpu_command_queue, high_prio_work); + list_del_init(&kcpuq->high_prio_work); + } + spin_unlock_irqrestore(&scheduler->kcpuq_work_queues_lock, flags); + + if (kcpuq != NULL) { + WARN_ON_ONCE(atomic_read(&kcpuq->pending_kick) == 0); + + mutex_lock(&kcpuq->lock); + kbase_csf_kcpu_queue_process(kcpuq, false); + mutex_unlock(&kcpuq->lock); + + atomic_dec(&kcpuq->pending_kick); + } + } while (kcpuq != NULL); +} + +static void handle_pending_queue_kicks(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; + struct kbase_queue *queue; + + if (atomic_cmpxchg(&kbdev->csf.pending_gpuq_kicks, true, false) == false) + return; + + do { + u8 prio; + + spin_lock(&kbdev->csf.pending_gpuq_kick_queues_lock); + queue = NULL; + for (prio = 0; prio != KBASE_QUEUE_GROUP_PRIORITY_COUNT; ++prio) { + if (!list_empty(&kbdev->csf.pending_gpuq_kick_queues[prio])) { + queue = list_first_entry(&kbdev->csf.pending_gpuq_kick_queues[prio], + struct kbase_queue, pending_kick_link); + list_del_init(&queue->pending_kick_link); + break; + } + } + spin_unlock(&kbdev->csf.pending_gpuq_kick_queues_lock); + + if (queue != NULL) { + WARN_ONCE( + prio != queue->group_priority, + "Queue %pK has priority %u but instead its kick was handled at priority %u", + (void *)queue, queue->group_priority, prio); + WARN_ON_ONCE(atomic_read(&queue->pending_kick) == 0); + + kbase_csf_process_queue_kick(queue); + + /* Perform a scheduling tock for high-priority queue groups if + * required. + */ + BUILD_BUG_ON(KBASE_QUEUE_GROUP_PRIORITY_REALTIME != 0); + BUILD_BUG_ON(KBASE_QUEUE_GROUP_PRIORITY_HIGH != 1); + if ((prio <= KBASE_QUEUE_GROUP_PRIORITY_HIGH) && + atomic_read(&scheduler->pending_tock_work)) + schedule_on_tock(kbdev); + } + } while (queue != NULL); +} + static int kbase_csf_scheduler_kthread(void *data) { struct kbase_device *const kbdev = data; struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; while (scheduler->kthread_running) { - struct kbase_queue *queue; - if (wait_for_completion_interruptible(&scheduler->kthread_signal) != 0) continue; reinit_completion(&scheduler->kthread_signal); - /* Iterate through queues with pending kicks */ - do { - u8 prio; + /* + * The order in which these requests are handled is based on + * how they would influence each other's decisions. As a + * result, the tick & tock requests must be handled after all + * other requests, but before the GPU IDLE work. + */ - spin_lock(&kbdev->csf.pending_gpuq_kicks_lock); - queue = NULL; - for (prio = 0; prio != KBASE_QUEUE_GROUP_PRIORITY_COUNT; ++prio) { - if (!list_empty(&kbdev->csf.pending_gpuq_kicks[prio])) { - queue = list_first_entry( - &kbdev->csf.pending_gpuq_kicks[prio], - struct kbase_queue, pending_kick_link); - list_del_init(&queue->pending_kick_link); - break; - } - } - spin_unlock(&kbdev->csf.pending_gpuq_kicks_lock); - - if (queue != NULL) { - WARN_ONCE( - prio != queue->group_priority, - "Queue %pK has priority %hhu but instead its kick was handled at priority %hhu", - (void *)queue, queue->group_priority, prio); - - kbase_csf_process_queue_kick(queue); - - /* Perform a scheduling tock for high-priority queue groups if - * required. - */ - BUILD_BUG_ON(KBASE_QUEUE_GROUP_PRIORITY_REALTIME != 0); - BUILD_BUG_ON(KBASE_QUEUE_GROUP_PRIORITY_HIGH != 1); - if ((prio <= KBASE_QUEUE_GROUP_PRIORITY_HIGH) && - atomic_read(&scheduler->pending_tock_work)) - schedule_on_tock(kbdev); - } - } while (queue != NULL); + handle_pending_sync_update_works(scheduler); + handle_pending_protm_requests(scheduler); + handle_pending_kcpuq_commands(scheduler); + handle_pending_queue_kicks(kbdev); /* Check if we need to perform a scheduling tick/tock. A tick * event shall override a tock event but not vice-versa. @@ -6665,6 +6722,10 @@ static int kbase_csf_scheduler_kthread(void *data) schedule_on_tock(kbdev); } + /* Drain pending GPU idle works */ + while (atomic_read(&scheduler->pending_gpu_idle_work) > 0) + gpu_idle_worker(kbdev); + dev_dbg(kbdev->dev, "Waking up for event after a scheduling iteration."); wake_up_all(&kbdev->csf.event_wait); } @@ -6694,7 +6755,7 @@ int kbase_csf_scheduler_init(struct kbase_device *kbdev) scheduler->kthread_running = true; scheduler->gpuq_kthread = kthread_run(&kbase_csf_scheduler_kthread, kbdev, "mali-gpuq-kthread"); - if (!scheduler->gpuq_kthread) { + if (IS_ERR_OR_NULL(scheduler->gpuq_kthread)) { kfree(scheduler->csg_slots); scheduler->csg_slots = NULL; @@ -6734,12 +6795,6 @@ int kbase_csf_scheduler_early_init(struct kbase_device *kbdev) atomic_set(&scheduler->timer_enabled, true); - scheduler->idle_wq = alloc_ordered_workqueue("csf_scheduler_gpu_idle_wq", WQ_HIGHPRI); - if (!scheduler->idle_wq) { - dev_err(kbdev->dev, "Failed to allocate GPU idle scheduler workqueue\n"); - return -ENOMEM; - } - INIT_DEFERRABLE_WORK(&scheduler->ping_work, firmware_aliveness_monitor); mutex_init(&scheduler->lock); @@ -6757,20 +6812,30 @@ int kbase_csf_scheduler_early_init(struct kbase_device *kbdev) KBASE_KTRACE_ADD(kbdev, SCHED_SUSPENDED, NULL, scheduler->state); scheduler->csg_scheduling_period_ms = CSF_SCHEDULER_TIME_TICK_MS; scheduler_doorbell_init(kbdev); - INIT_WORK(&scheduler->gpu_idle_work, gpu_idle_worker); hrtimer_init(&scheduler->tick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); scheduler->tick_timer.function = tick_timer_callback; - kbase_csf_tiler_heap_reclaim_mgr_init(kbdev); + atomic_set(&scheduler->pending_sync_update_works, false); + spin_lock_init(&scheduler->sync_update_work_ctxs_lock); + INIT_LIST_HEAD(&scheduler->sync_update_work_ctxs); + atomic_set(&scheduler->pending_protm_event_works, false); + spin_lock_init(&scheduler->protm_event_work_grps_lock); + INIT_LIST_HEAD(&scheduler->protm_event_work_grps); + atomic_set(&scheduler->pending_kcpuq_works, false); + spin_lock_init(&scheduler->kcpuq_work_queues_lock); + INIT_LIST_HEAD(&scheduler->kcpuq_work_queues); + atomic_set(&scheduler->pending_tick_work, false); + atomic_set(&scheduler->pending_tock_work, false); + atomic_set(&scheduler->pending_gpu_idle_work, 0); - return 0; + return kbase_csf_tiler_heap_reclaim_mgr_init(kbdev); } void kbase_csf_scheduler_term(struct kbase_device *kbdev) { struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; - if (scheduler->gpuq_kthread) { + if (!IS_ERR_OR_NULL(scheduler->gpuq_kthread)) { scheduler->kthread_running = false; complete(&scheduler->kthread_signal); kthread_stop(scheduler->gpuq_kthread); @@ -6784,7 +6849,6 @@ void kbase_csf_scheduler_term(struct kbase_device *kbdev) * to be active at the time of Driver unload. */ WARN_ON(kbase_csf_scheduler_get_nr_active_csgs(kbdev)); - flush_work(&kbdev->csf.scheduler.gpu_idle_work); mutex_lock(&kbdev->csf.scheduler.lock); if (kbdev->csf.scheduler.state != SCHED_SUSPENDED) { @@ -6811,9 +6875,6 @@ void kbase_csf_scheduler_term(struct kbase_device *kbdev) void kbase_csf_scheduler_early_term(struct kbase_device *kbdev) { - if (kbdev->csf.scheduler.idle_wq) - destroy_workqueue(kbdev->csf.scheduler.idle_wq); - kbase_csf_tiler_heap_reclaim_mgr_term(kbdev); mutex_destroy(&kbdev->csf.scheduler.lock); } @@ -6926,7 +6987,7 @@ int kbase_csf_scheduler_pm_suspend_no_lock(struct kbase_device *kbdev) dev_warn(kbdev->dev, "failed to suspend active groups"); goto exit; } else { - dev_info(kbdev->dev, "Scheduler PM suspend"); + dev_dbg(kbdev->dev, "Scheduler PM suspend"); scheduler_suspend(kbdev); cancel_tick_work(scheduler); } @@ -6968,7 +7029,7 @@ void kbase_csf_scheduler_pm_resume_no_lock(struct kbase_device *kbdev) lockdep_assert_held(&scheduler->lock); if ((scheduler->total_runnable_grps > 0) && (scheduler->state == SCHED_SUSPENDED)) { - dev_info(kbdev->dev, "Scheduler PM resume"); + dev_dbg(kbdev->dev, "Scheduler PM resume"); scheduler_wakeup(kbdev, true); } } @@ -6989,7 +7050,7 @@ void kbase_csf_scheduler_pm_active(struct kbase_device *kbdev) * the CSGs before powering down the GPU. */ mutex_lock(&kbdev->csf.scheduler.lock); - scheduler_pm_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE); + scheduler_pm_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE, false); mutex_unlock(&kbdev->csf.scheduler.lock); } KBASE_EXPORT_TEST_API(kbase_csf_scheduler_pm_active); @@ -7013,9 +7074,7 @@ static int scheduler_wait_mcu_active(struct kbase_device *kbdev, bool killable_w kbase_pm_lock(kbdev); WARN_ON(!kbdev->pm.active_count); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); WARN_ON(!scheduler->pm_active_count); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); kbase_pm_unlock(kbdev); if (killable_wait) @@ -7096,6 +7155,65 @@ int kbase_csf_scheduler_handle_runtime_suspend(struct kbase_device *kbdev) return 0; } +void kbase_csf_scheduler_enqueue_sync_update_work(struct kbase_context *kctx) +{ + struct kbase_csf_scheduler *const scheduler = &kctx->kbdev->csf.scheduler; + unsigned long flags; + + spin_lock_irqsave(&scheduler->sync_update_work_ctxs_lock, flags); + if (list_empty(&kctx->csf.sched.sync_update_work)) { + list_add_tail(&kctx->csf.sched.sync_update_work, &scheduler->sync_update_work_ctxs); + atomic_inc(&kctx->csf.pending_sync_update); + if (atomic_cmpxchg(&scheduler->pending_sync_update_works, false, true) == false) + complete(&scheduler->kthread_signal); + } + spin_unlock_irqrestore(&scheduler->sync_update_work_ctxs_lock, flags); +} + +void kbase_csf_scheduler_enqueue_protm_event_work(struct kbase_queue_group *group) +{ + struct kbase_context *const kctx = group->kctx; + struct kbase_csf_scheduler *const scheduler = &kctx->kbdev->csf.scheduler; + unsigned long flags; + + spin_lock_irqsave(&scheduler->protm_event_work_grps_lock, flags); + if (list_empty(&group->protm_event_work)) { + list_add_tail(&group->protm_event_work, &scheduler->protm_event_work_grps); + atomic_inc(&group->pending_protm_event_work); + if (atomic_cmpxchg(&scheduler->pending_protm_event_works, false, true) == false) + complete(&scheduler->kthread_signal); + } + spin_unlock_irqrestore(&scheduler->protm_event_work_grps_lock, flags); +} + +void kbase_csf_scheduler_enqueue_kcpuq_work(struct kbase_kcpu_command_queue *queue) +{ + struct kbase_csf_scheduler *const scheduler = &queue->kctx->kbdev->csf.scheduler; + unsigned long flags; + + spin_lock_irqsave(&scheduler->kcpuq_work_queues_lock, flags); + if (list_empty(&queue->high_prio_work)) { + list_add_tail(&queue->high_prio_work, &scheduler->kcpuq_work_queues); + atomic_inc(&queue->pending_kick); + if (atomic_cmpxchg(&scheduler->pending_kcpuq_works, false, true) == false) + complete(&scheduler->kthread_signal); + } + spin_unlock_irqrestore(&scheduler->kcpuq_work_queues_lock, flags); +} + +void kbase_csf_scheduler_wait_for_kthread_pending_work(struct kbase_device *kbdev, + atomic_t *pending) +{ + /* + * Signal kbase_csf_scheduler_kthread() to allow for the + * eventual completion of the current iteration. Once the work is + * done, the event_wait wait queue shall be signalled. + */ + + complete(&kbdev->csf.scheduler.kthread_signal); + wait_event(kbdev->csf.event_wait, atomic_read(pending) == 0); +} + void kbase_csf_scheduler_reval_idleness_post_sleep(struct kbase_device *kbdev) { u32 csg_nr; diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.h index 5047092d6650..e84994600809 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.h +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -235,7 +235,8 @@ void kbase_csf_scheduler_early_term(struct kbase_device *kbdev); * No explicit re-initialization is done for CSG & CS interface I/O pages; * instead, that happens implicitly on firmware reload. * - * Should be called only after initiating the GPU reset. + * Should be called either after initiating the GPU reset or when MCU reset is + * expected to follow such as GPU_LOST case. */ void kbase_csf_scheduler_reset(struct kbase_device *kbdev); @@ -487,6 +488,48 @@ static inline bool kbase_csf_scheduler_all_csgs_idle(struct kbase_device *kbdev) kbdev->csf.global_iface.group_num); } +/** + * kbase_csf_scheduler_enqueue_sync_update_work() - Add a context to the list + * of contexts to handle + * SYNC_UPDATE events. + * + * @kctx: The context to handle SYNC_UPDATE event + * + * This function wakes up kbase_csf_scheduler_kthread() to handle pending + * SYNC_UPDATE events for all contexts. + */ +void kbase_csf_scheduler_enqueue_sync_update_work(struct kbase_context *kctx); + +/** + * kbase_csf_scheduler_enqueue_protm_event_work() - Add a group to the list + * of groups to handle + * PROTM requests. + * + * @group: The group to handle protected mode request + * + * This function wakes up kbase_csf_scheduler_kthread() to handle pending + * protected mode requests for all groups. + */ +void kbase_csf_scheduler_enqueue_protm_event_work(struct kbase_queue_group *group); + +/** + * kbase_csf_scheduler_enqueue_kcpuq_work() - Wake up kbase_csf_scheduler_kthread() to process + * pending commands for a KCPU queue. + * + * @queue: The queue to process pending commands for + */ +void kbase_csf_scheduler_enqueue_kcpuq_work(struct kbase_kcpu_command_queue *queue); + +/** + * kbase_csf_scheduler_wait_for_kthread_pending_work - Wait until a pending work has completed in + * kbase_csf_scheduler_kthread(). + * + * @kbdev: Instance of a GPU platform device that implements a CSF interface + * @pending: The work to wait for + */ +void kbase_csf_scheduler_wait_for_kthread_pending_work(struct kbase_device *kbdev, + atomic_t *pending); + /** * kbase_csf_scheduler_invoke_tick() - Invoke the scheduling tick * @@ -591,11 +634,8 @@ int kbase_csf_scheduler_handle_runtime_suspend(struct kbase_device *kbdev); * @kbdev: Pointer to the device * * This function is called when a GPU idle IRQ has been raised. - * - * Return: true if the PM state machine needs to be invoked after the processing - * of GPU idle irq, otherwise false. */ -bool kbase_csf_scheduler_process_gpu_idle_event(struct kbase_device *kbdev); +void kbase_csf_scheduler_process_gpu_idle_event(struct kbase_device *kbdev); /** * kbase_csf_scheduler_get_nr_active_csgs() - Get the number of active CSGs diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_sync.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_sync.c index aa88b5f59d3b..27b792500bdf 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_sync.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_sync.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -117,13 +117,13 @@ static void kbasep_csf_sync_print_kcpu_fence_wait_or_signal(char *buffer, int *l timeline_name = fence->ops->get_timeline_name(fence); is_signaled = info.status > 0; - *length += snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, - "cmd:%s obj:0x%pK live_value:0x%.8x | ", cmd_name, fence, is_signaled); + *length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, + "cmd:%s obj:0x%pK live_value:0x%.8x | ", cmd_name, fence, is_signaled); /* Note: fence->seqno was u32 until 5.1 kernel, then u64 */ - *length += snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, - "timeline_name:%s timeline_context:0x%.16llx fence_seqno:0x%.16llx", - timeline_name, fence->context, (u64)fence->seqno); + *length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, + "timeline_name:%s timeline_context:0x%.16llx fence_seqno:0x%.16llx", + timeline_name, fence->context, (u64)fence->seqno); kbase_fence_put(fence); } @@ -149,19 +149,19 @@ static void kbasep_csf_sync_print_kcpu_cqs_wait(struct kbase_context *kctx, char int ret = kbasep_csf_sync_get_cqs_live_u32(kctx, cqs_obj->addr, &live_val); bool live_val_valid = (ret >= 0); - *length += - snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, - "cmd:CQS_WAIT_OPERATION obj:0x%.16llx live_value:", cqs_obj->addr); + *length += scnprintf( + buffer + *length, CSF_SYNC_DUMP_SIZE - *length, + "cmd:CQS_WAIT_OPERATION obj:0x%.16llx live_value:", cqs_obj->addr); if (live_val_valid) - *length += snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, - "0x%.16llx", (u64)live_val); + *length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, + "0x%.16llx", (u64)live_val); else - *length += snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, - CQS_UNREADABLE_LIVE_VALUE); + *length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, + CQS_UNREADABLE_LIVE_VALUE); - *length += snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, - " | op:gt arg_value:0x%.8x", cqs_obj->val); + *length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, + " | op:gt arg_value:0x%.8x", cqs_obj->val); } } @@ -187,18 +187,18 @@ static void kbasep_csf_sync_print_kcpu_cqs_set(struct kbase_context *kctx, char bool live_val_valid = (ret >= 0); *length += - snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, - "cmd:CQS_SET_OPERATION obj:0x%.16llx live_value:", cqs_obj->addr); + scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, + "cmd:CQS_SET_OPERATION obj:0x%.16llx live_value:", cqs_obj->addr); if (live_val_valid) - *length += snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, - "0x%.16llx", (u64)live_val); + *length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, + "0x%.16llx", (u64)live_val); else - *length += snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, - CQS_UNREADABLE_LIVE_VALUE); + *length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, + CQS_UNREADABLE_LIVE_VALUE); - *length += snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, - " | op:add arg_value:0x%.8x", 1); + *length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, + " | op:add arg_value:0x%.8x", 1); } } @@ -277,19 +277,19 @@ static void kbasep_csf_sync_print_kcpu_cqs_wait_op(struct kbase_context *kctx, c bool live_val_valid = (ret >= 0); - *length += - snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, - "cmd:CQS_WAIT_OPERATION obj:0x%.16llx live_value:", wait_op->addr); + *length += scnprintf( + buffer + *length, CSF_SYNC_DUMP_SIZE - *length, + "cmd:CQS_WAIT_OPERATION obj:0x%.16llx live_value:", wait_op->addr); if (live_val_valid) - *length += snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, - "0x%.16llx", live_val); + *length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, + "0x%.16llx", live_val); else - *length += snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, - CQS_UNREADABLE_LIVE_VALUE); + *length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, + CQS_UNREADABLE_LIVE_VALUE); - *length += snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, - " | op:%s arg_value:0x%.16llx", op_name, wait_op->val); + *length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, + " | op:%s arg_value:0x%.16llx", op_name, wait_op->val); } } @@ -319,18 +319,18 @@ static void kbasep_csf_sync_print_kcpu_cqs_set_op(struct kbase_context *kctx, ch bool live_val_valid = (ret >= 0); *length += - snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, - "cmd:CQS_SET_OPERATION obj:0x%.16llx live_value:", set_op->addr); + scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, + "cmd:CQS_SET_OPERATION obj:0x%.16llx live_value:", set_op->addr); if (live_val_valid) - *length += snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, - "0x%.16llx", live_val); + *length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, + "0x%.16llx", live_val); else - *length += snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, - CQS_UNREADABLE_LIVE_VALUE); + *length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, + CQS_UNREADABLE_LIVE_VALUE); - *length += snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, - " | op:%s arg_value:0x%.16llx", op_name, set_op->val); + *length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length, + " | op:%s arg_value:0x%.16llx", op_name, set_op->val); } } @@ -360,8 +360,8 @@ static void kbasep_csf_sync_kcpu_print_queue(struct kbase_context *kctx, int length = 0; started_or_pending = ((i == 0) && queue->command_started) ? 'S' : 'P'; - length += snprintf(buffer, CSF_SYNC_DUMP_SIZE, "queue:KCPU-%d-%d exec:%c ", - kctx->id, queue->id, started_or_pending); + length += scnprintf(buffer, CSF_SYNC_DUMP_SIZE, "queue:KCPU-%d-%d exec:%c ", + kctx->id, queue->id, started_or_pending); cmd = &queue->commands[(u8)(queue->start_offset + i)]; switch (cmd->type) { @@ -388,12 +388,12 @@ static void kbasep_csf_sync_kcpu_print_queue(struct kbase_context *kctx, kbasep_csf_sync_print_kcpu_cqs_set_op(kctx, buffer, &length, cmd); break; default: - length += snprintf(buffer + length, CSF_SYNC_DUMP_SIZE - length, - ", U, Unknown blocking command"); + length += scnprintf(buffer + length, CSF_SYNC_DUMP_SIZE - length, + ", U, Unknown blocking command"); break; } - length += snprintf(buffer + length, CSF_SYNC_DUMP_SIZE - length, "\n"); + length += scnprintf(buffer + length, CSF_SYNC_DUMP_SIZE - length, "\n"); kbasep_print(kbpr, buffer); } diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.c index 2d148eea025e..51d665f23970 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.c @@ -218,7 +218,7 @@ static void remove_unlinked_chunk(struct kbase_context *kctx, if (WARN_ON(!list_empty(&chunk->link))) return; - kbase_gpu_vm_lock(kctx); + kbase_gpu_vm_lock_with_pmode_sync(kctx); kbase_vunmap(kctx, &chunk->map); /* KBASE_REG_DONT_NEED regions will be confused with ephemeral regions (inc freed JIT * regions), and so we must clear that flag too before freeing. @@ -231,7 +231,7 @@ static void remove_unlinked_chunk(struct kbase_context *kctx, chunk->region->flags &= ~KBASE_REG_DONT_NEED; #endif kbase_mem_free_region(kctx, chunk->region); - kbase_gpu_vm_unlock(kctx); + kbase_gpu_vm_unlock_with_pmode_sync(kctx); kfree(chunk); } @@ -1058,6 +1058,7 @@ static bool delete_chunk_physical_pages(struct kbase_csf_tiler_heap *heap, u64 c struct kbase_csf_tiler_heap_chunk *chunk = NULL; lockdep_assert_held(&heap->kctx->csf.tiler_heaps.lock); + lockdep_assert_held(&kctx->kbdev->csf.scheduler.lock); chunk = find_chunk(heap, chunk_gpu_va); if (unlikely(!chunk)) { diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_reclaim.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_reclaim.c index a2bb49422e98..df4feb77f0cd 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_reclaim.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_reclaim.c @@ -331,8 +331,8 @@ static unsigned long kbase_csf_tiler_heap_reclaim_scan_free_pages(struct kbase_d static unsigned long kbase_csf_tiler_heap_reclaim_count_objects(struct shrinker *s, struct shrink_control *sc) { - struct kbase_device *kbdev = - container_of(s, struct kbase_device, csf.scheduler.reclaim_mgr.heap_reclaim); + struct kbase_device *kbdev = KBASE_GET_KBASE_DATA_FROM_SHRINKER( + s, struct kbase_device, csf.scheduler.reclaim_mgr.heap_reclaim); return kbase_csf_tiler_heap_reclaim_count_free_pages(kbdev, sc); } @@ -340,8 +340,8 @@ static unsigned long kbase_csf_tiler_heap_reclaim_count_objects(struct shrinker static unsigned long kbase_csf_tiler_heap_reclaim_scan_objects(struct shrinker *s, struct shrink_control *sc) { - struct kbase_device *kbdev = - container_of(s, struct kbase_device, csf.scheduler.reclaim_mgr.heap_reclaim); + struct kbase_device *kbdev = KBASE_GET_KBASE_DATA_FROM_SHRINKER( + s, struct kbase_device, csf.scheduler.reclaim_mgr.heap_reclaim); return kbase_csf_tiler_heap_reclaim_scan_free_pages(kbdev, sc); } @@ -352,11 +352,17 @@ void kbase_csf_tiler_heap_reclaim_ctx_init(struct kbase_context *kctx) INIT_LIST_HEAD(&kctx->csf.sched.heap_info.mgr_link); } -void kbase_csf_tiler_heap_reclaim_mgr_init(struct kbase_device *kbdev) +int kbase_csf_tiler_heap_reclaim_mgr_init(struct kbase_device *kbdev) { struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; - struct shrinker *reclaim = &scheduler->reclaim_mgr.heap_reclaim; u8 prio; + struct shrinker *reclaim; + + reclaim = + KBASE_INIT_RECLAIM(&(scheduler->reclaim_mgr), heap_reclaim, "mali-csf-tiler-heap"); + if (!reclaim) + return -ENOMEM; + KBASE_SET_RECLAIM(&(scheduler->reclaim_mgr), heap_reclaim, reclaim); for (prio = KBASE_QUEUE_GROUP_PRIORITY_REALTIME; prio < KBASE_QUEUE_GROUP_PRIORITY_COUNT; prio++) @@ -366,6 +372,11 @@ void kbase_csf_tiler_heap_reclaim_mgr_init(struct kbase_device *kbdev) reclaim->scan_objects = kbase_csf_tiler_heap_reclaim_scan_objects; reclaim->seeks = HEAP_SHRINKER_SEEKS; reclaim->batch = HEAP_SHRINKER_BATCH; + + if (!IS_ENABLED(CONFIG_MALI_VECTOR_DUMP)) + KBASE_REGISTER_SHRINKER(reclaim, "mali-csf-tiler-heap", kbdev); + + return 0; } void kbase_csf_tiler_heap_reclaim_mgr_term(struct kbase_device *kbdev) @@ -373,6 +384,9 @@ void kbase_csf_tiler_heap_reclaim_mgr_term(struct kbase_device *kbdev) struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; u8 prio; + if (!IS_ENABLED(CONFIG_MALI_VECTOR_DUMP)) + KBASE_UNREGISTER_SHRINKER(scheduler->reclaim_mgr.heap_reclaim); + for (prio = KBASE_QUEUE_GROUP_PRIORITY_REALTIME; prio < KBASE_QUEUE_GROUP_PRIORITY_COUNT; prio++) WARN_ON(!list_empty(&scheduler->reclaim_mgr.ctx_lists[prio])); diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_reclaim.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_reclaim.h index 7880de04c84f..d41b7baabd02 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_reclaim.h +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_reclaim.h @@ -66,8 +66,10 @@ void kbase_csf_tiler_heap_reclaim_ctx_init(struct kbase_context *kctx); * @kbdev: Pointer to the device. * * This function must be called only when a kbase device is initialized. + * + * Return: 0 if issuing reclaim_mgr init was successful, otherwise an error code. */ -void kbase_csf_tiler_heap_reclaim_mgr_init(struct kbase_device *kbdev); +int kbase_csf_tiler_heap_reclaim_mgr_init(struct kbase_device *kbdev); /** * kbase_csf_tiler_heap_reclaim_mgr_term - Termination call for the tiler heap reclaim manger. diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tl_reader.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tl_reader.c index 54054661f7a9..eb5c8a40b8c9 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tl_reader.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tl_reader.c @@ -151,13 +151,22 @@ static bool tl_reader_overflow_check(struct kbase_csf_tl_reader *self, u16 event * * Reset the reader to the default state, i.e. set all the * mutable fields to zero. + * + * NOTE: this function expects the irq spinlock to be held. */ static void tl_reader_reset(struct kbase_csf_tl_reader *self) { + lockdep_assert_held(&self->read_lock); + self->got_first_event = false; self->is_active = false; self->expected_event_id = 0; self->tl_header.btc = 0; + + /* There might be data left in the trace buffer from the previous + * tracing session. We don't want it to leak into this session. + */ + kbase_csf_firmware_trace_buffer_discard_all(self->trace_buffer); } int kbase_csf_tl_reader_flush_buffer(struct kbase_csf_tl_reader *self) @@ -324,21 +333,16 @@ static int tl_reader_update_enable_bit(struct kbase_csf_tl_reader *self, bool va void kbase_csf_tl_reader_init(struct kbase_csf_tl_reader *self, struct kbase_tlstream *stream) { - self->timer_interval = KBASE_CSF_TL_READ_INTERVAL_DEFAULT; + *self = (struct kbase_csf_tl_reader){ + .timer_interval = KBASE_CSF_TL_READ_INTERVAL_DEFAULT, + .stream = stream, + .kbdev = NULL, /* This will be initialized by tl_reader_init_late() */ + .is_active = false, + }; kbase_timer_setup(&self->read_timer, kbasep_csf_tl_reader_read_callback); - self->stream = stream; - - /* This will be initialized by tl_reader_init_late() */ - self->kbdev = NULL; - self->trace_buffer = NULL; - self->tl_header.data = NULL; - self->tl_header.size = 0; - spin_lock_init(&self->read_lock); - - tl_reader_reset(self); } void kbase_csf_tl_reader_term(struct kbase_csf_tl_reader *self) @@ -348,13 +352,19 @@ void kbase_csf_tl_reader_term(struct kbase_csf_tl_reader *self) int kbase_csf_tl_reader_start(struct kbase_csf_tl_reader *self, struct kbase_device *kbdev) { + unsigned long flags; int rcode; + spin_lock_irqsave(&self->read_lock, flags); + /* If already running, early exit. */ - if (self->is_active) + if (self->is_active) { + spin_unlock_irqrestore(&self->read_lock, flags); return 0; + } if (tl_reader_init_late(self, kbdev)) { + spin_unlock_irqrestore(&self->read_lock, flags); #if IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) dev_warn(kbdev->dev, "CSFFW timeline is not available for MALI_BIFROST_NO_MALI builds!"); return 0; @@ -366,6 +376,9 @@ int kbase_csf_tl_reader_start(struct kbase_csf_tl_reader *self, struct kbase_dev tl_reader_reset(self); self->is_active = true; + + spin_unlock_irqrestore(&self->read_lock, flags); + /* Set bytes to copy to the header size. This is to trigger copying * of the header to the user space. */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.c index 8ed7c91553a6..cdab5a17f70c 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.c @@ -519,6 +519,14 @@ void kbase_csf_firmware_trace_buffer_discard(struct firmware_trace_buffer *trace } EXPORT_SYMBOL(kbase_csf_firmware_trace_buffer_discard); +void kbase_csf_firmware_trace_buffer_discard_all(struct firmware_trace_buffer *trace_buffer) +{ + if (WARN_ON(!trace_buffer)) + return; + + *(trace_buffer->cpu_va.extract_cpu_va) = *(trace_buffer->cpu_va.insert_cpu_va); +} + static void update_trace_buffer_active_mask64(struct firmware_trace_buffer *tb, u64 mask) { unsigned int i; diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.h index 90dfcb2699bc..35988eaf8f5a 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.h +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.h @@ -179,6 +179,15 @@ unsigned int kbase_csf_firmware_trace_buffer_read_data(struct firmware_trace_buf */ void kbase_csf_firmware_trace_buffer_discard(struct firmware_trace_buffer *trace_buffer); +/** + * kbase_csf_firmware_trace_buffer_discard_all - Discard all data from a trace buffer + * + * @trace_buffer: Trace buffer handle + * + * Discard all the data in the trace buffer to make it empty. + */ +void kbase_csf_firmware_trace_buffer_discard_all(struct firmware_trace_buffer *trace_buffer); + /** * kbase_csf_firmware_trace_buffer_get_active_mask64 - Get trace buffer active mask * diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_util.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_util.c index 7dc32a11bb29..5f13672e70b8 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_util.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_util.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -115,7 +115,7 @@ struct kbasep_printer *kbasep_printer_buffer_init(struct kbase_device *kbdev, if (kbpr) { if (kfifo_alloc(&kbpr->fifo, KBASEP_PRINTER_BUFFER_MAX_SIZE, GFP_KERNEL)) { - kfree(kbpr); + vfree(kbpr); return NULL; } kbpr->kbdev = kbdev; @@ -224,7 +224,7 @@ __attribute__((format(__printf__, 2, 3))) void kbasep_print(struct kbasep_printe va_list arglist; va_start(arglist, fmt); - len = vsnprintf(buffer, KBASEP_PRINT_FORMAT_BUFFER_MAX_SIZE, fmt, arglist); + len = vscnprintf(buffer, KBASEP_PRINT_FORMAT_BUFFER_MAX_SIZE, fmt, arglist); if (len <= 0) { pr_err("message write to the buffer failed"); goto exit; diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_csf.c b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_csf.c index ec5ca10e135b..b14ffc69c54c 100644 --- a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_csf.c +++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_csf.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -27,8 +27,8 @@ void kbasep_ktrace_backend_format_header(char *buffer, int sz, s32 *written) { - *written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), - "group,slot,prio,csi,kcpu"), + *written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), + "group,slot,prio,csi,kcpu"), 0); } @@ -44,38 +44,39 @@ void kbasep_ktrace_backend_format_msg(struct kbase_ktrace_msg *trace_msg, char * if (be_msg->gpu.flags & KBASE_KTRACE_FLAG_CSF_GROUP) { const s8 slot = be_msg->gpu.csg_nr; /* group,slot, */ - *written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), "%u,%d,", - be_msg->gpu.group_handle, slot), + *written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), + "%u,%d,", be_msg->gpu.group_handle, slot), 0); /* prio */ if (slot >= 0) - *written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), - "%u", be_msg->gpu.slot_prio), + *written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), + "%u", be_msg->gpu.slot_prio), 0); /* , */ - *written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), ","), 0); + *written += + MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), ","), 0); } else { /* No group,slot,prio fields, but ensure ending with "," */ *written += - MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), ",,,"), 0); + MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), ",,,"), 0); } /* queue parts: csi */ if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_CSF_QUEUE) - *written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), "%d", - be_msg->gpu.csi_index), + *written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), "%d", + be_msg->gpu.csi_index), 0); /* , */ - *written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), ","), 0); + *written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), ","), 0); if (be_msg->gpu.flags & KBASE_KTRACE_FLAG_CSF_KCPU) { /* kcpu data */ - *written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), - "kcpu %d (0x%llx)", be_msg->kcpu.id, - be_msg->kcpu.extra_info_val), + *written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), + "kcpu %d (0x%llx)", be_msg->kcpu.id, + be_msg->kcpu.extra_info_val), 0); } diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_jm.c b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_jm.c index beac074f2035..39306e7d45e3 100644 --- a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_jm.c +++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_jm.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -27,8 +27,8 @@ void kbasep_ktrace_backend_format_header(char *buffer, int sz, s32 *written) { - *written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), - "katom,gpu_addr,jobslot,refcount"), + *written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), + "katom,gpu_addr,jobslot,refcount"), 0); } @@ -37,34 +37,34 @@ void kbasep_ktrace_backend_format_msg(struct kbase_ktrace_msg *trace_msg, char * { /* katom */ if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_JM_ATOM) - *written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), - "atom %u (ud: 0x%llx 0x%llx)", - trace_msg->backend.gpu.atom_number, - trace_msg->backend.gpu.atom_udata[0], - trace_msg->backend.gpu.atom_udata[1]), + *written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), + "atom %u (ud: 0x%llx 0x%llx)", + trace_msg->backend.gpu.atom_number, + trace_msg->backend.gpu.atom_udata[0], + trace_msg->backend.gpu.atom_udata[1]), 0); /* gpu_addr */ if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_BACKEND) - *written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), - ",%.8llx,", trace_msg->backend.gpu.gpu_addr), + *written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), + ",%.8llx,", trace_msg->backend.gpu.gpu_addr), 0); else *written += - MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), ",,"), 0); + MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), ",,"), 0); /* jobslot */ if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_JM_JOBSLOT) - *written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), "%d", - trace_msg->backend.gpu.jobslot), + *written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), "%d", + trace_msg->backend.gpu.jobslot), 0); - *written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), ","), 0); + *written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), ","), 0); /* refcount */ if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_JM_REFCOUNT) - *written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), "%d", - trace_msg->backend.gpu.refcount), + *written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), "%d", + trace_msg->backend.gpu.refcount), 0); } diff --git a/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace.c b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace.c index 0842460bc08a..036d1f5968f6 100644 --- a/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace.c +++ b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -71,15 +71,15 @@ static const char *const kbasep_ktrace_code_string[] = { static void kbasep_ktrace_format_header(char *buffer, int sz, s32 written) { - written += MAX(snprintf(buffer + written, (size_t)MAX(sz - written, 0), - "secs,thread_id,cpu,code,kctx,"), + written += MAX(scnprintf(buffer + written, (size_t)MAX(sz - written, 0), + "secs,thread_id,cpu,code,kctx,"), 0); kbasep_ktrace_backend_format_header(buffer, sz, &written); - written += MAX(snprintf(buffer + written, (size_t)MAX(sz - written, 0), - ",info_val,ktrace_version=%u.%u", KBASE_KTRACE_VERSION_MAJOR, - KBASE_KTRACE_VERSION_MINOR), + written += MAX(scnprintf(buffer + written, (size_t)MAX(sz - written, 0), + ",info_val,ktrace_version=%u.%u", KBASE_KTRACE_VERSION_MAJOR, + KBASE_KTRACE_VERSION_MINOR), 0); buffer[sz - 1] = 0; @@ -93,21 +93,21 @@ static void kbasep_ktrace_format_msg(struct kbase_ktrace_msg *trace_msg, char *b * * secs,thread_id,cpu,code, */ - written += MAX(snprintf(buffer + written, (size_t)MAX(sz - written, 0), "%d.%.6d,%d,%d,%s,", - (int)trace_msg->timestamp.tv_sec, - (int)(trace_msg->timestamp.tv_nsec / 1000), trace_msg->thread_id, - trace_msg->cpu, - kbasep_ktrace_code_string[trace_msg->backend.gpu.code]), + written += MAX(scnprintf(buffer + written, (size_t)MAX(sz - written, 0), + "%d.%.6d,%d,%d,%s,", (int)trace_msg->timestamp.tv_sec, + (int)(trace_msg->timestamp.tv_nsec / 1000), trace_msg->thread_id, + trace_msg->cpu, + kbasep_ktrace_code_string[trace_msg->backend.gpu.code]), 0); /* kctx part: */ if (trace_msg->kctx_tgid) { - written += MAX(snprintf(buffer + written, (size_t)MAX(sz - written, 0), "%d_%u", - trace_msg->kctx_tgid, trace_msg->kctx_id), + written += MAX(scnprintf(buffer + written, (size_t)MAX(sz - written, 0), "%d_%u", + trace_msg->kctx_tgid, trace_msg->kctx_id), 0); } /* Trailing comma */ - written += MAX(snprintf(buffer + written, (size_t)MAX(sz - written, 0), ","), 0); + written += MAX(scnprintf(buffer + written, (size_t)MAX(sz - written, 0), ","), 0); /* Backend parts */ kbasep_ktrace_backend_format_msg(trace_msg, buffer, sz, &written); @@ -119,8 +119,8 @@ static void kbasep_ktrace_format_msg(struct kbase_ktrace_msg *trace_msg, char *b * Note that the last column is empty, it's simply to hold the ktrace * version in the header */ - written += MAX(snprintf(buffer + written, (size_t)MAX(sz - written, 0), ",0x%.16llx", - (unsigned long long)trace_msg->info_val), + written += MAX(scnprintf(buffer + written, (size_t)MAX(sz - written, 0), ",0x%.16llx", + (unsigned long long)trace_msg->info_val), 0); buffer[sz - 1] = 0; } diff --git a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_csf.c b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_csf.c index 52aa63330afe..6fae88665d43 100644 --- a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_csf.c +++ b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_csf.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -159,9 +159,11 @@ fail_reset_gpu_init: */ static void kbase_backend_late_term(struct kbase_device *kbdev) { - kbase_backend_devfreq_term(kbdev); - kbasep_pm_metrics_term(kbdev); - kbase_ipa_control_term(kbdev); + { + kbase_backend_devfreq_term(kbdev); + kbasep_pm_metrics_term(kbdev); + kbase_ipa_control_term(kbdev); + } kbase_hwaccess_pm_halt(kbdev); kbase_reset_gpu_term(kbdev); kbase_hwaccess_pm_term(kbdev); @@ -279,10 +281,8 @@ static const struct kbase_device_init dev_init[] = { { kbase_gpu_device_create, kbase_gpu_device_destroy, "Dummy model initialization failed" }, #else /* !IS_ENABLED(CONFIG_MALI_REAL_HW) */ { kbase_get_irqs, NULL, "IRQ search failed" }, -#endif /* !IS_ENABLED(CONFIG_MALI_REAL_HW) */ -#if !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) { registers_map, registers_unmap, "Register map failed" }, -#endif /* !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) */ +#endif /* !IS_ENABLED(CONFIG_MALI_REAL_HW) */ #if IS_ENABLED(CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD) { kbase_gpu_metrics_init, kbase_gpu_metrics_term, "GPU metrics initialization failed" }, #endif /* IS_ENABLED(CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD) */ diff --git a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_csf.c b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_csf.c index ab9df01610ab..4706a4f15b65 100644 --- a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_csf.c +++ b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_csf.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -28,6 +28,16 @@ #include #include #include +#include + +bool kbase_is_gpu_removed(struct kbase_device *kbdev) +{ + if (!IS_ENABLED(CONFIG_MALI_ARBITER_SUPPORT)) + return false; + + + return (KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(GPU_ID)) == 0); +} /** * kbase_report_gpu_fault - Report a GPU fault of the device. @@ -173,6 +183,9 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) kbase_pm_power_changed(kbdev); } + if (val & MCU_STATUS_GPU_IRQ) + wake_up_all(&kbdev->csf.event_wait); + KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_DONE, NULL, val); } KBASE_EXPORT_TEST_API(kbase_gpu_interrupt); diff --git a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_jm.c b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_jm.c index f971b3b939df..c0cb835de69c 100644 --- a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_jm.c +++ b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_jm.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -28,6 +28,14 @@ #include #include +bool kbase_is_gpu_removed(struct kbase_device *kbdev) +{ + if (!IS_ENABLED(CONFIG_MALI_ARBITER_SUPPORT)) + return false; + + return (KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(GPU_ID)) == 0); +} + /** * kbase_report_gpu_fault - Report a GPU fault. * @kbdev: Kbase device pointer diff --git a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_jm.c b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_jm.c index ab46f858a542..0fe76918296a 100644 --- a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_jm.c +++ b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_jm.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -217,16 +217,14 @@ static const struct kbase_device_init dev_init[] = { { kbase_gpu_device_create, kbase_gpu_device_destroy, "Dummy model initialization failed" }, #else /* !IS_ENABLED(CONFIG_MALI_REAL_HW) */ { kbase_get_irqs, NULL, "IRQ search failed" }, -#endif /* !IS_ENABLED(CONFIG_MALI_REAL_HW) */ -#if !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) { registers_map, registers_unmap, "Register map failed" }, -#endif /* !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) */ +#endif /* !IS_ENABLED(CONFIG_MALI_REAL_HW) */ #if IS_ENABLED(CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD) { kbase_gpu_metrics_init, kbase_gpu_metrics_term, "GPU metrics initialization failed" }, #endif /* IS_ENABLED(CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD) */ + { power_control_init, power_control_term, "Power control initialization failed" }, { kbase_device_io_history_init, kbase_device_io_history_term, "Register access history initialization failed" }, - { kbase_device_pm_init, kbase_device_pm_term, "Power management initialization failed" }, { kbase_device_early_init, kbase_device_early_term, "Early device initialization failed" }, { kbase_backend_time_init, NULL, "Time backend initialization failed" }, { kbase_device_misc_init, kbase_device_misc_term, diff --git a/drivers/gpu/arm/bifrost/device/mali_kbase_device.c b/drivers/gpu/arm/bifrost/device/mali_kbase_device.c index b191c758c62f..ccb62c2a5cb5 100644 --- a/drivers/gpu/arm/bifrost/device/mali_kbase_device.c +++ b/drivers/gpu/arm/bifrost/device/mali_kbase_device.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -556,14 +556,27 @@ int kbase_device_early_init(struct kbase_device *kbdev) /* Ensure we can access the GPU registers */ kbase_pm_register_access_enable(kbdev); - /* Initialize GPU_ID props */ - kbase_gpuprops_parse_gpu_id(&kbdev->gpu_props.gpu_id, kbase_reg_get_gpu_id(kbdev)); - - /* Initialize register mapping LUTs */ - err = kbase_regmap_init(kbdev); - if (err) + /* + * If -EPERM is returned, it means the device backend is not supported, but + * device initialization can continue. + */ + err = kbase_device_backend_init(kbdev); + if (err != 0 && err != -EPERM) goto pm_runtime_term; + /* + * Initialize register mapping LUTs. This would have been initialized on HW + * Arbitration but not on PV or non-arbitration devices. + */ + if (!kbase_reg_is_init(kbdev)) { + /* Initialize GPU_ID props */ + kbase_gpuprops_parse_gpu_id(&kbdev->gpu_props.gpu_id, kbase_reg_get_gpu_id(kbdev)); + + err = kbase_regmap_init(kbdev); + if (err) + goto backend_term; + } + /* Set the list of features available on the current HW * (identified by the GPU_ID register) */ @@ -572,7 +585,7 @@ int kbase_device_early_init(struct kbase_device *kbdev) /* Find out GPU properties based on the GPU feature registers. */ err = kbase_gpuprops_init(kbdev); if (err) - goto regmap_term; + goto backend_term; /* Get the list of workarounds for issues on the current HW * (identified by the GPU_ID register and impl_tech in THREAD_FEATURES) @@ -585,13 +598,16 @@ int kbase_device_early_init(struct kbase_device *kbdev) kbase_pm_register_access_disable(kbdev); #ifdef CONFIG_MALI_ARBITER_SUPPORT - if (kbdev->arb.arb_if) - err = kbase_arbiter_pm_install_interrupts(kbdev); - else + if (kbdev->arb.arb_if) { + if (kbdev->pm.arb_vm_state) + err = kbase_arbiter_pm_install_interrupts(kbdev); + } else { err = kbase_install_interrupts(kbdev); + } #else err = kbase_install_interrupts(kbdev); #endif + if (err) goto gpuprops_term; @@ -599,9 +615,13 @@ int kbase_device_early_init(struct kbase_device *kbdev) gpuprops_term: kbase_gpuprops_term(kbdev); -regmap_term: +backend_term: + kbase_device_backend_term(kbdev); kbase_regmap_term(kbdev); pm_runtime_term: + if (kbdev->pm.backend.gpu_powered) + kbase_pm_register_access_disable(kbdev); + kbase_pm_runtime_term(kbdev); platform_device_term: kbasep_platform_device_term(kbdev); @@ -620,8 +640,11 @@ void kbase_device_early_term(struct kbase_device *kbdev) kbase_release_interrupts(kbdev); #else kbase_release_interrupts(kbdev); -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ +#endif + kbase_gpuprops_term(kbdev); + kbase_device_backend_term(kbdev); + kbase_regmap_term(kbdev); kbase_pm_runtime_term(kbdev); kbasep_platform_device_term(kbdev); kbase_ktrace_term(kbdev); diff --git a/drivers/gpu/arm/bifrost/device/mali_kbase_device.h b/drivers/gpu/arm/bifrost/device/mali_kbase_device.h index 9cca6aff4554..1b15ff059194 100644 --- a/drivers/gpu/arm/bifrost/device/mali_kbase_device.h +++ b/drivers/gpu/arm/bifrost/device/mali_kbase_device.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -58,6 +58,9 @@ void kbase_increment_device_id(void); * When a device file is opened for the first time, * load firmware and initialize hardware counter components. * + * It is safe for this function to be called multiple times without ill + * effects. Only the first call would be effective. + * * Return: 0 on success. An error code on failure. */ int kbase_device_firmware_init_once(struct kbase_device *kbdev); diff --git a/drivers/gpu/arm/bifrost/device/mali_kbase_device_hw.c b/drivers/gpu/arm/bifrost/device/mali_kbase_device_hw.c index da597af9c46e..91379ac6429d 100644 --- a/drivers/gpu/arm/bifrost/device/mali_kbase_device_hw.c +++ b/drivers/gpu/arm/bifrost/device/mali_kbase_device_hw.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -27,14 +27,6 @@ #include #include -bool kbase_is_gpu_removed(struct kbase_device *kbdev) -{ - if (!IS_ENABLED(CONFIG_MALI_ARBITER_SUPPORT)) - return false; - - return (kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_ID)) == 0); -} - /** * busy_wait_cache_operation - Wait for a pending cache flush to complete * diff --git a/drivers/gpu/arm/bifrost/hw_access/backend/mali_kbase_hw_access_model_linux.c b/drivers/gpu/arm/bifrost/hw_access/backend/mali_kbase_hw_access_model_linux.c index ca1ccbfb3dbe..9993b787ed21 100644 --- a/drivers/gpu/arm/bifrost/hw_access/backend/mali_kbase_hw_access_model_linux.c +++ b/drivers/gpu/arm/bifrost/hw_access/backend/mali_kbase_hw_access_model_linux.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -46,7 +46,7 @@ u32 kbase_reg_read32(struct kbase_device *kbdev, u32 reg_enum) u32 val = 0; u32 offset; - if (WARN_ON(!kbdev->pm.backend.gpu_powered)) + if (WARN_ON(!kbase_reg_is_powered_access_allowed(kbdev, reg_enum))) return 0; if (unlikely(!kbase_reg_is_accessible(kbdev, reg_enum, KBASE_REGMAP_PERM_READ | KBASE_REGMAP_WIDTH_32_BIT))) @@ -68,7 +68,7 @@ u64 kbase_reg_read64(struct kbase_device *kbdev, u32 reg_enum) u32 val32[2] = { 0 }; u32 offset; - if (WARN_ON(!kbdev->pm.backend.gpu_powered)) + if (WARN_ON(!kbase_reg_is_powered_access_allowed(kbdev, reg_enum))) return 0; if (unlikely(!kbase_reg_is_accessible(kbdev, reg_enum, KBASE_REGMAP_PERM_READ | KBASE_REGMAP_WIDTH_64_BIT))) @@ -91,7 +91,7 @@ u64 kbase_reg_read64_coherent(struct kbase_device *kbdev, u32 reg_enum) u32 hi1 = 0, hi2 = 0, lo = 0; u32 offset; - if (WARN_ON(!kbdev->pm.backend.gpu_powered)) + if (WARN_ON(!kbase_reg_is_powered_access_allowed(kbdev, reg_enum))) return 0; if (unlikely(!kbase_reg_is_accessible(kbdev, reg_enum, KBASE_REGMAP_PERM_READ | KBASE_REGMAP_WIDTH_64_BIT))) @@ -116,7 +116,7 @@ void kbase_reg_write32(struct kbase_device *kbdev, u32 reg_enum, u32 value) unsigned long flags; u32 offset; - if (WARN_ON(!kbdev->pm.backend.gpu_powered)) + if (WARN_ON(!kbase_reg_is_powered_access_allowed(kbdev, reg_enum))) return; if (unlikely(!kbase_reg_is_accessible(kbdev, reg_enum, KBASE_REGMAP_PERM_WRITE | KBASE_REGMAP_WIDTH_32_BIT))) @@ -135,7 +135,7 @@ void kbase_reg_write64(struct kbase_device *kbdev, u32 reg_enum, u64 value) unsigned long flags; u32 offset; - if (WARN_ON(!kbdev->pm.backend.gpu_powered)) + if (WARN_ON(!kbase_reg_is_powered_access_allowed(kbdev, reg_enum))) return; if (unlikely(!kbase_reg_is_accessible(kbdev, reg_enum, KBASE_REGMAP_PERM_WRITE | KBASE_REGMAP_WIDTH_64_BIT))) diff --git a/drivers/gpu/arm/bifrost/hw_access/backend/mali_kbase_hw_access_real_hw.c b/drivers/gpu/arm/bifrost/hw_access/backend/mali_kbase_hw_access_real_hw.c index f4afbf55e312..ecf58cb45d15 100644 --- a/drivers/gpu/arm/bifrost/hw_access/backend/mali_kbase_hw_access_real_hw.c +++ b/drivers/gpu/arm/bifrost/hw_access/backend/mali_kbase_hw_access_real_hw.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -24,12 +24,13 @@ #include #include +#include u64 kbase_reg_get_gpu_id(struct kbase_device *kbdev) { u32 val[2] = { 0 }; - val[0] = readl(kbdev->reg); + val[0] = mali_readl(kbdev->reg); return (u64)val[0] | ((u64)val[1] << 32); @@ -39,13 +40,13 @@ u32 kbase_reg_read32(struct kbase_device *kbdev, u32 reg_enum) { u32 val; - if (WARN_ON(!kbdev->pm.backend.gpu_powered)) + if (WARN_ON(!kbase_reg_is_powered_access_allowed(kbdev, reg_enum))) return 0; if (unlikely(!kbase_reg_is_accessible(kbdev, reg_enum, KBASE_REGMAP_PERM_READ | KBASE_REGMAP_WIDTH_32_BIT))) return 0; - val = readl(kbdev->regmap.regs[reg_enum]); + val = mali_readl(kbdev->regmap.regs[reg_enum]); #if IS_ENABLED(CONFIG_DEBUG_FS) if (unlikely(kbdev->io_history.enabled)) @@ -63,14 +64,13 @@ u64 kbase_reg_read64(struct kbase_device *kbdev, u32 reg_enum) { u64 val; - if (WARN_ON(!kbdev->pm.backend.gpu_powered)) + if (WARN_ON(!kbase_reg_is_powered_access_allowed(kbdev, reg_enum))) return 0; if (unlikely(!kbase_reg_is_accessible(kbdev, reg_enum, KBASE_REGMAP_PERM_READ | KBASE_REGMAP_WIDTH_64_BIT))) return 0; - val = (u64)readl(kbdev->regmap.regs[reg_enum]) | - ((u64)readl(kbdev->regmap.regs[reg_enum] + 4) << 32); + val = mali_readq(kbdev->regmap.regs[reg_enum]); #if IS_ENABLED(CONFIG_DEBUG_FS) if (unlikely(kbdev->io_history.enabled)) { @@ -90,23 +90,14 @@ KBASE_EXPORT_TEST_API(kbase_reg_read64); u64 kbase_reg_read64_coherent(struct kbase_device *kbdev, u32 reg_enum) { u64 val; -#if !IS_ENABLED(CONFIG_MALI_64BIT_HW_ACCESS) - u32 hi1, hi2, lo; -#endif - if (WARN_ON(!kbdev->pm.backend.gpu_powered)) + if (WARN_ON(!kbase_reg_is_powered_access_allowed(kbdev, reg_enum))) return 0; if (unlikely(!kbase_reg_is_accessible(kbdev, reg_enum, KBASE_REGMAP_PERM_READ | KBASE_REGMAP_WIDTH_64_BIT))) return 0; - do { - hi1 = readl(kbdev->regmap.regs[reg_enum] + 4); - lo = readl(kbdev->regmap.regs[reg_enum]); - hi2 = readl(kbdev->regmap.regs[reg_enum] + 4); - } while (hi1 != hi2); - - val = lo | (((u64)hi1) << 32); + val = mali_readq_coherent(kbdev->regmap.regs[reg_enum]); #if IS_ENABLED(CONFIG_DEBUG_FS) if (unlikely(kbdev->io_history.enabled)) { @@ -125,13 +116,13 @@ KBASE_EXPORT_TEST_API(kbase_reg_read64_coherent); void kbase_reg_write32(struct kbase_device *kbdev, u32 reg_enum, u32 value) { - if (WARN_ON(!kbdev->pm.backend.gpu_powered)) + if (WARN_ON(!kbase_reg_is_powered_access_allowed(kbdev, reg_enum))) return; if (unlikely(!kbase_reg_is_accessible(kbdev, reg_enum, KBASE_REGMAP_PERM_WRITE | KBASE_REGMAP_WIDTH_32_BIT))) return; - writel(value, kbdev->regmap.regs[reg_enum]); + mali_writel(value, kbdev->regmap.regs[reg_enum]); #if IS_ENABLED(CONFIG_DEBUG_FS) if (unlikely(kbdev->io_history.enabled)) @@ -145,14 +136,13 @@ KBASE_EXPORT_TEST_API(kbase_reg_write32); void kbase_reg_write64(struct kbase_device *kbdev, u32 reg_enum, u64 value) { - if (WARN_ON(!kbdev->pm.backend.gpu_powered)) + if (WARN_ON(!kbase_reg_is_powered_access_allowed(kbdev, reg_enum))) return; if (unlikely(!kbase_reg_is_accessible(kbdev, reg_enum, KBASE_REGMAP_PERM_WRITE | KBASE_REGMAP_WIDTH_64_BIT))) return; - writel(value & 0xFFFFFFFF, kbdev->regmap.regs[reg_enum]); - writel(value >> 32, kbdev->regmap.regs[reg_enum] + 4); + mali_writeq(value, kbdev->regmap.regs[reg_enum]); #if IS_ENABLED(CONFIG_DEBUG_FS) if (unlikely(kbdev->io_history.enabled)) { diff --git a/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access.c b/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access.c index 16a27c780d3b..2cce391606c3 100644 --- a/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access.c +++ b/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -24,9 +24,50 @@ #include #include "mali_kbase_hw_access.h" +#include "mali_kbase_hw_access_regmap.h" #include +#define KBASE_REGMAP_ACCESS_ALWAYS_POWERED (1U << 16) + +static u32 always_powered_regs[] = { +#if MALI_USE_CSF +#else /* MALI_USE_CSF */ + PTM_AW_IRQ_CLEAR, + PTM_AW_IRQ_INJECTION, + PTM_AW_IRQ_MASK, + PTM_AW_IRQ_RAWSTAT, + PTM_AW_IRQ_STATUS, + PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE0, + PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE1, + PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE0, + PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE1, + PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE_STATUS, + PTM_ID, +#endif /* MALI_USE_CSF */ +}; + +static void kbasep_reg_setup_always_powered_registers(struct kbase_device *kbdev) +{ + u32 i; + + for (i = 0; i < ARRAY_SIZE(always_powered_regs); i++) { + u32 reg_enum = always_powered_regs[i]; + + if (!kbase_reg_is_valid(kbdev, reg_enum)) + continue; + + kbdev->regmap.flags[reg_enum] |= KBASE_REGMAP_ACCESS_ALWAYS_POWERED; + } +} + +bool kbase_reg_is_powered_access_allowed(struct kbase_device *kbdev, u32 reg_enum) +{ + if (kbdev->regmap.flags[reg_enum] & KBASE_REGMAP_ACCESS_ALWAYS_POWERED) + return true; + return kbdev->pm.backend.gpu_powered; +} + bool kbase_reg_is_size64(struct kbase_device *kbdev, u32 reg_enum) { if (WARN_ON(reg_enum >= kbdev->regmap.size)) @@ -67,6 +108,11 @@ bool kbase_reg_is_accessible(struct kbase_device *kbdev, u32 reg_enum, u32 flags return true; } +bool kbase_reg_is_init(struct kbase_device *kbdev) +{ + return (kbdev->regmap.regs != NULL) && (kbdev->regmap.flags != NULL); +} + int kbase_reg_get_offset(struct kbase_device *kbdev, u32 reg_enum, u32 *offset) { if (unlikely(!kbase_reg_is_accessible(kbdev, reg_enum, 0))) @@ -108,12 +154,12 @@ int kbase_regmap_init(struct kbase_device *kbdev) return -ENOMEM; } + kbasep_reg_setup_always_powered_registers(kbdev); + dev_info(kbdev->dev, "Register LUT %08x initialized for GPU arch 0x%08x\n", lut_arch_id, kbdev->gpu_props.gpu_id.arch_id); -#if IS_ENABLED(CONFIG_MALI_64BIT_HW_ACCESS) && IS_ENABLED(CONFIG_MALI_REAL_HW) - dev_info(kbdev->dev, "64-bit HW access enabled\n"); -#endif + return 0; } diff --git a/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access.h b/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access.h index 40356596163d..654fb685fa06 100644 --- a/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access.h +++ b/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -128,6 +128,25 @@ bool kbase_reg_is_valid(struct kbase_device *kbdev, u32 reg_enum); */ bool kbase_reg_is_accessible(struct kbase_device *kbdev, u32 reg_enum, u32 flags); +/** + * kbase_reg_is_powered_access_allowed - check if registered is accessible given + * current power state + * + * @kbdev: Kbase device pointer + * @reg_enum: Register enum + * + * Return: boolean if register is accessible + */ +bool kbase_reg_is_powered_access_allowed(struct kbase_device *kbdev, u32 reg_enum); + +/** + * kbase_reg_is_init - check if regmap is initialized + * + * @kbdev: Kbase device pointer + * Return: boolean if regmap is initialized + */ +bool kbase_reg_is_init(struct kbase_device *kbdev); + /** * kbase_reg_get_offset - get register offset from enum * @kbdev: Kbase device pointer diff --git a/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access_regmap.h b/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access_regmap.h index 97adb1322a35..a8708fafc638 100644 --- a/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access_regmap.h +++ b/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access_regmap.h @@ -308,6 +308,16 @@ #define TC_CLOCK_GATE_OVERRIDE (1ul << 0) /* End TILER_CONFIG register */ +/* L2_FEATURES register */ +#define L2_FEATURES_CACHE_SIZE_SHIFT GPU_U(16) +#define L2_FEATURES_CACHE_SIZE_MASK (GPU_U(0xFF) << L2_FEATURES_CACHE_SIZE_SHIFT) +#define L2_FEATURES_CACHE_SIZE_GET(reg_val) \ + (((reg_val)&L2_FEATURES_CACHE_SIZE_MASK) >> L2_FEATURES_CACHE_SIZE_SHIFT) +#define L2_FEATURES_CACHE_SIZE_SET(reg_val, value) \ + (~(~(reg_val) | L2_FEATURES_CACHE_SIZE_MASK) | \ + (((value) << L2_FEATURES_CACHE_SIZE_SHIFT) & L2_FEATURES_CACHE_SIZE_MASK)) +/* End L2_FEATURES register */ + /* L2_CONFIG register */ #define L2_CONFIG_SIZE_SHIFT 16 #define L2_CONFIG_SIZE_MASK (0xFFul << L2_CONFIG_SIZE_SHIFT) diff --git a/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_csf_macros.h b/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_csf_macros.h index c3d12ad04c4e..56ec5e015d11 100644 --- a/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_csf_macros.h +++ b/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_csf_macros.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_jm.c b/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_jm.c index 178d45501916..4f41693ff3c2 100644 --- a/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_jm.c +++ b/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_jm.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -2240,6 +2240,56 @@ static void kbase_regmap_v9_2_init(struct kbase_device *kbdev) kbdev->regmap.regs[GPU_CONTROL__L2_CONFIG] = kbdev->reg + 0x48; } +static void kbase_regmap_v9_14_init(struct kbase_device *kbdev) +{ + if (kbdev->regmap.regs == NULL && kbdev->regmap.flags == NULL) { + kbdev->regmap.size = NR_V9_14_REGS; + kbdev->regmap.regs = + kcalloc(kbdev->regmap.size, sizeof(void __iomem *), GFP_KERNEL); + kbdev->regmap.flags = kcalloc(kbdev->regmap.size, sizeof(u32), GFP_KERNEL); + } + + if (WARN_ON(kbdev->regmap.regs == NULL)) + return; + if (WARN_ON(kbdev->regmap.flags == NULL)) + return; + + kbase_regmap_v9_2_init(kbdev); + + kbdev->regmap.flags[PTM_AW_IRQ_CLEAR] = KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | + KBASE_REGMAP_PERM_WRITE; + kbdev->regmap.flags[PTM_AW_IRQ_INJECTION] = + KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; + kbdev->regmap.flags[PTM_AW_IRQ_MASK] = KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | + KBASE_REGMAP_PERM_WRITE; + kbdev->regmap.flags[PTM_AW_IRQ_RAWSTAT] = KBASE_REGMAP_WIDTH_32_BIT | + KBASE_REGMAP_PERM_READ; + kbdev->regmap.flags[PTM_AW_IRQ_STATUS] = KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ; + kbdev->regmap.flags[PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE0] = KBASE_REGMAP_WIDTH_32_BIT | + KBASE_REGMAP_PERM_READ; + kbdev->regmap.flags[PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE1] = KBASE_REGMAP_WIDTH_32_BIT | + KBASE_REGMAP_PERM_READ; + kbdev->regmap.flags[PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE0] = + KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; + kbdev->regmap.flags[PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE1] = + KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; + kbdev->regmap.flags[PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE_STATUS] = + KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ; + kbdev->regmap.flags[PTM_ID] = KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ; + + kbdev->regmap.regs[PTM_AW_IRQ_CLEAR] = kbdev->reg + 0x1ffc8; + kbdev->regmap.regs[PTM_AW_IRQ_INJECTION] = kbdev->reg + 0x1ffd4; + kbdev->regmap.regs[PTM_AW_IRQ_MASK] = kbdev->reg + 0x1ffcc; + kbdev->regmap.regs[PTM_AW_IRQ_RAWSTAT] = kbdev->reg + 0x1ffc4; + kbdev->regmap.regs[PTM_AW_IRQ_STATUS] = kbdev->reg + 0x1ffd0; + kbdev->regmap.regs[PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE0] = kbdev->reg + 0x1ffd8; + kbdev->regmap.regs[PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE1] = kbdev->reg + 0x1ffdc; + kbdev->regmap.regs[PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE0] = kbdev->reg + 0x1ffe4; + kbdev->regmap.regs[PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE1] = kbdev->reg + 0x1ffe8; + kbdev->regmap.regs[PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE_STATUS] = kbdev->reg + 0x1ffe0; + kbdev->regmap.regs[PTM_ID] = kbdev->reg + 0x1ffc0; +} + u32 kbase_regmap_backend_init(struct kbase_device *kbdev) { int i = 0; @@ -2254,6 +2304,7 @@ u32 kbase_regmap_backend_init(struct kbase_device *kbdev) { GPU_ID_ARCH_MAKE(7, 2, 0), kbase_regmap_v7_2_init }, { GPU_ID_ARCH_MAKE(9, 0, 0), kbase_regmap_v9_0_init }, { GPU_ID_ARCH_MAKE(9, 2, 0), kbase_regmap_v9_2_init }, + { GPU_ID_ARCH_MAKE(9, 14, 0), kbase_regmap_v9_14_init }, }; for (i = 0; i < ARRAY_SIZE(init_array) - 1; i++) { @@ -2967,6 +3018,18 @@ static char *enum_strings[] = { [GPU_CONTROL__CORE_FEATURES] = "GPU_CONTROL__CORE_FEATURES", [GPU_CONTROL__THREAD_TLS_ALLOC] = "GPU_CONTROL__THREAD_TLS_ALLOC", [GPU_CONTROL__L2_CONFIG] = "GPU_CONTROL__L2_CONFIG", + [PTM_AW_IRQ_CLEAR] = "PTM_AW_IRQ_CLEAR", + [PTM_AW_IRQ_INJECTION] = "PTM_AW_IRQ_INJECTION", + [PTM_AW_IRQ_MASK] = "PTM_AW_IRQ_MASK", + [PTM_AW_IRQ_RAWSTAT] = "PTM_AW_IRQ_RAWSTAT", + [PTM_AW_IRQ_STATUS] = "PTM_AW_IRQ_STATUS", + [PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE0] = "PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE0", + [PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE1] = "PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE1", + [PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE0] = "PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE0", + [PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE1] = "PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE1", + [PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE_STATUS] = + "PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE_STATUS", + [PTM_ID] = "PTM_ID", }; const char *kbase_reg_get_enum_string(u32 reg_enum) diff --git a/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_jm_enums.h b/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_jm_enums.h index f5618c4794db..59d8745eaf4a 100644 --- a/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_jm_enums.h +++ b/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_jm_enums.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -759,4 +759,19 @@ enum kbase_regmap_enum_v9_2 { NR_V9_2_REGS, }; +enum kbase_regmap_enum_v9_14 { + PTM_AW_IRQ_CLEAR = NR_V9_2_REGS, /* (RW) 32-bit 0x1FFC8 */ + PTM_AW_IRQ_INJECTION, /* (RW) 32-bit 0x1FFD4 */ + PTM_AW_IRQ_MASK, /* (RW) 32-bit 0x1FFCC */ + PTM_AW_IRQ_RAWSTAT, /* (RO) 32-bit 0x1FFC4 */ + PTM_AW_IRQ_STATUS, /* (RO) 32-bit 0x1FFD0 */ + PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE0, /* (RO) 32-bit 0x1FFD8 */ + PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE1, /* (RO) 32-bit 0x1FFDC */ + PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE0, /* (RW) 32-bit 0x1FFE4 */ + PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE1, /* (RW) 32-bit 0x1FFE8 */ + PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE_STATUS, /* (RO) 32-bit 0x1FFE0 */ + PTM_ID, /* (RO) 32-bit 0x1FFC0 */ + NR_V9_14_REGS, +}; + #endif /* _MALI_KBASE_REGMAP_JM_ENUMS_H_ */ diff --git a/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_jm_macros.h b/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_jm_macros.h index c3bc0f3e9924..650ed9b31eea 100644 --- a/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_jm_macros.h +++ b/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_jm_macros.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -47,6 +47,8 @@ #define MMU_AS_OFFSET(n, regname) ENUM_OFFSET(n, MMU_AS_ENUM(0, regname), MMU_AS_ENUM(1, regname)) #define MMU_AS_BASE_OFFSET(n) MMU_AS_OFFSET(n, TRANSTAB) +#define PTM_AW_MESSAGE_ENUM(regname) PTM_AW_MESSAGE__##regname + /* register value macros */ /* GPU_STATUS values */ #define GPU_STATUS_PRFCNT_ACTIVE (1 << 2) /* Set if the performance counters are active. */ @@ -295,4 +297,11 @@ (GPU_FAULT | MULTIPLE_GPU_FAULTS | RESET_COMPLETED | POWER_CHANGED_ALL | \ PRFCNT_SAMPLE_COMPLETED) +#define WINDOW_IRQ_MESSAGE (1U << 0) +#define WINDOW_IRQ_INVALID_ACCESS (1U << 1) +#define WINDOW_IRQ_GPU (1U << 2) +#define WINDOW_IRQ_JOB (1U << 3) +#define WINDOW_IRQ_MMU (1U << 4) +#define WINDOW_IRQ_EVENT (1U << 5) + #endif /* _MALI_KBASE_REGMAP_JM_MACROS_H_ */ diff --git a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf.c b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf.c index d605253752ca..8bdfc9a0bfda 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf.c +++ b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2021-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2021-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -21,7 +21,6 @@ #include "hwcnt/backend/mali_kbase_hwcnt_backend_csf.h" #include "hwcnt/mali_kbase_hwcnt_gpu.h" -#include "hwcnt/mali_kbase_hwcnt_types.h" #include #include @@ -31,6 +30,7 @@ #include #include #include +#include #ifndef BASE_MAX_NR_CLOCKS_REGULATORS #define BASE_MAX_NR_CLOCKS_REGULATORS 4 @@ -255,7 +255,8 @@ struct kbase_hwcnt_csf_physical_layout { * @hwc_threshold_work: Worker for consuming available samples when * threshold interrupt raised. * @num_l2_slices: Current number of L2 slices allocated to the GPU. - * @shader_present_bitmap: Current shader-present bitmap that is allocated to the GPU. + * @powered_shader_core_mask: The common mask between the debug_core_mask + * and the shader_present_bitmap. */ struct kbase_hwcnt_backend_csf { struct kbase_hwcnt_backend_csf_info *info; @@ -283,7 +284,7 @@ struct kbase_hwcnt_backend_csf { struct work_struct hwc_dump_work; struct work_struct hwc_threshold_work; size_t num_l2_slices; - u64 shader_present_bitmap; + u64 powered_shader_core_mask; }; static bool kbasep_hwcnt_backend_csf_backend_exists(struct kbase_hwcnt_backend_csf_info *csf_info) @@ -296,9 +297,11 @@ static bool kbasep_hwcnt_backend_csf_backend_exists(struct kbase_hwcnt_backend_c } void kbase_hwcnt_backend_csf_set_hw_availability(struct kbase_hwcnt_backend_interface *iface, - size_t num_l2_slices, u64 shader_present_bitmap) + size_t num_l2_slices, u64 shader_present, + u64 power_core_mask) { struct kbase_hwcnt_backend_csf_info *csf_info; + u64 norm_shader_present = power_core_mask & shader_present; if (!iface) return; @@ -309,16 +312,17 @@ void kbase_hwcnt_backend_csf_set_hw_availability(struct kbase_hwcnt_backend_inte if (!csf_info || !csf_info->backend) return; + if (WARN_ON(csf_info->backend->enable_state != KBASE_HWCNT_BACKEND_CSF_DISABLED)) return; if (WARN_ON(num_l2_slices > csf_info->backend->phys_layout.mmu_l2_cnt) || - WARN_ON((shader_present_bitmap & csf_info->backend->phys_layout.shader_avail_mask) != - shader_present_bitmap)) + WARN_ON((norm_shader_present & csf_info->backend->phys_layout.shader_avail_mask) != + norm_shader_present)) return; csf_info->backend->num_l2_slices = num_l2_slices; - csf_info->backend->shader_present_bitmap = shader_present_bitmap; + csf_info->backend->powered_shader_core_mask = norm_shader_present; } /** @@ -424,7 +428,7 @@ static void kbasep_hwcnt_backend_csf_init_layout( WARN_ON(!prfcnt_info); WARN_ON(!phys_layout); - shader_core_cnt = (size_t)fls64(prfcnt_info->core_mask); + shader_core_cnt = (size_t)fls64(prfcnt_info->sc_core_mask); values_per_block = prfcnt_info->prfcnt_block_size / KBASE_HWCNT_VALUE_HW_BYTES; fw_block_cnt = div_u64(prfcnt_info->prfcnt_fw_size, prfcnt_info->prfcnt_block_size); hw_block_cnt = div_u64(prfcnt_info->prfcnt_hw_size, prfcnt_info->prfcnt_block_size); @@ -445,7 +449,7 @@ static void kbasep_hwcnt_backend_csf_init_layout( .fw_block_cnt = fw_block_cnt, .hw_block_cnt = hw_block_cnt, .block_cnt = fw_block_cnt + hw_block_cnt, - .shader_avail_mask = prfcnt_info->core_mask, + .shader_avail_mask = prfcnt_info->sc_core_mask, .headers_per_block = KBASE_HWCNT_V5_HEADERS_PER_BLOCK, .values_per_block = values_per_block, .counters_per_block = values_per_block - KBASE_HWCNT_V5_HEADERS_PER_BLOCK, @@ -454,17 +458,20 @@ static void kbasep_hwcnt_backend_csf_init_layout( } static void -kbasep_hwcnt_backend_csf_reset_internal_buffers(struct kbase_hwcnt_backend_csf *backend_csf) +kbasep_hwcnt_backend_csf_reset_internal_buffers(struct kbase_hwcnt_backend_csf *backend_csf, + bool user_bufs) { size_t user_buf_bytes = backend_csf->info->metadata->dump_buf_bytes; size_t block_state_bytes = backend_csf->phys_layout.block_cnt * KBASE_HWCNT_BLOCK_STATE_BYTES * KBASE_HWCNT_BLOCK_STATE_STRIDE; - memset(backend_csf->to_user_buf, 0, user_buf_bytes); memset(backend_csf->accum_buf, 0, user_buf_bytes); memset(backend_csf->old_sample_buf, 0, backend_csf->info->prfcnt_info.dump_bytes); memset(backend_csf->block_states, 0, block_state_bytes); - memset(backend_csf->to_user_block_states, 0, block_state_bytes); + if (user_bufs) { + memset(backend_csf->to_user_buf, 0, user_buf_bytes); + memset(backend_csf->to_user_block_states, 0, block_state_bytes); + } } static void @@ -517,34 +524,21 @@ static void kbasep_hwcnt_backend_csf_update_user_sample(struct kbase_hwcnt_backe memset(backend_csf->block_states, 0, block_state_bytes); } -/** - * kbasep_hwcnt_backend_csf_update_block_state - Update block state of a block instance with - * information from a sample. - * @phys_layout: Physical memory layout information of HWC - * sample buffer. - * @enable_mask: Counter enable mask for the block whose state is being updated. - * @enable_state: The CSF backend internal enabled state. - * @exiting_protm: Whether or not the sample is taken when the GPU is exiting - * protected mode. - * @block_idx: Index of block within the ringbuffer. - * @block_state: Pointer to existing block state of the block whose state is being - * updated. - * @fw_in_protected_mode: Whether or not GPU is in protected mode during sampling. - */ -static void kbasep_hwcnt_backend_csf_update_block_state( - const struct kbase_hwcnt_csf_physical_layout *phys_layout, const u32 enable_mask, - enum kbase_hwcnt_backend_csf_enable_state enable_state, bool exiting_protm, - size_t block_idx, blk_stt_t *const block_state, bool fw_in_protected_mode) +void kbasep_hwcnt_backend_csf_update_block_state(struct kbase_hwcnt_backend_csf *backend, + const u32 enable_mask, bool exiting_protm, + size_t block_idx, blk_stt_t *const block_state, + bool fw_in_protected_mode) { + const struct kbase_hwcnt_csf_physical_layout *phys_layout = &backend->phys_layout; /* Offset of shader core blocks from the start of the HW blocks in the sample */ size_t shader_core_block_offset = - (size_t)(phys_layout->hw_block_cnt - phys_layout->shader_cnt); + (size_t)(phys_layout->block_cnt - phys_layout->shader_cnt); bool is_shader_core_block; - is_shader_core_block = block_idx >= shader_core_block_offset; + is_shader_core_block = (block_idx >= shader_core_block_offset); /* Set power bits for the block state for the block, for the sample */ - switch (enable_state) { + switch (backend->enable_state) { /* Disabled states */ case KBASE_HWCNT_BACKEND_CSF_DISABLED: case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED: @@ -592,21 +586,45 @@ static void kbasep_hwcnt_backend_csf_update_block_state( KBASE_HWCNT_STATE_NORMAL); else kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_NORMAL); + + /* powered_shader_core_mask stored in the backend is a combination of + * the shader present and the debug core mask, so explicit checking of the + * core mask is not required here. + */ + if (is_shader_core_block) { + u64 current_shader_core = 1ULL << (block_idx - shader_core_block_offset); + + WARN_ON_ONCE(backend->phys_layout.shader_cnt > 64); + + if (current_shader_core & backend->info->backend->powered_shader_core_mask) + kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_AVAILABLE); + else if (current_shader_core & ~backend->info->backend->powered_shader_core_mask) + kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_UNAVAILABLE); + else + WARN_ON_ONCE(true); + } + else + kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_AVAILABLE); } -static void kbasep_hwcnt_backend_csf_accumulate_sample( - const struct kbase_hwcnt_csf_physical_layout *phys_layout, size_t dump_bytes, - u64 *accum_buf, const u32 *old_sample_buf, const u32 *new_sample_buf, - blk_stt_t *const block_states, bool clearing_samples, - enum kbase_hwcnt_backend_csf_enable_state enable_state, bool fw_in_protected_mode) +static void kbasep_hwcnt_backend_csf_accumulate_sample(struct kbase_hwcnt_backend_csf *backend, + const u32 *old_sample_buf, + const u32 *new_sample_buf) { + const struct kbase_hwcnt_csf_physical_layout *phys_layout = &backend->phys_layout; + const size_t dump_bytes = backend->info->prfcnt_info.dump_bytes; + const size_t values_per_block = phys_layout->values_per_block; + blk_stt_t *const block_states = backend->block_states; + const bool fw_in_protected_mode = backend->info->fw_in_protected_mode; + const bool clearing_samples = backend->info->prfcnt_info.clearing_samples; + u64 *accum_buf = backend->accum_buf; + size_t block_idx; const u32 *old_block = old_sample_buf; const u32 *new_block = new_sample_buf; u64 *acc_block = accum_buf; /* Flag to indicate whether current sample is exiting protected mode. */ bool exiting_protm = false; - const size_t values_per_block = phys_layout->values_per_block; /* The block pointers now point to the first HW block, which is always a CSHW/front-end * block. The counter enable mask for this block can be checked to determine whether this @@ -620,9 +638,8 @@ static void kbasep_hwcnt_backend_csf_accumulate_sample( const u32 old_enable_mask = old_block[phys_layout->enable_mask_offset]; const u32 new_enable_mask = new_block[phys_layout->enable_mask_offset]; /* Update block state with information of the current sample */ - kbasep_hwcnt_backend_csf_update_block_state(phys_layout, new_enable_mask, - enable_state, exiting_protm, block_idx, - &block_states[block_idx], + kbasep_hwcnt_backend_csf_update_block_state(backend, new_enable_mask, exiting_protm, + block_idx, &block_states[block_idx], fw_in_protected_mode); if (!(new_enable_mask & HWCNT_BLOCK_EMPTY_SAMPLE)) { @@ -706,7 +723,6 @@ static void kbasep_hwcnt_backend_csf_accumulate_samples(struct kbase_hwcnt_backe u8 *cpu_dump_base = (u8 *)backend_csf->ring_buf_cpu_base; const size_t ring_buf_cnt = backend_csf->info->ring_buf_cnt; const size_t buf_dump_bytes = backend_csf->info->prfcnt_info.dump_bytes; - bool clearing_samples = backend_csf->info->prfcnt_info.clearing_samples; u32 *old_sample_buf = backend_csf->old_sample_buf; u32 *new_sample_buf = old_sample_buf; const struct kbase_hwcnt_csf_physical_layout *phys_layout = &backend_csf->phys_layout; @@ -740,10 +756,8 @@ static void kbasep_hwcnt_backend_csf_accumulate_samples(struct kbase_hwcnt_backe const u32 buf_idx = raw_idx & (ring_buf_cnt - 1); new_sample_buf = (u32 *)&cpu_dump_base[buf_idx * buf_dump_bytes]; - kbasep_hwcnt_backend_csf_accumulate_sample( - phys_layout, buf_dump_bytes, backend_csf->accum_buf, old_sample_buf, - new_sample_buf, backend_csf->block_states, clearing_samples, - backend_csf->enable_state, backend_csf->info->fw_in_protected_mode); + kbasep_hwcnt_backend_csf_accumulate_sample(backend_csf, old_sample_buf, + new_sample_buf); old_sample_buf = new_sample_buf; } @@ -1215,11 +1229,6 @@ static void kbasep_hwcnt_backend_csf_dump_disable(struct kbase_hwcnt_backend *ba backend_csf->ring_buf, 0, backend_csf->info->ring_buf_cnt, false); - /* Reset accumulator, old_sample_buf and user_sample to all-0 to prepare - * for next enable. - */ - kbasep_hwcnt_backend_csf_reset_internal_buffers(backend_csf); - /* Disabling HWCNT is an indication that blocks have been powered off. This is important to * know for L2, CSHW, and Tiler blocks, as this is currently the only way a backend can * know if they are being powered off. @@ -1255,6 +1264,12 @@ static void kbasep_hwcnt_backend_csf_dump_disable(struct kbase_hwcnt_backend *ba kbase_hwcnt_block_state_set(&backend_csf->accum_all_blk_stt, KBASE_HWCNT_STATE_UNKNOWN); } + + /* Reset accumulator, old_sample_buf and block_states to all-0 to prepare for next enable. + * Reset user buffers if ownership is transferred to the caller (i.e. dump_buffer + * is provided). + */ + kbasep_hwcnt_backend_csf_reset_internal_buffers(backend_csf, dump_buffer); } /* CSF backend implementation of kbase_hwcnt_backend_dump_request_fn */ @@ -1279,6 +1294,11 @@ static int kbasep_hwcnt_backend_csf_dump_request(struct kbase_hwcnt_backend *bac backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_COMPLETED; *dump_time_ns = kbasep_hwcnt_backend_csf_timestamp_ns(backend); kbasep_hwcnt_backend_csf_cc_update(backend_csf); + /* There is a possibility that the transition to enabled state will remain + * during multiple dumps, hence append the OFF state. + */ + kbase_hwcnt_block_state_append(&backend_csf->accum_all_blk_stt, + KBASE_HWCNT_STATE_OFF); backend_csf->user_requested = true; backend_csf->info->csf_if->unlock(backend_csf->info->csf_if->ctx, flags); return 0; @@ -1457,7 +1477,7 @@ static int kbasep_hwcnt_backend_csf_dump_get(struct kbase_hwcnt_backend *backend ret = kbase_hwcnt_csf_dump_get(dst, backend_csf->to_user_buf, backend_csf->to_user_block_states, dst_enable_map, backend_csf->num_l2_slices, - backend_csf->shader_present_bitmap, accumulate); + backend_csf->powered_shader_core_mask, accumulate); /* If no error occurred (zero ret value), then update block state for all blocks in the * accumulation with the current sample's block state. @@ -1469,6 +1489,12 @@ static int kbasep_hwcnt_backend_csf_dump_get(struct kbase_hwcnt_backend *backend KBASE_HWCNT_STATE_UNKNOWN); } + /* Clear consumed user buffers. */ + memset(backend_csf->to_user_buf, 0, backend_csf->info->metadata->dump_buf_bytes); + memset(backend_csf->to_user_block_states, 0, + backend_csf->phys_layout.block_cnt * KBASE_HWCNT_BLOCK_STATE_BYTES * + KBASE_HWCNT_BLOCK_STATE_STRIDE); + return ret; } @@ -2098,7 +2124,7 @@ int kbase_hwcnt_backend_csf_metadata_init(struct kbase_hwcnt_backend_interface * gpu_info.has_fw_counters = csf_info->prfcnt_info.prfcnt_fw_size > 0; gpu_info.l2_count = csf_info->prfcnt_info.l2_count; gpu_info.csg_cnt = csf_info->prfcnt_info.csg_count; - gpu_info.core_mask = csf_info->prfcnt_info.core_mask; + gpu_info.sc_core_mask = csf_info->prfcnt_info.sc_core_mask; gpu_info.clk_cnt = csf_info->prfcnt_info.clk_cnt; gpu_info.prfcnt_values_per_block = csf_info->prfcnt_info.prfcnt_block_size / KBASE_HWCNT_VALUE_HW_BYTES; @@ -2115,7 +2141,7 @@ void kbase_hwcnt_backend_csf_metadata_term(struct kbase_hwcnt_backend_interface csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; if (csf_info->metadata) { - kbase_hwcnt_csf_metadata_destroy(csf_info->metadata); + kbase_hwcnt_metadata_destroy(csf_info->metadata); csf_info->metadata = NULL; } } diff --git a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf.h b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf.h index 2487db272a35..104f9c77a945 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf.h +++ b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2021-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2021-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -30,8 +30,10 @@ #include "hwcnt/backend/mali_kbase_hwcnt_backend.h" #include "hwcnt/backend/mali_kbase_hwcnt_backend_csf_if.h" #include "hwcnt/mali_kbase_hwcnt_watchdog_if.h" +#include "hwcnt/mali_kbase_hwcnt_types.h" struct kbase_hwcnt_physical_enable_map; +struct kbase_hwcnt_backend_csf; /** * kbase_hwcnt_backend_csf_create() - Create a CSF hardware counter backend @@ -123,11 +125,12 @@ void kbase_hwcnt_backend_csf_on_before_reset(struct kbase_hwcnt_backend_interfac * this function is called. * @iface: Non-NULL pointer to HWC backend interface. * @num_l2_slices: Current number of L2 slices allocated to the GPU. - * @shader_present_bitmap: Current shader-present bitmap that is allocated to the GPU. + * @shader_present: Shader_present of the current configuration. + * @power_core_mask: Mask containing changed shader core power state. */ void kbase_hwcnt_backend_csf_set_hw_availability(struct kbase_hwcnt_backend_interface *iface, - size_t num_l2_slices, - uint64_t shader_present_bitmap); + size_t num_l2_slices, u64 shader_present, + u64 power_core_mask); /** kbasep_hwcnt_backend_csf_process_enable_map() - Process the enable_map to * guarantee headers are @@ -174,4 +177,21 @@ void kbase_hwcnt_backend_csf_on_prfcnt_enable(struct kbase_hwcnt_backend_interfa */ void kbase_hwcnt_backend_csf_on_prfcnt_disable(struct kbase_hwcnt_backend_interface *iface); +/** + * kbasep_hwcnt_backend_csf_update_block_state - Update block state of a block instance with + * information from a sample. + * @backend: CSF hardware counter backend. + * @enable_mask: Counter enable mask for the block whose state is being updated. + * @exiting_protm: Whether or not the sample is taken when the GPU is exiting + * protected mode. + * @block_idx: Index of block within the ringbuffer. + * @block_state: Pointer to existing block state of the block whose state is being + * updated. + * @fw_in_protected_mode: Whether or not GPU is in protected mode during sampling. + */ +void kbasep_hwcnt_backend_csf_update_block_state(struct kbase_hwcnt_backend_csf *backend, + const u32 enable_mask, bool exiting_protm, + size_t block_idx, blk_stt_t *const block_state, + bool fw_in_protected_mode); + #endif /* _KBASE_HWCNT_BACKEND_CSF_H_ */ diff --git a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if.h b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if.h index 65bb965bcf9c..c982900cf755 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if.h +++ b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2021-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2021-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -68,7 +68,7 @@ struct kbase_hwcnt_backend_csf_if_enable { * @prfcnt_block_size: Bytes of each performance counter block. * @l2_count: The MMU L2 cache count. * @csg_count: The total number of CSGs in the system - * @core_mask: Shader core mask. + * @sc_core_mask: Shader core mask. * @clk_cnt: Clock domain count in the system. * @clearing_samples: Indicates whether counters are cleared after each sample * is taken. @@ -80,7 +80,7 @@ struct kbase_hwcnt_backend_csf_if_prfcnt_info { size_t prfcnt_block_size; size_t l2_count; u32 csg_count; - u64 core_mask; + u64 sc_core_mask; u8 clk_cnt; bool clearing_samples; }; diff --git a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.c b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.c index d79a99e5e89f..3cf039f70056 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.c +++ b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2021-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2021-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -229,7 +229,7 @@ static void kbasep_hwcnt_backend_csf_if_fw_get_prfcnt_info( *prfcnt_info = (struct kbase_hwcnt_backend_csf_if_prfcnt_info){ .l2_count = KBASE_DUMMY_MODEL_MAX_MEMSYS_BLOCKS, - .core_mask = (1ull << KBASE_DUMMY_MODEL_MAX_SHADER_CORES) - 1, + .sc_core_mask = (1ull << KBASE_DUMMY_MODEL_MAX_SHADER_CORES) - 1, .prfcnt_hw_size = KBASE_DUMMY_MODEL_MAX_NUM_HARDWARE_BLOCKS * KBASE_DUMMY_MODEL_BLOCK_SIZE, .prfcnt_fw_size = @@ -290,12 +290,13 @@ static void kbasep_hwcnt_backend_csf_if_fw_get_prfcnt_info( .dump_bytes = fw_ctx->buf_bytes, .prfcnt_block_size = prfcnt_block_size, .l2_count = kbdev->gpu_props.num_l2_slices, - .core_mask = kbasep_hwcnt_backend_csf_core_mask(&kbdev->gpu_props), + .sc_core_mask = kbasep_hwcnt_backend_csf_core_mask(&kbdev->gpu_props), .csg_count = fw_block_count > 1 ? csg_count : 0, .clk_cnt = fw_ctx->clk_cnt, .clearing_samples = true, }; + /* Block size must be multiple of counter size. */ WARN_ON((prfcnt_info->prfcnt_block_size % KBASE_HWCNT_VALUE_HW_BYTES) != 0); /* Total size must be multiple of block size. */ @@ -513,10 +514,15 @@ kbasep_hwcnt_backend_csf_if_fw_ring_buf_free(struct kbase_hwcnt_backend_csf_if_c fw_ring_buf->phys, fw_ring_buf->num_pages, fw_ring_buf->num_pages, MCU_AS_NR)); + /* Clear the dump ring_buf content to zeros */ + memset(fw_ring_buf->cpu_dump_base, 0, fw_ring_buf->num_pages * PAGE_SIZE); vunmap(fw_ring_buf->cpu_dump_base); + /* After zeroing, the ring_buf pages are dirty so need to pass the 'dirty' flag + * as true when freeing the pages to the Global pool. + */ kbase_mem_pool_free_pages(&fw_ctx->kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], - fw_ring_buf->num_pages, fw_ring_buf->phys, false, false); + fw_ring_buf->num_pages, fw_ring_buf->phys, true, false); kfree(fw_ring_buf->phys); diff --git a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_jm.c b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_jm.c index 7fbef163976a..1ee0e3b823ad 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_jm.c +++ b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_jm.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -165,7 +165,7 @@ static int kbasep_hwcnt_backend_jm_gpu_info_init(struct kbase_device *kbdev, #endif info->l2_count = l2_count; - info->core_mask = core_mask; + info->sc_core_mask = core_mask; info->prfcnt_values_per_block = KBASE_HWCNT_V5_DEFAULT_VALUES_PER_BLOCK; /* Determine the number of available clock domains. */ @@ -186,7 +186,7 @@ static void kbasep_hwcnt_backend_jm_init_layout(const struct kbase_hwcnt_gpu_inf WARN_ON(!gpu_info); WARN_ON(!phys_layout); - shader_core_cnt = fls64(gpu_info->core_mask); + shader_core_cnt = fls64(gpu_info->sc_core_mask); *phys_layout = (struct kbase_hwcnt_jm_physical_layout){ .fe_cnt = KBASE_HWCNT_V5_FE_BLOCK_COUNT, @@ -195,7 +195,7 @@ static void kbasep_hwcnt_backend_jm_init_layout(const struct kbase_hwcnt_gpu_inf .shader_cnt = shader_core_cnt, .block_cnt = KBASE_HWCNT_V5_FE_BLOCK_COUNT + KBASE_HWCNT_V5_TILER_BLOCK_COUNT + gpu_info->l2_count + shader_core_cnt, - .shader_avail_mask = gpu_info->core_mask, + .shader_avail_mask = gpu_info->sc_core_mask, .headers_per_block = KBASE_HWCNT_V5_HEADERS_PER_BLOCK, .values_per_block = gpu_info->prfcnt_values_per_block, .counters_per_block = @@ -384,14 +384,12 @@ kbasep_hwcnt_backend_jm_dump_enable_nolock(struct kbase_hwcnt_backend *backend, enable = (struct kbase_instr_hwcnt_enable) { - .fe_bm = phys_enable_map.fe_bm, - .shader_bm = phys_enable_map.shader_bm, - .tiler_bm = phys_enable_map.tiler_bm, - .mmu_l2_bm = phys_enable_map.mmu_l2_bm, + .fe_bm = phys_enable_map.fe_bm, .shader_bm = phys_enable_map.shader_bm, + .tiler_bm = phys_enable_map.tiler_bm, .mmu_l2_bm = phys_enable_map.mmu_l2_bm, .counter_set = phys_counter_set, #if IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) /* The dummy model needs the CPU mapping. */ - .dump_buffer = (uintptr_t)backend_jm->cpu_dump_va, + .dump_buffer = (uintptr_t)backend_jm->cpu_dump_va, #else .dump_buffer = backend_jm->gpu_dump_va, #endif /* CONFIG_MALI_BIFROST_NO_MALI */ @@ -411,7 +409,7 @@ kbasep_hwcnt_backend_jm_dump_enable_nolock(struct kbase_hwcnt_backend *backend, backend_jm->debug_core_mask = kbase_pm_ca_get_debug_core_mask(kbdev); backend_jm->max_l2_slices = backend_jm->info->hwcnt_gpu_info.l2_count; - backend_jm->max_core_mask = backend_jm->info->hwcnt_gpu_info.core_mask; + backend_jm->max_core_mask = backend_jm->info->hwcnt_gpu_info.sc_core_mask; backend_jm->pm_core_mask = kbase_pm_ca_get_instr_core_mask(kbdev); @@ -660,8 +658,8 @@ static int kbasep_hwcnt_backend_jm_dump_get(struct kbase_hwcnt_backend *backend, #endif /* CONFIG_MALI_BIFROST_NO_MALI */ errcode = kbase_hwcnt_jm_dump_get(dst, backend_jm->to_user_buf, dst_enable_map, backend_jm->pm_core_mask, backend_jm->debug_core_mask, - backend_jm->max_core_mask, backend_jm->max_l2_slices, - &backend_jm->curr_config, accumulate); + backend_jm->max_l2_slices, &backend_jm->curr_config, + accumulate); if (errcode) return errcode; @@ -864,7 +862,7 @@ static void kbasep_hwcnt_backend_jm_info_destroy(const struct kbase_hwcnt_backen if (!info) return; - kbase_hwcnt_jm_metadata_destroy(info->metadata); + kbase_hwcnt_metadata_destroy(info->metadata); kfree(info); } diff --git a/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.c b/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.c index 5da564546608..7cd16a0de4ce 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.c +++ b/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -169,7 +169,7 @@ static int kbasep_hwcnt_backend_gpu_metadata_create(const struct kbase_hwcnt_gpu /* Calculate number of block instances that aren't cores */ non_core_block_count = 2 + gpu_info->l2_count; /* Calculate number of block instances that are shader cores */ - sc_block_count = (size_t)fls64(gpu_info->core_mask); + sc_block_count = (size_t)fls64(gpu_info->sc_core_mask); /* Determine the total number of cores */ core_block_count = sc_block_count; @@ -277,7 +277,7 @@ static int kbasep_hwcnt_backend_gpu_metadata_create(const struct kbase_hwcnt_gpu kbase_hwcnt_set_avail_mask(&desc.avail_mask, 0, 0); kbase_hwcnt_set_avail_mask_bits(&desc.avail_mask, 0, non_core_block_count, U64_MAX); kbase_hwcnt_set_avail_mask_bits(&desc.avail_mask, non_core_block_count, sc_block_count, - gpu_info->core_mask); + gpu_info->sc_core_mask); return kbase_hwcnt_metadata_create(&desc, metadata); @@ -294,7 +294,7 @@ static size_t kbasep_hwcnt_backend_jm_dump_bytes(const struct kbase_hwcnt_gpu_in { WARN_ON(!gpu_info); - return (2 + gpu_info->l2_count + (size_t)fls64(gpu_info->core_mask)) * + return (2 + gpu_info->l2_count + (size_t)fls64(gpu_info->sc_core_mask)) * gpu_info->prfcnt_values_per_block * KBASE_HWCNT_VALUE_HW_BYTES; } @@ -338,14 +338,6 @@ int kbase_hwcnt_jm_metadata_create(const struct kbase_hwcnt_gpu_info *gpu_info, return 0; } -void kbase_hwcnt_jm_metadata_destroy(const struct kbase_hwcnt_metadata *metadata) -{ - if (!metadata) - return; - - kbase_hwcnt_metadata_destroy(metadata); -} - int kbase_hwcnt_csf_metadata_create(const struct kbase_hwcnt_gpu_info *gpu_info, enum kbase_hwcnt_set counter_set, const struct kbase_hwcnt_metadata **out_metadata) @@ -365,14 +357,6 @@ int kbase_hwcnt_csf_metadata_create(const struct kbase_hwcnt_gpu_info *gpu_info, return 0; } -void kbase_hwcnt_csf_metadata_destroy(const struct kbase_hwcnt_metadata *metadata) -{ - if (!metadata) - return; - - kbase_hwcnt_metadata_destroy(metadata); -} - bool kbase_hwcnt_is_block_type_shader(const enum kbase_hwcnt_gpu_v5_block_type blk_type) { if (blk_type == KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC || @@ -384,6 +368,7 @@ bool kbase_hwcnt_is_block_type_shader(const enum kbase_hwcnt_gpu_v5_block_type b return false; } + bool kbase_hwcnt_is_block_type_memsys(const enum kbase_hwcnt_gpu_v5_block_type blk_type) { if (blk_type == KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS || @@ -416,7 +401,7 @@ bool kbase_hwcnt_is_block_type_fe(const enum kbase_hwcnt_gpu_v5_block_type blk_t int kbase_hwcnt_jm_dump_get(struct kbase_hwcnt_dump_buffer *dst, u64 *src, const struct kbase_hwcnt_enable_map *dst_enable_map, u64 pm_core_mask, - u64 debug_core_mask, u64 max_core_mask, size_t max_l2_slices, + u64 debug_core_mask, size_t max_l2_slices, const struct kbase_hwcnt_curr_config *curr_config, bool accumulate) { const struct kbase_hwcnt_metadata *metadata; @@ -466,9 +451,7 @@ int kbase_hwcnt_jm_dump_get(struct kbase_hwcnt_dump_buffer *dst, u64 *src, else hw_res_available = true; - /* - * Skip block if no values in the destination block are enabled. - */ + /* Skip block if no values in the destination block are enabled. */ if (kbase_hwcnt_enable_map_block_enabled(dst_enable_map, blk, blk_inst)) { u64 *dst_blk = kbase_hwcnt_dump_buffer_block_instance(dst, blk, blk_inst); const u64 *src_blk = dump_src + src_offset; @@ -581,7 +564,6 @@ int kbase_hwcnt_jm_dump_get(struct kbase_hwcnt_dump_buffer *dst, u64 *src, /* Shift each core mask right by 1 */ core_mask >>= 1; debug_core_mask >>= 1; - max_core_mask >>= 1; shader_present >>= 1; } } @@ -592,7 +574,7 @@ int kbase_hwcnt_jm_dump_get(struct kbase_hwcnt_dump_buffer *dst, u64 *src, int kbase_hwcnt_csf_dump_get(struct kbase_hwcnt_dump_buffer *dst, u64 *src, blk_stt_t *src_block_stt, const struct kbase_hwcnt_enable_map *dst_enable_map, - size_t num_l2_slices, u64 shader_present_bitmap, bool accumulate) + size_t num_l2_slices, u64 powered_shader_core_mask, bool accumulate) { const struct kbase_hwcnt_metadata *metadata; const u64 *dump_src = src; @@ -614,9 +596,7 @@ int kbase_hwcnt_csf_dump_get(struct kbase_hwcnt_dump_buffer *dst, u64 *src, blk_stt_t *dst_blk_stt = kbase_hwcnt_dump_buffer_block_state_instance(dst, blk, blk_inst); - /* - * Skip block if no values in the destination block are enabled. - */ + /* Skip block if no values in the destination block are enabled. */ if (kbase_hwcnt_enable_map_block_enabled(dst_enable_map, blk, blk_inst)) { u64 *dst_blk = kbase_hwcnt_dump_buffer_block_instance(dst, blk, blk_inst); const u64 *src_blk = dump_src + src_offset; diff --git a/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.h b/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.h index 4339fddd64e2..1f25282d378a 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.h +++ b/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -169,7 +169,7 @@ enum kbase_hwcnt_physical_set { /** * struct kbase_hwcnt_gpu_info - Information about hwcnt blocks on the GPUs. * @l2_count: L2 cache count. - * @core_mask: Shader core mask. May be sparse. + * @sc_core_mask: Shader core mask. May be sparse. * @clk_cnt: Number of clock domains available. * @csg_cnt: Number of CSGs available. * @prfcnt_values_per_block: Total entries (header + counters) of performance @@ -178,7 +178,7 @@ enum kbase_hwcnt_physical_set { */ struct kbase_hwcnt_gpu_info { size_t l2_count; - u64 core_mask; + u64 sc_core_mask; u8 clk_cnt; u8 csg_cnt; size_t prfcnt_values_per_block; @@ -261,13 +261,6 @@ int kbase_hwcnt_jm_metadata_create(const struct kbase_hwcnt_gpu_info *info, const struct kbase_hwcnt_metadata **out_metadata, size_t *out_dump_bytes); -/** - * kbase_hwcnt_jm_metadata_destroy() - Destroy JM GPU hardware counter metadata. - * - * @metadata: Pointer to metadata to destroy. - */ -void kbase_hwcnt_jm_metadata_destroy(const struct kbase_hwcnt_metadata *metadata); - /** * kbase_hwcnt_csf_metadata_create() - Create hardware counter metadata for the * CSF GPUs. @@ -282,13 +275,6 @@ int kbase_hwcnt_csf_metadata_create(const struct kbase_hwcnt_gpu_info *info, enum kbase_hwcnt_set counter_set, const struct kbase_hwcnt_metadata **out_metadata); -/** - * kbase_hwcnt_csf_metadata_destroy() - Destroy CSF GPU hardware counter - * metadata. - * @metadata: Pointer to metadata to destroy. - */ -void kbase_hwcnt_csf_metadata_destroy(const struct kbase_hwcnt_metadata *metadata); - /** * kbase_hwcnt_jm_dump_get() - Copy or accumulate enabled counters from the raw * dump buffer in src into the dump buffer @@ -300,9 +286,6 @@ void kbase_hwcnt_csf_metadata_destroy(const struct kbase_hwcnt_metadata *metadat * @dst_enable_map: Non-NULL pointer to enable map specifying enabled values. * @pm_core_mask: PM state synchronized shaders core mask with the dump. * @debug_core_mask: User-set mask of cores to be used by the GPU. - * @max_core_mask: Core mask of all cores allocated to the GPU (non - * virtualized platforms) or resource group (virtualized - * platforms). * @max_l2_slices: Maximum number of L2 slices allocated to the GPU (non * virtualised platforms) or resource group (virtualized * platforms). @@ -319,23 +302,23 @@ void kbase_hwcnt_csf_metadata_destroy(const struct kbase_hwcnt_metadata *metadat */ int kbase_hwcnt_jm_dump_get(struct kbase_hwcnt_dump_buffer *dst, u64 *src, const struct kbase_hwcnt_enable_map *dst_enable_map, - const u64 pm_core_mask, u64 debug_core_mask, u64 max_core_mask, - size_t max_l2_slices, const struct kbase_hwcnt_curr_config *curr_config, - bool accumulate); + const u64 pm_core_mask, u64 debug_core_mask, size_t max_l2_slices, + const struct kbase_hwcnt_curr_config *curr_config, bool accumulate); /** * kbase_hwcnt_csf_dump_get() - Copy or accumulate enabled counters from the raw * dump buffer in src into the dump buffer * abstraction in dst. - * @dst: Non-NULL pointer to destination dump buffer. - * @src: Non-NULL pointer to source raw dump buffer, of same length - * as dump_buf_bytes in the metadata of dst dump buffer. - * @src_block_stt: Non-NULL pointer to source block state buffer. - * @dst_enable_map: Non-NULL pointer to enable map specifying enabled values. - * @num_l2_slices: Current number of L2 slices allocated to the GPU. - * @shader_present_bitmap: Current shader-present bitmap that is allocated to the GPU. - * @accumulate: True if counters in src should be accumulated into - * destination, rather than copied. + * @dst: Non-NULL pointer to destination dump buffer. + * @src: Non-NULL pointer to source raw dump buffer, of same length + * as dump_buf_bytes in the metadata of dst dump buffer. + * @src_block_stt: Non-NULL pointer to source block state buffer. + * @dst_enable_map: Non-NULL pointer to enable map specifying enabled values. + * @num_l2_slices: Current number of L2 slices allocated to the GPU. + * @powered_shader_core_mask: The common mask between the debug_core_mask + * and the shader_present_bitmap. + * @accumulate: True if counters in src should be accumulated into + * destination, rather than copied. * * The dst and dst_enable_map MUST have been created from the same metadata as * returned from the call to kbase_hwcnt_csf_metadata_create as was used to get @@ -346,7 +329,7 @@ int kbase_hwcnt_jm_dump_get(struct kbase_hwcnt_dump_buffer *dst, u64 *src, int kbase_hwcnt_csf_dump_get(struct kbase_hwcnt_dump_buffer *dst, u64 *src, blk_stt_t *src_block_stt, const struct kbase_hwcnt_enable_map *dst_enable_map, - size_t num_l2_slices, u64 shader_present_bitmap, bool accumulate); + size_t num_l2_slices, u64 powered_shader_core_mask, bool accumulate); /** * kbase_hwcnt_backend_gpu_block_map_to_physical() - Convert from a block @@ -453,6 +436,7 @@ bool kbase_hwcnt_is_block_type_memsys(const enum kbase_hwcnt_gpu_v5_block_type b bool kbase_hwcnt_is_block_type_tiler(const enum kbase_hwcnt_gpu_v5_block_type blk_type); bool kbase_hwcnt_is_block_type_fe(const enum kbase_hwcnt_gpu_v5_block_type blk_type); + /** * kbase_hwcnt_gpu_enable_map_from_cm() - Builds enable map abstraction from * counter selection bitmasks. diff --git a/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_types.c b/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_types.c index 3d0ad5af7263..3d2fd5e088da 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_types.c +++ b/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_types.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -125,6 +125,9 @@ int kbase_hwcnt_metadata_create(const struct kbase_hwcnt_description *desc, void kbase_hwcnt_metadata_destroy(const struct kbase_hwcnt_metadata *metadata) { + if (!metadata) + return; + kfree(metadata); } diff --git a/drivers/gpu/arm/bifrost/mali_base_hwconfig_features.h b/drivers/gpu/arm/bifrost/mali_base_hwconfig_features.h index 1f32fc9dd553..843094076d69 100644 --- a/drivers/gpu/arm/bifrost/mali_base_hwconfig_features.h +++ b/drivers/gpu/arm/bifrost/mali_base_hwconfig_features.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -27,6 +27,8 @@ #ifndef _BASE_HWCONFIG_FEATURES_H_ #define _BASE_HWCONFIG_FEATURES_H_ +#include + enum base_hw_feature { BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, @@ -45,55 +47,55 @@ enum base_hw_feature { BASE_HW_FEATURE_END }; -__attribute__((unused)) static const enum base_hw_feature base_hw_features_generic[] = { +__maybe_unused static const enum base_hw_feature base_hw_features_generic[] = { BASE_HW_FEATURE_END }; -__attribute__((unused)) static const enum base_hw_feature base_hw_features_tMIx[] = { +__maybe_unused static const enum base_hw_feature base_hw_features_tMIx[] = { BASE_HW_FEATURE_THREAD_GROUP_SPLIT, BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_END }; -__attribute__((unused)) static const enum base_hw_feature base_hw_features_tHEx[] = { +__maybe_unused static const enum base_hw_feature base_hw_features_tHEx[] = { BASE_HW_FEATURE_THREAD_GROUP_SPLIT, BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, BASE_HW_FEATURE_END }; -__attribute__((unused)) static const enum base_hw_feature base_hw_features_tSIx[] = { +__maybe_unused static const enum base_hw_feature base_hw_features_tSIx[] = { BASE_HW_FEATURE_THREAD_GROUP_SPLIT, BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, BASE_HW_FEATURE_END }; -__attribute__((unused)) static const enum base_hw_feature base_hw_features_tDVx[] = { +__maybe_unused static const enum base_hw_feature base_hw_features_tDVx[] = { BASE_HW_FEATURE_THREAD_GROUP_SPLIT, BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, BASE_HW_FEATURE_END }; -__attribute__((unused)) static const enum base_hw_feature base_hw_features_tNOx[] = { +__maybe_unused static const enum base_hw_feature base_hw_features_tNOx[] = { BASE_HW_FEATURE_THREAD_GROUP_SPLIT, BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, BASE_HW_FEATURE_TLS_HASHING, BASE_HW_FEATURE_IDVS_GROUP_SIZE, BASE_HW_FEATURE_END }; -__attribute__((unused)) static const enum base_hw_feature base_hw_features_tGOx[] = { +__maybe_unused static const enum base_hw_feature base_hw_features_tGOx[] = { BASE_HW_FEATURE_THREAD_GROUP_SPLIT, BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, BASE_HW_FEATURE_TLS_HASHING, BASE_HW_FEATURE_IDVS_GROUP_SIZE, BASE_HW_FEATURE_CORE_FEATURES, BASE_HW_FEATURE_THREAD_TLS_ALLOC, BASE_HW_FEATURE_END }; -__attribute__((unused)) static const enum base_hw_feature base_hw_features_tTRx[] = { +__maybe_unused static const enum base_hw_feature base_hw_features_tTRx[] = { BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, BASE_HW_FEATURE_IDVS_GROUP_SIZE, BASE_HW_FEATURE_CLEAN_ONLY_SAFE, BASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER, BASE_HW_FEATURE_END }; -__attribute__((unused)) static const enum base_hw_feature base_hw_features_tNAx[] = { +__maybe_unused static const enum base_hw_feature base_hw_features_tNAx[] = { BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, BASE_HW_FEATURE_IDVS_GROUP_SIZE, BASE_HW_FEATURE_CLEAN_ONLY_SAFE, BASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER, BASE_HW_FEATURE_END }; -__attribute__((unused)) static const enum base_hw_feature base_hw_features_tBEx[] = { +__maybe_unused static const enum base_hw_feature base_hw_features_tBEx[] = { BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, BASE_HW_FEATURE_IDVS_GROUP_SIZE, @@ -103,7 +105,7 @@ __attribute__((unused)) static const enum base_hw_feature base_hw_features_tBEx[ BASE_HW_FEATURE_END }; -__attribute__((unused)) static const enum base_hw_feature base_hw_features_tBAx[] = { +__maybe_unused static const enum base_hw_feature base_hw_features_tBAx[] = { BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, BASE_HW_FEATURE_IDVS_GROUP_SIZE, @@ -113,31 +115,31 @@ __attribute__((unused)) static const enum base_hw_feature base_hw_features_tBAx[ BASE_HW_FEATURE_END }; -__attribute__((unused)) static const enum base_hw_feature base_hw_features_tODx[] = { +__maybe_unused static const enum base_hw_feature base_hw_features_tODx[] = { BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, BASE_HW_FEATURE_L2_CONFIG, BASE_HW_FEATURE_CLEAN_ONLY_SAFE, BASE_HW_FEATURE_END }; -__attribute__((unused)) static const enum base_hw_feature base_hw_features_tGRx[] = { +__maybe_unused static const enum base_hw_feature base_hw_features_tGRx[] = { BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, BASE_HW_FEATURE_L2_CONFIG, BASE_HW_FEATURE_CLEAN_ONLY_SAFE, BASE_HW_FEATURE_CORE_FEATURES, BASE_HW_FEATURE_END }; -__attribute__((unused)) static const enum base_hw_feature base_hw_features_tVAx[] = { +__maybe_unused static const enum base_hw_feature base_hw_features_tVAx[] = { BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, BASE_HW_FEATURE_L2_CONFIG, BASE_HW_FEATURE_CLEAN_ONLY_SAFE, BASE_HW_FEATURE_CORE_FEATURES, BASE_HW_FEATURE_END }; -__attribute__((unused)) static const enum base_hw_feature base_hw_features_tTUx[] = { +__maybe_unused static const enum base_hw_feature base_hw_features_tTUx[] = { BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, BASE_HW_FEATURE_L2_CONFIG, BASE_HW_FEATURE_CLEAN_ONLY_SAFE, BASE_HW_FEATURE_L2_SLICE_HASH, BASE_HW_FEATURE_GPU_SLEEP, BASE_HW_FEATURE_CORE_FEATURES, BASE_HW_FEATURE_END }; -__attribute__((unused)) static const enum base_hw_feature base_hw_features_tTIx[] = { +__maybe_unused static const enum base_hw_feature base_hw_features_tTIx[] = { BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, BASE_HW_FEATURE_L2_CONFIG, @@ -149,7 +151,7 @@ __attribute__((unused)) static const enum base_hw_feature base_hw_features_tTIx[ BASE_HW_FEATURE_END }; -__attribute__((unused)) static const enum base_hw_feature base_hw_features_tKRx[] = { +__maybe_unused static const enum base_hw_feature base_hw_features_tKRx[] = { BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, BASE_HW_FEATURE_L2_CONFIG, BASE_HW_FEATURE_CLEAN_ONLY_SAFE, BASE_HW_FEATURE_L2_SLICE_HASH, BASE_HW_FEATURE_GPU_SLEEP, diff --git a/drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h b/drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h index 4426bd743b4e..409e2e8bedcf 100644 --- a/drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h +++ b/drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -27,6 +27,8 @@ #ifndef _BASE_HWCONFIG_ISSUES_H_ #define _BASE_HWCONFIG_ISSUES_H_ +#include + enum base_hw_issue { BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, @@ -72,13 +74,14 @@ enum base_hw_issue { BASE_HW_ISSUE_KRAKEHW_2151, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_KRAKEHW_2269, + BASE_HW_ISSUE_TURSEHW_2934, BASE_HW_ISSUE_END }; __attribute__(( unused)) static const enum base_hw_issue base_hw_issues_generic[] = { BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tMIx_r0p0_05dev0[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tMIx_r0p0_05dev0[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_10682, BASE_HW_ISSUE_11054, BASE_HW_ISSUE_T76X_3953, BASE_HW_ISSUE_TMIX_7891, BASE_HW_ISSUE_TMIX_8042, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TMIX_8138, BASE_HW_ISSUE_TMIX_8206, @@ -88,7 +91,7 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tMIx_r0p0 BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tMIx_r0p0[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tMIx_r0p0[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_10682, BASE_HW_ISSUE_11054, BASE_HW_ISSUE_TMIX_7891, BASE_HW_ISSUE_TMIX_7940, BASE_HW_ISSUE_TMIX_8042, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TMIX_8138, BASE_HW_ISSUE_TMIX_8206, @@ -98,7 +101,7 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tMIx_r0p0 BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tMIx_r0p1[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tMIx_r0p1[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_10682, BASE_HW_ISSUE_11054, BASE_HW_ISSUE_TMIX_7891, BASE_HW_ISSUE_TMIX_7940, BASE_HW_ISSUE_TMIX_8042, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TMIX_8138, BASE_HW_ISSUE_TMIX_8206, @@ -108,7 +111,7 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tMIx_r0p1 BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tMIx[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tMIx[] = { BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_7891, BASE_HW_ISSUE_TMIX_7940, BASE_HW_ISSUE_TMIX_8042, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TMIX_8138, BASE_HW_ISSUE_TMIX_8206, BASE_HW_ISSUE_TMIX_8343, @@ -116,7 +119,7 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tMI BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tHEx_r0p0[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tHEx_r0p0[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_10682, BASE_HW_ISSUE_11054, BASE_HW_ISSUE_TMIX_7891, BASE_HW_ISSUE_TMIX_8042, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, @@ -124,7 +127,7 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tHEx_r0p0 BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tHEx_r0p1[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tHEx_r0p1[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_10682, BASE_HW_ISSUE_11054, BASE_HW_ISSUE_TMIX_7891, BASE_HW_ISSUE_TMIX_8042, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, @@ -132,7 +135,7 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tHEx_r0p1 BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tHEx_r0p2[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tHEx_r0p2[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_10682, BASE_HW_ISSUE_11054, BASE_HW_ISSUE_TMIX_7891, BASE_HW_ISSUE_TMIX_8042, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, @@ -140,21 +143,21 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tHEx_r0p2 BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tHEx_r0p3[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tHEx_r0p3[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_10682, BASE_HW_ISSUE_TMIX_7891, BASE_HW_ISSUE_TMIX_8042, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tHEx[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tHEx[] = { BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_7891, BASE_HW_ISSUE_TMIX_8042, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tSIx_r0p0[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tSIx_r0p0[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_11054, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TSIX_1792, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TTRX_3464, @@ -162,7 +165,7 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tSIx_r0p0 BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tSIx_r0p1[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tSIx_r0p1[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_11054, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TSIX_1792, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TTRX_3464, @@ -170,77 +173,77 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tSIx_r0p1 BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tSIx_r1p0[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tSIx_r1p0[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_11054, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tSIx_r1p1[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tSIx_r1p1[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tSIx[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tSIx[] = { BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tDVx_r0p0[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tDVx_r0p0[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tDVx[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tDVx[] = { BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tNOx_r0p0[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tNOx_r0p0[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TNOX_1194, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tNOx[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tNOx[] = { BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tGOx_r0p0[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tGOx_r0p0[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TNOX_1194, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tGOx_r1p0[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tGOx_r1p0[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TGOX_R1_1234, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tGOx[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tGOx[] = { BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tTRx_r0p0[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tTRx_r0p0[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_2968_TTRX_3162, BASE_HW_ISSUE_TTRX_3076, BASE_HW_ISSUE_TTRX_921, @@ -251,7 +254,7 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tTRx_r0p0 BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tTRx_r0p1[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tTRx_r0p1[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_2968_TTRX_3162, BASE_HW_ISSUE_TTRX_3076, BASE_HW_ISSUE_TTRX_921, @@ -262,7 +265,7 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tTRx_r0p1 BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tTRx_r0p2[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tTRx_r0p2[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, @@ -280,14 +283,14 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tTRx_r0p2 BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tTRx[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tTRx[] = { BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_3414, BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tNAx_r0p0[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tNAx_r0p0[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_2968_TTRX_3162, BASE_HW_ISSUE_TTRX_3076, BASE_HW_ISSUE_TTRX_921, @@ -298,7 +301,7 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tNAx_r0p0 BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tNAx_r0p1[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tNAx_r0p1[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, @@ -316,14 +319,14 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tNAx_r0p1 BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tNAx[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tNAx[] = { BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_3414, BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tBEx_r0p0[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tBEx_r0p0[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_2968_TTRX_3162, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_TTRX_3414, @@ -333,7 +336,7 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tBEx_r0p0 BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tBEx_r0p1[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tBEx_r0p1[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, @@ -349,7 +352,7 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tBEx_r0p1 BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tBEx_r1p0[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tBEx_r1p0[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, @@ -365,7 +368,7 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tBEx_r1p0 BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tBEx_r1p1[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tBEx_r1p1[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, @@ -381,14 +384,14 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tBEx_r1p1 BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tBEx[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tBEx[] = { BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_3414, BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_lBEx_r1p0[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_lBEx_r1p0[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_2968_TTRX_3162, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_TTRX_3414, @@ -398,7 +401,7 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_lBEx_r1p0 BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_lBEx_r1p1[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_lBEx_r1p1[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, @@ -414,7 +417,7 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_lBEx_r1p1 BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tBAx_r0p0[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tBAx_r0p0[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, @@ -430,7 +433,7 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tBAx_r0p0 BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tBAx_r0p1[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tBAx_r0p1[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, @@ -446,7 +449,7 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tBAx_r0p1 BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tBAx_r0p2[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tBAx_r0p2[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, @@ -462,73 +465,56 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tBAx_r0p2 BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tBAx_r1p0[] = { - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tBAx[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tBAx[] = { BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_3414, BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tODx_r0p0[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tODx_r0p0[] = { BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3212, BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tODx[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tODx[] = { BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3212, BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tGRx_r0p0[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tGRx_r0p0[] = { BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tGRx[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tGRx[] = { BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tVAx_r0p0[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, - BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tVAx[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tVAx_r0p0[] = { BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tTUx_r0p0[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_1997, - BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2019_3901, - BASE_HW_ISSUE_GPU2021PRO_290, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END +__maybe_unused static const enum base_hw_issue base_hw_issues_tVAx_r0p1[] = { + BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, + BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, + BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tTUx_r0p1[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tVAx[] = { + BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, + BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, + BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r0p0[] = { BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_1997, @@ -539,79 +525,96 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tTUx_r0p1 BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2938, + BASE_HW_ISSUE_TURSEHW_2934, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tTUx[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r0p1[] = { + BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_1997, + BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2019_3901, + BASE_HW_ISSUE_GPU2021PRO_290, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, + BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2938, + BASE_HW_ISSUE_TURSEHW_2934, BASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tTUx[] = { BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_GPU2021PRO_290, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_TURSEHW_2934, + BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tTUx_r1p0[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r1p0[] = { BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_GPU2021PRO_290, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_TURSEHW_2934, + BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tTUx_r1p1[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r1p1[] = { BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_GPU2021PRO_290, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_TURSEHW_2934, + BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tTUx_r1p2[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r1p2[] = { BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_GPU2021PRO_290, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_TURSEHW_2934, + BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tTUx_r1p3[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r1p3[] = { BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_GPU2021PRO_290, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_TURSEHW_2934, + BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tTIx[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tTIx[] = { BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2021PRO_290, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2952, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_TURSEHW_2934, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tTIx_r0p0[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tTIx_r0p0[] = { BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2021PRO_290, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2952, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_TURSEHW_2934, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tTIx_r0p1[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_2716, - BASE_HW_ISSUE_GPU2021PRO_290, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END +__maybe_unused static const enum base_hw_issue base_hw_issues_tTIx_r0p1[] = { + BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2021PRO_290, + BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, + BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, + BASE_HW_ISSUE_TURSEHW_2934, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tKRx_r0p0[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tKRx_r0p0[] = { BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_KRAKEHW_2151, BASE_HW_ISSUE_KRAKEHW_2269, BASE_HW_ISSUE_END + BASE_HW_ISSUE_KRAKEHW_2151, BASE_HW_ISSUE_KRAKEHW_2269, BASE_HW_ISSUE_TITANHW_2922, + BASE_HW_ISSUE_TURSEHW_2934, BASE_HW_ISSUE_END }; -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tKRx_r0p1[] = { - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_KRAKEHW_2269, BASE_HW_ISSUE_END -}; - -__attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tKRx[] = { +__maybe_unused static const enum base_hw_issue base_hw_issues_tKRx_r0p1[] = { BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_KRAKEHW_2151, BASE_HW_ISSUE_KRAKEHW_2269, BASE_HW_ISSUE_END + BASE_HW_ISSUE_KRAKEHW_2269, BASE_HW_ISSUE_TURSEHW_2934, BASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tKRx[] = { + BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2022PRO_148, + BASE_HW_ISSUE_KRAKEHW_2151, BASE_HW_ISSUE_KRAKEHW_2269, BASE_HW_ISSUE_TURSEHW_2934, + BASE_HW_ISSUE_END }; diff --git a/drivers/gpu/arm/bifrost/mali_csffw.bin b/drivers/gpu/arm/bifrost/mali_csffw.bin index 1f8413ba14d7310dd3c6b254e16c3244305b1980..a1168ffdc7436dd69b31f643f2a77b9fc993d6e0 100644 GIT binary patch delta 61395 zcmb5W3w%>W`ae8#k|rm;(6p3NN`;f&NTEPLE=5sJ(-I0rTP|Wzlpvr1QNu;8u4_&a zrQ)r45R0e_F1qNt+M=$Mi!F)@ikbi_3c6uefwsHuX(;Jw(#!julN4}o@9*EwC!I5M z=JL!l&pgjF&-2WY{ANi?s>zPYP9kL6#=OCXZFoq@**GITTC{AiAti^9bqhCcNH

peIC>W9bK6Mt}J;`U(}~ySNc_yu5`QMziGuc(%$G-2j=>4WapX zx^jrEe<7iSkUQM>F1%~$npMjduO>z_k$y@jBl9ujF1D`oJuS{Pt>zOutn^r@#Jt?^fyg1K-e*y1O3G9>y4#~2i;+Hr9aEzBoFMi~mjRO|)JZfuya@!RX z|5uEuB8;HXD_PS&l4sORh(z31QSCjWY5Z~C_c$$27(p|X z2$A{~1M~#%IO{M%9zuUa#Crknw?*=&IEk$MqeRXi{dtGvArHk!&HJS zk`Eze`TOD|QvZ@iA|Hqnx&LK}JX|P|FJ6~OJ5rWUE{b%tJesbHrfJ2I@^7MPiX+l- z1ya`Wo@l}_M$)H|GC;i1{Jfcw@)x7& zhtV|ix@fy-`XW-+?t7&7z9Es^>mvidhLnwCD+96rP9l7{L_9wOfNSxV#lwsuiXujX#NInB8s~3AmBwdgaDX&Dz%3qA8w!x9|Ymu_@ z1rJMP*;^9Xy*84jJ`zc5qiKp~gG2_^0rg1l!Y(+0^a3rDcYPSaNc=~UCE0vXA}_rsktaWmlzTD`MP95x z%4YuJXOXlWDeE}%^GL-F(Xu9Z5?N)4q|ZjvebF@46e+(c znpQ{C&!TC1Y~;Nony!kbN1|yknr6mD>b;R%MV=emfc+14&DL6gP+}<$!Nis#6)Br3 zmsh2sIMb%RXEJb*yeWI;@Vs$gkKnaK6n$b^qkGyv&-2pA=C(3jNs~&v$;zL)Om$>G zY4pA)$L4JJ#2gHen4PYQcgljb6{SI^qpY1wDpN@8SHa%OrT~}Aw@Go5F74}&RZ=s~ z^@&QhZF33r)SBosDf2qx{U&8jXNt-v=q!J170sCrp;#>5SwNL5I(NE=^|bH0;BN5` zZSi_yL2m+ezUtV2PI+czvr?vedt;qvh2x&{oc>iuX>iFRDv)ecvK$O>dfr_cTp_Y1 zxh_&ktwiaP$S9)xO^V&=2#Vv&+BcSxws@+yH3dixso5NQO`#j>LQ9`LfWWls-FnF zV(cPTQMpTc!e11qiO~rwD>tj8DYk}T-qY$zckFbvx@3FEN49>{Ot?EyVs+iAmC&u% z1)rRXVRG1;FPak%lm&ZN?_5S)=bYCC-_r2Yc)sizTFp61GScj2N`{Vatj;h?R!8g` zNlLfM-tKfQTgE#F`?sisl(i=BdaL1BtX1zNN=^^jD5c$W9G`#GX99MrMJtfXFt{5yVp#OF zFL>Kf8}l-?&=64`D`Ja8MzKZmv`=_@MzGxNVJJLMnY5RY39&FjVI|ZFdT?q_?g+tL zACSy?OG!|gO|rfW5YiO5nM18WIi5Iyx0w}huX2W14!CJ0$@|tTRTArQUzs8~sfAbe zhQ2%-A-Sr$K8m|8_;;~s02>Sqcn8@y&Fcv8rqX~ECy&crPM?u~mpe4&KM}8dJoj#W zX@J;d$1oeWLX<0q-CZ!QtJq)e=%>7_RyY!rA3_xl%sP`Q)-F~mtEB@?ceQ*>zjcq#;j^y>R( zz<2m14wh%s$XZ9Z2AhPWCQD`;Ro>`i_?_9CU!0&cb;&bF2)1o11L54A#L~RDYVp%& z7^));o0>@y!_O`Z!j@x+gDN+L=QlPlLas*Ao< z=W=cUcU_Nw|M&E4QY$$6GMkjBP_j;~kQe4pqTA#x`Bz*k_arEM7v6~n)fdUnKSw6J zz|prj;xZ6P(ihFKNnk>SD7Ba>X`ap(`?_A4)hoB>PmX`4(=7bn@iJ>VMxHz}pIhB2 zFCUqm?(8gU-{u_UU>&ENRo*KNoVr2pC=L8nF7F&U+fxNz8{w=3dqhw9O5dR{Fsq(OKypPp6Y#4K#PUJ>^_gx7Va$INm-ZIBPRY)`| zT%r*?shbeXmb5)ANE$WqP z_R^94eI!=^Hy*3)1)*|LCCN-umh^zsoq|Tp`_Hku?fcyO!P*Mf)#)A5ax7rHKFFuZ znYPEcgdSOXI@QzLu>%CDuLD?xdgc550{C{4Dh33xDNsmi&h;ZrE?v!2NS8TKPr0RA zt6Rl-1>IaE7Je^)CC62M{&6*aS)4V=x9zS9kOo(IV+#*mPXN}lx|;?sYUL?~iu>5k2h0FTqakE*kq_^ ztlK}JUT^^VUi1uJwEtWk=KJagjFCT4R~qQj7Jx}8qHL9FLfe?+4)1N_6Nw^AhMmBL ziMQ!%N(1p8C)-!$>IjJH`@PYsp*^VT25H1=Z%OMDB&G&?!G@hs2swG%7tQ>ZFPaTo z!~oHWLD!Ag7Xp4+abZeh!<@n|!2guv>LIH+Y%I`UNVwy;dZeIMwUAf7I4b;Ekt!l|1;$tNRtNd3C|i zk-U#b-z}=V5D2WeQTkaw3e9b)by<6mRKE8elOi5 zl0f=|v685~Bw^Rv$py!648uuaqx6?^oHAh8*c+DVVGw9O0O6ZPsUP1xqrtN@jB5DJMbQB<1hjclgmwR*h=av62`? z?2WPZ_b%`g3lNpWb`)35D zRgfF1wLkEYNiLME?C+S3#b{Eqv*zQ!a3!y-Rfqd6m7oX?6fwuE3OZ0kW>Q#Z*`)54 zZy7)4>MO#6W0K=0Y_bt*Oo&(VT8N2ZtRbQlsh;+!Z_|m!(Vq6vZ59pJbCX15+H_ItI%}!3Z$D@p;&hx)p5?8CKz^&$`x{=0XM6fu%XyypF zv!}z+t@Cp`Ee`fnvp2T(hh{!MChMJ6vnS50OQa))ctwxYs5q>iDoOYA zTzo6FEC88LcL8JnRl}3Zx2MQQt{&AdqESN0N!Bz;ezM5j+$GPLTsiO!^8*cD9BzrF zzO}|<@})VpfFSeb?z%aJVq03J_Y6W(_3lH$uI$X%>j^-i%R+#foLi(z_ z@|pp3zr6mME7CdTP`I~)hKcZxvl@YxU7X;)Yn+_8QXX>c?U>-dOb=5#fhnFUp+esT z+0-JFWHj}kR9H5_bf(2&S}RL48GSI?!9;g64OdpLo&2wJw@(>IuafVdGK_n?SKc-y zjf-C_@0pUB*^m~aaGgS>Ic$=8JB+H;5yJ%GBUErtqn1Qv!*2Ou@yoR8-W6_Ir!Yxc zw!UsHv?XFI4f=1CmlO&d>6RZW92fgD&T;2(?nCLp!fR8$>0-y6Qs37qZ~D5K&-z-8 zA0(&R^B3OV^C;BO=hTM@lW~tCwK|?t86bMqS_!r>%N%A0oJ(q**{4$q!-;H!Ku$0V%ESz@zXm=GjnLaUmwQgtIB3WEtjZVQ#MM7Q*Kqu z6dJGdA5@(ZY!}kk%{t4+&AMu9WohvE4f2m=GudG%zdYM8zl$9o<$2{p4AUZy50+;d zKIzsy*R))Y^9VPMVV2yqq0bFRX`IGl|NiF0ay*Op~*N0Gl4 z`F=lQLsr`j-hOR-zcW)ij!}Uqaq5q%9=oe{5gUW`L7ft;DiPfT{KqT?CGiSJ3KaT6 zVpcZxULUmH_|Nk(5)QA%rd`*%60M$oClUjPR{rH2QErOk&Ev$GanZku#9#`&-Y*94 zwj~B}RE#_Yj<1Y#)*AA-214b{XiN+oahsLDh2C%LouWI1-jF*Lsw5M>jy!)8vVlat z4YU6n!b79|NRfU#;QH*v!xvvHhz_ta@~Un7w>n?qSw^!r>`#>&2pmOUBLx25R%eQp<)v*JnSNY$#)*UkF= zJC_OLhdWp~A1%K(>_E?n5<53yZnUM{+lIwjuXZH0LSr{6?)ci@|%M^iqQSHnSMuqZLHtcOZX03V8wYJCV0W-`kLX z6Zu=*k?yU?*CGFKG%p~(TlhxZXA^_JkAjy9tUUv~qeTo*oY*OcZWuCui;ax3-~`4u z&WhqQsNR5utiVFHsdU2o_7)$bi>2Mg0b)29&>48Y71$1}1@adp zSJhNcd+04O__4fl-T-<|UO#W>;CPW8xC=?SYTX94Qj9*=iGgQTz~t!cpTt5BL2EuDO^ z^bV8?k@i7$NG|kX8n>WuTePs66{2ho%63G{^wF~EC~JzAO^TLHLD|7bSwM~gG#+I~ zqGj(pBmIp)nK#<@rf6FW$~?!T72k_C9*DwI(Xx}##tA6%N6TJ_uCfkgKSj%0L^epG z(%EZsoYUqkIO4$h+Iq2{1GiknaH4Xqmrkh1DqZt7GjHr}(|Hm-?TK$gXlq@Vm5MGb zQi>S-S%CQ#w#q`X*p=x@RF3p6UXh9Yk!gzzb+GrisvmP)B^EKbM2M8_L0K`%M3fn# zWp%w`usquL@UEhOpcez5yZidKxIBrm~7~Y>M22!H$S0nElC>EW$^_r=3dpxa`q;cEFBmms!K!SK4F#tg0B61FvVpOsI{|9B`s z;sm`bW?PQy^<|05`#rBO;KV?j> z7)TSZ3)aDkR!Gp3g_#V}AVsbVzAMUUHw$KiQmPV#sMiO~1oGVePRY&RP!E&kkJJ@< zIFHW3)zC?W_aELx^SJMHOfz~$=i`QV`ncK@KQ}zps~b*~N=ZLl@o_KebAD~PAy6~( zLb>+~4y|R`Rj0XH4t4afwp_CBuv)31(t!Vl-1`A!j@46Hm_e+0?i37C8aOu_g_+p+ zuE$-(Wp?s7A^bFp&4z~+{8aRbtF3`;ikK00o18POW0d+w_l`6$=S8Ygl z#&7(eC}+r~TMpE7wO%!!t5L@crdS9GX`l0gR?r_6^lW;4p&z|4eK87X=(u+Ex99ShvBYuO%1aSK1tOBT;HXj2{?+1rtbi zWbsLkZZli-Q4Z8&%Z?)u_c&YjK{2f+qbUc=ysaSWdtx58f!?k1GxrSi^eu9pTF|%1 zB31VGoK&4;yo0YdDWCRkRbMB%m#LNa6=3OIs=59N^>vZ=vvO;_v27={ft}QJ5&s%@ zf#dV$gT*FgZEt~tvw%p%^=dXm+1m&O&immPKh00sRx8w1GOH%GpU^){H4h*s^I;lyQmr8V zdalycrZ+Bjn7L~z@*MM3;{162Q}#|+xXHwlVhlLU*vci(e#on?@x#;Mro+=3N41cC zsX%arhhxU@lQ;sOZ03iZ#1VM0;{nkwCKbI4`!%sNoIR<|{p-o*{6&i$Lqkt@Ep*(d z@`(VZnJhB4Z6{<4$xCZAwpmFHJj;99^ocp%J#9wqe|%!L?^@SM^$>qjJ!0Le{w3`s zOeN(LuABtKaa`U>)ob+>xYjN$Q{1<`+!%KuVbOF~;q6?@fMk11JZD$POuc`mA}yf! zPZkuoh^2%Ld|<2EfbJe5Q(Oc#Cr(M^>@BpAC>)2gDk*H?3JY6^s4jV#irp|*t4lrYsc+NT!hqR`-i`8~SD)b^0bg9>5F$<;cugw5Xr?h=s0DtL zm-%xu9SU@vUsRd9ceoy>K~=WhyABVsD(}8`6doQ_<(7M|$HOdDo_t?E9ww>siu=ZS z>fXED{_n0g6FI4GQhMVjO>_7{3)>F(sAsmuk<7KoEZ3NPBsobTvi6-Y;)u{Z^`vU+7EIP zuX+Feb7rxtqY=zr%tYuDDlsSfm~R2SYwYpA3(!m64r)!t z7C2RoHjI%%dNIYpoZC{<0luG$zx%KOjN_785>dp1&ZlZ9j&wU2J^|7_ON)|6{^5 zc9tVa-H~VqZerb>_xtADETjMXW}21k`b@1A2D?6M9(LtC2eHmmB@gd@q=loyT40=2 zq&sw@eC3)uxFs~P-sTeEU$;3?qWrO&55@eet^2}$`7a9xcv{1`_tQ+Fk|;xI(|Nm@ z!%l*>s4WFZ-j;>V?I3+7PX_b|9?m-6N4BuspTjo4)5GQHa%q7R3e+WaNoVP&G@{=t z{5c-ouN-CU&92Kw^!)z}c_zdtC8FVgR!BypW(j*HY7r~z>Ue)I1BR+Cn)rOY?hyy^ znCnfN9*AYuEOlTnSR$R(>7~{-AIYXU6;M}T%4qbEMr{DVbw~IKH3MtgH5+R@mz9D( zn0+(54>t@SF~HBDA`K(1Nx-oUBF|RH7YWXjFgnq@sFFyZP%UPKfas>7{E|ZX?AgoVzXU#i z5&S;T{}pUZOH)}Q;4{y3Q^3PnN-C+X#BKf6Y^+W(a2!&J9euOSbet}Kqxz=6wNb7L zX*-09p2S^kA_n}o3$yt~8LYKq&c?ufFQ9cH7C;hnfXS8ECH`n-7Wfg(n-G%Fq^>hpGm)0W>eTc-nV1~@yqSk(H=;{#RXt+}wLJl^bDx?b!izlx z>^KrXLmg5zwN*96ef`IF}ofkyr7R55g(L>L0~SQdfis0Fa&1y$Zr`L-9y0RPG6VaZf*p64idlQT9< z+#BuM{P5P`WnK!*%#1FCSSLg2HehPU#l5FOBVX}xUEBff$(8JW!cbzp;DnH*y8M1_U02!yp9;prsV0^A0#h1kB;bVY1*ZJ%og zMTGyfPx(_dqp|j@|D-X>GL!O~eGz4riE7b9a;PoB)45Duxlx-+2O_pe9QgS~(Pijc zh(TD0I#X*LpO99oPOxq3!#q1J5ao6WwlOy=i|naEY8p3(u)W7Id=D5=K2lEphx zJw(~oGZk(Xg7BjYI4$A?tqkg%94+~Kw8Y$5i``Pamvzf{1nYXXwuF6MlgtDqjN0fY z6P<4(%2Pdq{1}wbOvDr{20pSFdt}lMZ3HIybggUUukT0%3|qV7E+pN#S0ldPyxk{i zdO8k8T~O5yi1!+z3|xCjC1KGziEG)W0HMdQKsyNc=^*%dG#mc-4v=ilEvLT+-(jMN zqW1Sya3^7&YG;*$N$G1k=Q-eq_9V_zUrbyADzQQB_=^O{-`V$ISZt{IV!6l{iOK^~LiT?i_S#-HWVyqvK0GuDb`As#;W$m* z<&GENVuO=X?E@tWh&kMGsCO=aC|9MS-eyDAP2yS)f)ZlIU2vMjoU%Y?rJ4#e88Aiw z6RQ|7TsA}Rzk|WCh3{ctyuiYEfM#pezYfY&myFY_7y2K1@v!O?-JF0+FM`77x?)bp z{f9Znq1MORU<|#i-KBZmf+Zlz=AJWtrjrrnhi)4bxNYP=V|B3aB2GR3vX;;F4G_bK zDLP1wIcz8ocZ}%0!vEg@d*FYxOVofX(I`WgjlSmfQABCbvqEI3#SXz?mRh;SIuEtg z-S5VK_x)@Aq?WN^Hcw_NFllP55o#-D3&4*#2CKF>1{{H=7z1%ih^YV(!huSOK7XTw zW@8dJ41|Kql>bR?qYtVrwxOtS~6G_h~HvuzFx3VLjNYj2e!0_fbPlnBfQUWZ37C{S9p7(1HRt{T(yJ=`<~zC}+C# zA*1pyVYeNg_Jwa7HRTisqiImdD%e>>56dik=mE}p!oyIe9b{|<0mj$}F2ul#CWHYh z=2&n6Q*6k1VZ1%oJ6|0J#k_i&P=5lgT;d`8f2)X$;u~QFUC*NNnvb!kK05`|`65F_ zls+hVV+f8hg%rnSf!cJ|XI1&j74aU6#|ar=&(P=Hc7H}A2U&0&#f-9)45LS$X z$Q?wEY5f!u;;Il6b`c3dH|1(J4PdDxhDit^#*rH4>$#1HHZhe%g)&OY5jx6)hr_gx zkodUz`Zf|+izuX67*`73qsxU_y$w zro-+AinLK}Byj;2eUWUt+nRe*=n@mZ5lpau1o%I2T|FWrEMZes zgI(`6hhyD1-7BMd>9xiSNp~ndwyq2ex!;4Sp7g#x9xN5KUZ$S|MQQy!NNTLusj-C( zb{^fB+%@X=X@`ya#zJ)0(-Y}V->8KZu5V_CCDk>qa&S!X{9{ z1{Nyw9Of|ZEGR}Gf2dD=I?tf4VwMa2Wyf;8)#%&ip#|G;h-4NDMDjFHD{XY5%z1y^ z{_|($^xp`@)4Tt>k;5$-l*bzX{j`~&T!qsn>((vyc09aCe)u<8bdmhZZ)Ogew5K${ zJE^5KpkKk59e(+w($YdW(7_@({vmI2WK9tdD#cU^4x_R=EdTOQF1NW?PW`Pou_28rw;<#!b~NYXm6~48&smJhzEJEa z{sa}oKzt#rGxiJwixFkC$_OLRhj(oER&k@^w4l==O5xHoP*%%RO;Dcd zl&^m{+q1}yGeRk%ty&^&*+p)Q+q{iQ=9Qnd^bt2G)C<7S zy-;JnR_mSQY5fbFa{^*whtaAQ5|g7C4MXY$d9k#Jd87*00=EdtCJA5)3vt4DgouWl zkF(WnK-5qliy+onzEO^ODjzr9YGX<@|4lQ251hnf%5!i$9*9_@YCx8ckUC>#WpY^e zAypnKdRDavOu_t(bGMVjx$Jj~T#l=pd?qUk&F?4(r`1mvSkoB)ejH^4r56W-={Rg` zcO6jIk(x0WhC_@W>|B8@s$EPaC=nnwA)s)E5kAUv1_MH`ic2x2N%ONLfy zM1rw;KnrIWdQm+% zXBhZ3i#XlYix6(Js~&nc7U)qdP`KkJZI4Nc8T;%-qVpq`Q+vk(yX6^=J{Y?*q_++B zl?eJ8*?YA$4uQGJdK{j6J4$8xSbxJCW~@k|oc@@Ydp(QqhIo)%U~xB~gDminoGjn~ zqS2mSgaHJSDc3&s;3z(i9mE-Rxl{?42%2~+!pjfvk)!GX1hW!F*JXg4)u_~V%i{Vp z$lr1zFIb-*r#Fr9!}1R(cgj`kM~~7!a8Nym81BTy2i3p39yw<*A5^;z9aKr)f%7_; zW;k46TBjOJeEd2!#w7o^enT#w*Q8bpP0jkJs}?mi^YfbkOq2Sg4ZAJAX*sMiS$L0j7G&$r~nfd&6@1hE%;#`Es|sd_%6`g)pOO`K{lk z#x$f6C0nlFFd!ofeFLyX6k+skam!CYa@nIWWNL8zb;GUPV?DC{i3!~D9{Is1BrdN< z?s{TedcU6MRe#~WcZ?U>SH@jvcV|*ApF0t+a6R&jC#~E+a3$%jOC|juVIpyy zSGZehFd_`V;#lW=#39^nREBlV41Npyq(nY%*2PwU%D?T5UJnfFTOEja52Y+>Xj1uC z$%So;9&{xkLR_Eo0REGEIpwu(6Wl&bGhZ00hDUT;zcx4{Lsh}kK04kwvPtfI=E`B~ z;62M|Jg!O~vfayMWW@5!JW2r3l2G~Edi{koc}`7UUVazqaRRLWRM;s(GA7%H5Mi^O zZzOp>OaxvfZ9}e;U#*#FN_!AfjUxyh|G6e_WT<#%1H3C!kE!}cp}&iG9{-W5m!7RR zv_7WWi?K;-QGZ?QwEnW1Az!`m@qTZZ8|sNgQllI5=*Ho3Y+J6Y>*t`3S~;g6%QNN_ zow&z73a0LL3dP+-xa4-~GD96_Lfn>;43k$^OY?Z|poXUAn59i>hD3&?`021@KN+^J z0c)`e8|S{JWWRfqGyLN^rX>Vt>mHhG}k zn})7)NM*R=7;d4Rs7QeNg)|LkS8X}IOIwFFjBgf)X^z$5-oNdx2;Ez^7;3 z@71Uc6<})yz2hx=f;YD%hBD#U0N+HOngiNlEOCkoNm{`q>Z5jjBha}&Ggg3J1t@zu zw#&_%uFfzhi`AV*9r!(GF(CeHfD%uc3TaSoRAuqG^ce=_hR#bB$~rGqm<|d4Te7og z9sbv8#;mG3guzey=)5r-QF1U7bHm`djGt2Z@Q~N+&hO+`p4($HDo?1o|Ca#qCpWL6 zZ^*ry2W0$&DBF+D>sbFxQZQ|^pw^!G9rlnYk9vOI|F|hBzx;f~1QwFNPCZ_+PW`&_ zpn3>e=#lD!s<-xwv+R1Y$)$6VOdaqqx>@hw>XBIc{92swJ9t$tuN5gTZ>%lkj`YgM zYnP<|x?%}CaSQCGD&p2>Z?SS}NPcL`WY2%ErPaT98$w^l!iw2Vhp+MEJ`&s!ZbyL#SZ)6G4Px&F+U@A`x~?pp-#m>;q}? z3s?Owh-+TR=e#}g2QS#@Zuy@tjKP1))*FW2qq$rkuZwZ$nO3K1<3Fh7HA5T&5$e8M zer{_XO_m$CPUI%_$m-TT+})jivWc#d@yX5jNTM~TpOHpcdgZ;Zx0 zsDSN*Ep!B1$?N)w;RYIpz7IFKY?Jokr_R2v@#L;ryKv(FZFp^O$Tw_HVsmh9cO>sF z;77#yF280ZH*LQv=TJ4{XKZ8p0nty`)ZW^IxNCC|9QvSoYWFrsk(cGKt{(BP*GMW} z5jcNHe(;rBv1;GFaz(boj7u~JNSAf;3ELDyWbkF}i< ztX_?NvY_tkNsGaBtT`L*rlF0;n$yjKNa1{%1rq~PC-mrGga4!ETr(~$)XZA;X4;xClP?q+ou5$c)uWdtP54Jxw>w)-B#z0g6p{`M$ReGAdEA=%Ks z9u8p9kMW%dMFgT4l3*uU&Smyplq3RCK2kB~HF%YMygAj(4h1?Kjky-eaK{but`sp8 z=MuLsJ>Hx)FdrF;?BV&rbSUFXxcxrs^9fpahrbxK(;sf{RG^1k0c$d#;? z3wFF_+5A_{M)izl%w#qy>vAt!1u>AA#M_DTzTCUR5-$dtvWR8QD==#67(*{JMW}r?^s?>q6M8D&xHRBILz%;o)oWH5lE8H zhTwuMO^enmAMe}DpfnhcUtaqDK6&{YfAVPBb@h-02Sa-dE?679QyT;emuvKk7JaVX zZ?@1}>L3kd~>8QEB&W>ZD zIeyy&qn}+-OzsjUaN1LB7XyGB39{D6p7&A<9mzL>((D{2ZX;KEzk>z3Wa=>AjG(xZ z(ID@twh#0UQmgXK5EHW9{2f7w>Pzrj9JAD&p3xiZSsDNg*4hvoV6dM$?5+wFPIjb8aE%-UdZTqL&@5yXtKC|WjG&R1A(wgB`@M35yl z?c&91dp`|tz!|AgKQtFA@(X`7dm=Co`4!ClE`!+^g<0>9AaU>6{{&$4d{ETe5#Gyq zzTQziBvxBi!wC>21Us{`XONB z$xs|*cDS#GX>6Nqp);-VV6!#T(qL|v+|=h`ck80i!$|tlUMTbD>F~0aIsMC8iu&`7 z=By%YWCqwxqFDn200ERH;%y!U=GG4cWdyN7+0N3IMeOW&3lV25uUCEvaU;LL8R#@T z*oOh~XJ@ey9|CRUpModmwy7q0^jniWh)D}>n#-7oP|4R5!MvZsK1z}Wx1EV<$U!Iw zVw#OHo;j{$*B}IqD@A4zEwb?W724bJyH4uBESR;^P0^lx!31xG!pbo?Ey0ooeKDz? z;v*#PX?>p}>C`?&67-dMv~Q90o-GYl%w(Wl9vC;+XMPVr^eYXP7Xygv8Ia5vl;tdI zmx@Y*bBk|PbvTQ1*;%msE#UzFuNW|=V$}?8*BIBT6&x1Jwt}=!{ak1^Dd?M6bjk;~cBQA^KpVA+7I^tPe4nUI z;wS3C@|fH{*D$L|Cuh=Zo$>;#*)EM-^oUWoCR`M-5d-!Zu!hNS=x}os0e%2DSr*oYywtOu8E-;m22(9 z=bl#Lj;&caW@d zl^>0c)R@VAZGwBiP4K=NA!=X2mM=M}rRryiF$}`|NC2PV$wKgvr>>3#xT+&TVP!bM zo5C5(Y(BNxj93YpiLcF+wH#dqlNi-ERwYOrryTE%RM#mFV%4SnAUjdc{VTsfmY zKnYXsV}7R0qeoqFAT1Wn<4c2^5%owIgBhl4vvSAOYzZpu)btfYmTy_{-HQ1u2HkZ` zom+QIt*AMsF7Owru}!7HCN1_CymTtBBKNKpxbmaL`X%d^xXv4s3Kcu0g^OYK zDFaKx#pbZDG}wm`5g0*k7P`Y0U>C`8)kq`pT$Le{F+{AgD6e^c#v-EFI|~CDHe4ni z1Ac^kQ%FteogHUE-ZfDaQ+!H=)nV_6%UWL;5TehJxI$^r%@_f=&vn5!wGF{yCK8}9 z?{pLfvgM(>bMn@ACA9JJ;Cp<0TYSHG#I2K>>w;&r+GYgCvDn61iTRuF+C86pa;ki2 zcLw*CD4*Fa4B?fHo%qUFA=^wJINo%yv8r9uK}@U%IlEz8UKLf<_%|4SyHq+xFVH zLw0%7Ubkn()UtNBa6~=AFqwmEY=CWC6=RxUqi&R0_zK<`?nv$>qhTVgas-+vwsI!B zxFL)!RP}Lt?4e^BrzwIumg(?{>cZ4uEu!V|ZhHUhl?NyzgVFE7k zanBN~#r;YfyMIu^;eg%7z%6T;m6YWEe^sHV00EiHDf7C3MnWz4q6=a=BRYV9UDexR z?PRW35^GXYI(cUK0v*N4{QiQt6Fg%BcS)Pn@8r_`BRnQ$Fr4AGnoY1ai`bvc>dYp= zCIH}#FqrWF!Kua!e!uc-cXVup6L=c zn1*WT?9(-rZIUnS&+##gY>)?# z6PFr`fhRHA3MXs8f8J$nC=FIS<#A2b+~{8UgQjfHk^LHmp6TqE&;|JpGXduB0tLXY zkA~B+b=Y3Cf*uS=^Qtisws~z%FZ322^sp@`p@^*c_YJkS3kDmFvwqtguc??BjXHr- zD%Ec_l@Z_18LIq@OQOfi`!>AxJw{-RR355Fq!K6JePFC-d3PU^p)L-ezYDmMarD-t zv37kVx$>y-O7GYe>^M3aGI}=z|GPIb;SVL?2Ivv9p{X!Q&D02lZ$(rIq?!0x2D_%X zvRxxx$(5$cdzZYsq+`kIrC99M@ULesI&Vz2F9PQpv!pGm0#Fy5@VN#qZ?G4~Knp`i zpCl^80wGuhoS3Pnyi(Ca1cQZB)iN!i{h*L|4Eq5Xz~*7!BdusW;& z?RnVh50>jq%B)Vg{G-XN!qXpFxiuy7?vI>lCZ$ysgDYn3DK=)twi!TYP)3x~xDa-* z0Bt_Pt*?U_(N4KF)8uy#-h^VeDkp!ebyfcHa8~@#$Ic9sVnSC7W__WOy1nQwF4`Th zboa_L4`s_4pTu%M_sZ5!=5Y-WV^G*~|QWJv51tV$up2eo?TUp|?fwiwsV zSImlQjWHNA4Q(b3G+?n;wtsr9P|zDI@M;WkWkmKgRrHzz;?X(-9N6f0+ou{}M?SsO zZ*Py>{^_;pf9d%(lzlxP>j9LbxV=5{Lx<)Lsp*LoSonD4!X^T$_A-U{kve0f`(F+L zY=@k7cnNn^kNoiA0`7_)dG}#En^N>$?q^H5ce>>#Kbw&Hd^ej)1S!VoM4sxFPkd(Q z?(CK`KG!f2ZFbw|OXvXkw}&TjYM0C%(Ry?o(I&I%$nD&dUGn!wCZxK%*d*$JX1=Yj z$J@K)+%FuQy-U9P3;PhED^{?&;R<2lG-Xuz$L>oLs417f{6d?6`o$9Nv#@;Ymj$_h z3dgn@iD0o6A6OZOCwO7?qNAhZ&s~gJ7s9s4D_g_zzAv-QZV}XlZP{d4(5QnQN8a{P zDnkeV)evrJShjvOnRA5Y+rP?Y=y>w0Tt;p0ekE}I!tys?UCaFvl2g47?z515tJiv2 zl{&A$)!?H-sN#-PnHiFA@(HZUL%wWoO|krnPa`|O4+wF{@ka&jubuL!qs6Ai&J)E8 zg=#fU_I*B7e&MK@`+cYU=FwZ(CotoWm2lU0%JYtCgGtBo89Z+tTczifv#OkZyciFR z(vQ#1;}wMK`w6bkf-Q?|)gmq&Mwy=s!k07@Z1J2OVfgoc7zRbzt;Pz@8!89!wg{ey zGI>993u`p^^4FQ%nyGU6*E87+qPY*hE@!!vZ?qYR-w52HUfFguRj&LdpL?xWe(f83 z+B%K&`6%i4*C@t_`1)tRwR1Q2%EjLfXI;3z#YCpdFMpd6Tdr*#qcX5pzWuvmuDwTo z^E*3tv`1FHb0%*G!LOL54rMbbq()^+kKA-(LdI`F;g<+x#@`ruJlaR!(Z0p+F7@r` z(fZDc^i8$Clf0np1()@GNNzkaG3Cwf%e#KyuNpSq{61S)1NNn9NJ?4=R8`DaTYRRV za)b5TjY?IwtbDK0OzKIED~Ko4gc2Ly8fyaN;Ci`EnnXaLuA%atcJ zZ0tChpSecU=0*4$YCG3DM6G*cCaqxuRAK;pwkvS*dhSG*Y(M4X{?H}wI90(t*d_Oz za^_C&+SwWdqVI=qJpb#;lh#-`6^+WoE_vM#6S%=$^1dI2uq6n7waZmod5e_dEJkhtlynKV$T%I-XAl#wFv+GaTm8{uKdbhMrE!k2XyT4C*Ws^ zQJT8hA#Iy$5e{c&oHLR?1KN3+TP_r~XpU*1f%F>)wO#eMk(5{m~2oenfW) zSILgS(sB$ImkO7)$SkJ2GK-R6h2g#NuR8&t4uZ_Km(R3k4XnrqJ4J--b#){=d(!G7Rm*$&4lp8gvkb(W)`^e*t!_UP zeE{!1=2yyaXc3~hhE_80+vaq-rKqOpQ>5%P4F_1~Rd+FYS2OJ!gptt}Jx5w%IhJO27GT`GV4*D0QjdqCIInCg+>FlZm&;)Z?OoJz}EXlw@T-EI;C zs|`~@nBVBr=Io_6dT$$oZ-Hd;Me7mFHN{u3LYh|P*uA3KAx$H>BnzJ%9hCpzxP}!E z{$!+$`{rjA-Q^lyWS$_73ixy_CIbgcB0ygHSP^M4dB{^mLW|CX&h-?gMxlk%8=)$& z*)*x473?;4%{1nQ>nnXvz3Q$KZH@%ICfu>JhuvpNZ;S#>Sl|i3&0?VAd0LDj>>^xBn|VHW;5k~$Ht8&~23uFMu3%n>cf7r_;v!88Zj z7@?LTE3ghl4sZnISN}HE({GRay`Bbh z3`-9w8hmF{`e*-Nd*2=(RdMz|bM~4|Ae#`#g^+AQNH!2)0|A48Y%VO}q67^Z6m`*{ z!J;C&Le7>1;o^#IZobQ=4XXcsbc?QykVjY>a*6@r#j|xt@i}ph+REf(21AG%K z%XgHstz_y78THI`+yNC-pjQqRzeKN6j0K*vCB+>~+F*anKFcm;khFGH*K_tYwl3e- zgQP16Dq#+-WYmYuJ8_0(BqkM|MgKQ1-p%r^!wA`=qL(bas=eUz$qLZ<*%wzSKqw)U8vxm z8I}>VP~_=KQb}0@R;}+YI|@$0cFoVy(u%KzpZ>)o%W?1Ya}y?4PbqyOxd=8#8V@vf z)SJ#|0}IChJSgM!DeF^TJG(44`eXK^g)p3AEGebFSp6{#u{j!z+PxmWM8g7IC*{&rfM_}ebe>)HyL+q8>FZsC15pgh)yNpvK`*PQ!s&-+$xV)u&iZ@sOraT}F$&MP_jfqc#G!-r4ARuh z1>bhWFfZq>yl+!#VSh58Wn$>epnG1%o&f76sgJ`qB^7Vp6>W`htvoK9i}BQ=)NOji zQZb=7hEU8G3ky)s4H!kXU^H)`>&Eus(}j-|>|JVJZMEE`hb`{gy)>6UstxDl<9O4< z@{8btW#~&80-!(z)#KYdsP@oFWb+;t}HgIgADA-pbPR^iFdrx!{f|B-zWY3J`Ggc z!N%c&&lQLoy{Y9YEF*wCV$|@-V3%8KJKJXkR}+x(kO>-~q9|^ETIQHi;%p02xI981 zw;6w#X$91hG`{~wKXu#<=~=Ge8F;=hzrm9x&vEFa*c7Qx4 zQR7(MYUqg5ek!YAb&>B-AzBDWBpl;RKDWD#rZQL--N0sSk(!NeEk#z_SIAj4*CHa0 zUSqCKYR#~5btW|EuJ6x`t3@4)M(MAI4jC4*D0_~k&w!;*6$*Ip-o-njQhcmp|&vMIgDtCpmiZSrxmG!gzlup{I( zlE-rg>}H2Op8wRwGiH#!M?d*+aMFFL^&9G-KGK3`h0iGLLW*3+zbmR$4{47Q5CF=m zvvwFB%rKQ@BFrK*6wGK^Stn<(CKGLEB+;am*1NqQkpTk|6HI^1QhIQErRRQYXI@}? zLM63ShvmGhEi0KW_LJ>$el&fbPHa+Nh=E6x&sWt)GUlzNaIX2$lE|^~>_j>B=Fo@~ z9F~%|!E2bYD{3?uOu(NW+a`zBdqac3Kd>d*=skU)U4np2E487CIed@RH zPGgcY7+Ju2v7OOvm@D6d8qRvrO?$32=f*#AF0J;|@S#&UZE{{b1r=$KXN_gK%qR4!(OXLq>b#T1m6sN+L})Nr(8Y1uw_@*2J;GI; zHRx@gx9+^9b(8~hjB04UYSk%{`mAdFuPSe-rVO~}GX#{3Phse$8Dl>nUxzz1*(_~11jdT6deV}N7BJUsWD3lnwErj zW)(sYAZ#~zy);-AJ+3N?j^LijODVAsy+CT~FR8i7L3b})bNAlwz+cDx=2|GD)(+C0 zllvo|8T4Y`_zFjKvT~gkD_-a^gp+di+DuTnu|&P&Et~XQuvypRvk{FMdR6rBobNMv z4%)sx@Us?&3Im;HSXllXUp$}hdq2NSk67s^|C`>dU>>fD^9k|pea<(!eg7pNIPFBd zp~amJ?)vHZiDRduLkw#Ltf%5d16OA^-Nw8YOsqrk?K*fqG3WsD#d+((E&H49WactA zx!u8)1?~~tYZ{QUOFK6$Jgto$<8NHI zdNb7BzUDz!y?5fcd-kh4$}sB@^g-g@!hTu>Bxnm>>H3p*9AE1C-kToXg;+n-N#Nd> zU5OnV_{5{tro#V!};7dT3J~>8o z_+S}jfFIaKCQXfj6;O-o_no6~_9vY;M*Xc9T5`C=ee0UtHE)!ASsl)z3KdoTeDO$J z^QP|Cz0>#JISi5{LE2i5cq1 zw$bv&^zHIo^JYva%D$vNQsQjXm>o?e=9-oRuD!l1GW>fb{7|3FU<8`QjC4CibSnrw zLG>ObPLyYlpYG=~Ra4HvRPqR_7y5yLRWJVOyv+3j|MR?>H%8dhHYpc1VX_{pnYi3? zYOMywN4Wu;yguLOdz;hn3DUxLz-x`NerLtv1&3hHq{RTt;Qp{xnry_S2F0kU+vkO4 zv@|&yG6|M*EF&Ivc}5RvE=J~qzVZFmj3_uyo`i`LxxfXbWBW}RL4&y>*!=9eMfMm+ zA|L{n;-Dp$+WyjN%!F|oY^$XBqpyuZ40^080?$hR9gAx{%hGj-ie*Os6((D($PR<8 z+bHmUjkVYJ6J=GA%>i%2H}xZMr@`lZvge+0&Bb^aaLHJN*_>P$h+T!pm2WeU?y=}D z*WJDIQ@1S8z67JB`yM>a%W-Z_)YiRNv`WU5GG?9Ky=nYu*ZJP`+@Xq`6|BfsvI)Q3 z-+X6dtCWYYr3CP24_Z6-yDsayUurILwe*!6w*=HmO~}s^%+J`LX6_GBjT}J7eP}iT z-`GRU)N*!Z`Unha+LyVm?KewEoc#sp0y@#uqkTQ=RFkg{%|tlaDiDF$KU=pi$Ms47 zq)C6bzG25%rfCl5*nGRWm3I_M#j^SaUf9<*!tOl~>UuM2;AUe|O-ic?ZP?>|$sSBw z7Q1#16y`W@vBKz`S7&@@;50!%wN3Ic(^2^JP((QDtvx2!xq1fDXV)Q~7mw8+^d2BT zpoxeir^j46EsLg#iFO>59j3*tF&%{(cn00GOjd8;v*+V)PaFJ$T-W$>d7bM4fBxvY z_&?}Ix+w>`Hp)@d%HT=#u75%WS~UzKpwT_6HqPJ3vARLa0i#9A==R+R4@U;I0#@jO zvj_8D+eccPI|ItXM^O`;cWNBle|ARa8jroE2_ zVXcQvSqNWfH;UU{xJ^EiMB~)P{sPpHd=GA;7WWi9NE1QJ{oKY8!B%6!!es~bHj`(l zINN$saEG!lW^fGJ3U?tc#g)g*RMcPlV<@WtBb0sXpk&^8&_+=VnZ`7w75u{i*U#&a zYRpsIYIBY|X&*J8hxE17@sBoF&`h6FcY@_#5-2~ugBbjD`Q8qEXxGT*)_^$!?j0-f zph0KeJ9>gn9&{$hnJ*3oGlsW;GR~n@(X^N$53Y(vb(ZRAl!H|xkbt;Z;uDhEdyajp zev!t-;|4zXb||ZLgW_q4r%d$sydF3z`!af7`4*(lGTUyzTbwTE!wnN^oWMVfd@#4;7XH8-vwPF8$kF7 z(30bDm|FKbsL|M4<2=bRKspX3+KCk+Neeq^PqA33<>c(@V8Ws0Hnem>TM|;m>fA$n zT#5RHkD>Tl%aap%AP7@uo9QlNYnLykUyaJ-_stQpI-VW~T_L9FxO0}r8fZQO5!0HS zIt24aI}Ph2W*FsJ`tf!IV}C+!Hf=7hKlooAYJ<&_M>5m1aR!+KsYrWu0= zI&s`b3?VQ`Ek!pd+h8(R_Tc!jo+UB-l*F|nKbEIUC;G(I zvHaS^*?k&JtEoHI>9|MTu|yZWzQikOwc=nbpQtH@!NPR$_gG#v-qLs5`kZP+w2RVC z2u_Ji3iO$0JM8P>r-!k8gZQzLPmj{{-M&5>Cu&8jk&lkoO4J8QI#?7n9cQY%HxG*A zM$~w%=rf`i%Vp6L$H$0iaXgiODVE0ZNz#Zxu`!M>5=Y{2-Vt#&j!);Wi1c_qiT_3{ zj_2w8dhw%pJ}x$;-?D{kb)N(l?en$@5zp78_4?X+EL)6TV=SznyhdiJGoj+C50=?F ze9SM>Msa(5jGz2YH_PgBtM{an9*Jle#m5j$7V*+3{;Py@fiD+~DY1sl)#)+x`Myrr z6JY+*evPP4;A_pr{o_4aNsYN|wPum5HQl>HJ+U8#yEFSWQ^z2tkIt(aZHO6JC{joB zIYmzfRBcIZPs-ykka!e-hh7AFEs_on_`blBi$RUi77xq&!i|{JtnT(tzb!*ElG6LF z`1xp_Kk};qdM&77AMkxUAP$e_Ch5kv#W$n*Zr(1Q8p9V&3cMeB4@^v`^J7wBt0mBE zVr1fh60*7pz9hG7BKtaFOyt*${+AaBM8lI zoV!{WllYDi%TOpQ!sJkk`?z>LiBHdiIS)eW{%Zc+Zba8&dpo4q208r@b>fDiE}8?X zZIq%%C;GlrtG#n>K1cMl!gJcE1Jn}!Q{HHMBXo!(=(SZFk$mSRQ9&hBypYAW&) z87X{x?I*3Xw!G72WfZ*2#0u*suD>QtJ&ASdH}TA`PpiDDmlG3JjO7)#>2+h<%hBTY z_z=n8Lnd*|d%?LL)-HwBMRv?X-2Z(*OKJ6f=OiUB_uhh`5{%4>+?Y!4eUXD+jbDmtdg=Lt$di?ZfXC(W8ugkA8VG!r-mEEw>siDpj<93Y+8F{@p zyGS>#eQI@vjf^JL{X!bcv-RZ|oxsduP*}$DEWTVUAIoRNHTnxtYxngGoxDe&Sr{c= z8Ovkxi=)PRB{)s7i5kR)l70P%c_sU9>mx1eEcg72>=wEMHvVX*JCl2F7}zP!j^*?C zZ^fiko*Vl~j|LTq?u_H9?H0GD@)CZh_zjLBq_1$N^3mKbzDecjV*?#f?x28Y*S9?( zmzv^rneEF_t@LY!X&l~7yQmn4c51Eo$v9+U7jKQ@mAp=jO5@A<1AEq_p~r+b%NZqV z(qTxm-zvUH=P&bp;t>-cuYDM=-+r5DG4ZmI>_Xm;c}Ion8uVTFmk4bJUpzvk+X>aq z=$xSFrVMU1=^8ap(rK)-dpUC?S6`LNyze;KcC7J)C2RK=i9;DYuNH=?*~Oh!3P`P| zr^e5sv)xfS5?96)biHu+Tgjdzd!k3*Xl>QQ{7moGH_`&hMq9E??a?%1>0poDb`O4>Iorry2F9Va zzJ|aO6Snz(ZHr>hm;>5uvzRxYUlymv(z+R#@Rr8Iy2k1E{Xg-@cs?uR>w%nmsM8Y& znNoDYH{vY`Yn`!7gc)qG;5+?{I60m_AjJ*}dlt9yyT$VeLaX=pSZrQj>VWuT7BAy_ zMdAchuiy8G;tBYizR)8a2rH(0G;q>y#GVQ0r)(ErPT;eo>S{42n_ILW4)*vmY!@eU_=1=#F>oMlhS#@dpC~Z%mAp@EHS^ii(C}T3ryq^U zQ4?CNiJAsOGfYkN+Gf3l6kn=_l1?=&4pr>9MMAJ1=RP5uFL;;xAH7470mQVwJC5s_^mEdZkO9UmJ|7- z0^bPfpU%vqtcp9^1I%!n(Grs%Rju91s_k8mXK1$`>e3ZG%9iyud1(kM(kAlM(MPbb zo{>H;8JeAe;(aHvBVy@9zCc=Q6^#>lY3;p(6lLsGTdZlBP5YAJw}ys7eAs?6K&OvE z_-Lb(9v9s=Jt}?RUaemHgF|H=3FGcykqjO>o9>mapvOm7xGfmqf?{r;{})79N+rYj zJK8K1scjlw;K?OW24-SyN7D}3J{fQ9t%y1VC%L_5{lrmiR+HgH1L`sEV_&!L`2q3Z zB;HtCENeU$ebWalsQ2BzKMdUb-R2tlN&2}FpMd^yRe*kSg(S|Xug)_4zTJK*oHfww zAClXRg{Vn4I_cKN1a51(@(A}*2uiC4v6@#P#%1skLBhIx&%@(HUCZVMp3uiODO)pv zdbspN>ktuQ*`?Ddu1fUKx87OZ<*Vx(;z?~W;d;CEW7|76j_Wmn)paQcQZZ)QN&$1`=6fkLXqFDLVHT5}-79h1>XE0+YH%twpj$vi9S<^dJz zuitl@xO*~BRt-A85RXmf+0ZZsY9ak%;1)&PWS+vy#fiz>I!X;az1No`m*K*{^~sp@ zct%W}!V{+;jAF*sHtNElA_qodI-3M{gW$W082(3aJ%Sr}D))I#dga@`ujpc)#x(Kh@xQ-Ssci#q+vv3qQ=6r5U%03gQC&GZ^rjgwxErANG#9iMd-ANe4fj5#qaZZy5Y)RvV$1(ecU6y z%japlN<>ZL3wf!yW*WbYuMm$-S6Qt{*IeCdd0Bg@(a&&j%^Z4k9h z=RZQfe{KPvz+1$v1#pKx(<`1XK*+fx;=KZXho*dS=aFmo6wJVfS#$m3&O?=A&P;wl zvkc;EgnuUA#%;oN8Gj(wGw{|~t)y}{C*yX#pNsTD{-#uai}<3D&y?0yiTGK3?a17L zzl>G6&D9;r982(uEwlI{o+&RpOPod}m?9s-la{H0r?2e<>pWp6sQe6^^flgD_2G z(DRsp7F$oNe^=X~vPbXllvOyd;BrdCYM!sU;JlU2xube(as9_do%L3X5NzU!c|3n! zXY88d&Mq{9PE7D@GQ20pI8!`3WXUNtmUIrTDDKpyTso$A#^4xtavVF4V`EeHNLFaB z68>J~maBDC#wf>oa1J6}tyo!vj%mZ{J?|HBqonrv=DjJ(7w`n}_YywE=!9_}({R(3 z7we0zyjZtD`E?b}K%+@cQ3g0g)d%)?HBJl4tK$PcNjQR$6eC%tTkcPf zvzK1%zt>rb^8zC*qx}yfB-^U4&#s1~NW{$Nr6X_cyXbqjFX)ZAvPj%CpWjsc00weJ zjCn5`^uVfZL=*E)8*I{*2DT;-ny@tzGV`EE8{8T{*o3vLjR!_HQ3r{6#~u{^`TXjl zp?tpgdo;nTPy3sct-tzB*rI&C@_W?5tPvpR zvh{1fN!e=hdwA)=SR8%I-^A(iSmkn;7_oqV!n4I+7w{?3M%b~iBDm_*ED$5h_)6(i zu~=KirDnsy5P1h_3bZcal^u&h$49w|0`jxa46{p8e-#Ip-|T5YaO%MK?&Ie7%( zMkv$=LVO_1N4p~#%jl}m44@IZH>vX9M>6@x(vyFBaKcTQl%M&K4P6#1dMUS~)?_F5 zF+1Mc1({M%U3WQZ227T4buNLLOi1Cj;ygp8RZ5C}Rpq^dI*r?X#-ekk+z)%TDIX1;b5t&D!VE-9^aMIvosBzVht3$s zSJG${y(k3#FQ@Hk${q`>PoU03?G3C?@KiPeGg>k9#R5vK($?#<__+yP&!D-@FNcVH zNpvj=>iaC>7mLx6c@j%?_4ytWdlvJeB^0}>GSDdFQT)pYUZIajd^^0*YBkSK%jJkL zP_qClxS>60St3f8V5HK6wO^Zy#7~#-Ib*hzY8q$)3VCE*nko{+#D&i*-rADCyapNGxShpw z4oamvp^x1mYjT&5-yv)NAf8>yOA%=8-@i%1zm%J!=^cC@Z{z#&F_B*mUBm`)O*zlu z=f#F{K2Opt6#L5&M)6u%c*?mwjm`Q}md#(vr?I%xxg=O=Ccr{d&1a`rpOV$H#7)b1 z_Vvf`E!5&OO6T<#ekpq^zm&U^oJsiVN=Ic`e^l-_YrGqaZ|iz)HLi-HCvc1Xd3!YG zmQ8lEjrIxO``(r*HIIwF*w4TB?UDChLEclaoF7t2KlSgqSjnH2q=bPzJ8s}h__R*nhPR;~ z`<)~iVZXpVoxc102{MmXBe<%J&@o1Ark+$mFAzw_T10aV)+-mWJdD?Od_->7-E~!!;lei)ae`JWhH_E%Kzy$>^qa7aNY+3k=KPAE8qln^2Nhm+m%1Xb^IsNP3@$s~eXK`7&= zAS0Yp(2bN3Vf=?PzU%_3_whz15j+Y)$?k@X(62!KB$EgcvVS<)&i|(B?ML-K{XCNh zUIn3ytA7QCffxk|NC^?fe>mgIbkc$uh?Fl8*l!htu$=%IAyq*ZQbL66A4+zqC^t%F zGxWQeL@+7{;hJSZM#xi8fRqp+I}EauL3YvsCJ{^uLdh@djA1li{I zG4}#k6oisp3K?OEf(oRB2-#tfJqctN{)tHhtAbFnDlj?y61)*H; zf{gI8g55|75w64FdKD^mqlQa_CIz8n?}m)9U%_Fdgb3MTkbMWpZXU@cLW_b>vJXQ> zIHsT-DIr337-ZiEviBx(iEv0kDB10h5l$#Lg_IB>I}EZng6tzEE)m=cLdiY_8R49Q zZlr_=*5%%AtUrFP*3C%AwqT-Wd8zWpPt4gf>%K(+3M*a7l=`i zfRqp+I}EY~$d+ewiNNM42w^(`GD50?ETn`8*xkNB32;rJ#K}N_^P=J&W zAv+ARUj*4nS8<78QV>dZ0c3xkv76qYXmqJEZqM!mPAwqT- zWbXsng?26xtO`QOu7HfNQb83`LWJxv$UY3R%hq7AH=tZWDA`qz5!NbLhm;T@I}EZv z1lg7Ma*1G95K8ts$Os!1)FCBA$PR<-PeFF|1}+h56oisp2N_|Df^A3%5wgP|`)iQx zsN)jBsUVc>ZIBV3R2f#}u?9B}B*$gKU)w>g;By zvw=ejLdkB2jBrB1DWrr5*Hyq5wgP|do;*C{ub2P zfJZ?n+1-#4`W2}6LE{QU$PR<-G?0DzU8u7GuYypr)rUbY5ThUgDIr337-Z*wZ21_} z*#P@cK?vIkkP%W9WFaL)$PR;S3&_@g3UxMMR1m^7%YuxMr=S2SAwqT-WX}TGNnb;q z4VV;!l3f59VUB`Qq=X3BVUS%6vdyQV&IT+BLdh@dh)46+M5pw0%Y z3PQ=QfQ+zGK^0O$gzPZLt^nC(y-;TZ3-?gb3MTkbNV_u2e}9!LA^b z>~)Y4HY%t?N{Emh2HCfP?CKavBGf1dCA$tX!WISFkP;$fhe7t;Alorok_b)(p=57^ zjPSIAT}TNLvcn+Tp;}mwCQ0=`gMv`5cR@yYS;20kgb3GRaJ>l?yD>+S2u%t?$=(ea zVZVaINC^?L!ytPz$Zobs5}`#wDA|W0BOFuEj+77~I}Ea2AbamDNg^Cl5K4ACWP}q6 zP9Y^k$PR<-r$P3SVo4&n6@-#~3Npeu1>Hyq5wgP|`_~}*_+m*Scoc+^-3=L`Ux9jw zBoQKHhe7r$Ap3NMBoVv{LdjMy2f09uf&`?52-#tfy$58=H%bzL-J~Fd?F7gOsS2`? z5+Y=WLG~e#t-oE82u1}VT(d052zd$$kP;$fhe7rqKz34%BoRysLdh@djw2xObrOA^7NAe8J<$OuamR3Ifp$PR<-FF|(UgOWtBDhMUJ0y4r%1yx815wgP| z`xMA7drXoD3-?gb3MTko`T#uG|K7Hego}O7=R)2pbjDAtgk}4ukBA zAiMe*sI!3@1)*fuK}Oi3U>j0GgzPZL9t7Er7og4toC-q8-Ub=rX$8BG5+Y>(V6w+s zU&VJ3C{w_Akm*Y769(#?;zvjMIK2})1o8Ym(kzPQtV{fL)+RP%!X}mjypHVwpk%@( z@%vACa+Hj72^&OQl-jIsL4OB`5=GcEV5<_C^^Q|K{~;f(cVLG=dkKL+(Jo7zT2K_< zzy`(Srj0!a$vs2Ty^cXGFnspbbnLYNL$%}(wh8yCH?h4)1HUo~w{{vk1UlduK7i|t zyD`iF=ztz+@Cq!wSBM<~9k9>=++C8%jxEB_57uCta3{{HMmqe1jo2a3frjA&xb6^k z2y|fY@B#ebY3vZ_fQJs4V$?|{EF6Xi=a+uagl)pIvrd+Tbod9$utT5&*5L#A!5Zuk z=s@+*fdu?uBX$UMpaCfp`7!E5;~{Jkjwsl3Qbv delta 42245 zcmb4r349Y}-v9GVCdu@oX(^?Y3X@)>oZ&9W$~1*gxmpmlx*kD61EL0HwH|AxP^6w) z8N|Y>TU~Ww*Q3Q_Ic?Rdcqf30ifnXQrLDU=jU}C?NB-YuCKX)wegB`gpHKSCbIf!6 zp5ODkpONJ>Ib^HLH)}RRHat=^(juo3axDLmh1v06042K$5KiNI=RI(afo!B%gyr>tH* zP3bXBqVK<4#J`eEhbf(@8H$)@QS#Hure5ig$wgyjGLy<=vnZ2C#>wOc{3!SH-(OLd zG+riGj+Y->U&rkp{n0X+Jh50N?|Njc;pcx-SE5k|onRVwxBf8MdSiYydC4a$C21or zoiMDL%+IVQM{kz(*H)&>4}olK_QA(Ksa z;Wrk2oPF=Fm{t*;l!rIj--UHDdGtP+T#vN$5t%H(@1dF8?mVNwbCJAPToCFCzi6ObN6+JoOt{2T!MLj0!kCCcz&S1o>t<2;QVXKUiPXT@sr z6}kZS5DR`u_|3xWlkuC2-*0oP$s_nZi=VkzCjXczlQZ}Y{?V8B4Z9lXF=~eL@TlR+ zjidhLXP8KcMDE7_;z`wnAtWIsWW`V;&#;jYnanAsdOgEQ{BeOMPR|pDkc99=G}?!T z@C0>S@fbqN&>wN4UPOI^k^iztCcFP4lT%2~dsg-njUm7bGC9r5kRwR1 z|C3BAEJpc*NLl`PJipy4lQ*80i3jU^=}DP{1(~!xAroXAQHGS|tK#X&csj&wl+Q!TR(3zq z7qHS(NZCZzEime@Dl_uik+OcCkEf^NX*8bZUMLekYp@Fs3^a`kjThfX%F10A8Tm_) zviz4TjDcUd*hn)kG2W{{$|mHCr(5IYWtSP{>yWbdxyy}w@vn_^>#wo@sn5vdxkO{2 zE~KoZ>?Grb!|`NPp&cAC8rzh97x&w zi{oiahEd*_ZKSXJ@xU5P!$h8XQ6}3q8EH74Zm2Qx(ny&sYm~|356a}(=Vfx$Lo#^_ z`I{b=$*k98a`$60x$6nMA5UxRjRCeJW&P*uF!Isg8|~&b8}Iq+-!LGs3n{|^$6Fv* zugYZMJ4X4JNcHk}jSec`Gt%dfvW`z7We{BWzL8cUW$n9=GDuzbfpL64{Mg8UiHSE8Tqd$=B<3@U0JiXuxBX9fC=wMXHNH0NqcJ5DnV{~}%lu^FoJR^S%QZ~TW z4p8t{Wb$1+eeee(fA7DHR94x90rro>#>{7T8x^hb^yGk%e=%mHck&>vdt`Ep)kxC? zBfTJ=u8pUy@$_HuG&{j)_ol6ye6&v{10c?9%cgksnv{hF zPc5-5iD(H+B7I5aDoJb&6H7~&E5dKwXPO{qb&F~-v^gOT8L6nn3Z4MxXyj^j~xH3dr72=pevD9ks`rO9$64@hlPZN z5qXgkb*!b&D65M`XSR?(btdz~t~c~N>ppZ^{T*pm>WgmT7eJ9oN)y6)hc)ZhC(Yo=jEOSpQ z`ek(l3%IdQVoOmU=vCese+i8!=S?`zXnVng;zV%kZp8QTHm(P8hxTnH+mlD!z4ljBK~u+ZCV*epJbDlR7z^JG9_=)EP7bE za?*Jjg|QU%_kF07TD8Z?KmNfMr93<-EB$VY&g}_Rw8#`p+-+7*_b7WO%}EYMZQ>u@ zPqKFFw>rmjuSS)L&iw4Bqvc%Dm9otVGrU2(y7Zf_F zKxfdI<}CdN#-NPIefTlIe%vSJ$&be1io`I;XCsa$(S)JDdoml1Jpmx z>^29;xa8)G-K2nombf3)MB9bd(qnq{xU_ifdh9s~t#l_;y=R>sWNqHFOg=_ua+ALJ zLd4eG9<4Yq#ywmUQ*A1jY7G#zpAdVECKAzYQ+xXw=R1PMRW_BTP7f>!Yr}{-K(NGR ze%1uwEO2;fvBM*}Ikhn6@MayLB980=K|QhmoNj}X$`Sfgw}HOmsupWX5FOgs(V$*B znR9KYD&`Zg^`L9wxzVVMw?77#1c(` zB=ue8>M6sfWd+45DNLvo@h;~Jupf3?FP9;@i*!U6{k(%~Zt&CMhTBCD8K1Qq#!0<+WISG9S?dB{1b)ZtQ^a@Rm$}ta}F#lW)YN zOU&woy`r0_X_^$yl3Kzgr0(<((&9DMJ%n_P8|~D#nBKOU^?EKM2?w@mPI64sDvxP> z-Xup_u;Io^AP!e`X}dry0k$Wr`ddb>><}mo5s}y@WGyWbm48m1>d!{4S-q>Lv1yyO znXrATLAofMB{C3`s>-HC;bd`9*rro11~%eN@baO?L&w@EN4++0nbcybY;HU>tVwjo zx_8Df8tBoX(~Xi@{nuX1wfup`Md3bu%ZQgah`Lj*i#Di@QMvbkzf(vh>Q>pZ3jj(G zoP6D)aE?B7g*J{5$8CXl)98M*ycx)3)RK+|h`A1DXL3@<2PCoMeeR<+VaG>pmL1qV z6L#-Xws-jD(T*0ojT5e*9bEn~ZLh=DYc1x_Bwv3_J6K$=rQ6BVpB&RZ^geX@X=2(= z)PudssOe6+S1F%9T)4WI0q0k}%1zU=CjT&y*h#5SWDc0?samIPbPJB{!6dyb0TVM; zNYzc+Ce--5ZdLv|{rn-)hG(xBJwXTx=;)f>tuv&QKc&q%k)0{eN%#@4Q^3a{y zTJr*J?_>tr4p)5CM*NwkIvmJHX#r8Ay(WhcppDPlp+OhykZt75Lmq;* zX=-o3H^jQB)3Qvg#G-!LV|EM;Tp1#EY)Be5BrQHnPCye)7$%3)+kG>@I~N9*MjXq_ z#E-*-v(#N`b604#k!f9|d}>gzOgctHX9$9YL4LiYXl_@{OA8~?6mn+XdLUx%Q1^0g zn&&c4OIz!du^~G(+v3)L`7an$-XF_vLDWz*gci=uy?ms)XD$ z^|W$m*44HM$Dop4rMPmBfGcv!_}QYLs1voam#HH^m@gjI_EyzujRM6}OXXp$wTumv z$Zf179TaE0VzT6A%YLJcAop`=1?3RIN>oaz&&-H*HPQd8uWoJ};->B#_gg=jq+#x% z$@esoF@>t%)f%SOxylE>OYu)UL1*rFjSG@7TU3gJN;uOlQ*Nx=({I{QmSYX@^|Z(m zn0&ypvZc+kA~!@udnmS#ZM{*#=9J)lstKDvnRE)+T+Y7GJJhoXThFB@G9)pU?%fmJ z7``;&gYSuI(uinoR1Y=2TIbhD3+FD6)K`2GiTTpj$j*k2#WveKQq*uJ+t<)p&!+~6 zXjlEEm@*+=QT0okT;@PiXKe13C#Z9i>v9ODeT3IZAm_nwgn@(iY+b{3`8;)9#LqYD zwe-y&O*ZP_115*iT+$^iqoI;6zr2i7Qwd|++Rb13+65GBa^` zP2TL%-e76iNvs8|gnFpBO;cw1&G>(ULgPfEU4c$$2G)I3LvJY>&n18b0ltEuOj@aitbdZe>IHWpgi`g8^`7J zY5N!e*0@dMGmkg`hOE9}p?bF$dDD?1HW3ld`Q6m2H=PSJX0uNU)Svh&Zb^b=-KNg6^9mXO*ETiJ!64~O6 zWn#nst1iMY=>+``|PONqEqFh#spB<~;%QTlnkI|m01Ik--C;umGIB6O!QD#bG zxR(YLkCesbOlsbHkhm_Z)kF#zX zSm&cS^b8%gG;VY=8GcdZhvCXIB_fyHr@T`#EwQtgI8F!jQr>kz#+W|F0u}{7)imML zHX-j*tuITt#5MlvEBiM?S9w&sM;D4VBXvMnW^1^;$5917v&_~@J&@)N{si^F7Bs09 zeW@W3I9dTt^Ml4l!4k#@A%{CjEgj%P1Klq`&rDKtly6+4#l3wF_cGA(RJFB_KxEkd z(syj5ORF1YeeQ}_D$y!sW$ENhtr)Uh<$L74tofb9)T(J&ty*8Pva2+=aC+Z67*TR3 zt3#pWQMP%d$!g^RT^QAK1KnS}{<2z{Pq_T7y}WF30rmgxCtaNpGDz&;)qPv$bnIS*7 z$S5^SmP0d_OO!8{Oym>Q*9Mg2rP*9)nBrKvnAtvVTbgfK8`a-#SUSp5X*{+r&9$WU z>7_p`U2Hkst3O^)zSQ!%@wl~ouHRM?UNML53ACVO>MaRp0Ch@;XMUnL>Btc+D}<7% zC^2Puee=x$Vd}N>dyZ0(s0mu7m)Pr8vQF9i)FbPp;DJW2PKx-XI(M?0^<-1!en_5| z{QF0<41BS)?3wOObqqP48a}rEZ>J$LU7jS^rb&yF;(wK?5s@wpNs;TEsgWGI$ao5e zAceBdzSM5>W%C-w4#s5O)tcC2U9Gs?GWiK0zS=&y(7cw9>+ z1gn1%5Ml4sXnE%i`9NI5%07vg-Kp`N877pjWu?+&923^`X6<+zKlC_`i_j^W$GTZ1 ziPx>vJZn?KIqPjI&!>iyeL^#Jr-n1I`W2c|w`{EcBQ06p?V5J{CD)XryQ~Z}KX`2F z%YDPt(W%aW6uH`2ppJ%}5FRCHr(-m8=VmXcXw9*QU-Td>v6UtLFh(q{y0u zOjk_}&sl|^^2G8y|32wLcX@jP>;)wxRoxT&^JNp~Ns%1!R*2KAWOrqPEvKBML4*yJ0eusiBu)Ce_K4Ci2PphGwq;Liu^u4Jp!n0 z0o*Ymcy`pDf%tZH+y|@4*II_WWzS!mNa1|9&zcyNB9A!9M7w*6TWDSu2XtuQVr+dS zw*IRDI^(ad_8`Ng@?N&d*m#Q&lEUka0nq<@1FTQJKB1E)2xnw(^?ZL<^hGIhD8F1C zN*HeO4%iMk2MK>COX1_nCo4w#`<$eZ0jxrEG)Q698Gmk+!lwqGE1YiL9#~*?GnUkd zH~%0NVVbOEs=9FiG#<6oErqAWVOKREg$sSN;w{#rg%EFXWrgw9^(eF1k1yc-ZYk0u zCdHdxf#NJyjM8b&!O~?Y6^#xeS@G>#fU<^onH(?s70RB857ZammZ>OfiMN{^Z&!q} z)_6Pr1MwM+M&XC?!k6Of7>2Sy{N?5Gmy=L-EZ**g_`4KkC*oyC+iAe~#r|U&KlzwO>{8?k(I|TcW%E!bp=`irlxefi_Am{NI zUEkP$?b_V6Qsj>K@T)O=C{qe2$Lp^|-aA|h-yP3i-2c=%V03((rI=WyWK8UEkDrP| z)bpZ)Ml9m3eK~G-Qw7JkfN2M$xEZL;fD8wW=JFp}1BrxMI^M`7_N|)JdBZ+BnLA^< z##+? zm2Zx;F`mtP`Gx}T?lr0EANn}a?j8$s%EP@|FB|9At07y8iQZWE`d;}}qH5X@RqB_* z8Pbx-UdZ7k1YN*DNgPlVp$r~md69aShHYeNxFwAVhb`e#w8TT~)gF@_f}&a%BP!8Y z3y2LIv77RYR$MOiRD04q&SsP6n%A0teEIpy4pM(3?8tGs6Jn>YDWPCcC1;8_)hz_o zK4$zB)Q=8|P~%7m%f-I8!4(k7!F9D&Wx8WUD zC$g5z5L_nN{OWL+aL=zc!4TinJL4O;Fn%(ZQ%KR@$ao1a9%rf$dvOc|`$jf_jow-> zt>d=Z-1mEmy>fdJ4vEt|8mMDfIi&F8iKMVxwK4LVC`2tiBihZWiGjiS#&`&>j@sechMGiqBcoNYx5Gu0{U3joLX!bvqOb^(&baagSUd zfTfLt97C=Ne60zMFny0LS7`z!I`;R`zc7nyH}sG53m=IT(08{ zzunsLE(h~+K_WKOW{16;Y0L~2audAi-t^F?Z4_1mTOLaC61*b-;EcRBK!n>r7rY%3ZY+RIf;za8huv`(*0ZslQ1)V5Z*hn#F#T+B z69#_-evZQ1iCWO>2o?B$SzrBsn4Ad~6gJk&y~Y&%46kXiO^u}%`*NBMV+sskO!F=8 z>#lsAHKy7Rx(VgJ#!YVGw>2fO+TUS4!>~qpqoCdrljF(&b#w+vA190Vx?BM-2SC}n_jddnl9tas+NMq=wffN5Hgio9u( zQ0nx2rSXK`NYzyOfM&PRbsN`x&~|_NX75*;K(=@{d*A|9{ZXgaY+v;wN1Pru*br6i zuLN3m_B+o(>n-hOI=0fSucmz5-$_0#tb~MX?EB!%kM(2bK(xN}t!p#ot(%`ww9Ll}SRsKH&b#t4z(>E)_PVDj}x z8s8Y_Heq z@a2}xLp~Q&)t2Cg+W_Yd1dJ`e*dut*Xl!Cff>5vR&}{?TWa_v$Sg$#ya{-#IC6IcJ z+Gy?&wYiV^#mXc9UT@2*h3C(jkG{mW!m-I8M&N996M!Y|HG>6f1oj;!e$R32aEs=b zZ7o`jt=7rB-mIuD{-g=VVSYN?Ccvq0*Vj|88+vQcG4x9CfzAsIntiETuBFae`TP*` zn{#Sj51Vw#7z>>9cK2+(kHKlDVA`wtFjDzfqaIZkWH)&F`k_qye@r zgOTxA#Y$ixwTn7Ve%dd3Gedg4ZWE-azXtV!XO_1XHab?rj28?NSe6Yi@Du~mS^8F- zJ*1|f?&Mcr=x0-dz{T9H{^e}v875%YK|D#J;dwRo^(@b&h6mQ!aJ$FdO6H-@eEW&_0b6GJHXglX+|o9NMs_NfdEmzEOr8`J==uS?@Q%sM&F)BUwkrp2ECS$* zV7vQ9qti_jzSOy_NtYuA!EXlm4HEDI@Vu}4#a{6o67YNGQMWSg{By53uTGD#I!UGE z0i^8B*L%M*Xm*@rSAsHr0{2M1hzb0*pB?+2;5lSLC#r}L=^FDwEzoXmSKbQ-sYy^D z(?yJGh!~Qe6tGQT3(V zvJd==sMpCT1bdqt$KH`RA3JgAOaSoTqq{pPxnFbok{nBq|LrtUEs0CmU&WH4OJ;yk zs#|*J!`4c05)%8tBJF}Yzi&>wgvLwe^zFoXsy)EQ!6f|&Y_Ns#y)*_R80>{8>xkxJ zxH_tC=p6xL93?atE@+)&Gpf&+C?+T}hHC2A$eqhb*>rA;3TJHIy*gUn^^ymH1B^|1 zZqI4f&9uKO@RLISxtyAS(;SQz0+@P(a-tpsWDI+DvRk*Gs?LpPPXjYkIySOHCxuA@ zj@r0w(PvZ~r;8!CXaKq%hUNVRq=<=LB4lH5IwqDys*Bn3)HXjm@p&|tIM5((1Uu0b z!lc?Evq*%LP9eEmO-W>|!G!b9PaX2XgQru;^r~X)_i`O-Y`8nBimMob85%8ibGz@= z}ieQ1l5s1{7nPGq1Y1ivJl5c4|4W%W};-wNZo?->>;ckqUHwwKnk^)-6q9-Cq7P zz%qv8W!*CAbX>6;vmQs)g+0T*0V(bJcMpGe;dcz!en<)Ng9l7E1&Tq0heXd0F-Y(0Ois;0loTZG z9Q%G~iZqE0MiE4stsc=)@P zzhu?LWP<>0YGj;&XUKO}WK^EoaGw8|R+++hKr2#@A6^vVx&4@SMT*o(NGnoC;ApO( zUf$1GoX#1N=r*ycz-`XQrdhCs&ZLfa7}JMy2wRt6b(h7t@OZbi;~f?p3E9lqzKcW65l zdfPBA$F5}E=2-ZvSkf8W4F`0>eDOJic?fhT@-y{?*i+Pd5!A%gmz+4Oyu0Ttt5mML zEq}F5~zMh;(Uhi~MIKK8r0jcShNbw8;t0VJUkpk29iEgwR3>vg2b+i;wKx>4#Bf*+b zz<|c+2|EeXq14!~IN*CAlf;p9UBgAh`@kM7=3?T=S6WZlOs~_Y5#$*z_cI z2*46Sy(ngDs?~N#LWq?+nyj7m6w^{~=_jELptgtGTILDrjp&cF*J}4jTeRI}R`tgi zS6{1DxhnP{@TBZ9ZI>W8a+(o|w*`;2+D?gWgC*j%7+X>I5-H^CCqs@9)W7yB`>!q> z^^>GqMQ5dfc~ajELEUd$ZdJ!0PzJ7c`Fp$PM!owioi>t&U_qe{?$8}W)Z2RKOxrZa z3}MpeNBR0Xh(IhJ(E=$|)6E-+UOGF`!|z1Q8|Cs?9OFi4ZBTULwb%a0afu!)LJ0dP zUoWGM*pt-{d--|`h)s%G2)-F&>acRNXXL2f?%}Two)cOZuYrlEqKPTZ$}Z1%f5vS| zp&^|%>%E`Qxy4dymaCIcwvD840qB@cwb(YA*m2O0f(P=2_w%c^=rqI&Cr7yobplxV`&(TN6W;HY0KxXJk#J#ro5juGi z%=KiOFyOgliIvJC)*tBr@BW+ABcqAmuo^QCelXMCw4wU z$PQAJ)tuPrAaxpPIoQdk76uM>TJ`_QsRh9=F`bjzUg4y6(D9`9N!Ce(SyarNI_q6H zLAj!nT8qPlCS>F4a@BXulg&fUB(J>CTXG%OJ}iAosY@kpKIB&AODO)6OIS-{FJptR zd{Wzu4(}!?Yfx<0uCZ&|=MMCXA7Y zY{lo5ZgHK_18#TiVeqPk+rs8wU}FfOJn zUR8sK?+27)t0v;%;{nC8dMO^B9Z)J)kH^D914=D!u|0U3k3~4&zV-GQRzY13r$(#t ztJ`nzC#cuO7`6Nu#w<>K7y&#mSZp!OdKTRT2{f6hqmQn60%69$4_?P@cUC19GFE6s z;3MDBGM0Amg0&;j3{p^cuBe?6K#4AB8^XqLEm7BQ8Ch8A*M(o5E;JI=0mTNAbQI^h zakI92FV6B_{)}zaql4#J+>EDJKuvMh_uyDJa!(!zS~@jzdDoB)WQ@a|W}kWS{5Jmf z6!j1NIF!?0IWy7!V~~jNfMvV`7ICj8CqJh-k?u;~j_vgSx3kJl>b=+TMbz3D+kJs)Rby5o zkdW?e_pE#~G;*~p-ZkRhn21pQ9YWoiGD!R>?y{H9 zU0doe)@CeC*KZ_?n)^Ci3iB!uD2uvv2SKK*TEyach~n#w>(J$$K^EAt6cC7ovzQN~Tf2Yj3L!39e?|-)9`!K(^ z5B<_$nWCokFddC`Ine#bLrTLPdDO17-m!R;4-ry=NMQx$$FqgvH=b14+X=S_LaH+H z&Oo{m|Bnzp*<2;ML6V}1?XE)Z{(hzSE@{-R9!tnK3Ido~k^n7G6{0BzLTj170^urY z%J#b|{JXQL`UXOi6DM&&K|RvXg*dwvA*G2Eh2vC&2dD(@2UiY4QDK+Y?Z(rCsB2~$ z#Z4Uke+qF|OBQ{HK-LOu4#bGrOps8P+->C%U=&j-@6Pwzrw|BkGS|{(vNMR-q~7ig z2xYgbJu%W`JA`0FM@O(zx4tqAVYo32Xa-#rVLKV10c1j}Cc`@aihMRA_Ls5uJ79Rq zVk)Ym9^AnNKSnJJ@$`eAC&yLW^bm_lhs15@=iV%j4W~Lom*{Oc)w*3iSAQxJnL{8~ z7mAB;8wql%EY1zoX!nRUU^q2eqeBXlM2OB6W0(pxc6cv&4#C8__`)di1dY*N(LIgb zPuK|>d6@=XN>`;YM<@zTceHz2tOoROJ7`;r6^1r+c1;yJYEUl!Z8pSyL%T@y)*;xG zsqn0WrUrAR=L8M^5=1}0o?sXdmjA=BH2C0tO3q~tp$r{b zEu*03n7odmZru-4)0UmHxQLnH~G^xjzjn`M??fLme5?l2F?UYXA|FEo$z1zU>SxL2v$J!ZE3F@3f%02$#8Gk|~H@j+KQU3_3 zyh1A*U9F?Kpgz}o7Jsm1@rY`l6zxuUUHQRZ3_h(C%4;pN``PT3?XsK8=vPk4qb$5p z!c~u5c&;1IqJDvZI*a;ucrT2Bc=)E+p%>Is`+bMlxk&h#vcSTzlhyb8^Z{D+xx#$!G|za3 z%7`%G3dS>3jz3)v@zR!iv$}XbYH~BYR9)=KZRhzEoEBUD@YT8Hk<-6HRsLakAl{9{ zev|J_lrp5%P$pR?Y&h2C1TTdrp`@K>rA(g4w|h8sUP(#2iMAjV4^wJk(;XMu|wYr^HhqrT41-ATVtwrn~- zdu6|XP~bXklkGxF8DL87vbNKP4WP=2P0|G)G0lwKmNBgLx7rFpMw!!x+3KglR?kRL z0q4z@Fl(J|jPy>GvP(G+$!$PDU(KC<&4JL%Vur|xwd||kX9ZFi%lqF0wLM<4jvzF=M(YwLhggt!jaE;RI|ba$AbG6cNt&#jzj^e; zuBuk;IPR?sxwch1;oWrFZfn(kINYkKjyG7uc`g?v2#R`;>px&fqM4O5uGnhbxr3?wiK_xmP)PUncituM)Y>!QI!Z zi1$z7Zthi9+quVIekqB30ZNYha0aD_L}&CKoZ zC<5jtbeIu>*4uq^pViHrVzXg)pB=3lk!VjqEp?jJ3p61#P-HpII>cMKD^{>eZgF$) zCbn>0%(|%xnoW+~9S(8OZ8+KLze!OLOMv30lZg$=MGwzR$hebP!w7EVC=VB9w9i|-@2IBEJF0cn90n!D z^E4&!a2CE$()RFuLw4HsHIcIBadwRR?0Y2|7 zQ!46~&a0SWW~BK48$CZgL<*;d%G(3NPOv^_lfWF4xuh`K{aj4>wl3Fyb#IYK;8bW7 zF4HWe^Dg?yXVst^ zMsY{>(gKIucj%|r3fXJ(VK%!fGoPKlTY04bp^~p&?E-$p4|Va$oF)w=~rHSRH6yWzaK5(em$T} z*}f|K=MBr4@mjhj-~}n%F6B2{r_`g{k5O&dhw02eiOoc0PLNLE(Co^kjL4*_p>;{?#?VS zre!wf_4s46|F>zYkB#T1^eLn2ophH{UOyTCtLrZwJ@O!>{8Cu7g3usddE!ey*{OK$A0e8sFJa>a4b_wEFls8T*2qS8L-6&&VXU%zTc{S zO4WX9_a&&FsIqG3nAx3BO{ZkRRQi}!4Q#?Kdlmo$T^nIPFw~PmCvfufp6yyATvblQ zCcLZFB$ov&XOfinc3yo&MsP~iptq2D2X6%)x`vz7(L-q7a&1hDDLUlG@$Dv)OB>@P zxrsjB!8beIiE1XI7GvF$v>c{1>6bwmrUA!vuRE#Q=vs9|dHVN-0LFX2f0P@Jd-9Ku z;wq*n^5dB~|Jx-#WzXYYyfpHO5e5Ib`;EE+FXYx~yRrbyi%7lz`9IjN^D%qZ2sFRET`sIT;Pmc1R z>MO~?$Bm8|A0O;)&PotCmeLgmV5Zy1vS=R7yzXmGA~!TgVOvycc=s|`O1 zKMp^NAFur5*;!nrNfDo0ZmFEIC|o)9<*Mf{r(C5|IrgV{bdxe-*9~;Z%bRzlQ4_De zrR>;!Ue^8t+u4jPmSavtZRY6$6R$phK>2w03;qQuZr;rmB?Rb%hS1Uxwz58u zrB#o&Y1MNS*V-=-`RdYw*N3sYe zo{V}BjK-^>s5p~_rSa_NE>Wlw2h~qff5@Y+AOw}TkP&-+OA3WyK?l2 zvdR2TpG-!G_2B={F7yk$3x=NJ)s1>H43|0#pd&aM%pJn!&Ij4O>lH8A>{wl{tDUB` z>=C3|m%nS_OP79cfSC<#pe44ciYuB1dBhYTMRx_cLR@t%9OIs;g;I%MHE|H(g0yH7 zDl?jePKm?C5^ih=qGFihj_309@^)9c&}_?fBS4eYFOvj`O#^_(M=)#wI2Bh|fh(J70)_Z()O&L)^p*0&_z`rxT&Pq@6RDw3o0=sNg&NEzqir{j*^IV%=OX zcB+EnIxfB;(J9gOQjwIVrl!u4Q-ZnfodOdJ;$^L=hdF9=GHbxu5Djh zm+VUk=IfoT(E~4xX;E%{Y9t@){&iH@`qWrF z%!?{5Pi6S=22E&(vd}IhecG0fIgD=pv@P34@HTBvqRU{uq>$LT(A#Z=HbAWI*QhC1 zW3$TwJEFT)M4xuD=u;eHE3;gZP7zmiv8`%Dr$}3{$<`=2L*Usag=3;z7b)7+Y>m<` z5SGcinu(uT33SId#y5b)r4@eO=FhZ!-d31rY9~Ag*U-i7 zJU1e+M#7MsN7}dg^R}!x+0yLmFf^yF7D~DHWbV%EOhId>j|ljj4iOo-GpToLq@La4 z9C(q4P9GJc-z|!)V?yZ9BEClkZLeZahb)S$;<2AgQSZ+`sS#(Mr@TupW!GS@(o9U^ z%g=+!*M~;ziP^w*H=))9*WP-R<$VxIr6av75qUd8LT4scHqu*PO7dF6>aD{RE8!vDStZ?lL72>2w48BS+;VO+8#!3k_>$7dJx zITw3TV=I$dtmg&EcuD<2&V)IMsGrK1JDnF6xXkt>W%8c$E}Pg(@<)DCkA8}q&Kh1B zEE)&c-GBv={`LC30L_<%s43hGAR<22(gS3g#@2C6KQ);kyxBU3uz)&n>aSy>va)HI zAMQl4wJj57&I+wTw^xF>VX;~d5D{qE+Ez%=i~hX{cKR|AlxeeJWn_WZ;tgng8bapn zv!p3-*LDY6v=%7etv)kESHsG5tSu97^br$w@ z0gIh=O|b-%x>RG9XLb>z#VV-}Gr$lA7`Ca^Ub(Uk2O>u|5QBsyJQ{tL1(#h;$Kc#W z(59%Nio-`*EWL?!(eC7aYt*7{E@iFA1lHo;L00d-E>IScm$MBfP?LVm3F?TRD?)=q zkqO3Nn<_W8Q%iM4L4buf(8Q$9RI4o{g9kSf*cEL01rxk6FKF3ycDb2@x z&#)Dld#zZ@pZfTyS-rQE)su;=E*#`ToY>$qVe&Mep~}w@4U@*gNTn`bEQF*^ zg1!aHmwOWZo~B@1qK+E~{?k$Y(Y>&dd#L^We+cXpOxg%Lwo%iu*SA1sRDv+45rYoH z4uTuOece~~Qc=Iw3dMl600zz=AY^)*!g*@~S}n;}$$VG}2;clt$_w0Qz>W$($a_w+ zMMsc>+Gjx5i`%W-J8k)wN&&>?lx}Q4cFVo67mj5PnK2yr7Re_C>Ii!0EMoP`Jp3iB;&kwJJ)63c-4)6xbdiVg2+rKX8s@TwX&yH|ak40KI5XR`K&6Z*7v_1&RLW2DEQ-96 z$@1@@uj*^fUDEnDuQjczS#{Ho)-;&6Y00&a7fL}z@zK5B;G)Rjk|iuT7g>F90gQzq zhfZD)6`+F%18yKn;;Rc+_IIDxUlQ&UaYY&^Sb~FJLKCvJ?yaDm4RH`sd?1RoVfDo8 zXetR4up9QwB%wtSIZm;cL|(!BhMz74OtsNcg82PcUo9xgiW*^|R7mbj9+KS26H>P% z@-Mx$4I$kuTEAXqp$w+iF5|YDm8)LM;a(%k-LHwGtneMg?uYOzU2af)2h}DkbI|3L`WW5%pv*lLonX> zct8z7?52wy1EE<-5t_GhOmzPchtT-6%i(64|DYyHk3+w=H#wS^+1;vg19W=683By! zWYYXt_q5)@;dy}Gc4#msRf$bTt5|pL0UYw2{aKX;i^Ge+peh4+UJk(Mzi zEntNF1U#Q7v|Z$cmh3a3aYP@AQ|MzRA2Ss!iJUempY5MxYvp+6GR-Y+HbI5%B1-N7 zCl}z96$gC&yAhn>6F<~GWG@#u;Ea{jsnv|&U|Bwt*@a5M)7$+`j7&lZNVPlM0#-$G zSuBk>r9|VC_Pe4-M}9-W#F>CV)J5RkgG|2oTgKdt^0~P9=pYb=c*b|obACYc_{WG< zumEf&4!iGZd_)VXkqZ)IC!210`?9n&-~Uy@0QD&3)Tep@L_+Ow_4k~O$Y7k@4~ha} zktnDK5b$7;WfXoh?re?SMQiFQSFH)OaBRrhA@Jz>HcYpVbni*ar zUR|u2_towb5!=iFwoUmfVh$RGBFEgWhhGYN#w2aWxfz8m@2b6bSAAiB|v^SxXm3K)xcEAXem(@IU9_hf% zKN#~)rvbcOPcTQMeT629C7M`4ohHYydEIc2blDVVb0$|~Q$o$z#qGVj_MIg`WjYBm z2oED!>}nx;1i_*P!uzD~6%<5+u$RZb(OZR&L%o7MW^1KBP$eg*6R6hNm!94EwzmHg-VSlQ25FxnP=)|MP+@Foa<7wKG9D9vi-)Flw~Lbk!YCNqt6QMb*gXb|A>yhKK4-w#OL5h4z~n5A;%5a}6^E)+ z$Xv1r2{MeUKM)cHmGKk;eUFr|6M)ZBAr!}m_razS#q(zV;u&12gF;Y||VT6Y6%iR|Xg6nBDpg9hn>1rJG7^}Wsv2sMo?vJD*u-gIy^Rwer8 z72HToseEe=hr;b|p-@m--tuG#>Z>u_yI!_`o;5eI(*jtQP(l4mOetcu}~R8G}aCMe%HHW$JSWU9irN) z`ukoh?!M7F254(;rCZdvky2!^=lA-RzIQLk-i;gH=PqEd?)h^+D1U)raH-gk|f-e&;Epf*%3W-)NMm25ROIX=wV4_0x1>s10D6mxI(Dv2NJ z<1GD9--$avypFr1SNZnC8JV+t*=icG6+-9WI8JzPKXP-)y~=GLxw!VIvj3y(lm=UE z9PHGQqtN%x!&+vRg10r|1 z(PXyXWL##?&k>TSSBRY&!C)E*trruy2^TX?J56jR0@ zn}>&oV#e7e&*tQ1Imk^xw4k( z*e=AeeW;EvR(u#s5k7Zu_5o$_=VRIX*8Hr5 ze$%hq^u;{x^?v1*FI?Q?{fhR5C+!Ac{uRr#(QIKQ)T&;qtod>#x1fLP7g_P%xtau} z?2DYl!ES6uH~;L@yZNAxb(6jmsJ-}{ZnpL*YrmW^@;0Eg!w1g~qg~~9d^@;#Qk9I& z1G@3bmtW>^i_poJI_8Y}$}zh}z}a<3aDRS-Td*4_>Esd}uuVweIZmM?5kUitNERY$ zEYdw^fL(uJ_us6lwNH`1(qZ!aSNf)Y`PC@y{az*d)l%;1US;7454XNodHzHtH?LQ* zeC;V5+Pka6j4xRYN&MyZl8!_R+hE$OJoxpD+5%JC312PSCgUq`EL_* zYiyVZc(ZPbvzze|YbNu_lgYB43v=x4PRBI(iwHDhIUa{0BWqZ>~Y5NeEK7>Fr3@S?_9^Eo*lg#Rx{Uo0k ziT%U2!g+n&k3;?O=tqqALF$v(bR-ByIf#k0Nx^SzIy=)<=??hE1X$sHtfru$OOn|g;gx1E^s+bNe_njSa- zlTw~E7BNXM#$(X=+UKxkI!f)B5>jx}J<6-6=KDA9^SwG~Hk$9{e=-{tNx5;eku2fb zaRm>rrCk>^_3sle7}PA;x1Ndmf($iv6!c#PcNUtK2iv4L!uz{ysK0tpbb8feu4HEX zvp~#s&$!{fksoWsv(NA1*p*Ko;xr?H9Lm&=@h93yNjh%2{u&>(6iInk^>3K`z^^J< zWnO9a4(pQV_Tt0v^7Q$hh4|1KC}AP1-@NYLRqi#Jx|l)>-&ti@?Oy%P>dmX>uNkJ7 zv3`Ephkw2Y5)+H;D)5=c&i&vlTX^yf)lCP%Rr69JvF22yag(|1-`SGQ;@>RMng zXhGfJU|C+Q)C@ZVTkU-6+_h{`s>#IsYR zk2N<&^j4>}!w#<~y+=8Mu;&X<@DhsHWIfT*^?SI?BqZ&DZRXCFPvHsTgiYa*0k(O9 zDLQ`LrtPYLZ8_YR*v4>Mzcq~CYR9oDN>vSelH*xe*^~U^4rw>da~Av-b20bFy0^X} zV|Lh$h-Vj~LxVhCBgGym8E|7XE4lOk&!bjelsG$V$6Z^UdH#XUoZ1B%E#vpm)!{+~ zHWYOY@tffH5W6Y@U*LJl7oX?Fg5mY}*vh9Ia{v+#6BM>0nyPA#e&of=TvLCskpVM0X|Mj&v75MNi;gNvULi z42<&P!$sh2E+e>=;XPiDw>XF2DZfpu0S{UvZ!P|fD4W=lVr*C23`r@>kS@4j6j)TV zL#r+JE}QS3mW1vU z`^TDvmydiFIPXxTeblvxuIlaBrD1J5*MqVA6F9>33R+w_f0Aacs?>68Zh|+P(}p#-U3*dVJQz)vTFwY9G>m0kyy&X3zun&$B_+Hn|{VVsje*&k?igKRS zt0e!%Go`gy8Hvp@jEB3A$KmNt`a!l*o@47)thltxz3E3k@Y*A{fe9vExmrzw7BK)Q{&>at-#jR(K~~s$<+o&uYaqR)!Qy4*A!#mCx%!C z3)&Is$2gG1TrcHK)H2t_MuYqrp^#_WBWE?9SzlIT%a#NJTj1F3vq!!_q(hXc5`G7g zHtkzd9x~`A)$M!9JAr*jlk8pZ&2)A-aC@<~&42Lhv_%Ic{hpBPgD??lz4%uv{^Ge! z+%J9$;jiY4e>FS(ThEr%?u5NZKBh~i*YDI~(?ip6bS#7NJ&{TEvU^rXhUP%`O_H5m z?-d&5JSVx!Zmsfh;BiRg4y`bi!+NXQ;we%-_DHCc`bvsz-Hp=wT1H!cv}eSP>b4e^ z&%PX~psM|Z){yf6)~*}LR(m0-%KkKUy_a2F)}iIPJ2a&n7s4W`-6>_C)Y?j@#`4Tb zSZZ*zkfWW}6-N=L5s#>7UGK?nUvM9Utd&KTGgl=?aWEh?|D-Ltc>H| zv2D<@3uVCacBi}4+L+FZ<=mJZ?JU5AmW%k=+Z%B*6SzytN>NPMPACN(J1ND3zg}|M z8sC~!9)4l-oh{F3QXAHgESM>@totfCXLqHgD|z~@)f7IV?Nv_U-cu)IxxCvo_6WSr zpE~W}0R9qU+Lmxk+asMh^OI}4^gAG>$o{kqZfg{or@2~Ab=u}l3|n_Na?RIl-SxB{ zA*DVRA*B=_C!CX6uMOd_8yYw^3Kj(D_(fBRCGB{QmbXshTT%wf4G_N-*ahl4{fQGjrT8EX2yb)EY?rIH2ZmZkP?oK;YN)xv!8vL zcmnTQFQS4VP|0JzpV*jEA@>7b+7hDCGF7ZI>aDb)8{Mo4@3&4 z?xRf=Rs?CIp)7(t{W7|AL6Z9VkWBP{&hW5>70LaPgc@j;d#Mb1&Tc97d?+s(4Tfb zKa;X8k={3C{M-Eo0=7l|#5P5$hGCGv&gX9gxD{hd@< zGE9psbIxEy0wZe)4Ng6mC^;XmJl_fX9{=|TGHYY*GR^P^v%e0bc-*wpgOzXerAjf{ z&6cDtkA$6WV3%V@i=Nn?vQN%W2SK7?d1Mm`H(Vw zP!#)yE=}+uQ zm?y&S**OMxufJELsLgQsw+>cMZI)4*a#E|;V17RzR!6tuB@}xTxs5LUzCU5;YI%pl zUon({^)d1MdYWu2fhoz2{wIdA^36h<>p}_1CR9|t(H#SOH#fG(g=h(7fSx~$_>lgN zq3_Dgt7 z^qR>n3MwXLa8eixRI-1;D4Hz)=Fv%4*2sHc!S6v|Y#Y+pl6eFH-{G2?XN9kDHl~M{ z#8yJ^J6pUv4@VBMpMbw@G&3dr!S$N;QCe~}{_XF8qk#YHXg)o>b0QUwe>nA+V{ALC z>YuQ$v_-m*zVmA|<0e#6ylR25fz@^OV}z{=upvT`oZTBugmVc-)q|vO-XTS9805G6 z%|FGw`Jcest=nw#?SRj>=K$AIZ3wo6y>{3s$hq@lQfimVC$QtxiO$05H1nsYe+VlN zDRLLOe-|q>c*9{M6zTm=tHi5DSOmmpIU4;18vbTCdlJ3SodNd^vfGrY`BKjwgtkbD z{7!Pv-~ayy$U3BH`2QK{3+Z%BqmFPiRLnPH0Dc*xlwRZ+ycW-XMD} z?w!6g^f7-tkxhD>a}5TO=F_oFsQJ38`NT1zg?6n0H|Y~rnaU_Ztd=RiwiPda9vJF| zHiwf6zS#j+1brNp*?&*?gA7l%qvEt|3YS0p95Z@{1={ys(!jNgt~ z86N%gM(@S)pM;1c?iT*|`5EBeQ5U0t^v6OFvL-=uy@P))mA+4(h;vrDD}NyZVxeBiA^7mz(Z6L2BHyDuc%~nwQW$>P+rPI(|2=X-JZht> zt-VoP;fRf@XSvf#*I!@t33g(8y6SkKXHGh zCz^erM&A?0yo1v%Wj#$;HGRw??oXx?&e|hlZ!#tfPKaa4G=2-BDA&d4062jFsA(p-z~3=Mt>;w zjiLE`{x6G_YB(4P8jM#+vEWO!S;;Y&M&moNCXNa^^nxD`%aNm|ZDB8rj zR2rXZlVQ(=C79BL`OrjPazuP56`z4k>`Fy8ZqmecX*5Y3Po+uJF3zUXEct~IF*%La z$kJhPcN$cGAbyfY^Ql@KNTXTwocJ=0vhaKy=`?lRRl`Nw$?o__k4YK%cQHSmzHW_1 zUmg`r>69VgKP+BPr>erHVK$k4QnOZYj9i}1K&P7h^mx5F5zTV=4g9_m1yrgL-tqK8 zdavH9CHkc7maYw1$=qB|6hq@-^$<=IxfAGqDiu#ppoy-9L$r8ji{% z;^B#;$n}TCvlA&qN5t9;S~E*8{?0Ir2J|T>3qG`Fy`{zYB8&+|Y%hcwu%3;3L|X=3 zKmH${Al#G4r?l&Rr45=}+1-M2DAR?+hxl7g^+^RVFh~_`ljzwo>xQji4+2VYWx7q= zHJRoY{!ruaFfUyFt@b}+zk}9pd7P?cwQ;3>1hpx5Hr=SyB?Y-4@z!MeY<_ISknFXi z*M>f8VPbpWHT#9a8hdxH!x=`5C6^uL&l0xx^j6qW2EinbPoe3xZ?`Po{?>Vq#4+g< zsbp)xmgCdpSsBOWf}D0L@I4`oYsg_Qz?JkfA`x^g80S=uWO$eKaEyW zvbbd$y755yi|| zC{9enr>6*K7Am7cQIG{jMLdv2mGm?5ZWi50EB8-T$U>-M>Q;-~Y^s)zNaCq%3er1b z2{K~;3BI=@c5!zOm5-(F8|>1B!UwNMKlO-3yq-gA$JiaaF;$Z?GbT!%PM#dwE=QxT z+S9w9`^-hv-b^X-CbpNK5_V$Gmm?&;J)H_``8Ud2(c@uz2`9e{sSE8xccn$@zF}-> z&r(8(YG0LkOr z#h;9G7mvxDySiU<>~dC331?JI3cIQ%hMkR`7AGu?7CYOWjpQEJVQWl5aHmt~%TTz+ zI|N@oJccaT|qffldo0)pA{!@pv%c)OaJH`22x*{b2CpaFb%hO?lf&HRB5=&;# z;_TYtnGbS@N7e^W7or~@kP#&wg8IJeuLm8!Ylh zqC1Af5Avv-UKj7>QNG+cAo}vqjD9d63TIM@qh)}VvcxxMqQ^2u{A?yImTTPN_cN); z{)f@NXwt}U-L#cb9y=gqk3_c|aJ?EE@Es95^J!J`BN#{UTty`M`Rn3zK5e35adiPL zK@5myAId@!yeatEyRB;CUaYvPcKmC>+qdc^;$)Gmmj4O1OvYb!m^Pl!kDWzQSZ;Jw zvl(954}Fd+Px$Pf?w{A2XSQoiO_TARNRh|uv19Rp_d630$TJXt7eRY(@y@ID##+Tj z)Qnd2%6sZ65!p-Cnp2(KPCKTT5kILp6_r)w&0Uk62dVUg>rgq;+_d#Fr9LU~vZrdZQ1GihE~M zW7bB^7Ve8aGK#o*7?}QYlrP8^yZx&tYsP$ZQcRsglWu!|g!AIj>-jeAgsa3p>m$C0 zF?#+d;$Lo7L_G?CbUu2W#@g@(G1}+4>hsYu%`95hq9pthTN&qHuF!-whw?1d^(RI4 zT*@O!l+Q&6$0Z(~OIObL^{@ra%1E?%v=u7{8|(P8v-Qi4H!eF4CmxTTI4~DeH7=1` zM4m+(Fw!^|?L>^mk!bM{CQh-=%qM5+C-ak+#?8heu~k}yTVLVj{zR1|b{C{nWtMxa;Aw>++ioIifR-uGi(dA*_2X zVj0FfPSBu5v_ZZw=Us+ZJ-_};SVmi=vHBT$KwQjU!s5XTpNS>N`!(X`Vp=;B_L|s( zbus!NG<_th=!I!{@8KZxEbo04DIxupI95#W%WL~Z<2=f>4n}{eiMDxkI~prxKB|X9 z;_>-Zihk7L`FK+Ih|c+xmH!h2UdzVqjDgMU2}~pNr;=E?BI1z~E1D0%u|wZkcQpFE zC|*FTX@Pij0bM~&V&4LKh(^TBh4i3}M-mIfa|>zRm`0bB8$j9Hny~|-{)O~SdRN@D zh-ToFZP-CtE*ca^7Ll7;#o0yl4IAPg9-Xj%(-mkvZIjpbv}K69OX!d-1>!W3x0rSy z^9L5w9tukrrio)qX=To#A&ZiZ0u|fNa7_-%4cEijiP%0=`Z0UQi#b=( zWOpn}>3Wu4K$cFu@^34gmu1OS$GT25q@K;S@IVmE33AX(sB;nH2~HCryY% z{{>d=7-rnGoILqSRTN1YtbpC}E7~4Rb+}vh7}0LFA?1a183EjfqA0k_X#$E&Ut_lT z<8s7I>K*%!EbZFw|~wG%nPOWQM*>nHj~)A{cS|2;u%lT00#@XamN0hmE@9bz0rFP zuM|(OrkO%rP3aTZsP)>u1}ocqkaBIG3ajJo4uKKU0*nNI>ZO|t8o2t$c%R&f@QG0M zzo;5&F*&yTz{$9lDvPnJXhQmSy`a0Qj*XGMcbxxZ12jpLtfI29FAkrJu0xNE-Q8+P zJh_T)Tk(iKT#+JaqhS~)jcJl1V@I1DW%^OVsDh)hkXc8=_Sn(rNE0^Sb{!hq#9a@J zG2Rdb<+P#HqX z9d<$W$0Jq7QT>Qw96dG?rn1B1@auO+nut#yXPo{mv3NCoM6wvShUSR1Yv>mFj4WPU z1B26l7N3Bh?4@l;>oDgV3wux&3+ADqGEBDkcCSeJP$+AuP^l>1q`{D@n&0t1+O$~Q zzm_sLoMA7V=~+Io=u48V__pOpmd!bP@^PnZ!8eV%VO~b%|2tqoe7>fH#fxicTJ2bk z+zA~Sm=0`3v;qzffUp1v4RFNA_R+BcCI-R2r+Mjm9*2%-^j_>3pkKiK_T%1LwWZ^v zeNCH@J3W8`N|r2R^>clt2NAz6hR#66{C68N*Ty}pYr=m-kk~dv1u}2b(!GP!`Lg^} zi~!8yg6BSohCgNRN8(H&y_wKN_xfA?J* zo93z!@^zHs*oOvgF#0z!>pCjEhNJ3K>XjfK@&7lrj`;H8&W_boJT0H#kX^G1+dSbb zZA})f*P)}H0_UP;5`VjnmQL7VwW+-MCb%zO5b>;|X>_x=eI1RLn=Rry>!@7bVHIub zC|$;S$;bFPBl)t`5+{ZlE)QR*Sr*9^RGzWJ%2TudfcpLlP`@d*RnR24*(P4BAlL_6 z#osD0?-N^!nKt2}%M2h7!c6c<)Hu*xy8eIbqOL!*p0XB2+)wV~MHGHWslFrUEi3&i z+`PxP<%?G!n?42K1j=;TZcH9Nt6gy4h*0i*M@8xeD$D$5fjGpsH;_AtOYS=;!|!MZ z#lLMpkLy11?gpAkXT`Y-r`l=vr`kz0G@oT)OP6#w z3Rr2$WX~S0w?uq+J>_luYyTbuBa7*sfquC1Q|-?gdP69`r=3QlbjEFqJhtNA^FO}_ zXE}Lk>dvZPR3-JDCZ)>l<-ZwcqbK`0bx%#|J40uq9sT<^+<^Hmd&k%7q@}Nj@83kt zsZR{ld7seNj9eIAo{QEG2+z$l`MQS&R(RLq?{5v*!snv@W&HKW0jqZ`1~u-r=dc!M zISDWJBJw)01Uq9bugsl`Hus;4E*f;@CSX1+@l)6wPd^uZq<{b3o2ktr?-q`B_0IaDZi=#`}#0c1923P1%RM|5K@Mz-V*&_D<&!_I zz|_>)APWFN10kdgQzxG7Kkt@TUDaSEnV}jmRdqVZ0zl9}2r0wVxr|P+U)3-MOn_>@ z)Y6$C3jjd_A*2jb$7E6-l$mJ)m;lv)siiYP765_%N?J)H^r{xkrp0aHt7 zf-C?84TO*~OdXTS^Fy7<4JJS}U~1`1kOhFCfe=!Lsbex#Y}c8J!33xVOf8)WvH%b? z5JJi@bxfwJojTK%U;@)Y6$C3jjd_A*2jb$7E`H zO=r3dOn_>@)Y6$C3jjd_A*2jb$7E`LOJ}+dOn_>@)Y6$C3jjd_A*2jb$7E`IPiLwD z6QCL}wR9%P0zl9}2r0wVF`3#==uBI{1gHi~Eu9In01z|~Ldr08Os4QDo#_W)0#pO0 zmd*rO0010A3mFjaLr$O1snKnN+r)QP9-cUVC8BIp3sfT^m} zK^6dl20};~rcOLvLJHHp3_3tHV5;hLkOhFCfe=!LsS{5(X)@Ebf(}p(n5sG*WC0*( zAcT}*>crFKPiML}K?kS?OjVr@vH%b?5JJi@b>it37c$-NK?kS?OjVr@vH%b?5JJi@ zbxgWg&s=?htm2x5ma?lFj)N0W4Vd~mCu9L2Xdr}?Vd}(lRxY!k-Tn-8fNH>0)#)G$ z06_yGqzqFhp00Wo)BOc>fNH>0)#)G$06_yGqzqFhp3YanbbkXKpc*h$bvnobK+r%4 zDZ|u>r)#){>7t+mR0F1}P6t^42pR|>Wtckgbd7g1og~W))qtt0(?J#hf(AlJ8KzD= z-QI69T>|I;)qtt0(?J#hf(AlJ8KzD=UF*Y4mkK&SHDIdhbdUvrpn(ulhN%-zSH~lk z_aC>EEu8{NKs8{h>y(fMfS`d8QiiD$PkHjkoWoqu0jdF0Ri}e200a$$kTOi2c)HU+ zW4hU(15^X1s!j)400r*po*bjv{p zs0K_`oer`95Ht`%$}n}}=`w!BbmgD}R0F1}P6t^42pR|>WtckgbnXz-tp^>T8ZcFL zI>-V*&_D<&!_=k;8$k!C2253*4zd6cG!R0{Fm>YTst$3*_YPNlcYza74Vd~m zCu9L2Xdr}?Vd}(lR{nvrct7X>)qtt0(?N!yrE<_f2r0wViKnYR$#jo^4p0r4syZEH z0U&4~gp^_G#MAjcXS(l$4p0r4syZEH0U&4~gp^_G#M3o=#dJRb9iSR8RdqVZ0zl9} z2r0wViKlBk&vYv20M&r0s?$Lh0D=ZWNExP1Jl)U59=fS`d8QiiD$ zPuFU*g6;*-0jdF0Ri}e200a$$kTOi2IJ#M$U&%6~+&~3VhN)eJW5$C!>%=&hHCxoF z@IrY+#>GjgPNK=rmo?h=SO_`dcL7lPhK%IE7i)JLbgR;yhJy-{Y@qD%` zsS*OlsWx5c^x>ERjda#X7s>~e{_KZ1sKSBZ?!o>VmvyS_cv+UMfT`(XO&2s>tm)D< z1vHZdYIVr?^GZ&$&d$O)&477pZ8&C(%Jot!QqEf%Nz!Q?Gguxrlock); - fops_count = kfile->fops_count; - spin_unlock(&kfile->lock); - - return fops_count; -} - -/** - * kbase_file_inc_fops_count_unless_closed() - Increment the kfile::fops_count value if the - * kfile::owner is still set. - * - * @kfile: Pointer to the object representing the /dev/malixx device file instance. - * - * Return: true if the increment was done otherwise false. - */ -static inline bool kbase_file_inc_fops_count_unless_closed(struct kbase_file *kfile) -{ - bool count_incremented = false; - - spin_lock(&kfile->lock); - if (kfile->owner) { - kfile->fops_count++; - count_incremented = true; - } - spin_unlock(&kfile->lock); - - return count_incremented; -} - -/** - * kbase_file_dec_fops_count() - Decrement the kfile::fops_count value - * - * @kfile: Pointer to the object representing the /dev/malixx device file instance. - * - * This function shall only be called to decrement kfile::fops_count if a successful call - * to kbase_file_inc_fops_count_unless_closed() was made previously by the current thread. - * - * The function would enqueue the kfile::destroy_kctx_work if the process that originally - * created the file instance has closed its copy and no Kbase handled file operations are - * in progress and no memory mappings are present for the file instance. - */ -static inline void kbase_file_dec_fops_count(struct kbase_file *kfile) -{ - spin_lock(&kfile->lock); - WARN_ON_ONCE(kfile->fops_count <= 0); - kfile->fops_count--; - if (unlikely(!kfile->fops_count && !kfile->owner && !kfile->map_count)) { - queue_work(system_wq, &kfile->destroy_kctx_work); -#if IS_ENABLED(CONFIG_DEBUG_FS) - wake_up(&kfile->zero_fops_count_wait); +#if !defined(UINT32_MAX) +#define UINT32_MAX ((uint32_t)0xFFFFFFFFU) #endif - } - spin_unlock(&kfile->lock); -} - -/** - * kbase_file_inc_cpu_mapping_count() - Increment the kfile::map_count value. - * - * @kfile: Pointer to the object representing the /dev/malixx device file instance. - * - * This function shall be called when the memory mapping on /dev/malixx device file - * instance is created. The kbase_file::setup_state shall be KBASE_FILE_COMPLETE. - */ -static inline void kbase_file_inc_cpu_mapping_count(struct kbase_file *kfile) -{ - spin_lock(&kfile->lock); - kfile->map_count++; - spin_unlock(&kfile->lock); -} - -/** - * kbase_file_dec_cpu_mapping_count() - Decrement the kfile::map_count value - * - * @kfile: Pointer to the object representing the /dev/malixx device file instance. - * - * This function is called to decrement kfile::map_count value when the memory mapping - * on /dev/malixx device file is closed. - * The function would enqueue the kfile::destroy_kctx_work if the process that originally - * created the file instance has closed its copy and there are no mappings present and no - * Kbase handled file operations are in progress for the file instance. - */ -static inline void kbase_file_dec_cpu_mapping_count(struct kbase_file *kfile) -{ - spin_lock(&kfile->lock); - WARN_ON_ONCE(kfile->map_count <= 0); - kfile->map_count--; - if (unlikely(!kfile->map_count && !kfile->owner && !kfile->fops_count)) - queue_work(system_wq, &kfile->destroy_kctx_work); - spin_unlock(&kfile->lock); -} #endif diff --git a/drivers/gpu/arm/bifrost/mali_kbase_caps.h b/drivers/gpu/arm/bifrost/mali_kbase_caps.h index a92569d31f06..f6bcdd06e1aa 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_caps.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_caps.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -33,15 +33,26 @@ * * @MALI_KBASE_CAP_SYSTEM_MONITOR: System Monitor * @MALI_KBASE_CAP_JIT_PRESSURE_LIMIT: JIT Pressure limit + * @MALI_KBASE_CAP_MEM_DONT_NEED: Not needed physical memory * @MALI_KBASE_CAP_MEM_GROW_ON_GPF: Memory grow on page fault * @MALI_KBASE_CAP_MEM_PROTECTED: Protected memory + * @MALI_KBASE_CAP_MEM_IMPORT_SYNC_ON_MAP_UNMAP: CPU cache maintenance required when + * imported GPU memory is mapped/unmapped + * @MALI_KBASE_CAP_MEM_KERNEL_SYNC: Kernel side cache sync ops required + * @MALI_KBASE_CAP_MEM_SAME_VA: Same VA on CPU and GPU * @MALI_KBASE_NUM_CAPS: Delimiter + * + * New enumerator must not be negative and smaller than @MALI_KBASE_NUM_CAPS. */ enum mali_kbase_cap { MALI_KBASE_CAP_SYSTEM_MONITOR = 0, MALI_KBASE_CAP_JIT_PRESSURE_LIMIT, + MALI_KBASE_CAP_MEM_DONT_NEED, MALI_KBASE_CAP_MEM_GROW_ON_GPF, MALI_KBASE_CAP_MEM_PROTECTED, + MALI_KBASE_CAP_MEM_IMPORT_SYNC_ON_MAP_UNMAP, + MALI_KBASE_CAP_MEM_KERNEL_SYNC, + MALI_KBASE_CAP_MEM_SAME_VA, MALI_KBASE_NUM_CAPS }; @@ -57,6 +68,11 @@ static inline bool mali_kbase_supports_jit_pressure_limit(unsigned long api_vers return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_JIT_PRESSURE_LIMIT); } +static inline bool mali_kbase_supports_mem_dont_need(unsigned long api_version) +{ + return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_MEM_DONT_NEED); +} + static inline bool mali_kbase_supports_mem_grow_on_gpf(unsigned long api_version) { return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_MEM_GROW_ON_GPF); @@ -67,4 +83,19 @@ static inline bool mali_kbase_supports_mem_protected(unsigned long api_version) return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_MEM_PROTECTED); } +static inline bool mali_kbase_supports_mem_import_sync_on_map_unmap(unsigned long api_version) +{ + return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_MEM_IMPORT_SYNC_ON_MAP_UNMAP); +} + +static inline bool mali_kbase_supports_mem_kernel_sync(unsigned long api_version) +{ + return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_MEM_KERNEL_SYNC); +} + +static inline bool mali_kbase_supports_mem_same_va(unsigned long api_version) +{ + return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_MEM_SAME_VA); +} + #endif /* __KBASE_CAPS_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_config.h b/drivers/gpu/arm/bifrost/mali_kbase_config.h index 7233e2dd3920..2f9e28aaec9a 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_config.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_config.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -166,8 +166,9 @@ struct kbase_pm_callback_conf { * * The system integrator can decide whether to either do nothing, just switch off * the clocks to the GPU, or to completely power down the GPU. - * The platform specific private pointer kbase_device::platform_context can be accessed and modified in here. It is the - * platform \em callbacks responsibility to initialize and terminate this pointer if used (see @ref kbase_platform_funcs_conf). + * The platform specific private pointer kbase_device::platform_context can be + * accessed and modified in here. It is the platform \em callbacks responsibility + * to initialize and terminate this pointer if used (see @ref kbase_platform_funcs_conf). * * If runtime PM is enabled and @power_runtime_gpu_idle_callback is used * then this callback should power off the GPU (or switch off the clocks @@ -179,15 +180,18 @@ struct kbase_pm_callback_conf { /** Callback for when the GPU is about to become active and power must be supplied. * - * This function must not return until the GPU is powered and clocked sufficiently for register access to - * succeed. The return value specifies whether the GPU was powered down since the call to power_off_callback. - * If the GPU state has been lost then this function must return 1, otherwise it should return 0. - * The platform specific private pointer kbase_device::platform_context can be accessed and modified in here. It is the - * platform \em callbacks responsibility to initialize and terminate this pointer if used (see @ref kbase_platform_funcs_conf). + * This function must not return until the GPU is powered and clocked sufficiently + * for register access to succeed. The return value specifies whether the GPU was + * powered down since the call to power_off_callback. + * If the GPU is in reset state it should return 2, if the GPU state has been lost + * then this function must return 1, otherwise it should return 0. + * The platform specific private pointer kbase_device::platform_context can be + * accessed and modified in here. It is the platform \em callbacks responsibility + * to initialize and terminate this pointer if used (see @ref kbase_platform_funcs_conf). * * The return value of the first call to this function is ignored. * - * @return 1 if the GPU state may have been lost, 0 otherwise. + * @return 2 if GPU in reset state, 1 if the GPU state may have been lost, 0 otherwise. */ int (*power_on_callback)(struct kbase_device *kbdev); @@ -223,9 +227,11 @@ struct kbase_pm_callback_conf { /** Callback for handling runtime power management initialization. * - * The runtime power management callbacks @ref power_runtime_off_callback and @ref power_runtime_on_callback - * will become active from calls made to the OS from within this function. - * The runtime calls can be triggered by calls from @ref power_off_callback and @ref power_on_callback. + * The runtime power management callbacks @ref power_runtime_off_callback + * and @ref power_runtime_on_callback will become active from calls made + * to the OS from within this function. + * The runtime calls can be triggered by calls from @ref power_off_callback + * and @ref power_on_callback. * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. * * @return 0 on success, else int error code. @@ -234,8 +240,9 @@ struct kbase_pm_callback_conf { /** Callback for handling runtime power management termination. * - * The runtime power management callbacks @ref power_runtime_off_callback and @ref power_runtime_on_callback - * should no longer be called by the OS on completion of this function. + * The runtime power management callbacks @ref power_runtime_off_callback + * and @ref power_runtime_on_callback should no longer be called by the + * OS on completion of this function. * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. */ void (*power_runtime_term_callback)(struct kbase_device *kbdev); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h b/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h index 20003c852863..baca78679f0b 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2013-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2013-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -194,9 +194,22 @@ enum { */ #define CSF_CSG_SUSPEND_TIMEOUT_CYCLES (3100000000ull) +/* Waiting timeout in clock cycles for GPU suspend to complete. */ +#define CSF_GPU_SUSPEND_TIMEOUT_CYCLES (CSF_CSG_SUSPEND_TIMEOUT_CYCLES) + /* Waiting timeout in clock cycles for GPU reset to complete. */ #define CSF_GPU_RESET_TIMEOUT_CYCLES (CSF_CSG_SUSPEND_TIMEOUT_CYCLES * 2) +/* Waiting timeout in clock cycles for a CSG to be terminated. + * + * Based on 0.6s timeout at 100MHZ, scaled from 0.1s at 600Mhz GPU frequency + * which is the timeout defined in FW to wait for iterator to complete the + * transitioning to DISABLED state. + * More cycles (0.4s @ 100Mhz = 40000000) are added up to ensure that + * host timeout is always bigger than FW timeout. + */ +#define CSF_CSG_TERM_TIMEOUT_CYCLES (100000000) + /* Waiting timeout in clock cycles for GPU firmware to boot. * * Based on 250ms timeout at 100MHz, scaled from a 50MHz GPU system. @@ -213,7 +226,10 @@ enum { * * Based on 10s timeout at 100MHz, scaled from a 50MHz GPU system. */ -#if IS_ENABLED(CONFIG_MALI_IS_FPGA) +#if IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) +/* Set a large value to avoid timing out while vector dumping */ +#define KCPU_FENCE_SIGNAL_TIMEOUT_CYCLES (250000000000ull) +#elif IS_ENABLED(CONFIG_MALI_IS_FPGA) #define KCPU_FENCE_SIGNAL_TIMEOUT_CYCLES (2500000000ull) #else #define KCPU_FENCE_SIGNAL_TIMEOUT_CYCLES (1000000000ull) diff --git a/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c b/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c index 237a3b829be9..1e7e823f44c4 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -106,6 +106,7 @@ #include #include #include +#include #include @@ -152,13 +153,21 @@ static const struct mali_kbase_capability_def kbase_caps_table[MALI_KBASE_NUM_CA #if MALI_USE_CSF { 1, 0 }, /* SYSTEM_MONITOR */ { 1, 0 }, /* JIT_PRESSURE_LIMIT */ + { 1, 22 }, /* MEM_DONT_NEED */ { 1, 0 }, /* MEM_GROW_ON_GPF */ - { 1, 0 } /* MEM_PROTECTED */ + { 1, 0 }, /* MEM_PROTECTED */ + { 1, 26 }, /* MEM_IMPORT_SYNC_ON_MAP_UNMAP */ + { 1, 26 }, /* MEM_KERNEL_SYNC */ + { 1, 28 } /* MEM_SAME_VA */ #else { 11, 15 }, /* SYSTEM_MONITOR */ { 11, 25 }, /* JIT_PRESSURE_LIMIT */ + { 11, 40 }, /* MEM_DONT_NEED */ { 11, 2 }, /* MEM_GROW_ON_GPF */ - { 11, 2 } /* MEM_PROTECTED */ + { 11, 2 }, /* MEM_PROTECTED */ + { 11, 43 }, /* MEM_IMPORT_SYNC_ON_MAP_UNMAP */ + { 11, 43 }, /* MEM_KERNEL_SYNC */ + { 11, 44 } /* MEM_SAME_VA */ #endif }; @@ -167,13 +176,11 @@ static const struct mali_kbase_capability_def kbase_caps_table[MALI_KBASE_NUM_CA static struct mutex kbase_probe_mutex; #endif -static void kbase_file_destroy_kctx_worker(struct work_struct *work); - /** * mali_kbase_supports_cap - Query whether a kbase capability is supported * * @api_version: API version to convert - * @cap: Capability to query for - see mali_kbase_caps.h + * @cap: Capability to query for - see mali_kbase_caps.h. Shouldn't be negative. * * Return: true if the capability is supported */ @@ -184,13 +191,10 @@ bool mali_kbase_supports_cap(unsigned long api_version, enum mali_kbase_cap cap) struct mali_kbase_capability_def const *cap_def; - if (WARN_ON(cap < 0)) - return false; - if (WARN_ON(cap >= MALI_KBASE_NUM_CAPS)) return false; - cap_def = &kbase_caps_table[(int)cap]; + cap_def = &kbase_caps_table[cap]; required_ver = KBASE_API_VERSION(cap_def->required_major, cap_def->required_minor); supported = (api_version >= required_ver); @@ -212,7 +216,7 @@ bool mali_kbase_supports_cap(unsigned long api_version, enum mali_kbase_cap cap) * Return: Address of an object representing a simulated device file, or NULL * on failure. * - * Note: This function always gets called in Userspace context. + * Note: This function shall always be called in Userspace context. */ static struct kbase_file *kbase_file_new(struct kbase_device *const kbdev, struct file *const filp) { @@ -224,17 +228,6 @@ static struct kbase_file *kbase_file_new(struct kbase_device *const kbdev, struc kfile->kctx = NULL; kfile->api_version = 0; atomic_set(&kfile->setup_state, KBASE_FILE_NEED_VSN); - /* Store the pointer to the file table structure of current process. */ - kfile->owner = current->files; - INIT_WORK(&kfile->destroy_kctx_work, kbase_file_destroy_kctx_worker); - spin_lock_init(&kfile->lock); - kfile->fops_count = 0; - kfile->map_count = 0; - typecheck(typeof(kfile->map_count), typeof(current->mm->map_count)); -#if IS_ENABLED(CONFIG_DEBUG_FS) - init_waitqueue_head(&kfile->zero_fops_count_wait); -#endif - init_waitqueue_head(&kfile->event_queue); } return kfile; } @@ -313,33 +306,6 @@ static unsigned long kbase_file_get_api_version(struct kbase_file *const kfile) */ static int kbase_file_create_kctx(struct kbase_file *kfile, base_context_create_flags flags); -/** - * kbase_file_inc_fops_count_if_allowed - Increment the kfile::fops_count value if the file - * operation is allowed for the current process. - * - * @kfile: Pointer to the object representing the /dev/malixx device file instance. - * - * The function shall be called at the beginning of certain file operation methods - * implemented for @kbase_fops, like ioctl, poll, read and mmap. - * - * kbase_file_dec_fops_count() shall be called if the increment was done. - * - * Return: true if the increment was done otherwise false. - * - * Note: This function shall always be called in Userspace context. - */ -static bool kbase_file_inc_fops_count_if_allowed(struct kbase_file *const kfile) -{ - /* Disallow file operations from the other process that shares the instance - * of /dev/malixx file i.e. 'kfile' or disallow file operations if parent - * process has closed the file instance. - */ - if (unlikely(kfile->owner != current->files)) - return false; - - return kbase_file_inc_fops_count_unless_closed(kfile); -} - /** * kbase_file_get_kctx_if_setup_complete - Get a kernel base context * pointer from a device file @@ -352,8 +318,6 @@ static bool kbase_file_inc_fops_count_if_allowed(struct kbase_file *const kfile) * * Return: Address of the kernel base context associated with the @kfile, or * NULL if no context exists. - * - * Note: This function shall always be called in Userspace context. */ static struct kbase_context *kbase_file_get_kctx_if_setup_complete(struct kbase_file *const kfile) { @@ -364,103 +328,38 @@ static struct kbase_context *kbase_file_get_kctx_if_setup_complete(struct kbase_ return kfile->kctx; } -/** - * kbase_file_destroy_kctx - Destroy the Kbase context created for @kfile. - * - * @kfile: A device file created by kbase_file_new() - */ -static void kbase_file_destroy_kctx(struct kbase_file *const kfile) -{ - if (atomic_cmpxchg(&kfile->setup_state, KBASE_FILE_COMPLETE, KBASE_FILE_DESTROY_CTX) != - KBASE_FILE_COMPLETE) - return; - -#if IS_ENABLED(CONFIG_DEBUG_FS) - kbasep_mem_profile_debugfs_remove(kfile->kctx); - kbase_context_debugfs_term(kfile->kctx); -#endif - - kbase_destroy_context(kfile->kctx); - dev_dbg(kfile->kbdev->dev, "Deleted kbase context"); -} - -/** - * kbase_file_destroy_kctx_worker - Work item to destroy the Kbase context. - * - * @work: Pointer to the kfile::destroy_kctx_work. - * - * The work item shall only be enqueued if the context termination could not - * be done from @kbase_flush(). - */ -static void kbase_file_destroy_kctx_worker(struct work_struct *work) -{ - struct kbase_file *kfile = container_of(work, struct kbase_file, destroy_kctx_work); - - WARN_ON_ONCE(kfile->owner); - WARN_ON_ONCE(kfile->map_count); - WARN_ON_ONCE(kfile->fops_count); - - kbase_file_destroy_kctx(kfile); -} - -/** - * kbase_file_destroy_kctx_on_flush - Try destroy the Kbase context from the flush() - * method of @kbase_fops. - * - * @kfile: A device file created by kbase_file_new() - */ -static void kbase_file_destroy_kctx_on_flush(struct kbase_file *const kfile) -{ - bool can_destroy_context = false; - - spin_lock(&kfile->lock); - kfile->owner = NULL; - /* To destroy the context from flush() method, unlike the release() - * method, need to synchronize manually against the other threads in - * the current process that could be operating on the /dev/malixx file. - * - * Only destroy the context if all the memory mappings on the - * /dev/malixx file instance have been closed. If there are mappings - * present then the context would be destroyed later when the last - * mapping is closed. - * Also, only destroy the context if no file operations are in progress. - */ - can_destroy_context = !kfile->map_count && !kfile->fops_count; - spin_unlock(&kfile->lock); - - if (likely(can_destroy_context)) { - WARN_ON_ONCE(work_pending(&kfile->destroy_kctx_work)); - kbase_file_destroy_kctx(kfile); - } -} - /** * kbase_file_delete - Destroy an object representing a device file * * @kfile: A device file created by kbase_file_new() * - * If any context was created for the @kfile and is still alive, then it is destroyed. + * If any context was created for the @kfile then it is destroyed. */ static void kbase_file_delete(struct kbase_file *const kfile) { + struct kbase_device *kbdev = NULL; + if (WARN_ON(!kfile)) return; - /* All the CPU mappings on the device file should have been closed */ - WARN_ON_ONCE(kfile->map_count); -#if IS_ENABLED(CONFIG_DEBUG_FS) - /* There could still be file operations due to the debugfs file (mem_view) */ - wait_event(kfile->zero_fops_count_wait, !kbase_file_fops_count(kfile)); -#else - /* There shall not be any file operations in progress on the device file */ - WARN_ON_ONCE(kfile->fops_count); -#endif - kfile->filp->private_data = NULL; - cancel_work_sync(&kfile->destroy_kctx_work); - /* Destroy the context if it wasn't done earlier from the flush() method. */ - kbase_file_destroy_kctx(kfile); - kbase_release_device(kfile->kbdev); + kbdev = kfile->kbdev; + + if (atomic_read(&kfile->setup_state) == KBASE_FILE_COMPLETE) { + struct kbase_context *kctx = kfile->kctx; + +#if IS_ENABLED(CONFIG_DEBUG_FS) + kbasep_mem_profile_debugfs_remove(kctx); +#endif + kbase_context_debugfs_term(kctx); + + kbase_destroy_context(kctx); + + dev_dbg(kbdev->dev, "deleted base context\n"); + } + + kbase_release_device(kbdev); + kfree(kfile); } @@ -585,6 +484,9 @@ int kbase_get_irqs(struct kbase_device *kbdev) kbdev->nr_irqs = 0; result = get_irqs(kbdev, pdev); + if (!result) + return result; + if (result) dev_err(kbdev->dev, "Invalid or No interrupt resources"); @@ -736,7 +638,8 @@ static int kbase_file_create_kctx(struct kbase_file *const kfile, kbdev = kfile->kbdev; - kctx = kbase_create_context(kbdev, in_compat_syscall(), flags, kfile->api_version, kfile); + kctx = kbase_create_context(kbdev, in_compat_syscall(), flags, kfile->api_version, + kfile->filp); /* if bad flags, will stay stuck in setup mode */ if (!kctx) @@ -823,36 +726,6 @@ static int kbase_release(struct inode *inode, struct file *filp) return 0; } -/** - * kbase_flush - Function implementing the flush() method of @kbase_fops. - * - * @filp: Pointer to the /dev/malixx device file instance. - * @id: Pointer to the file table structure of current process. - * If @filp is being shared by multiple processes then @id can differ - * from kfile::owner. - * - * This function is called everytime the copy of @filp is closed. So if 3 processes - * are sharing the @filp then this function would be called 3 times and only after - * that kbase_release() would get called. - * - * Return: 0 if successful, otherwise a negative error code. - * - * Note: This function always gets called in Userspace context when the - * file is closed. - */ -static int kbase_flush(struct file *filp, fl_owner_t id) -{ - struct kbase_file *const kfile = filp->private_data; - - /* Try to destroy the context if the flush() method has been called for the - * process that created the instance of /dev/malixx file i.e. 'kfile'. - */ - if (kfile->owner == id) - kbase_file_destroy_kctx_on_flush(kfile); - - return 0; -} - static int kbase_api_set_flags(struct kbase_file *kfile, struct kbase_ioctl_set_flags *flags) { int err = 0; @@ -1413,10 +1286,11 @@ static int kbase_api_sticky_resource_map(struct kbase_context *kctx, if (ret != 0) return -EFAULT; - kbase_gpu_vm_lock(kctx); + down_read(kbase_mem_get_process_mmap_lock()); + kbase_gpu_vm_lock_with_pmode_sync(kctx); for (i = 0; i < map->count; i++) { - if (!kbase_sticky_resource_acquire(kctx, gpu_addr[i])) { + if (!kbase_sticky_resource_acquire(kctx, gpu_addr[i], current->mm)) { /* Invalid resource */ ret = -EINVAL; break; @@ -1430,7 +1304,8 @@ static int kbase_api_sticky_resource_map(struct kbase_context *kctx, } } - kbase_gpu_vm_unlock(kctx); + kbase_gpu_vm_unlock_with_pmode_sync(kctx); + up_read(kbase_mem_get_process_mmap_lock()); return ret; } @@ -1450,7 +1325,7 @@ static int kbase_api_sticky_resource_unmap(struct kbase_context *kctx, if (ret != 0) return -EFAULT; - kbase_gpu_vm_lock(kctx); + kbase_gpu_vm_lock_with_pmode_sync(kctx); for (i = 0; i < unmap->count; i++) { if (!kbase_sticky_resource_release_force(kctx, NULL, gpu_addr[i])) { @@ -1459,7 +1334,7 @@ static int kbase_api_sticky_resource_unmap(struct kbase_context *kctx, } } - kbase_gpu_vm_unlock(kctx); + kbase_gpu_vm_unlock_with_pmode_sync(kctx); return ret; } @@ -1517,6 +1392,12 @@ static int kbasep_cs_queue_kick(struct kbase_context *kctx, struct kbase_ioctl_c return kbase_csf_queue_kick(kctx, kick); } +static int kbasep_queue_group_clear_faults(struct kbase_context *kctx, + struct kbase_ioctl_queue_group_clear_faults *faults) +{ + return kbase_csf_queue_group_clear_faults(kctx, faults); +} + static int kbasep_cs_queue_group_create_1_6(struct kbase_context *kctx, union kbase_ioctl_cs_queue_group_create_1_6 *create) { @@ -1535,10 +1416,8 @@ static int kbasep_cs_queue_group_create_1_6(struct kbase_context *kctx, } }; for (i = 0; i < ARRAY_SIZE(create->in.padding); i++) { - if (create->in.padding[i] != 0) { - dev_warn(kctx->kbdev->dev, "Invalid padding not 0 in queue group create\n"); + if (create->in.padding[i] != 0) return -EINVAL; - } } ret = kbase_csf_queue_group_create(kctx, &new_create); @@ -1569,10 +1448,8 @@ static int kbasep_cs_queue_group_create_1_18(struct kbase_context *kctx, } }; for (i = 0; i < ARRAY_SIZE(create->in.padding); i++) { - if (create->in.padding[i] != 0) { - dev_warn(kctx->kbdev->dev, "Invalid padding not 0 in queue group create\n"); + if (create->in.padding[i] != 0) return -EINVAL; - } } ret = kbase_csf_queue_group_create(kctx, &new_create); @@ -1586,6 +1463,8 @@ static int kbasep_cs_queue_group_create_1_18(struct kbase_context *kctx, static int kbasep_cs_queue_group_create(struct kbase_context *kctx, union kbase_ioctl_cs_queue_group_create *create) { + /* create->in.reserved only present pre-TDRX configuration. */ + if (create->in.reserved != 0) { dev_warn(kctx->kbdev->dev, "Invalid reserved field not 0 in queue group create\n"); return -EINVAL; @@ -1826,19 +1705,28 @@ static int kbasep_ioctl_set_limited_core_count( struct kbase_ioctl_set_limited_core_count *set_limited_core_count) { const u64 shader_core_mask = kbase_pm_get_present_cores(kctx->kbdev, KBASE_PM_CORE_SHADER); - const u64 limited_core_mask = ((u64)1 << (set_limited_core_count->max_core_count)) - 1; + const u8 max_core_count = set_limited_core_count->max_core_count; + u64 limited_core_mask = 0; - if ((shader_core_mask & limited_core_mask) == 0) { - /* At least one shader core must be available after applying the mask */ + /* Sanity check to avoid shift-out-of-bounds */ + if (max_core_count > 64) + return -EINVAL; + else if (max_core_count == 64) + limited_core_mask = UINT64_MAX; + else + limited_core_mask = ((u64)1 << max_core_count) - 1; + + /* At least one shader core must be available after applying the mask */ + if ((shader_core_mask & limited_core_mask) == 0) return -EINVAL; - } kctx->limited_core_mask = limited_core_mask; return 0; } -static long kbase_kfile_ioctl(struct kbase_file *kfile, unsigned int cmd, unsigned long arg) +static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) { + struct kbase_file *const kfile = filp->private_data; struct kbase_context *kctx = NULL; struct kbase_device *kbdev = kfile->kbdev; void __user *uarg = (void __user *)arg; @@ -2087,6 +1975,11 @@ static long kbase_kfile_ioctl(struct kbase_file *kfile, unsigned int cmd, unsign KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_KCPU_QUEUE_ENQUEUE, kbasep_kcpu_queue_enqueue, struct kbase_ioctl_kcpu_queue_enqueue, kctx); break; + case KBASE_IOCTL_QUEUE_GROUP_CLEAR_FAULTS: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_QUEUE_GROUP_CLEAR_FAULTS, + kbasep_queue_group_clear_faults, + struct kbase_ioctl_queue_group_clear_faults, kctx); + break; case KBASE_IOCTL_CS_TILER_HEAP_INIT: KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_CS_TILER_HEAP_INIT, kbasep_cs_tiler_heap_init, union kbase_ioctl_cs_tiler_heap_init, kctx); @@ -2137,45 +2030,22 @@ static long kbase_kfile_ioctl(struct kbase_file *kfile, unsigned int cmd, unsign return -ENOIOCTLCMD; } -static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) -{ - struct kbase_file *const kfile = filp->private_data; - long ioctl_ret; - - if (unlikely(!kbase_file_inc_fops_count_if_allowed(kfile))) - return -EPERM; - - ioctl_ret = kbase_kfile_ioctl(kfile, cmd, arg); - kbase_file_dec_fops_count(kfile); - - return ioctl_ret; -} - #if MALI_USE_CSF static ssize_t kbase_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos) { struct kbase_file *const kfile = filp->private_data; - struct kbase_context *kctx; + struct kbase_context *const kctx = kbase_file_get_kctx_if_setup_complete(kfile); struct base_csf_notification event_data = { .type = BASE_CSF_NOTIFICATION_EVENT }; const size_t data_size = sizeof(event_data); bool read_event = false, read_error = false; - ssize_t err = 0; CSTD_UNUSED(f_pos); - if (unlikely(!kbase_file_inc_fops_count_if_allowed(kfile))) + if (unlikely(!kctx)) return -EPERM; - kctx = kbase_file_get_kctx_if_setup_complete(kfile); - if (unlikely(!kctx)) { - err = -EPERM; - goto out; - } - - if (count < data_size) { - err = -ENOBUFS; - goto out; - } + if (count < data_size) + return -ENOBUFS; if (atomic_read(&kctx->event_count)) read_event = true; @@ -2196,41 +2066,29 @@ static ssize_t kbase_read(struct file *filp, char __user *buf, size_t count, lof if (copy_to_user(buf, &event_data, data_size) != 0) { dev_warn(kctx->kbdev->dev, "Failed to copy data\n"); - err = -EFAULT; - goto out; + return -EFAULT; } if (read_event) atomic_set(&kctx->event_count, 0); -out: - kbase_file_dec_fops_count(kfile); - return err ? err : (ssize_t)data_size; + return data_size; } #else /* MALI_USE_CSF */ static ssize_t kbase_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos) { struct kbase_file *const kfile = filp->private_data; - struct kbase_context *kctx; + struct kbase_context *const kctx = kbase_file_get_kctx_if_setup_complete(kfile); struct base_jd_event_v2 uevent; - size_t out_count = 0; - ssize_t err = 0; + int out_count = 0; CSTD_UNUSED(f_pos); - if (unlikely(!kbase_file_inc_fops_count_if_allowed(kfile))) + if (unlikely(!kctx)) return -EPERM; - kctx = kbase_file_get_kctx_if_setup_complete(kfile); - if (unlikely(!kctx)) { - err = -EPERM; - goto out; - } - - if (count < sizeof(uevent)) { - err = -ENOBUFS; - goto out; - } + if (count < sizeof(uevent)) + return -ENOBUFS; memset(&uevent, 0, sizeof(uevent)); @@ -2239,29 +2097,21 @@ static ssize_t kbase_read(struct file *filp, char __user *buf, size_t count, lof if (out_count > 0) goto out; - if (filp->f_flags & O_NONBLOCK) { - err = -EAGAIN; - goto out; - } + if (filp->f_flags & O_NONBLOCK) + return -EAGAIN; - if (wait_event_interruptible(kfile->event_queue, - kbase_event_pending(kctx)) != 0) { - err = -ERESTARTSYS; - goto out; - } + if (wait_event_interruptible(kctx->event_queue, + kbase_event_pending(kctx)) != 0) + return -ERESTARTSYS; } if (uevent.event_code == BASE_JD_EVENT_DRV_TERMINATED) { - if (out_count == 0) { - err = -EPIPE; - goto out; - } + if (out_count == 0) + return -EPIPE; goto out; } - if (copy_to_user(buf, &uevent, sizeof(uevent)) != 0) { - err = -EFAULT; - goto out; - } + if (copy_to_user(buf, &uevent, sizeof(uevent)) != 0) + return -EFAULT; buf += sizeof(uevent); out_count++; @@ -2269,59 +2119,40 @@ static ssize_t kbase_read(struct file *filp, char __user *buf, size_t count, lof } while (count >= sizeof(uevent)); out: - kbase_file_dec_fops_count(kfile); - return err ? err : (ssize_t)(out_count * sizeof(uevent)); + return out_count * sizeof(uevent); } #endif /* MALI_USE_CSF */ static __poll_t kbase_poll(struct file *filp, poll_table *wait) { struct kbase_file *const kfile = filp->private_data; - struct kbase_context *kctx; - __poll_t ret = 0; + struct kbase_context *const kctx = kbase_file_get_kctx_if_setup_complete(kfile); - if (unlikely(!kbase_file_inc_fops_count_if_allowed(kfile))) { -#if (KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE) - ret = POLLNVAL; -#else - ret = EPOLLNVAL; -#endif - return ret; - } - - kctx = kbase_file_get_kctx_if_setup_complete(kfile); if (unlikely(!kctx)) { #if (KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE) - ret = POLLERR; + return POLLERR; #else - ret = EPOLLERR; + return EPOLLERR; #endif - goto out; } - poll_wait(filp, &kfile->event_queue, wait); + poll_wait(filp, &kctx->event_queue, wait); if (kbase_event_pending(kctx)) { #if (KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE) - ret = POLLIN | POLLRDNORM; + return POLLIN | POLLRDNORM; #else - ret = EPOLLIN | EPOLLRDNORM; + return EPOLLIN | EPOLLRDNORM; #endif } -out: - kbase_file_dec_fops_count(kfile); - return ret; + return 0; } void kbase_event_wakeup(struct kbase_context *kctx) { KBASE_DEBUG_ASSERT(kctx); dev_dbg(kctx->kbdev->dev, "Waking event queue for context %pK\n", (void *)kctx); -#ifdef CONFIG_MALI_BIFROST_DEBUG - if (WARN_ON_ONCE(!kctx->kfile)) - return; -#endif - wake_up_interruptible(&kctx->kfile->event_queue); + wake_up_interruptible(&kctx->event_queue); } KBASE_EXPORT_TEST_API(kbase_event_wakeup); @@ -2354,20 +2185,12 @@ KBASE_EXPORT_TEST_API(kbase_event_pending); static int kbase_mmap(struct file *const filp, struct vm_area_struct *const vma) { struct kbase_file *const kfile = filp->private_data; - struct kbase_context *kctx; - int ret; + struct kbase_context *const kctx = kbase_file_get_kctx_if_setup_complete(kfile); - if (unlikely(!kbase_file_inc_fops_count_if_allowed(kfile))) + if (unlikely(!kctx)) return -EPERM; - kctx = kbase_file_get_kctx_if_setup_complete(kfile); - if (likely(kctx)) - ret = kbase_context_mmap(kctx, vma); - else - ret = -EPERM; - - kbase_file_dec_fops_count(kfile); - return ret; + return kbase_context_mmap(kctx, vma); } static int kbase_check_flags(int flags) @@ -2386,26 +2209,17 @@ static unsigned long kbase_get_unmapped_area(struct file *const filp, const unsi const unsigned long flags) { struct kbase_file *const kfile = filp->private_data; - struct kbase_context *kctx; - unsigned long address; + struct kbase_context *const kctx = kbase_file_get_kctx_if_setup_complete(kfile); - if (unlikely(!kbase_file_inc_fops_count_if_allowed(kfile))) + if (unlikely(!kctx)) return -EPERM; - kctx = kbase_file_get_kctx_if_setup_complete(kfile); - if (likely(kctx)) - address = kbase_context_get_unmapped_area(kctx, addr, len, pgoff, flags); - else - address = -EPERM; - - kbase_file_dec_fops_count(kfile); - return address; + return kbase_context_get_unmapped_area(kctx, addr, len, pgoff, flags); } static const struct file_operations kbase_fops = { .owner = THIS_MODULE, .open = kbase_open, - .flush = kbase_flush, .release = kbase_release, .read = kbase_read, .poll = kbase_poll, @@ -2544,6 +2358,9 @@ static ssize_t core_mask_show(struct device *dev, struct device_attribute *attr, struct kbase_device *kbdev; unsigned long flags; ssize_t ret = 0; +#if !MALI_USE_CSF + size_t i; +#endif CSTD_UNUSED(attr); @@ -2562,22 +2379,147 @@ static ssize_t core_mask_show(struct device *dev, struct device_attribute *attr, ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), "Current in use core mask : 0x%llX\n", kbdev->pm.backend.shaders_avail); #else - ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), "Current core mask (JS0) : 0x%llX\n", - kbdev->pm.debug_core_mask[0]); - ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), "Current core mask (JS1) : 0x%llX\n", - kbdev->pm.debug_core_mask[1]); - ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), "Current core mask (JS2) : 0x%llX\n", - kbdev->pm.debug_core_mask[2]); + for (i = 0; i < BASE_JM_MAX_NR_SLOTS; i++) { + if (PAGE_SIZE < ret) + goto out_unlock; + + ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), + "Current core mask (JS%zu) : 0x%llX\n", i, + kbdev->pm.debug_core_mask[i]); + } #endif /* MALI_USE_CSF */ ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), "Available core mask : 0x%llX\n", kbdev->gpu_props.shader_present); - +#if !MALI_USE_CSF +out_unlock: +#endif spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); return ret; } +#if MALI_USE_CSF +struct kbase_core_mask { + u64 new_core_mask; +}; + +static int core_mask_parse(struct kbase_device *const kbdev, const char *const buf, + struct kbase_core_mask *const mask) +{ + int err = kstrtou64(buf, 0, &mask->new_core_mask); + + if (err) + dev_err(kbdev->dev, "Couldn't process core mask write operation.\n"); + + return err; +} + +static int core_mask_set(struct kbase_device *kbdev, struct kbase_core_mask *const new_mask) +{ + u64 new_core_mask = new_mask->new_core_mask; + u64 shader_present = kbdev->gpu_props.shader_present; + + lockdep_assert_held(&kbdev->pm.lock); + lockdep_assert_held(&kbdev->hwaccess_lock); + + if ((new_core_mask & shader_present) != new_core_mask) { + dev_err(kbdev->dev, + "Invalid core mask 0x%llX: Includes non-existent cores (present = 0x%llX)", + new_core_mask, shader_present); + return -EINVAL; + + } else if (!(new_core_mask & shader_present & kbdev->pm.backend.ca_cores_enabled)) { + dev_err(kbdev->dev, + "Invalid core mask 0x%llX: No intersection with currently available cores (present = 0x%llX, CA enabled = 0x%llX)", + new_core_mask, kbdev->gpu_props.shader_present, + kbdev->pm.backend.ca_cores_enabled); + return -EINVAL; + } + + + if (kbdev->pm.debug_core_mask != new_core_mask) + kbase_pm_set_debug_core_mask(kbdev, new_core_mask); + + return 0; +} +#else +struct kbase_core_mask { + u64 new_core_mask[BASE_JM_MAX_NR_SLOTS]; +}; + +static int core_mask_parse(struct kbase_device *const kbdev, const char *const buf, + struct kbase_core_mask *const mask) +{ + int items; + + items = sscanf(buf, "%llx %llx %llx", &mask->new_core_mask[0], &mask->new_core_mask[1], + &mask->new_core_mask[2]); + + if (items != 1 && items != BASE_JM_MAX_NR_SLOTS) { + dev_err(kbdev->dev, "Couldn't process core mask write operation.\n" + "Use format \n" + "or \n"); + return -EINVAL; + } + + /* If only one value was provided, set all other core masks equal to the value. */ + if (items == 1) { + size_t i; + + for (i = 1; i < BASE_JM_MAX_NR_SLOTS; i++) + mask->new_core_mask[i] = mask->new_core_mask[0]; + } + + return 0; +} + +static int core_mask_set(struct kbase_device *kbdev, struct kbase_core_mask *const new_mask) +{ + u64 shader_present = kbdev->gpu_props.shader_present; + u64 group_core_mask = kbdev->gpu_props.coherency_info.group.core_mask; + u64 *new_core_mask = &new_mask->new_core_mask[0]; + size_t i; + + for (i = 0; i < BASE_JM_MAX_NR_SLOTS; ++i) { + if ((new_core_mask[i] & shader_present) != new_core_mask[i]) { + dev_err(kbdev->dev, + "Invalid core mask 0x%llX for JS %zu: Includes non-existent cores (present = 0x%llX)", + new_core_mask[i], i, shader_present); + return -EINVAL; + + } else if (!(new_core_mask[i] & shader_present & + kbdev->pm.backend.ca_cores_enabled)) { + dev_err(kbdev->dev, + "Invalid core mask 0x%llX for JS %zu: No intersection with currently available cores (present = 0x%llX, CA enabled = 0x%llX)", + new_core_mask[i], i, kbdev->gpu_props.shader_present, + kbdev->pm.backend.ca_cores_enabled); + return -EINVAL; + } else if (!(new_core_mask[i] & group_core_mask)) { + dev_err(kbdev->dev, + "Invalid core mask 0x%llX for JS %zu: No intersection with group 0 core mask 0x%llX", + new_core_mask[i], i, group_core_mask); + return -EINVAL; + } else if (!(new_core_mask[i] & kbdev->gpu_props.curr_config.shader_present)) { + dev_err(kbdev->dev, + "Invalid core mask 0x%llX for JS %zu: No intersection with current core mask 0x%llX", + new_core_mask[i], i, kbdev->gpu_props.curr_config.shader_present); + return -EINVAL; + } + } + + for (i = 0; i < BASE_JM_MAX_NR_SLOTS; i++) { + if (kbdev->pm.debug_core_mask[i] != new_core_mask[i]) { + kbase_pm_set_debug_core_mask(kbdev, new_core_mask, BASE_JM_MAX_NR_SLOTS); + break; + } + } + + return 0; +} + +#endif + /** * core_mask_store - Store callback for the core_mask sysfs file. * @@ -2594,18 +2536,10 @@ static ssize_t core_mask_store(struct device *dev, struct device_attribute *attr size_t count) { struct kbase_device *kbdev; -#if MALI_USE_CSF - u64 new_core_mask; -#else - u64 new_core_mask[3]; - u64 group_core_mask; - int i; -#endif /* MALI_USE_CSF */ + struct kbase_core_mask core_mask = {}; - int items; - ssize_t err = (ssize_t)count; + int err; unsigned long flags; - u64 shader_present; CSTD_UNUSED(attr); @@ -2614,102 +2548,22 @@ static ssize_t core_mask_store(struct device *dev, struct device_attribute *attr if (!kbdev) return -ENODEV; -#if MALI_USE_CSF - items = sscanf(buf, "%llx", &new_core_mask); - - if (items != 1) { - dev_err(kbdev->dev, "Couldn't process core mask write operation.\n" - "Use format \n"); - err = -EINVAL; - goto end; - } -#else - items = sscanf(buf, "%llx %llx %llx", &new_core_mask[0], &new_core_mask[1], - &new_core_mask[2]); - - if (items != 1 && items != 3) { - dev_err(kbdev->dev, "Couldn't process core mask write operation.\n" - "Use format \n" - "or \n"); - err = -EINVAL; - goto end; - } - - if (items == 1) - new_core_mask[1] = new_core_mask[2] = new_core_mask[0]; -#endif + err = core_mask_parse(kbdev, buf, &core_mask); + if (err) + return err; mutex_lock(&kbdev->pm.lock); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - shader_present = kbdev->gpu_props.shader_present; + err = core_mask_set(kbdev, &core_mask); -#if MALI_USE_CSF - if ((new_core_mask & shader_present) != new_core_mask) { - dev_err(dev, - "Invalid core mask 0x%llX: Includes non-existent cores (present = 0x%llX)", - new_core_mask, shader_present); - err = -EINVAL; - goto unlock; - - } else if (!(new_core_mask & shader_present & kbdev->pm.backend.ca_cores_enabled)) { - dev_err(dev, - "Invalid core mask 0x%llX: No intersection with currently available cores (present = 0x%llX, CA enabled = 0x%llX\n", - new_core_mask, kbdev->gpu_props.shader_present, - kbdev->pm.backend.ca_cores_enabled); - err = -EINVAL; - goto unlock; - } - - if (kbdev->pm.debug_core_mask != new_core_mask) - kbase_pm_set_debug_core_mask(kbdev, new_core_mask); -#else - group_core_mask = kbdev->gpu_props.coherency_info.group.core_mask; - - for (i = 0; i < 3; ++i) { - if ((new_core_mask[i] & shader_present) != new_core_mask[i]) { - dev_err(dev, - "Invalid core mask 0x%llX for JS %d: Includes non-existent cores (present = 0x%llX)", - new_core_mask[i], i, shader_present); - err = -EINVAL; - goto unlock; - - } else if (!(new_core_mask[i] & shader_present & - kbdev->pm.backend.ca_cores_enabled)) { - dev_err(dev, - "Invalid core mask 0x%llX for JS %d: No intersection with currently available cores (present = 0x%llX, CA enabled = 0x%llX\n", - new_core_mask[i], i, kbdev->gpu_props.shader_present, - kbdev->pm.backend.ca_cores_enabled); - err = -EINVAL; - goto unlock; - } else if (!(new_core_mask[i] & group_core_mask)) { - dev_err(dev, - "Invalid core mask 0x%llX for JS %d: No intersection with group 0 core mask 0x%llX\n", - new_core_mask[i], i, group_core_mask); - err = -EINVAL; - goto unlock; - } else if (!(new_core_mask[i] & kbdev->gpu_props.curr_config.shader_present)) { - dev_err(dev, - "Invalid core mask 0x%llX for JS %d: No intersection with current core mask 0x%llX\n", - new_core_mask[i], i, kbdev->gpu_props.curr_config.shader_present); - err = -EINVAL; - goto unlock; - } - } - - if (kbdev->pm.debug_core_mask[0] != new_core_mask[0] || - kbdev->pm.debug_core_mask[1] != new_core_mask[1] || - kbdev->pm.debug_core_mask[2] != new_core_mask[2]) { - kbase_pm_set_debug_core_mask(kbdev, new_core_mask[0], new_core_mask[1], - new_core_mask[2]); - } -#endif /* MALI_USE_CSF */ - -unlock: spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); mutex_unlock(&kbdev->pm.lock); -end: - return err; + + if (err) + return err; + + return count; } /* @@ -3478,12 +3332,8 @@ int kbase_pm_gpu_freq_init(struct kbase_device *kbdev) /* convert found frequency to KHz */ found_freq /= 1000; - /* If lowest frequency in OPP table is still higher - * than the reference, then keep the reference frequency - * as the one to use for scaling . - */ - if (found_freq < lowest_freq_khz) - lowest_freq_khz = found_freq; + /* always use the lowest freqency from opp table */ + lowest_freq_khz = found_freq; } #else dev_err(kbdev->dev, "No operating-points-v2 node or operating-points property in DT"); @@ -4466,7 +4316,7 @@ static int kbase_common_reg_map(struct kbase_device *kbdev) goto out_region; } - kbdev->reg = ioremap(kbdev->reg_start, kbdev->reg_size); + kbdev->reg = mali_ioremap(kbdev->reg_start, kbdev->reg_size); if (!kbdev->reg) { dev_err(kbdev->dev, "Can't remap register window\n"); err = -EINVAL; @@ -4484,7 +4334,7 @@ out_region: static void kbase_common_reg_unmap(struct kbase_device *const kbdev) { if (kbdev->reg) { - iounmap(kbdev->reg); + mali_iounmap(kbdev->reg); release_mem_region(kbdev->reg_start, kbdev->reg_size); kbdev->reg = NULL; kbdev->reg_start = 0; @@ -4536,7 +4386,6 @@ void registers_unmap(struct kbase_device *kbdev) } #if defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF) - static bool kbase_is_pm_enabled(const struct device_node *gpu_node) { const struct device_node *power_model_node; @@ -4562,17 +4411,6 @@ static bool kbase_is_pm_enabled(const struct device_node *gpu_node) return is_pm_enable; } -static bool kbase_is_pv_enabled(const struct device_node *gpu_node) -{ - const void *arbiter_if_node; - - arbiter_if_node = of_get_property(gpu_node, "arbiter-if", NULL); - if (!arbiter_if_node) - arbiter_if_node = of_get_property(gpu_node, "arbiter_if", NULL); - - return arbiter_if_node ? true : false; -} - static bool kbase_is_full_coherency_enabled(const struct device_node *gpu_node) { const void *coherency_dts; @@ -4586,71 +4424,62 @@ static bool kbase_is_full_coherency_enabled(const struct device_node *gpu_node) } return false; } +#endif /* defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF) */ -#endif /* CONFIG_MALI_ARBITER_SUPPORT && CONFIG_OF */ - -int kbase_device_pm_init(struct kbase_device *kbdev) +int kbase_device_backend_init(struct kbase_device *kbdev) { int err = 0; #if defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF) - u32 product_model; + /* + * Attempt to initialize arbitration. + * If the platform is not suitable for arbitration, return -EPERM. + * The device initialization should not fail but kbase will + * not support arbitration. + */ + if (kbase_is_pm_enabled(kbdev->dev->of_node)) { + /* Arbitration AND power management invalid */ + dev_err(kbdev->dev, "Invalid combination of arbitration AND power management\n"); + return -EPERM; + } - if (kbase_is_pv_enabled(kbdev->dev->of_node)) { - dev_info(kbdev->dev, "Arbitration interface enabled\n"); - if (kbase_is_pm_enabled(kbdev->dev->of_node)) { - /* Arbitration AND power management invalid */ - dev_err(kbdev->dev, - "Invalid combination of arbitration AND power management\n"); - return -EPERM; - } - if (kbase_is_full_coherency_enabled(kbdev->dev->of_node)) { - /* Arbitration AND full coherency invalid */ - dev_err(kbdev->dev, - "Invalid combination of arbitration AND full coherency\n"); - return -EPERM; - } - err = kbase_arbiter_pm_early_init(kbdev); - if (err == 0) { - /* Check if Arbitration is running on - * supported GPU platform - */ - kbase_pm_register_access_enable(kbdev); + if (kbase_is_full_coherency_enabled(kbdev->dev->of_node)) { + /* Arbitration AND full coherency invalid */ + dev_err(kbdev->dev, "Invalid combination of arbitration AND full coherency\n"); + return -EPERM; + } + + err = kbase_arbiter_pm_early_init(kbdev); + if (err == 0) { +#if !MALI_USE_CSF + u32 product_model; + + /* + * Attempt to obtain and parse gpu_id in the event an external AW module + * is used for messaging. We should have access to GPU at this point. + */ + if (kbdev->gpu_props.gpu_id.arch_major == 0) kbase_gpuprops_parse_gpu_id(&kbdev->gpu_props.gpu_id, kbase_reg_get_gpu_id(kbdev)); - kbase_pm_register_access_disable(kbdev); - product_model = kbdev->gpu_props.gpu_id.product_model; - if (product_model != GPU_ID_PRODUCT_TGOX && - product_model != GPU_ID_PRODUCT_TNOX && - product_model != GPU_ID_PRODUCT_TBAX) { - kbase_arbiter_pm_early_term(kbdev); - dev_err(kbdev->dev, "GPU platform not suitable for arbitration\n"); - return -EPERM; - } + product_model = kbdev->gpu_props.gpu_id.product_model; + if (product_model != GPU_ID_PRODUCT_TGOX && product_model != GPU_ID_PRODUCT_TNOX && + product_model != GPU_ID_PRODUCT_TBAX) { + kbase_arbiter_pm_early_term(kbdev); + dev_err(kbdev->dev, "GPU platform not suitable for arbitration\n"); + return -EPERM; } - } else { - kbdev->arb.arb_if = NULL; - kbdev->arb.arb_dev = NULL; - err = power_control_init(kbdev); +#endif /* !MALI_USE_CSF */ + dev_info(kbdev->dev, "Arbitration interface enabled\n"); } -#else - err = power_control_init(kbdev); -#endif /* CONFIG_MALI_ARBITER_SUPPORT && CONFIG_OF */ +#endif /* defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF) */ return err; } -void kbase_device_pm_term(struct kbase_device *kbdev) +void kbase_device_backend_term(struct kbase_device *kbdev) { #ifdef CONFIG_MALI_ARBITER_SUPPORT -#if IS_ENABLED(CONFIG_OF) - if (kbase_is_pv_enabled(kbdev->dev->of_node)) - kbase_arbiter_pm_early_term(kbdev); - else - power_control_term(kbdev); -#endif /* CONFIG_OF */ -#else - power_control_term(kbdev); + kbase_arbiter_pm_early_term(kbdev); #endif } @@ -5064,6 +4893,7 @@ static struct dentry *init_debugfs(struct kbase_device *kbdev) return dentry; } + dentry = debugfs_ctx_defaults_init(kbdev); if (IS_ERR_OR_NULL(dentry)) return dentry; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_view.c b/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_view.c index dd8f8ff6fe79..48469cdcc34e 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_view.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_view.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2013-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2013-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -237,7 +237,11 @@ static int debug_mem_open(struct inode *i, struct file *file) int ret; enum kbase_memory_zone idx; - if (!kbase_file_inc_fops_count_unless_closed(kctx->kfile)) +#if (KERNEL_VERSION(6, 7, 0) > LINUX_VERSION_CODE) + if (get_file_rcu(kctx->filp) == 0) +#else + if (get_file_rcu(&kctx->filp) == 0) +#endif return -ENOENT; /* Check if file was opened in write mode. GPU memory contents @@ -297,7 +301,7 @@ out: } seq_release(i, file); open_fail: - kbase_file_dec_fops_count(kctx->kfile); + fput(kctx->filp); return ret; } @@ -327,7 +331,7 @@ static int debug_mem_release(struct inode *inode, struct file *file) kfree(mem_data); } - kbase_file_dec_fops_count(kctx->kfile); + fput(kctx->filp); return 0; } diff --git a/drivers/gpu/arm/bifrost/mali_kbase_defs.h b/drivers/gpu/arm/bifrost/mali_kbase_defs.h index 13a5c30dcb61..2335e0b8e449 100755 --- a/drivers/gpu/arm/bifrost/mali_kbase_defs.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_defs.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2011-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -178,16 +178,11 @@ struct kbase_gpu_metrics { * * @link: Links the object in kbase_device::gpu_metrics::active_list * or kbase_device::gpu_metrics::inactive_list. - * @first_active_start_time: Records the time at which the application first became + * @active_start_time: Records the time at which the application first became * active in the current work period. - * @last_active_start_time: Records the time at which the application last became - * active in the current work period. - * @last_active_end_time: Records the time at which the application last became - * inactive in the current work period. - * @total_active: Tracks the time for which application has been active - * in the current work period. - * @prev_wp_active_end_time: Records the time at which the application last became - * inactive in the previous work period. + * @active_end_time: Records the time at which the application last became + * inactive in the current work period, or the time of the end of + * previous work period if the application remained active. * @aid: Unique identifier for an application. * @kctx_count: Counter to keep a track of the number of Kbase contexts * created for an application. There may be multiple Kbase @@ -195,19 +190,14 @@ struct kbase_gpu_metrics { * metrics context. * @active_cnt: Counter that is updated every time the GPU activity starts * and ends in the current work period for an application. - * @flags: Flags to track the state of GPU metrics context. */ struct kbase_gpu_metrics_ctx { struct list_head link; - u64 first_active_start_time; - u64 last_active_start_time; - u64 last_active_end_time; - u64 total_active; - u64 prev_wp_active_end_time; + u64 active_start_time; + u64 active_end_time; unsigned int aid; unsigned int kctx_count; u8 active_cnt; - u8 flags; }; #endif @@ -555,7 +545,7 @@ struct kbase_mem_pool { u8 group_id; spinlock_t pool_lock; struct list_head page_list; - struct shrinker reclaim; + DEFINE_KBASE_SHRINKER reclaim; atomic_t isolation_in_progress_cnt; struct kbase_mem_pool *next_pool; @@ -847,8 +837,6 @@ struct kbase_mem_migrate { * @as_free: Bitpattern of free/available GPU address spaces. * @mmu_mask_change: Lock to serialize the access to MMU interrupt mask * register used in the handling of Bus & Page faults. - * @pagesize_2mb: Boolean to determine whether 2MiB page sizes are - * supported and used where possible. * @gpu_props: Object containing complete information about the * configuration/properties of GPU HW device in use. * @hw_issues_mask: List of SW workarounds for HW issues @@ -1144,8 +1132,6 @@ struct kbase_device { spinlock_t mmu_mask_change; - bool pagesize_2mb; - struct kbase_gpu_props gpu_props; unsigned long hw_issues_mask[(BASE_HW_ISSUE_END + BITS_PER_LONG - 1) / BITS_PER_LONG]; @@ -1424,9 +1410,6 @@ struct kbase_device { * @KBASE_FILE_COMPLETE: Indicates if the setup for context has * completed, i.e. flags have been set for the * context. - * @KBASE_FILE_DESTROY_CTX: Indicates that destroying of context has begun or - * is complete. This state can only be reached after - * @KBASE_FILE_COMPLETE. * * The driver allows only limited interaction with user-space until setup * is complete. @@ -1436,8 +1419,7 @@ enum kbase_file_state { KBASE_FILE_VSN_IN_PROGRESS, KBASE_FILE_NEED_CTX, KBASE_FILE_CTX_IN_PROGRESS, - KBASE_FILE_COMPLETE, - KBASE_FILE_DESTROY_CTX + KBASE_FILE_COMPLETE }; /** @@ -1447,12 +1429,6 @@ enum kbase_file_state { * allocated from the probe method of the Mali driver. * @filp: Pointer to the struct file corresponding to device file * /dev/malixx instance, passed to the file's open method. - * @owner: Pointer to the file table structure of a process that - * created the instance of /dev/malixx device file. Set to - * NULL when that process closes the file instance. No more - * file operations would be allowed once set to NULL. - * It would be updated only in the Userspace context, i.e. - * when @kbase_open or @kbase_flush is called. * @kctx: Object representing an entity, among which GPU is * scheduled and which gets its own GPU address space. * Invalid until @setup_state is KBASE_FILE_COMPLETE. @@ -1461,44 +1437,13 @@ enum kbase_file_state { * @setup_state is KBASE_FILE_NEED_CTX. * @setup_state: Initialization state of the file. Values come from * the kbase_file_state enumeration. - * @destroy_kctx_work: Work item for destroying the @kctx, enqueued only when - * @fops_count and @map_count becomes zero after /dev/malixx - * file was previously closed by the @owner. - * @lock: Lock to serialize the access to members like @owner, @fops_count, - * @map_count. - * @fops_count: Counter that is incremented at the beginning of a method - * defined for @kbase_fops and is decremented at the end. - * So the counter keeps a track of the file operations in progress - * for /dev/malixx file, that are being handled by the Kbase. - * The counter is needed to defer the context termination as - * Userspace can close the /dev/malixx file and flush() method - * can get called when some other file operation is in progress. - * @map_count: Counter to keep a track of the memory mappings present on - * /dev/malixx file instance. The counter is needed to defer the - * context termination as Userspace can close the /dev/malixx - * file and flush() method can get called when mappings are still - * present. - * @zero_fops_count_wait: Waitqueue used to wait for the @fops_count to become 0. - * Currently needed only for the "mem_view" debugfs file. - * @event_queue: Wait queue used for blocking the thread, which consumes - * the base_jd_event corresponding to an atom, when there - * are no more posted events. */ struct kbase_file { struct kbase_device *kbdev; struct file *filp; - fl_owner_t owner; struct kbase_context *kctx; unsigned long api_version; atomic_t setup_state; - struct work_struct destroy_kctx_work; - spinlock_t lock; - int fops_count; - int map_count; -#if IS_ENABLED(CONFIG_DEBUG_FS) - wait_queue_head_t zero_fops_count_wait; -#endif - wait_queue_head_t event_queue; }; #if MALI_JIT_PRESSURE_LIMIT_BASE /** @@ -1680,8 +1625,8 @@ struct kbase_sub_alloc { /** * struct kbase_context - Kernel base context * - * @kfile: Pointer to the object representing the /dev/malixx device - * file instance. + * @filp: Pointer to the struct file corresponding to device file + * /dev/malixx instance, passed to the file's open method. * @kbdev: Pointer to the Kbase device for which the context is created. * @kctx_list_link: Node into Kbase device list of contexts. * @mmu: Structure holding details of the MMU tables for this @@ -1734,6 +1679,9 @@ struct kbase_sub_alloc { * used in conjunction with @cookies bitmask mainly for * providing a mechansim to have the same value for CPU & * GPU virtual address. + * @event_queue: Wait queue used for blocking the thread, which consumes + * the base_jd_event corresponding to an atom, when there + * are no more posted events. * @tgid: Thread group ID of the process whose thread created * the context (by calling KBASE_IOCTL_VERSION_CHECK or * KBASE_IOCTL_SET_FLAGS, depending on the @api_version). @@ -1945,7 +1893,7 @@ struct kbase_sub_alloc { * is made on the device file. */ struct kbase_context { - struct kbase_file *kfile; + struct file *filp; struct kbase_device *kbdev; struct list_head kctx_list_link; struct kbase_mmu_table mmu; @@ -1997,6 +1945,7 @@ struct kbase_context { DECLARE_BITMAP(cookies, BITS_PER_LONG); struct kbase_va_region *pending_regions[BITS_PER_LONG]; + wait_queue_head_t event_queue; pid_t tgid; pid_t pid; atomic_t prioritized; @@ -2006,7 +1955,8 @@ struct kbase_context { struct kbase_mem_pool_group mem_pools; - struct shrinker reclaim; + DEFINE_KBASE_SHRINKER reclaim; + struct list_head evict_list; atomic_t evict_nents; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_fence.h b/drivers/gpu/arm/bifrost/mali_kbase_fence.h index 06690d4f17bb..d45a0fec4104 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_fence.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_fence.h @@ -35,8 +35,37 @@ #include #if MALI_USE_CSF +/* Number of digits needed to express the max value of given unsigned type. + * + * Details: The number of digits needed to express the max value of given type is log10(t_max) + 1 + * sizeof(t) == log2(t_max)/8 + * log10(t_max) == log2(t_max) / log2(10) + * log2(t_max) == sizeof(type) * 8 + * 1/log2(10) is approx (1233 >> 12) + * Hence, number of digits for given type == log10(t_max) + 1 == sizeof(type) * 8 * (1233 >> 12) + 1 + */ +#define MAX_DIGITS_FOR_UNSIGNED_TYPE(t) ((((sizeof(t) * BITS_PER_BYTE) * 1233) >> 12) + 1) + +/* Number of digits needed to express the max value of given signed type, + * including the sign character, + */ +#define MAX_DIGITS_FOR_SIGNED_TYPE(t) (MAX_DIGITS_FOR_UNSIGNED_TYPE(t) + 1) + +/* Max number of characters for id member of kbase_device struct. */ +#define MAX_KBDEV_ID_LEN MAX_DIGITS_FOR_UNSIGNED_TYPE(u32) +/* Max number of characters for tgid member of kbase_context struct. */ +#define MAX_KCTX_TGID_LEN MAX_DIGITS_FOR_SIGNED_TYPE(pid_t) +/* Max number of characters for id member of kbase_context struct. */ +#define MAX_KCTX_ID_LEN MAX_DIGITS_FOR_UNSIGNED_TYPE(u32) +/* Max number of characters for fence_context member of kbase_kcpu_command_queue struct. */ +#define MAX_KCTX_QUEUE_FENCE_CTX_LEN MAX_DIGITS_FOR_UNSIGNED_TYPE(u64) +/* Max number of characters for timeline name fixed format, including null character. */ +#define FIXED_FORMAT_LEN (9) + /* Maximum number of characters in DMA fence timeline name. */ -#define MAX_TIMELINE_NAME (32) +#define MAX_TIMELINE_NAME \ + (MAX_KBDEV_ID_LEN + MAX_KCTX_TGID_LEN + MAX_KCTX_ID_LEN + MAX_KCTX_QUEUE_FENCE_CTX_LEN + \ + FIXED_FORMAT_LEN) /** * struct kbase_kcpu_dma_fence_meta - Metadata structure for dma fence objects containing diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gpu_metrics.c b/drivers/gpu/arm/bifrost/mali_kbase_gpu_metrics.c index 3a5b97db7c04..8a2e13f03683 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_gpu_metrics.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_gpu_metrics.c @@ -29,46 +29,12 @@ #include #include -/** - * enum gpu_metrics_ctx_flags - Flags for the GPU metrics context - * - * @ACTIVE_INTERVAL_IN_WP: Flag set when the application first becomes active in - * the current work period. - * - * @INSIDE_ACTIVE_LIST: Flag to track if object is in kbase_device::gpu_metrics::active_list - * - * All members need to be separate bits. This enum is intended for use in a - * bitmask where multiple values get OR-ed together. - */ -enum gpu_metrics_ctx_flags { - ACTIVE_INTERVAL_IN_WP = 1 << 0, - INSIDE_ACTIVE_LIST = 1 << 1, -}; - static unsigned long gpu_metrics_tp_emit_interval_ns = DEFAULT_GPU_METRICS_TP_EMIT_INTERVAL_NS; module_param(gpu_metrics_tp_emit_interval_ns, ulong, 0444); MODULE_PARM_DESC(gpu_metrics_tp_emit_interval_ns, "Time interval in nano seconds at which GPU metrics tracepoints are emitted"); -static inline bool gpu_metrics_ctx_flag(struct kbase_gpu_metrics_ctx *gpu_metrics_ctx, - enum gpu_metrics_ctx_flags flag) -{ - return (gpu_metrics_ctx->flags & flag); -} - -static inline void gpu_metrics_ctx_flag_set(struct kbase_gpu_metrics_ctx *gpu_metrics_ctx, - enum gpu_metrics_ctx_flags flag) -{ - gpu_metrics_ctx->flags |= flag; -} - -static inline void gpu_metrics_ctx_flag_clear(struct kbase_gpu_metrics_ctx *gpu_metrics_ctx, - enum gpu_metrics_ctx_flags flag) -{ - gpu_metrics_ctx->flags &= ~flag; -} - static inline void validate_tracepoint_data(struct kbase_gpu_metrics_ctx *gpu_metrics_ctx, u64 start_time, u64 end_time, u64 total_active) { @@ -82,43 +48,30 @@ static inline void validate_tracepoint_data(struct kbase_gpu_metrics_ctx *gpu_me WARN(total_active > (end_time - start_time), "total_active %llu > end_time %llu - start_time %llu for aid %u active_cnt %u", total_active, end_time, start_time, gpu_metrics_ctx->aid, gpu_metrics_ctx->active_cnt); - - WARN(gpu_metrics_ctx->prev_wp_active_end_time > start_time, - "prev_wp_active_end_time %llu > start_time %llu for aid %u active_cnt %u", - gpu_metrics_ctx->prev_wp_active_end_time, start_time, gpu_metrics_ctx->aid, - gpu_metrics_ctx->active_cnt); #endif } static void emit_tracepoint_for_active_gpu_metrics_ctx( struct kbase_device *kbdev, struct kbase_gpu_metrics_ctx *gpu_metrics_ctx, u64 current_time) { - const u64 start_time = gpu_metrics_ctx->first_active_start_time; - u64 total_active = gpu_metrics_ctx->total_active; - u64 end_time; + const u64 start_time = gpu_metrics_ctx->active_start_time; + u64 total_active, end_time = current_time; /* Check if the GPU activity is currently ongoing */ if (gpu_metrics_ctx->active_cnt) { /* The following check is to handle the race on CSF GPUs that can happen between * the draining of trace buffer and FW emitting the ACT=1 event . */ - if (unlikely(current_time == gpu_metrics_ctx->last_active_start_time)) - current_time++; - end_time = current_time; - total_active += end_time - gpu_metrics_ctx->last_active_start_time; - - gpu_metrics_ctx->first_active_start_time = current_time; - gpu_metrics_ctx->last_active_start_time = current_time; - } else { - end_time = gpu_metrics_ctx->last_active_end_time; - gpu_metrics_ctx_flag_clear(gpu_metrics_ctx, ACTIVE_INTERVAL_IN_WP); + if (unlikely(end_time == start_time)) + end_time++; + gpu_metrics_ctx->active_start_time = end_time; } + total_active = end_time - start_time; trace_gpu_work_period(kbdev->id, gpu_metrics_ctx->aid, start_time, end_time, total_active); validate_tracepoint_data(gpu_metrics_ctx, start_time, end_time, total_active); - gpu_metrics_ctx->prev_wp_active_end_time = end_time; - gpu_metrics_ctx->total_active = 0; + gpu_metrics_ctx->active_end_time = end_time; } void kbase_gpu_metrics_ctx_put(struct kbase_device *kbdev, @@ -131,7 +84,8 @@ void kbase_gpu_metrics_ctx_put(struct kbase_device *kbdev, if (gpu_metrics_ctx->kctx_count) return; - if (gpu_metrics_ctx_flag(gpu_metrics_ctx, ACTIVE_INTERVAL_IN_WP)) + /* Generate a tracepoint if there's still activity */ + if (gpu_metrics_ctx->active_cnt) emit_tracepoint_for_active_gpu_metrics_ctx(kbdev, gpu_metrics_ctx, ktime_get_raw_ns()); @@ -166,12 +120,11 @@ struct kbase_gpu_metrics_ctx *kbase_gpu_metrics_ctx_get(struct kbase_device *kbd void kbase_gpu_metrics_ctx_init(struct kbase_device *kbdev, struct kbase_gpu_metrics_ctx *gpu_metrics_ctx, unsigned int aid) { + gpu_metrics_ctx->active_start_time = 0; + gpu_metrics_ctx->active_end_time = 0; gpu_metrics_ctx->aid = aid; - gpu_metrics_ctx->total_active = 0; gpu_metrics_ctx->kctx_count = 1; gpu_metrics_ctx->active_cnt = 0; - gpu_metrics_ctx->prev_wp_active_end_time = 0; - gpu_metrics_ctx->flags = 0; list_add_tail(&gpu_metrics_ctx->link, &kbdev->gpu_metrics.inactive_list); } @@ -180,17 +133,9 @@ void kbase_gpu_metrics_ctx_start_activity(struct kbase_context *kctx, u64 timest struct kbase_gpu_metrics_ctx *gpu_metrics_ctx = kctx->gpu_metrics_ctx; gpu_metrics_ctx->active_cnt++; - if (gpu_metrics_ctx->active_cnt == 1) - gpu_metrics_ctx->last_active_start_time = timestamp_ns; - - if (!gpu_metrics_ctx_flag(gpu_metrics_ctx, ACTIVE_INTERVAL_IN_WP)) { - gpu_metrics_ctx->first_active_start_time = timestamp_ns; - gpu_metrics_ctx_flag_set(gpu_metrics_ctx, ACTIVE_INTERVAL_IN_WP); - } - - if (!gpu_metrics_ctx_flag(gpu_metrics_ctx, INSIDE_ACTIVE_LIST)) { + if (gpu_metrics_ctx->active_cnt == 1) { + gpu_metrics_ctx->active_start_time = timestamp_ns; list_move_tail(&gpu_metrics_ctx->link, &kctx->kbdev->gpu_metrics.active_list); - gpu_metrics_ctx_flag_set(gpu_metrics_ctx, INSIDE_ACTIVE_LIST); } } @@ -201,22 +146,22 @@ void kbase_gpu_metrics_ctx_end_activity(struct kbase_context *kctx, u64 timestam if (WARN_ON_ONCE(!gpu_metrics_ctx->active_cnt)) return; + /* Do not emit tracepoint if GPU activity still continues. */ if (--gpu_metrics_ctx->active_cnt) return; - if (likely(timestamp_ns > gpu_metrics_ctx->last_active_start_time)) { - gpu_metrics_ctx->last_active_end_time = timestamp_ns; - gpu_metrics_ctx->total_active += - timestamp_ns - gpu_metrics_ctx->last_active_start_time; + if (likely(timestamp_ns > gpu_metrics_ctx->active_start_time)) { + emit_tracepoint_for_active_gpu_metrics_ctx(kctx->kbdev, gpu_metrics_ctx, + timestamp_ns); return; } /* Due to conversion from system timestamp to CPU timestamp (which involves rounding) * the value for start and end timestamp could come as same on CSF GPUs. */ - if (timestamp_ns == gpu_metrics_ctx->last_active_start_time) { - gpu_metrics_ctx->last_active_end_time = timestamp_ns + 1; - gpu_metrics_ctx->total_active += 1; + if (timestamp_ns == gpu_metrics_ctx->active_start_time) { + emit_tracepoint_for_active_gpu_metrics_ctx(kctx->kbdev, gpu_metrics_ctx, + timestamp_ns + 1); return; } @@ -224,12 +169,9 @@ void kbase_gpu_metrics_ctx_end_activity(struct kbase_context *kctx, u64 timestam * visible to the Kbase even though the system timestamp value sampled by FW was less than * the system timestamp value sampled by Kbase just before the draining of trace buffer. */ - if (gpu_metrics_ctx->last_active_start_time == gpu_metrics_ctx->first_active_start_time && - gpu_metrics_ctx->prev_wp_active_end_time == gpu_metrics_ctx->first_active_start_time) { - WARN_ON_ONCE(gpu_metrics_ctx->total_active); - gpu_metrics_ctx->last_active_end_time = - gpu_metrics_ctx->prev_wp_active_end_time + 1; - gpu_metrics_ctx->total_active = 1; + if (gpu_metrics_ctx->active_end_time == gpu_metrics_ctx->active_start_time) { + emit_tracepoint_for_active_gpu_metrics_ctx(kctx->kbdev, gpu_metrics_ctx, + gpu_metrics_ctx->active_end_time + 1); return; } @@ -242,15 +184,12 @@ void kbase_gpu_metrics_emit_tracepoint(struct kbase_device *kbdev, u64 ts) struct kbase_gpu_metrics_ctx *gpu_metrics_ctx, *tmp; list_for_each_entry_safe(gpu_metrics_ctx, tmp, &gpu_metrics->active_list, link) { - if (!gpu_metrics_ctx_flag(gpu_metrics_ctx, ACTIVE_INTERVAL_IN_WP)) { - WARN_ON(!gpu_metrics_ctx_flag(gpu_metrics_ctx, INSIDE_ACTIVE_LIST)); - WARN_ON(gpu_metrics_ctx->active_cnt); - list_move_tail(&gpu_metrics_ctx->link, &gpu_metrics->inactive_list); - gpu_metrics_ctx_flag_clear(gpu_metrics_ctx, INSIDE_ACTIVE_LIST); + if (gpu_metrics_ctx->active_cnt) { + emit_tracepoint_for_active_gpu_metrics_ctx(kbdev, gpu_metrics_ctx, ts); continue; } - emit_tracepoint_for_active_gpu_metrics_ctx(kbdev, gpu_metrics_ctx, ts); + list_move_tail(&gpu_metrics_ctx->link, &gpu_metrics->inactive_list); } } diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gpu_metrics.h b/drivers/gpu/arm/bifrost/mali_kbase_gpu_metrics.h index c445dff32dc9..658cf1c164c5 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_gpu_metrics.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_gpu_metrics.h @@ -106,7 +106,7 @@ void kbase_gpu_metrics_ctx_init(struct kbase_device *kbdev, * @kctx: Pointer to the Kbase context contributing data to the GPU metrics context. * @timestamp_ns: CPU timestamp at which the GPU activity started. * - * The provided timestamp would be later used as the "start_time_ns" for the + * The provided timestamp is used as the "start_time_ns" for the * power/gpu_work_period tracepoint if this is the first GPU activity for the GPU * metrics context in the current work period. * @@ -122,9 +122,9 @@ void kbase_gpu_metrics_ctx_start_activity(struct kbase_context *kctx, u64 timest * @kctx: Pointer to the Kbase context contributing data to the GPU metrics context. * @timestamp_ns: CPU timestamp at which the GPU activity ended. * - * The provided timestamp would be later used as the "end_time_ns" for the - * power/gpu_work_period tracepoint if this is the last GPU activity for the GPU - * metrics context in the current work period. + * The provided timestamp is used as the "end_time_ns" for the power/gpu_work_period + * tracepoint if this is the last GPU activity for the GPU metrics context + * in the current work period. * * Note: The caller must appropriately serialize the call to this function with the * call to other GPU metrics functions declared in this file. @@ -138,8 +138,8 @@ void kbase_gpu_metrics_ctx_end_activity(struct kbase_context *kctx, u64 timestam * @kbdev: Pointer to the GPU device. * @ts: Timestamp at which the tracepoint is being emitted. * - * This function would loop through all the active GPU metrics contexts and emit a - * power/gpu_work_period tracepoint for them. + * This function would loop through all GPU metrics contexts in the active list and + * emit a power/gpu_work_period tracepoint if the GPU work in the context still active. * The GPU metrics context that is found to be inactive since the last tracepoint * was emitted would be moved to the inactive list. * The current work period would be considered as over and a new work period would diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.c b/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.c index 10b3b506e84e..25ee8c1042a2 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.c @@ -357,6 +357,7 @@ enum l2_config_override_result { /** * kbase_read_l2_config_from_dt - Read L2 configuration * @kbdev: The kbase device for which to get the L2 configuration. + * @regdump: Pointer to struct kbase_gpuprops_regdump structure. * * Check for L2 configuration overrides in module parameters and device tree. * Override values in module parameters take priority over override values in @@ -366,9 +367,16 @@ enum l2_config_override_result { * overridden, L2_CONFIG_OVERRIDE_NONE if no overrides are provided. * L2_CONFIG_OVERRIDE_FAIL otherwise. */ -static enum l2_config_override_result kbase_read_l2_config_from_dt(struct kbase_device *const kbdev) +static enum l2_config_override_result +kbase_read_l2_config_from_dt(struct kbase_device *const kbdev, + struct kbasep_gpuprops_regdump *regdump) { struct device_node *np = kbdev->dev->of_node; + /* + * CACHE_SIZE bit fields in L2_FEATURES register, default value after the reset/powerup + * holds the maximum size of the cache that can be programmed in L2_CONFIG register. + */ + const u8 l2_size_max = L2_FEATURES_CACHE_SIZE_GET(regdump->l2_features); if (!np) return L2_CONFIG_OVERRIDE_NONE; @@ -378,8 +386,12 @@ static enum l2_config_override_result kbase_read_l2_config_from_dt(struct kbase_ else if (of_property_read_u8(np, "l2-size", &kbdev->l2_size_override)) kbdev->l2_size_override = 0; - if (kbdev->l2_size_override != 0 && kbdev->l2_size_override < OVERRIDE_L2_SIZE_MIN_LOG2) + if (kbdev->l2_size_override != 0 && (kbdev->l2_size_override < OVERRIDE_L2_SIZE_MIN_LOG2 || + kbdev->l2_size_override > l2_size_max)) { + dev_err(kbdev->dev, "Invalid Cache Size in %s", + override_l2_size ? "Module parameters" : "Device tree node"); return L2_CONFIG_OVERRIDE_FAIL; + } /* Check overriding value is supported, if not will result in * undefined behavior. @@ -429,7 +441,7 @@ int kbase_gpuprops_update_l2_features(struct kbase_device *kbdev) struct kbasep_gpuprops_regdump *regdump = &PRIV_DATA_REGDUMP(kbdev); /* Check for L2 cache size & hash overrides */ - switch (kbase_read_l2_config_from_dt(kbdev)) { + switch (kbase_read_l2_config_from_dt(kbdev, regdump)) { case L2_CONFIG_OVERRIDE_FAIL: err = -EIO; goto exit; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gwt.c b/drivers/gpu/arm/bifrost/mali_kbase_gwt.c index c92d54c9e663..5e59bf60aa38 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_gwt.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_gwt.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -30,9 +30,10 @@ #include static inline void kbase_gpu_gwt_setup_page_permission(struct kbase_context *kctx, - unsigned long flag, struct rb_node *node) + unsigned long flag, + struct kbase_reg_zone *zone) { - struct rb_node *rbnode = node; + struct rb_node *rbnode = rb_first(&zone->reg_rbtree); while (rbnode) { struct kbase_va_region *reg; @@ -55,17 +56,15 @@ static inline void kbase_gpu_gwt_setup_page_permission(struct kbase_context *kct static void kbase_gpu_gwt_setup_pages(struct kbase_context *kctx, unsigned long flag) { - kbase_gpu_gwt_setup_page_permission(kctx, flag, - rb_first(&kctx->reg_zone[SAME_VA_ZONE].reg_rbtree)); - kbase_gpu_gwt_setup_page_permission(kctx, flag, - rb_first(&kctx->reg_zone[CUSTOM_VA_ZONE].reg_rbtree)); + kbase_gpu_gwt_setup_page_permission(kctx, flag, &kctx->reg_zone[SAME_VA_ZONE]); + kbase_gpu_gwt_setup_page_permission(kctx, flag, &kctx->reg_zone[CUSTOM_VA_ZONE]); } int kbase_gpu_gwt_start(struct kbase_context *kctx) { - kbase_gpu_vm_lock(kctx); + kbase_gpu_vm_lock_with_pmode_sync(kctx); if (kctx->gwt_enabled) { - kbase_gpu_vm_unlock(kctx); + kbase_gpu_vm_unlock_with_pmode_sync(kctx); return -EBUSY; } @@ -91,7 +90,7 @@ int kbase_gpu_gwt_start(struct kbase_context *kctx) kbase_gpu_gwt_setup_pages(kctx, ~KBASE_REG_GPU_WR); - kbase_gpu_vm_unlock(kctx); + kbase_gpu_vm_unlock_with_pmode_sync(kctx); return 0; } diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hw.c b/drivers/gpu/arm/bifrost/mali_kbase_hw.c index 7d4200e96fd3..1fde75b996c4 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_hw.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_hw.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2012-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2012-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -225,6 +225,8 @@ static const enum base_hw_issue *kbase_hw_get_issues_for_new_id(struct kbase_dev { GPU_ID_PRODUCT_TVAX, { { GPU_ID_VERSION_MAKE(0, 0, 0), base_hw_issues_tVAx_r0p0 }, + { GPU_ID_VERSION_MAKE(0, 0, 5), base_hw_issues_tVAx_r0p0 }, + { GPU_ID_VERSION_MAKE(0, 1, 0), base_hw_issues_tVAx_r0p1 }, { U32_MAX, NULL } } }, { GPU_ID_PRODUCT_TTUX, @@ -334,6 +336,8 @@ static const enum base_hw_issue *kbase_hw_get_issues_for_new_id(struct kbase_dev gpu_id->version_id = fallback_version; } } + + return issues; } diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_pm.h b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_pm.h index 7a0ea49099ba..982547d16022 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_pm.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_pm.h @@ -129,14 +129,14 @@ void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, u64 new_core_mask) * kbase_pm_set_debug_core_mask - Set the debug core mask. * * @kbdev: The kbase device structure for the device (must be a valid pointer) - * @new_core_mask_js0: The core mask to use for job slot 0 - * @new_core_mask_js1: The core mask to use for job slot 1 - * @new_core_mask_js2: The core mask to use for job slot 2 + * @new_core_mask: The core mask to use, as an array where each element refers + * to a job slot. + * @new_core_mask_size: Number of elements in the core mask array. * * This determines which cores the power manager is allowed to use. */ -void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, u64 new_core_mask_js0, - u64 new_core_mask_js1, u64 new_core_mask_js2); +void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, u64 *new_core_mask, + size_t new_core_mask_size); #endif /* MALI_USE_CSF */ /** diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_time.h b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_time.h index 0630dfa6db3a..222ff2001e56 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_time.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_time.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2014, 2018-2021, 2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014, 2018-2021, 2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -27,7 +27,8 @@ * * @multiplier: Numerator of the converter's fraction. * @divisor: Denominator of the converter's fraction. - * @offset: Converter's offset term. + * @gpu_timestamp_offset: Cached CPU to GPU TS offset computed whenever whole system + * enters into standby mode where CPU Monotonic time is suspend. * @device_scaled_timeouts: Timeouts in milliseconds that were scaled to be * consistent with the minimum MCU frequency. This * array caches the results of all of the conversions @@ -55,7 +56,7 @@ struct kbase_backend_time { #if MALI_USE_CSF u64 multiplier; u64 divisor; - s64 offset; + s64 gpu_timestamp_offset; #endif unsigned int device_scaled_timeouts[KBASE_TIMEOUT_SELECTOR_COUNT]; }; @@ -70,6 +71,40 @@ struct kbase_backend_time { * Return: The CPU timestamp. */ u64 __maybe_unused kbase_backend_time_convert_gpu_to_cpu(struct kbase_device *kbdev, u64 gpu_ts); + +/** + * kbase_backend_update_gpu_timestamp_offset() - Updates GPU timestamp offset register with the + * cached value. + * + * @kbdev: Kbase device pointer + * + * Compute the new cached value for GPU timestamp offset if the previously cached value has been + * invalidated and update the GPU timestamp offset register with the cached value. + */ +void kbase_backend_update_gpu_timestamp_offset(struct kbase_device *kbdev); + +/** + * kbase_backend_invalidate_gpu_timestamp_offset() - Invalidate cached GPU timestamp offset value + * + * @kbdev: Kbase device pointer + * + * This function invalidates cached GPU timestamp offset value whenever system suspend + * is about to happen where CPU TS counter will be stopped. + */ +void kbase_backend_invalidate_gpu_timestamp_offset(struct kbase_device *kbdev); + +#if MALI_UNIT_TEST +/** + * kbase_backend_read_gpu_timestamp_offset_reg() - Read GPU TIMESTAMP OFFSET Register + * + * @kbdev: Kbase device pointer + * + * This function read GPU TIMESTAMP OFFSET Register with proper register access + * + * Return: GPU TIMESTAMP OFFSET Register value, as unsigned 64 bit value + */ +u64 kbase_backend_read_gpu_timestamp_offset_reg(struct kbase_device *kbdev); +#endif #endif /** diff --git a/drivers/gpu/arm/bifrost/mali_kbase_js.c b/drivers/gpu/arm/bifrost/mali_kbase_js.c index 55c1f4be25d5..3e6b6b5eb066 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_js.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_js.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2011-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -161,7 +161,7 @@ static inline int gpu_metrics_ctx_init(struct kbase_context *kctx) put_cred(cred); /* Return early if this is not a Userspace created context */ - if (unlikely(!kctx->kfile)) + if (unlikely(!kctx->filp)) return 0; /* Serialize against the other threads trying to create/destroy Kbase contexts. */ @@ -200,7 +200,7 @@ static inline void gpu_metrics_ctx_term(struct kbase_context *kctx) unsigned long flags; /* Return early if this is not a Userspace created context */ - if (unlikely(!kctx->kfile)) + if (unlikely(!kctx->filp)) return; /* Serialize against the other threads trying to create/destroy Kbase contexts. */ @@ -2615,7 +2615,7 @@ static void kbase_js_move_to_tree(struct kbase_jd_atom *katom) * * Remove all post dependencies of an atom from the context ringbuffers. * - * The original atom's event_code will be propogated to all dependent atoms. + * The original atom's event_code will be propagated to all dependent atoms. * * Context: Caller must hold the HW access lock */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_linux.h b/drivers/gpu/arm/bifrost/mali_kbase_linux.h index 9195be347e2b..cb55d4b417c4 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_linux.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_linux.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -35,8 +35,13 @@ #if IS_ENABLED(MALI_KERNEL_TEST_API) #define KBASE_EXPORT_TEST_API(func) EXPORT_SYMBOL(func) +/* Note: due to the 2-layer macro translation, using the NULL _etype does not + * compile, and one workaround is to use ERRNO_NULL instead. + */ +#define KBASE_ALLOW_ERROR_INJECTION_TEST_API(func, etype) ALLOW_ERROR_INJECTION(func, etype) #else #define KBASE_EXPORT_TEST_API(func) +#define KBASE_ALLOW_ERROR_INJECTION_TEST_API(func, etype) #endif #define KBASE_EXPORT_SYMBOL(func) EXPORT_SYMBOL(func) diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem.c b/drivers/gpu/arm/bifrost/mali_kbase_mem.c index ddf6ea352e72..7470a94a44ad 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -46,6 +46,9 @@ #include #include +/* Static key used to determine if large pages are enabled or not */ +static DEFINE_STATIC_KEY_FALSE(large_pages_static_key); + #define VA_REGION_SLAB_NAME_PREFIX "va-region-slab-" #define VA_REGION_SLAB_NAME_SIZE (DEVNAME_SIZE + sizeof(VA_REGION_SLAB_NAME_PREFIX) + 1) @@ -143,20 +146,20 @@ MODULE_PARM_DESC(large_page_conf, "User override for large page usage on support static void kbasep_mem_page_size_init(struct kbase_device *kbdev) { if (!IS_ENABLED(CONFIG_LARGE_PAGE_SUPPORT)) { - kbdev->pagesize_2mb = false; dev_info(kbdev->dev, "Large page support was disabled at compile-time!"); return; } switch (large_page_conf) { case LARGE_PAGE_AUTO: { - kbdev->pagesize_2mb = kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_LARGE_PAGE_ALLOC); + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_LARGE_PAGE_ALLOC)) + static_branch_inc(&large_pages_static_key); dev_info(kbdev->dev, "Large page allocation set to %s after hardware feature check", - kbdev->pagesize_2mb ? "true" : "false"); + static_branch_unlikely(&large_pages_static_key) ? "true" : "false"); break; } case LARGE_PAGE_ON: { - kbdev->pagesize_2mb = true; + static_branch_inc(&large_pages_static_key); if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_LARGE_PAGE_ALLOC)) dev_warn(kbdev->dev, "Enabling large page allocations on unsupporting GPU!"); @@ -165,12 +168,10 @@ static void kbasep_mem_page_size_init(struct kbase_device *kbdev) break; } case LARGE_PAGE_OFF: { - kbdev->pagesize_2mb = false; dev_info(kbdev->dev, "Large page allocation override: turned off\n"); break; } default: { - kbdev->pagesize_2mb = false; dev_info(kbdev->dev, "Invalid large page override, turning off large pages\n"); break; } @@ -180,12 +181,18 @@ static void kbasep_mem_page_size_init(struct kbase_device *kbdev) * so that userspace could read it to figure out the state of the configuration * if necessary. */ - if (kbdev->pagesize_2mb) + if (static_branch_unlikely(&large_pages_static_key)) large_page_conf = LARGE_PAGE_ON; else large_page_conf = LARGE_PAGE_OFF; } +inline bool kbase_is_large_pages_enabled(void) +{ + return static_branch_unlikely(&large_pages_static_key); +} +KBASE_EXPORT_TEST_API(kbase_is_large_pages_enabled); + int kbase_mem_init(struct kbase_device *kbdev) { int err = 0; @@ -524,15 +531,20 @@ int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg) switch (alloc->imported.user_buf.state) { case KBASE_USER_BUF_STATE_GPU_MAPPED: { alloc->imported.user_buf.current_mapping_usage_count = 0; - kbase_user_buf_from_gpu_mapped_to_empty(kctx, reg); + kbase_mem_phy_alloc_ref_read(alloc) ? + kbase_user_buf_from_gpu_mapped_to_pinned(kctx, reg) : + kbase_user_buf_from_gpu_mapped_to_empty(kctx, reg); break; } case KBASE_USER_BUF_STATE_DMA_MAPPED: { - kbase_user_buf_from_dma_mapped_to_empty(kctx, reg); + kbase_mem_phy_alloc_ref_read(alloc) ? + kbase_user_buf_from_dma_mapped_to_pinned(kctx, reg) : + kbase_user_buf_from_dma_mapped_to_empty(kctx, reg); break; } case KBASE_USER_BUF_STATE_PINNED: { - kbase_user_buf_from_pinned_to_empty(kctx, reg); + if (!kbase_mem_phy_alloc_ref_read(alloc)) + kbase_user_buf_from_pinned_to_empty(kctx, reg); break; } case KBASE_USER_BUF_STATE_EMPTY: { @@ -672,7 +684,9 @@ void kbase_sync_single(struct kbase_context *kctx, struct tagged_addr t_cpu_pa, dma_addr_t dma_addr; WARN_ON(!cpu_page); - WARN_ON((size_t)offset + size > PAGE_SIZE); + + if ((size_t)offset + size > PAGE_SIZE) + dev_warn(kctx->kbdev->dev, "Size and offset exceed page size"); dma_addr = kbase_dma_addr_from_tagged(t_cpu_pa) + (dma_addr_t)offset; @@ -713,19 +727,105 @@ void kbase_sync_single(struct kbase_context *kctx, struct tagged_addr t_cpu_pa, } } +static int kbase_get_sync_scope_params(struct kbase_context *kctx, unsigned long start, size_t size, + u64 *page_off, u64 *page_cnt, u64 *offset) +{ + u64 tmp_off; + struct kbase_cpu_mapping *map = + kbasep_find_enclosing_cpu_mapping(kctx, start, size, &tmp_off); + + if (!map) { + dev_dbg(kctx->kbdev->dev, "%s: Can't find CPU mapping 0x%016lX", __func__, start); + return -EINVAL; + } + + *page_off = tmp_off >> PAGE_SHIFT; + tmp_off &= ~PAGE_MASK; + *page_cnt = (size + tmp_off + (PAGE_SIZE - 1)) >> PAGE_SHIFT; + *offset = tmp_off; + + return 0; +} + +static int kbase_sync_imported_user_buf(struct kbase_context *kctx, struct kbase_va_region *reg, + struct basep_syncset *sset, enum kbase_sync_type sync_fn) +{ + unsigned long start = (uintptr_t)sset->user_addr; + size_t size = (size_t)sset->size; + dma_addr_t *dma_addr = reg->gpu_alloc->imported.user_buf.dma_addrs; + u64 page_off = 0, page_count = 0, offset = 0; + u64 i; + size_t sz; + int err; + + lockdep_assert_held(&kctx->reg_lock); + + if (sync_fn != KBASE_SYNC_TO_CPU && sync_fn != KBASE_SYNC_TO_DEVICE) { + dev_dbg(kctx->kbdev->dev, "%s: Unknown kbase sync_fn type!", __func__); + return -EINVAL; + } + + /* Early return if the imported user_buffer is not yet mapped to GPU */ + if (reg->gpu_alloc->imported.user_buf.state != KBASE_USER_BUF_STATE_GPU_MAPPED) + return -EINVAL; + + err = kbase_get_sync_scope_params(kctx, start, size, &page_off, &page_count, &offset); + if (err) + return err; + + /* Check the sync is inside the imported range */ + if ((page_off >= reg->gpu_alloc->nents) || + ((page_off + page_count) > reg->gpu_alloc->nents)) + return -EINVAL; + + dma_addr = reg->gpu_alloc->imported.user_buf.dma_addrs; + /* Sync first page */ + sz = MIN(((size_t)PAGE_SIZE - offset), size); + if (sync_fn == KBASE_SYNC_TO_CPU) + dma_sync_single_for_cpu(kctx->kbdev->dev, dma_addr[page_off] + offset, sz, + DMA_BIDIRECTIONAL); + else + dma_sync_single_for_device(kctx->kbdev->dev, dma_addr[page_off] + offset, sz, + DMA_BIDIRECTIONAL); + + /* Calculate the size for last page */ + sz = ((start + size - 1) & ~PAGE_MASK) + 1; + + /* Sync middle pages (if any) */ + for (i = 1; page_count > 2 && i < page_count - 1; i++) { + if (sync_fn == KBASE_SYNC_TO_CPU) + dma_sync_single_for_cpu(kctx->kbdev->dev, dma_addr[page_off + i], PAGE_SIZE, + DMA_BIDIRECTIONAL); + else + dma_sync_single_for_device(kctx->kbdev->dev, dma_addr[page_off + i], + PAGE_SIZE, DMA_BIDIRECTIONAL); + } + + /* Sync last page (if any) */ + if (page_count > 1) { + i = page_off + page_count - 1; + if (sync_fn == KBASE_SYNC_TO_CPU) + dma_sync_single_for_cpu(kctx->kbdev->dev, dma_addr[i], sz, + DMA_BIDIRECTIONAL); + else + dma_sync_single_for_device(kctx->kbdev->dev, dma_addr[i], sz, + DMA_BIDIRECTIONAL); + } + + return 0; +} + static int kbase_do_syncset(struct kbase_context *kctx, struct basep_syncset *sset, enum kbase_sync_type sync_fn) { int err = 0; struct kbase_va_region *reg; - struct kbase_cpu_mapping *map; unsigned long start; size_t size; struct tagged_addr *cpu_pa; struct tagged_addr *gpu_pa; - u64 page_off, page_count; + u64 page_off = 0, page_count = 0, offset = 0; u64 i; - u64 offset; size_t sz; kbase_os_mem_map_lock(kctx); @@ -748,7 +848,10 @@ static int kbase_do_syncset(struct kbase_context *kctx, struct basep_syncset *ss * memory may be cached. */ if (kbase_mem_is_imported(reg->gpu_alloc->type)) { - err = kbase_mem_do_sync_imported(kctx, reg, sync_fn); + if (reg->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_USER_BUF) + err = kbase_sync_imported_user_buf(kctx, reg, sset, sync_fn); + else + err = kbase_sync_imported_umm(kctx, reg, sync_fn); goto out_unlock; } @@ -758,17 +861,10 @@ static int kbase_do_syncset(struct kbase_context *kctx, struct basep_syncset *ss start = (uintptr_t)sset->user_addr; size = (size_t)sset->size; - map = kbasep_find_enclosing_cpu_mapping(kctx, start, size, &offset); - if (!map) { - dev_warn(kctx->kbdev->dev, "Can't find CPU mapping 0x%016lX for VA 0x%016llX", - start, sset->mem_handle.basep.handle); - err = -EINVAL; + err = kbase_get_sync_scope_params(kctx, start, size, &page_off, &page_count, &offset); + if (err) goto out_unlock; - } - page_off = offset >> PAGE_SHIFT; - offset &= ~PAGE_MASK; - page_count = (size + offset + (PAGE_SIZE - 1)) >> PAGE_SHIFT; cpu_pa = kbase_get_cpu_phy_pages(reg); gpu_pa = kbase_get_gpu_phy_pages(reg); @@ -777,7 +873,6 @@ static int kbase_do_syncset(struct kbase_context *kctx, struct basep_syncset *ss err = -EINVAL; goto out_unlock; } - if (page_off >= reg->gpu_alloc->nents) { /* Start of sync range is outside the physically backed region * so nothing to do @@ -942,7 +1037,7 @@ int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr) __func__); return -EINVAL; } - kbase_gpu_vm_lock(kctx); + kbase_gpu_vm_lock_with_pmode_sync(kctx); if (gpu_addr >= BASE_MEM_COOKIE_BASE && gpu_addr < BASE_MEM_FIRST_FREE_ADDRESS) { unsigned int cookie = PFN_DOWN(gpu_addr - BASE_MEM_COOKIE_BASE); @@ -981,7 +1076,7 @@ int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr) } out_unlock: - kbase_gpu_vm_unlock(kctx); + kbase_gpu_vm_unlock_with_pmode_sync(kctx); return err; } @@ -1156,7 +1251,7 @@ int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pa /* Check if we have enough pages requested so we can allocate a large * page (512 * 4KB = 2MB ) */ - if (kbdev->pagesize_2mb && nr_left >= NUM_PAGES_IN_2MB_LARGE_PAGE) { + if (kbase_is_large_pages_enabled() && nr_left >= NUM_PAGES_IN_2MB_LARGE_PAGE) { size_t nr_lp = nr_left / NUM_PAGES_IN_2MB_LARGE_PAGE; res = kbase_mem_pool_alloc_pages(&kctx->mem_pools.large[alloc->group_id], @@ -1307,6 +1402,7 @@ alloc_failed: invalid_request: return -ENOMEM; } +KBASE_EXPORT_TEST_API(kbase_alloc_phy_pages_helper); static size_t free_partial_locked(struct kbase_context *kctx, struct kbase_mem_pool *pool, struct tagged_addr tp) @@ -1363,7 +1459,7 @@ struct tagged_addr *kbase_alloc_phy_pages_helper_locked(struct kbase_mem_phy_all kctx = alloc->imported.native.kctx; kbdev = kctx->kbdev; - if (!kbdev->pagesize_2mb) + if (!kbase_is_large_pages_enabled()) WARN_ON(pool->order); if (alloc->reg) { @@ -1386,7 +1482,7 @@ struct tagged_addr *kbase_alloc_phy_pages_helper_locked(struct kbase_mem_phy_all tp = alloc->pages + alloc->nents; new_pages = tp; - if (kbdev->pagesize_2mb && pool->order) { + if (kbase_is_large_pages_enabled() && pool->order) { size_t nr_lp = nr_left / NUM_PAGES_IN_2MB_LARGE_PAGE; res = kbase_mem_pool_alloc_pages_locked(pool, nr_lp * NUM_PAGES_IN_2MB_LARGE_PAGE, @@ -1503,7 +1599,7 @@ alloc_failed: struct tagged_addr *start_free = alloc->pages + alloc->nents; - if (kbdev->pagesize_2mb && pool->order) { + if (kbase_is_large_pages_enabled() && pool->order) { while (nr_pages_to_free) { if (is_huge_head(*start_free)) { kbase_mem_pool_free_pages_locked( @@ -1659,6 +1755,7 @@ int kbase_free_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pag return 0; } +KBASE_EXPORT_TEST_API(kbase_free_phy_pages_helper); void kbase_free_phy_pages_helper_locked(struct kbase_mem_phy_alloc *alloc, struct kbase_mem_pool *pool, struct tagged_addr *pages, @@ -2156,17 +2253,31 @@ void kbase_gpu_vm_lock(struct kbase_context *kctx) KBASE_DEBUG_ASSERT(kctx != NULL); mutex_lock(&kctx->reg_lock); } - KBASE_EXPORT_TEST_API(kbase_gpu_vm_lock); +void kbase_gpu_vm_lock_with_pmode_sync(struct kbase_context *kctx) +{ +#if MALI_USE_CSF + down_read(&kctx->kbdev->csf.pmode_sync_sem); +#endif + kbase_gpu_vm_lock(kctx); +} + void kbase_gpu_vm_unlock(struct kbase_context *kctx) { KBASE_DEBUG_ASSERT(kctx != NULL); mutex_unlock(&kctx->reg_lock); } - KBASE_EXPORT_TEST_API(kbase_gpu_vm_unlock); +void kbase_gpu_vm_unlock_with_pmode_sync(struct kbase_context *kctx) +{ + kbase_gpu_vm_unlock(kctx); +#if MALI_USE_CSF + up_read(&kctx->kbdev->csf.pmode_sync_sem); +#endif +} + #if IS_ENABLED(CONFIG_DEBUG_FS) struct kbase_jit_debugfs_data { int (*func)(struct kbase_jit_debugfs_data *data); @@ -2708,7 +2819,7 @@ static int kbase_jit_grow(struct kbase_context *kctx, const struct base_jit_allo delta = info->commit_pages - reg->gpu_alloc->nents; pages_required = delta; - if (kctx->kbdev->pagesize_2mb && pages_required >= NUM_PAGES_IN_2MB_LARGE_PAGE) { + if (kbase_is_large_pages_enabled() && pages_required >= NUM_PAGES_IN_2MB_LARGE_PAGE) { pool = &kctx->mem_pools.large[kctx->jit_group_id]; /* Round up to number of 2 MB pages required */ pages_required += (NUM_PAGES_IN_2MB_LARGE_PAGE - 1); @@ -2746,10 +2857,10 @@ static int kbase_jit_grow(struct kbase_context *kctx, const struct base_jit_allo kbase_mem_pool_lock(pool); } - if (reg->gpu_alloc->nents > info->commit_pages) { + if (reg->gpu_alloc->nents >= info->commit_pages) { kbase_mem_pool_unlock(pool); spin_unlock(&kctx->mem_partials_lock); - dev_warn( + dev_info( kctx->kbdev->dev, "JIT alloc grown beyond the required number of initially required pages, this grow no longer needed."); goto done; @@ -2999,7 +3110,7 @@ struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, if (!jit_allow_allocate(kctx, info, ignore_pressure_limit)) return NULL; - if (kctx->kbdev->pagesize_2mb) { + if (kbase_is_large_pages_enabled()) { /* Preallocate memory for the sub-allocation structs */ for (i = 0; i != ARRAY_SIZE(prealloc_sas); ++i) { prealloc_sas[i] = kmalloc(sizeof(*prealloc_sas[i]), GFP_KERNEL); @@ -3008,7 +3119,7 @@ struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, } } - kbase_gpu_vm_lock(kctx); + kbase_gpu_vm_lock_with_pmode_sync(kctx); mutex_lock(&kctx->jit_evict_lock); /* @@ -3086,7 +3197,7 @@ struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, kbase_jit_done_phys_increase(kctx, needed_pages); #endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - kbase_gpu_vm_unlock(kctx); + kbase_gpu_vm_unlock_with_pmode_sync(kctx); if (ret) { /* @@ -3147,7 +3258,7 @@ struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, #endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ mutex_unlock(&kctx->jit_evict_lock); - kbase_gpu_vm_unlock(kctx); + kbase_gpu_vm_unlock_with_pmode_sync(kctx); reg = kbase_mem_alloc(kctx, info->va_pages, info->commit_pages, info->extension, &flags, &gpu_addr, mmu_sync_info); @@ -3249,9 +3360,9 @@ void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg) u64 delta = old_pages - new_size; if (delta) { - mutex_lock(&kctx->reg_lock); + kbase_gpu_vm_lock_with_pmode_sync(kctx); kbase_mem_shrink(kctx, reg, old_pages - delta); - mutex_unlock(&kctx->reg_lock); + kbase_gpu_vm_unlock_with_pmode_sync(kctx); } } @@ -3356,8 +3467,7 @@ void kbase_jit_term(struct kbase_context *kctx) struct kbase_va_region *walker; /* Free all allocations for this context */ - - kbase_gpu_vm_lock(kctx); + kbase_gpu_vm_lock_with_pmode_sync(kctx); mutex_lock(&kctx->jit_evict_lock); /* Free all allocations from the pool */ while (!list_empty(&kctx->jit_pool_head)) { @@ -3398,7 +3508,7 @@ void kbase_jit_term(struct kbase_context *kctx) WARN_ON(kctx->jit_phys_pages_to_be_allocated); #endif mutex_unlock(&kctx->jit_evict_lock); - kbase_gpu_vm_unlock(kctx); + kbase_gpu_vm_unlock_with_pmode_sync(kctx); /* * Flush the freeing of allocations whose backing has been freed @@ -3916,9 +4026,6 @@ int kbase_map_external_resource(struct kbase_context *kctx, struct kbase_va_regi case KBASE_MEM_TYPE_IMPORTED_USER_BUF: { user_buf_original_state = reg->gpu_alloc->imported.user_buf.state; - if ((reg->gpu_alloc->imported.user_buf.mm != locked_mm) && (!reg->gpu_alloc->nents)) - return -EINVAL; - /* This function is reachable through many code paths, and the imported * memory handle could be in any of the possible states: consider all * of them as a valid starting point, and progress through all stages @@ -3928,19 +4035,31 @@ int kbase_map_external_resource(struct kbase_context *kctx, struct kbase_va_regi * Error recovery restores the original state and goes no further. */ switch (user_buf_original_state) { - case KBASE_USER_BUF_STATE_EMPTY: - case KBASE_USER_BUF_STATE_PINNED: + case KBASE_USER_BUF_STATE_EMPTY: { + if (reg->gpu_alloc->imported.user_buf.mm != locked_mm) + return -EINVAL; + err = kbase_user_buf_from_empty_to_gpu_mapped(kctx, reg); + break; + } + case KBASE_USER_BUF_STATE_PINNED: { + if (!reg->gpu_alloc->nents) + return -EINVAL; + err = kbase_user_buf_from_pinned_to_gpu_mapped(kctx, reg); + break; + } case KBASE_USER_BUF_STATE_DMA_MAPPED: { - if (user_buf_original_state == KBASE_USER_BUF_STATE_EMPTY) - err = kbase_user_buf_from_empty_to_gpu_mapped(kctx, reg); - else if (user_buf_original_state == KBASE_USER_BUF_STATE_PINNED) - err = kbase_user_buf_from_pinned_to_gpu_mapped(kctx, reg); - else - err = kbase_user_buf_from_dma_mapped_to_gpu_mapped(kctx, reg); - - if (err) - return err; - + /* If the imported handle has not pinned any physical pages yet: + * this function can only be called within the context of a user + * process, which must be the same process as the one that + * originally created the memory handle. + * + * In all other transitions: make sure that the imported handle + * has already pinned physical pages before proceeding to mapping + * operations. + */ + if (!reg->gpu_alloc->nents) + return -EINVAL; + err = kbase_user_buf_from_dma_mapped_to_gpu_mapped(kctx, reg); break; } case KBASE_USER_BUF_STATE_GPU_MAPPED: { @@ -3954,6 +4073,8 @@ int kbase_map_external_resource(struct kbase_context *kctx, struct kbase_va_regi reg->gpu_alloc->imported.user_buf.state); return -EINVAL; } + if (err) + return err; /* If the state was valid and the transition is happening, then the handle * must be in GPU_MAPPED state now and the reference counter of GPU mappings @@ -4021,13 +4142,8 @@ void kbase_unmap_external_resource(struct kbase_context *kctx, struct kbase_va_r kbase_va_region_alloc_put(kctx, reg); } -static inline u64 kbasep_get_va_gpu_addr(struct kbase_va_region *reg) -{ - return reg->start_pfn << PAGE_SHIFT; -} - -struct kbase_ctx_ext_res_meta *kbase_sticky_resource_acquire(struct kbase_context *kctx, - u64 gpu_addr) +struct kbase_ctx_ext_res_meta * +kbase_sticky_resource_acquire(struct kbase_context *kctx, u64 gpu_addr, struct mm_struct *locked_mm) { struct kbase_ctx_ext_res_meta *meta = NULL; struct kbase_ctx_ext_res_meta *walker; @@ -4066,7 +4182,7 @@ struct kbase_ctx_ext_res_meta *kbase_sticky_resource_acquire(struct kbase_contex /* Map the external resource to the GPU allocation of the region * and acquire the reference to the VA region */ - if (kbase_map_external_resource(kctx, meta->reg, NULL)) + if (kbase_map_external_resource(kctx, meta->reg, locked_mm)) goto fail_map; meta->ref = 1; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem.h b/drivers/gpu/arm/bifrost/mali_kbase_mem.h index e4a7d6bd0a30..19b50f5bf08b 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -589,6 +589,11 @@ int kbase_mem_init(struct kbase_device *kbdev); void kbase_mem_halt(struct kbase_device *kbdev); void kbase_mem_term(struct kbase_device *kbdev); +static inline unsigned int kbase_mem_phy_alloc_ref_read(struct kbase_mem_phy_alloc *alloc) +{ + return kref_read(&alloc->kref); +} + static inline struct kbase_mem_phy_alloc *kbase_mem_phy_alloc_get(struct kbase_mem_phy_alloc *alloc) { kref_get(&alloc->kref); @@ -1408,12 +1413,30 @@ int kbase_update_region_flags(struct kbase_context *kctx, struct kbase_va_region */ void kbase_gpu_vm_lock(struct kbase_context *kctx); +/** + * kbase_gpu_vm_lock_with_pmode_sync() - Wrapper of kbase_gpu_vm_lock. + * @kctx: KBase context + * + * Same as kbase_gpu_vm_lock for JM GPU. + * Additionally acquire P.mode read-write semaphore for CSF GPU. + */ +void kbase_gpu_vm_lock_with_pmode_sync(struct kbase_context *kctx); + /** * kbase_gpu_vm_unlock() - Release the per-context region list lock * @kctx: KBase context */ void kbase_gpu_vm_unlock(struct kbase_context *kctx); +/** + * kbase_gpu_vm_unlock_with_pmode_sync() - Wrapper of kbase_gpu_vm_unlock. + * @kctx: KBase context + * + * Same as kbase_gpu_vm_unlock for JM GPU. + * Additionally release P.mode read-write semaphore for CSF GPU. + */ +void kbase_gpu_vm_unlock_with_pmode_sync(struct kbase_context *kctx); + int kbase_alloc_phy_pages(struct kbase_va_region *reg, size_t vsize, size_t size); /** @@ -1651,7 +1674,7 @@ int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pa * * @prealloc_sa: Information about the partial allocation if the amount of memory requested * is not a multiple of 2MB. One instance of struct kbase_sub_alloc must be - * allocated by the caller if kbdev->pagesize_2mb is enabled. + * allocated by the caller if large pages are enabled. * * Allocates @nr_pages_requested and updates the alloc object. This function does not allocate new * pages from the kernel, and therefore will never trigger the OoM killer. Therefore, it can be @@ -1679,9 +1702,9 @@ int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pa * This ensures that the pool can be grown to the required size and that the allocation can * complete without another thread using the newly grown pages. * - * If kbdev->pagesize_2mb is enabled and the allocation is >= 2MB, then @pool must be one of the - * pools from alloc->imported.native.kctx->mem_pools.large[]. Otherwise it must be one of the - * mempools from alloc->imported.native.kctx->mem_pools.small[]. + * If large (2MiB) pages are enabled and the allocation is >= 2MiB, then @pool + * must be one of the pools from alloc->imported.native.kctx->mem_pools.large[]. Otherwise it + * must be one of the mempools from alloc->imported.native.kctx->mem_pools.small[]. * * @prealloc_sa is used to manage the non-2MB sub-allocation. It has to be pre-allocated because we * must not sleep (due to the usage of kmalloc()) whilst holding pool->pool_lock. @prealloc_sa @@ -2070,7 +2093,8 @@ bool kbase_has_exec_va_zone(struct kbase_context *kctx); * kbase_map_external_resource - Map an external resource to the GPU. * @kctx: kbase context. * @reg: External resource to map. - * @locked_mm: The mm_struct which has been locked for this operation. + * @locked_mm: The mm_struct which has been locked for this operation, + * or NULL if none is available. * * On successful mapping, the VA region and the gpu_alloc refcounts will be * increased, making it safe to use and store both values directly. @@ -2335,12 +2359,15 @@ int kbase_sticky_resource_init(struct kbase_context *kctx); * kbase_sticky_resource_acquire - Acquire a reference on a sticky resource. * @kctx: kbase context. * @gpu_addr: The GPU address of the external resource. + * @locked_mm: The mm_struct which has been locked for this operation, + * or NULL if none is available. * * Return: The metadata object which represents the binding between the * external resource and the kbase context on success or NULL on failure. */ struct kbase_ctx_ext_res_meta *kbase_sticky_resource_acquire(struct kbase_context *kctx, - u64 gpu_addr); + u64 gpu_addr, + struct mm_struct *locked_mm); /** * kbase_sticky_resource_release - Release a reference on a sticky resource. @@ -2494,19 +2521,19 @@ void kbase_mem_umm_unmap(struct kbase_context *kctx, struct kbase_va_region *reg struct kbase_mem_phy_alloc *alloc); /** - * kbase_mem_do_sync_imported - Sync caches for imported memory + * kbase_sync_imported_umm - Sync caches for imported UMM memory * @kctx: Pointer to the kbase context * @reg: Pointer to the region with imported memory to sync * @sync_fn: The type of sync operation to perform * - * Sync CPU caches for supported (currently only dma-buf (UMM)) memory. + * Sync CPU caches for supported dma-buf (UMM) memory. * Attempting to sync unsupported imported memory types will result in an error * code, -EINVAL. * * Return: 0 on success, or a negative error code. */ -int kbase_mem_do_sync_imported(struct kbase_context *kctx, struct kbase_va_region *reg, - enum kbase_sync_type sync_fn); +int kbase_sync_imported_umm(struct kbase_context *kctx, struct kbase_va_region *reg, + enum kbase_sync_type sync_fn); /** * kbase_mem_copy_to_pinned_user_pages - Memcpy from source input page to @@ -2595,4 +2622,7 @@ static inline base_mem_alloc_flags kbase_mem_group_id_set(int id) { return BASE_MEM_GROUP_ID_SET(id); } + +bool kbase_is_large_pages_enabled(void); + #endif /* _KBASE_MEM_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c b/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c index 9a30001634f1..7c319abf381b 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -46,6 +46,7 @@ #include #include #include +#include #if ((KERNEL_VERSION(5, 3, 0) <= LINUX_VERSION_CODE) || \ (KERNEL_VERSION(5, 0, 0) > LINUX_VERSION_CODE)) @@ -433,7 +434,7 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages } reg->initial_commit = commit_pages; - kbase_gpu_vm_lock(kctx); + kbase_gpu_vm_lock_with_pmode_sync(kctx); if (reg->flags & KBASE_REG_PERMANENT_KERNEL_MAPPING) { /* Permanent kernel mappings must happen as soon as @@ -443,7 +444,7 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages */ int err = kbase_phy_alloc_mapping_init(kctx, reg, va_pages, commit_pages); if (err < 0) { - kbase_gpu_vm_unlock(kctx); + kbase_gpu_vm_unlock_with_pmode_sync(kctx); goto no_kern_mapping; } } @@ -455,7 +456,7 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages /* Bind to a cookie */ if (bitmap_empty(kctx->cookies, BITS_PER_LONG)) { dev_err(dev, "No cookies available for allocation!"); - kbase_gpu_vm_unlock(kctx); + kbase_gpu_vm_unlock_with_pmode_sync(kctx); goto no_cookie; } /* return a cookie */ @@ -472,7 +473,7 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages } else /* we control the VA */ { size_t align = 1; - if (kctx->kbdev->pagesize_2mb) { + if (kbase_is_large_pages_enabled()) { /* If there's enough (> 33 bits) of GPU VA space, align to 2MB * boundaries. The similar condition is used for mapping from * the SAME_VA zone inside kbase_context_get_unmapped_area(). @@ -490,7 +491,7 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages } if (kbase_gpu_mmap(kctx, reg, *gpu_va, va_pages, align, mmu_sync_info) != 0) { dev_warn(dev, "Failed to map memory on GPU"); - kbase_gpu_vm_unlock(kctx); + kbase_gpu_vm_unlock_with_pmode_sync(kctx); goto no_mmap; } /* return real GPU VA */ @@ -508,7 +509,7 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages } #endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - kbase_gpu_vm_unlock(kctx); + kbase_gpu_vm_unlock_with_pmode_sync(kctx); #if MALI_USE_CSF if (*flags & BASE_MEM_FIXABLE) @@ -596,8 +597,10 @@ int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, u64 query, u64 *co *out |= BASE_MEM_COHERENT_SYSTEM; if (KBASE_REG_SHARE_IN & reg->flags) *out |= BASE_MEM_COHERENT_LOCAL; - if (KBASE_REG_DONT_NEED & reg->flags) - *out |= BASE_MEM_DONT_NEED; + if (mali_kbase_supports_mem_dont_need(kctx->api_version)) { + if (KBASE_REG_DONT_NEED & reg->flags) + *out |= BASE_MEM_DONT_NEED; + } if (mali_kbase_supports_mem_grow_on_gpf(kctx->api_version)) { /* Prior to this version, this was known about by * user-side but we did not return them. Returning @@ -634,9 +637,30 @@ int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, u64 query, u64 *co else *out |= BASE_MEM_FIXABLE; } -#endif +#endif /* MALI_USE_CSF */ if (KBASE_REG_GPU_VA_SAME_4GB_PAGE & reg->flags) *out |= BASE_MEM_GPU_VA_SAME_4GB_PAGE; + if (mali_kbase_supports_mem_import_sync_on_map_unmap(kctx->api_version)) { + if (reg->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM) { + if (reg->gpu_alloc->imported.umm.need_sync) + *out |= BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP; + } + } + if (mali_kbase_supports_mem_kernel_sync(kctx->api_version)) { + if (unlikely(reg->cpu_alloc != reg->gpu_alloc)) + *out |= BASE_MEM_KERNEL_SYNC; + } + if (mali_kbase_supports_mem_same_va(kctx->api_version)) { + if (kbase_bits_to_zone(reg->flags) == SAME_VA_ZONE) { + /* Imported memory is an edge case, where declaring it SAME_VA + * would be ambiguous. + */ + if (reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM && + reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_USER_BUF) { + *out |= BASE_MEM_SAME_VA; + } + } + } *out |= kbase_mem_group_id_set(reg->cpu_alloc->group_id); @@ -667,7 +691,9 @@ out_unlock: static unsigned long kbase_mem_evictable_reclaim_count_objects(struct shrinker *s, struct shrink_control *sc) { - struct kbase_context *kctx = container_of(s, struct kbase_context, reclaim); + struct kbase_context *kctx = + KBASE_GET_KBASE_DATA_FROM_SHRINKER(s, struct kbase_context, reclaim); + int evict_nents = atomic_read(&kctx->evict_nents); unsigned long nr_freeable_items; @@ -717,8 +743,15 @@ static unsigned long kbase_mem_evictable_reclaim_scan_objects(struct shrinker *s struct kbase_mem_phy_alloc *tmp; unsigned long freed = 0; - kctx = container_of(s, struct kbase_context, reclaim); + kctx = KBASE_GET_KBASE_DATA_FROM_SHRINKER(s, struct kbase_context, reclaim); +#if MALI_USE_CSF + if (!down_read_trylock(&kctx->kbdev->csf.pmode_sync_sem)) { + dev_warn(kctx->kbdev->dev, + "Can't shrink GPU memory when P.Mode entrance is in progress"); + return 0; + } +#endif mutex_lock(&kctx->jit_evict_lock); list_for_each_entry_safe(alloc, tmp, &kctx->evict_list, evict_node) { @@ -757,32 +790,36 @@ static unsigned long kbase_mem_evictable_reclaim_scan_objects(struct shrinker *s } mutex_unlock(&kctx->jit_evict_lock); - +#if MALI_USE_CSF + up_read(&kctx->kbdev->csf.pmode_sync_sem); +#endif return freed; } int kbase_mem_evictable_init(struct kbase_context *kctx) { + struct shrinker *reclaim; + INIT_LIST_HEAD(&kctx->evict_list); mutex_init(&kctx->jit_evict_lock); - kctx->reclaim.count_objects = kbase_mem_evictable_reclaim_count_objects; - kctx->reclaim.scan_objects = kbase_mem_evictable_reclaim_scan_objects; - kctx->reclaim.seeks = DEFAULT_SEEKS; - /* Kernel versions prior to 3.1 : - * struct shrinker does not define batch - */ -#if KERNEL_VERSION(6, 0, 0) > LINUX_VERSION_CODE - register_shrinker(&kctx->reclaim); -#else - register_shrinker(&kctx->reclaim, "mali-mem"); -#endif + reclaim = KBASE_INIT_RECLAIM(kctx, reclaim, "mali-mem"); + if (!reclaim) + return -ENOMEM; + KBASE_SET_RECLAIM(kctx, reclaim, reclaim); + + reclaim->count_objects = kbase_mem_evictable_reclaim_count_objects; + reclaim->scan_objects = kbase_mem_evictable_reclaim_scan_objects; + reclaim->seeks = DEFAULT_SEEKS; + + KBASE_REGISTER_SHRINKER(reclaim, "mali-mem", kctx); + return 0; } void kbase_mem_evictable_deinit(struct kbase_context *kctx) { - unregister_shrinker(&kctx->reclaim); + KBASE_UNREGISTER_SHRINKER(kctx->reclaim); } /** @@ -1058,7 +1095,7 @@ int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned in /* Lock down the context, and find the region */ down_write(kbase_mem_get_process_mmap_lock()); - kbase_gpu_vm_lock(kctx); + kbase_gpu_vm_lock_with_pmode_sync(kctx); /* Validate the region */ reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); @@ -1110,7 +1147,7 @@ int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned in } out_unlock: - kbase_gpu_vm_unlock(kctx); + kbase_gpu_vm_unlock_with_pmode_sync(kctx); up_write(kbase_mem_get_process_mmap_lock()); return ret; @@ -1118,8 +1155,8 @@ out_unlock: #define KBASE_MEM_IMPORT_HAVE_PAGES (1UL << BASE_MEM_FLAGS_NR_BITS) -int kbase_mem_do_sync_imported(struct kbase_context *kctx, struct kbase_va_region *reg, - enum kbase_sync_type sync_fn) +int kbase_sync_imported_umm(struct kbase_context *kctx, struct kbase_va_region *reg, + enum kbase_sync_type sync_fn) { int ret = -EINVAL; struct dma_buf __maybe_unused *dma_buf; @@ -1317,7 +1354,7 @@ int kbase_mem_umm_map(struct kbase_context *kctx, struct kbase_va_region *reg) if (IS_ENABLED(CONFIG_MALI_DMA_BUF_LEGACY_COMPAT) || alloc->imported.umm.need_sync) { if (!kbase_is_region_invalid_or_free(reg)) { - err = kbase_mem_do_sync_imported(kctx, reg, KBASE_SYNC_TO_DEVICE); + err = kbase_sync_imported_umm(kctx, reg, KBASE_SYNC_TO_DEVICE); WARN_ON_ONCE(err); } } @@ -1379,7 +1416,7 @@ void kbase_mem_umm_unmap(struct kbase_context *kctx, struct kbase_va_region *reg if (IS_ENABLED(CONFIG_MALI_DMA_BUF_LEGACY_COMPAT) || alloc->imported.umm.need_sync) { if (!kbase_is_region_invalid_or_free(reg)) { - int err = kbase_mem_do_sync_imported(kctx, reg, KBASE_SYNC_TO_CPU); + int err = kbase_sync_imported_umm(kctx, reg, KBASE_SYNC_TO_CPU); WARN_ON_ONCE(err); } } @@ -1794,7 +1831,7 @@ u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, u64 nent if (!reg->gpu_alloc->imported.alias.aliased) goto no_aliased_array; - kbase_gpu_vm_lock(kctx); + kbase_gpu_vm_lock_with_pmode_sync(kctx); /* validate and add src handles */ for (i = 0; i < nents; i++) { @@ -1904,7 +1941,7 @@ u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, u64 nent reg->flags &= ~KBASE_REG_FREE; reg->flags &= ~KBASE_REG_GROWABLE; - kbase_gpu_vm_unlock(kctx); + kbase_gpu_vm_unlock_with_pmode_sync(kctx); return gpu_va; @@ -1915,7 +1952,7 @@ bad_handle: * them is handled by putting reg's allocs, so no rollback of those * actions is done here. */ - kbase_gpu_vm_unlock(kctx); + kbase_gpu_vm_unlock_with_pmode_sync(kctx); no_aliased_array: invalid_flags: kbase_mem_phy_alloc_put(reg->cpu_alloc); @@ -2016,7 +2053,7 @@ int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, if (!reg) goto no_reg; - kbase_gpu_vm_lock(kctx); + kbase_gpu_vm_lock_with_pmode_sync(kctx); /* mmap needed to setup VA? */ if (*flags & (BASE_MEM_SAME_VA | BASE_MEM_NEED_MMAP)) { @@ -2050,13 +2087,13 @@ int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, /* clear out private flags */ *flags &= ((1UL << BASE_MEM_FLAGS_NR_BITS) - 1); - kbase_gpu_vm_unlock(kctx); + kbase_gpu_vm_unlock_with_pmode_sync(kctx); return 0; no_gpu_va: no_cookie: - kbase_gpu_vm_unlock(kctx); + kbase_gpu_vm_unlock_with_pmode_sync(kctx); kbase_mem_phy_alloc_put(reg->cpu_alloc); kbase_mem_phy_alloc_put(reg->gpu_alloc); kfree(reg); @@ -2096,7 +2133,7 @@ void kbase_mem_shrink_cpu_mapping(struct kbase_context *kctx, struct kbase_va_re /* Nothing to do */ return; - unmap_mapping_range(kctx->kfile->filp->f_inode->i_mapping, + unmap_mapping_range(kctx->filp->f_inode->i_mapping, (loff_t)(gpu_va_start + new_pages) << PAGE_SHIFT, (loff_t)(old_pages - new_pages) << PAGE_SHIFT, 1); } @@ -2142,7 +2179,7 @@ int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages) } down_write(kbase_mem_get_process_mmap_lock()); - kbase_gpu_vm_lock(kctx); + kbase_gpu_vm_lock_with_pmode_sync(kctx); /* Validate the region */ reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); @@ -2250,7 +2287,7 @@ int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages) } out_unlock: - kbase_gpu_vm_unlock(kctx); + kbase_gpu_vm_unlock_with_pmode_sync(kctx); if (read_locked) up_read(kbase_mem_get_process_mmap_lock()); else @@ -2274,11 +2311,16 @@ int kbase_mem_shrink(struct kbase_context *const kctx, struct kbase_va_region *c return -EINVAL; old_pages = kbase_reg_current_backed_size(reg); - if (WARN_ON(old_pages < new_pages)) + if (old_pages < new_pages) { + dev_warn( + kctx->kbdev->dev, + "Requested number of pages (%llu) is larger than the current number of pages (%llu)", + new_pages, old_pages); return -EINVAL; + } delta = old_pages - new_pages; - if (kctx->kbdev->pagesize_2mb) { + if (kbase_is_large_pages_enabled()) { struct tagged_addr *start_free = reg->gpu_alloc->pages + new_pages; /* Move the end of new commited range to a valid location. @@ -2332,7 +2374,7 @@ static void kbase_cpu_vm_close(struct vm_area_struct *vma) KBASE_DEBUG_ASSERT(map->kctx); KBASE_DEBUG_ASSERT(map->alloc); - kbase_gpu_vm_lock(map->kctx); + kbase_gpu_vm_lock_with_pmode_sync(map->kctx); if (map->free_on_close) { KBASE_DEBUG_ASSERT(kbase_bits_to_zone(map->region->flags) == SAME_VA_ZONE); @@ -2346,10 +2388,9 @@ static void kbase_cpu_vm_close(struct vm_area_struct *vma) list_del(&map->mappings_list); kbase_va_region_alloc_put(map->kctx, map->region); - kbase_gpu_vm_unlock(map->kctx); + kbase_gpu_vm_unlock_with_pmode_sync(map->kctx); kbase_mem_phy_alloc_put(map->alloc); - kbase_file_dec_cpu_mapping_count(map->kctx->kfile); kfree(map); } @@ -2549,7 +2590,6 @@ static int kbase_cpu_mmap(struct kbase_context *kctx, struct kbase_va_region *re map->alloc->properties |= KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED; list_add(&map->mappings_list, &map->alloc->mappings); - kbase_file_inc_cpu_mapping_count(kctx->kfile); out: return err; @@ -2749,7 +2789,7 @@ int kbase_context_mmap(struct kbase_context *const kctx, struct vm_area_struct * goto out; } - kbase_gpu_vm_lock(kctx); + kbase_gpu_vm_lock_with_pmode_sync(kctx); if (vma->vm_pgoff == PFN_DOWN(BASE_MEM_MAP_TRACKING_HANDLE)) { /* The non-mapped tracking helper page */ @@ -2784,11 +2824,11 @@ int kbase_context_mmap(struct kbase_context *const kctx, struct vm_area_struct * #endif /* defined(CONFIG_MALI_VECTOR_DUMP) */ #if MALI_USE_CSF case PFN_DOWN(BASEP_MEM_CSF_USER_REG_PAGE_HANDLE): - kbase_gpu_vm_unlock(kctx); + kbase_gpu_vm_unlock_with_pmode_sync(kctx); err = kbase_csf_cpu_mmap_user_reg_page(kctx, vma); goto out; case PFN_DOWN(BASEP_MEM_CSF_USER_IO_PAGES_HANDLE)... PFN_DOWN(BASE_MEM_COOKIE_BASE) - 1: { - kbase_gpu_vm_unlock(kctx); + kbase_gpu_vm_unlock_with_pmode_sync(kctx); mutex_lock(&kctx->csf.lock); err = kbase_csf_cpu_mmap_user_io_pages(kctx, vma); mutex_unlock(&kctx->csf.lock); @@ -2882,7 +2922,7 @@ int kbase_context_mmap(struct kbase_context *const kctx, struct vm_area_struct * } #endif /* defined(CONFIG_MALI_VECTOR_DUMP) */ out_unlock: - kbase_gpu_vm_unlock(kctx); + kbase_gpu_vm_unlock_with_pmode_sync(kctx); out: if (err) dev_err(dev, "mmap failed %d\n", err); @@ -3276,25 +3316,6 @@ void kbasep_os_process_page_usage_update(struct kbase_context *kctx, int pages) #endif } -static void kbase_special_vm_open(struct vm_area_struct *vma) -{ - struct kbase_context *kctx = vma->vm_private_data; - - kbase_file_inc_cpu_mapping_count(kctx->kfile); -} - -static void kbase_special_vm_close(struct vm_area_struct *vma) -{ - struct kbase_context *kctx = vma->vm_private_data; - - kbase_file_dec_cpu_mapping_count(kctx->kfile); -} - -static const struct vm_operations_struct kbase_vm_special_ops = { - .open = kbase_special_vm_open, - .close = kbase_special_vm_close, -}; - static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_struct *vma) { if (vma_pages(vma) != 1) @@ -3303,10 +3324,7 @@ static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_ /* no real access */ vma->vm_flags &= ~(VM_READ | VM_MAYREAD | VM_WRITE | VM_MAYWRITE | VM_EXEC | VM_MAYEXEC); vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP | VM_IO; - vma->vm_ops = &kbase_vm_special_ops; - vma->vm_private_data = kctx; - kbase_file_inc_cpu_mapping_count(kctx->kfile); return 0; } @@ -3367,7 +3385,6 @@ static void kbase_csf_user_io_pages_vm_close(struct vm_area_struct *vma) struct kbase_device *kbdev; int err; bool reset_prevented = false; - struct kbase_file *kfile; if (!queue) { pr_debug("Close method called for the new User IO pages mapping vma\n"); @@ -3376,7 +3393,6 @@ static void kbase_csf_user_io_pages_vm_close(struct vm_area_struct *vma) kctx = queue->kctx; kbdev = kctx->kbdev; - kfile = kctx->kfile; err = kbase_reset_gpu_prevent_and_wait(kbdev); if (err) @@ -3394,9 +3410,8 @@ static void kbase_csf_user_io_pages_vm_close(struct vm_area_struct *vma) if (reset_prevented) kbase_reset_gpu_allow(kbdev); - kbase_file_dec_cpu_mapping_count(kfile); /* Now as the vma is closed, drop the reference on mali device file */ - fput(kfile->filp); + fput(kctx->filp); } #if (KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE) @@ -3546,7 +3561,6 @@ static int kbase_csf_cpu_mmap_user_io_pages(struct kbase_context *kctx, struct v /* Also adjust the vm_pgoff */ vma->vm_pgoff = queue->db_file_offset; - kbase_file_inc_cpu_mapping_count(kctx->kfile); return 0; map_failed: @@ -3586,7 +3600,6 @@ static void kbase_csf_user_reg_vm_close(struct vm_area_struct *vma) { struct kbase_context *kctx = vma->vm_private_data; struct kbase_device *kbdev; - struct kbase_file *kfile; if (unlikely(!kctx)) { pr_debug("Close function called for the unexpected mapping"); @@ -3594,7 +3607,6 @@ static void kbase_csf_user_reg_vm_close(struct vm_area_struct *vma) } kbdev = kctx->kbdev; - kfile = kctx->kfile; if (unlikely(!kctx->csf.user_reg.vma)) dev_warn(kbdev->dev, "user_reg VMA pointer unexpectedly NULL for ctx %d_%d", @@ -3606,9 +3618,8 @@ static void kbase_csf_user_reg_vm_close(struct vm_area_struct *vma) kctx->csf.user_reg.vma = NULL; - kbase_file_dec_cpu_mapping_count(kfile); /* Now as the VMA is closed, drop the reference on mali device file */ - fput(kfile->filp); + fput(kctx->filp); } /** @@ -3738,7 +3749,6 @@ static int kbase_csf_cpu_mmap_user_reg_page(struct kbase_context *kctx, struct v vma->vm_ops = &kbase_csf_user_reg_vm_ops; vma->vm_private_data = kctx; - kbase_file_inc_cpu_mapping_count(kctx->kfile); return 0; } diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_migrate.c b/drivers/gpu/arm/bifrost/mali_kbase_mem_migrate.c index 93a07e7db4fa..f0ce0cf4e56c 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem_migrate.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_migrate.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2022-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2022-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -28,6 +28,9 @@ #include #include +/* Static key used to determine if page migration is enabled or not */ +static DEFINE_STATIC_KEY_FALSE(page_migration_static_key); + /* Global integer used to determine if module parameter value has been * provided and if page migration feature is enabled. * Feature is disabled on all platforms by default. @@ -50,15 +53,6 @@ MODULE_PARM_DESC(kbase_page_migration_enabled, KBASE_EXPORT_TEST_API(kbase_page_migration_enabled); -bool kbase_is_page_migration_enabled(void) -{ - /* Handle uninitialised int case */ - if (kbase_page_migration_enabled < 0) - return false; - return IS_ENABLED(CONFIG_PAGE_MIGRATION_SUPPORT) && kbase_page_migration_enabled; -} -KBASE_EXPORT_SYMBOL(kbase_is_page_migration_enabled); - #if (KERNEL_VERSION(6, 0, 0) <= LINUX_VERSION_CODE) static const struct movable_operations movable_ops; #endif @@ -225,7 +219,7 @@ static int kbasep_migrate_page_pt_mapped(struct page *old_page, struct page *new * This blocks the CPU page fault handler from remapping pages. * Only MCU's mmut is device wide, i.e. no corresponding kctx. */ - kbase_gpu_vm_lock(kctx); + kbase_gpu_vm_lock_with_pmode_sync(kctx); ret = kbase_mmu_migrate_page( as_tagged(page_to_phys(old_page)), as_tagged(page_to_phys(new_page)), old_dma_addr, @@ -254,7 +248,7 @@ static int kbasep_migrate_page_pt_mapped(struct page *old_page, struct page *new dma_unmap_page(kbdev->dev, new_dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL); /* Page fault handler for CPU mapping unblocked. */ - kbase_gpu_vm_unlock(kctx); + kbase_gpu_vm_unlock_with_pmode_sync(kctx); return ret; } @@ -293,10 +287,10 @@ static int kbasep_migrate_page_allocated_mapped(struct page *old_page, struct pa /* Lock context to protect access to array of pages in physical allocation. * This blocks the CPU page fault handler from remapping pages. */ - kbase_gpu_vm_lock(kctx); + kbase_gpu_vm_lock_with_pmode_sync(kctx); /* Unmap the old physical range. */ - unmap_mapping_range(kctx->kfile->filp->f_inode->i_mapping, + unmap_mapping_range(kctx->filp->f_inode->i_mapping, (loff_t)(page_md->data.mapped.vpfn / GPU_PAGES_PER_CPU_PAGE) << PAGE_SHIFT, PAGE_SIZE, 1); @@ -332,7 +326,7 @@ static int kbasep_migrate_page_allocated_mapped(struct page *old_page, struct pa dma_unmap_page(kctx->kbdev->dev, new_dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL); /* Page fault handler for CPU mapping unblocked. */ - kbase_gpu_vm_unlock(kctx); + kbase_gpu_vm_unlock_with_pmode_sync(kctx); return ret; } @@ -685,11 +679,15 @@ void kbase_mem_migrate_init(struct kbase_device *kbdev) * integer for a negative value to see if insmod parameter was * passed in at all (it will override the default negative value). */ - if (kbase_page_migration_enabled < 0) - kbase_page_migration_enabled = kbdev->pagesize_2mb ? 1 : 0; - else + if (kbase_page_migration_enabled < 0) { + if (kbase_is_large_pages_enabled()) + static_branch_inc(&page_migration_static_key); + } else { dev_info(kbdev->dev, "Page migration support explicitly %s at insmod.", kbase_page_migration_enabled ? "enabled" : "disabled"); + if (kbase_page_migration_enabled) + static_branch_inc(&page_migration_static_key); + } spin_lock_init(&mem_migrate->free_pages_lock); INIT_LIST_HEAD(&mem_migrate->free_pages_list); @@ -714,3 +712,9 @@ void kbase_mem_migrate_term(struct kbase_device *kbdev) iput(mem_migrate->inode); #endif } + +bool kbase_is_page_migration_enabled(void) +{ + return static_branch_unlikely(&page_migration_static_key); +} +KBASE_EXPORT_TEST_API(kbase_is_page_migration_enabled); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_migrate.h b/drivers/gpu/arm/bifrost/mali_kbase_mem_migrate.h index ece8734de792..70c3135a7829 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem_migrate.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_migrate.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2022-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2022-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_pool.c b/drivers/gpu/arm/bifrost/mali_kbase_mem_pool.c index cb862d5b029c..5984730c337c 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem_pool.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_pool.c @@ -480,7 +480,7 @@ static unsigned long kbase_mem_pool_reclaim_count_objects(struct shrinker *s, CSTD_UNUSED(sc); - pool = container_of(s, struct kbase_mem_pool, reclaim); + pool = KBASE_GET_KBASE_DATA_FROM_SHRINKER(s, struct kbase_mem_pool, reclaim); kbase_mem_pool_lock(pool); if (pool->dont_reclaim && !pool->dying) { @@ -502,7 +502,7 @@ static unsigned long kbase_mem_pool_reclaim_scan_objects(struct shrinker *s, struct kbase_mem_pool *pool; unsigned long freed; - pool = container_of(s, struct kbase_mem_pool, reclaim); + pool = KBASE_GET_KBASE_DATA_FROM_SHRINKER(s, struct kbase_mem_pool, reclaim); kbase_mem_pool_lock(pool); if (pool->dont_reclaim && !pool->dying) { @@ -528,6 +528,8 @@ int kbase_mem_pool_init(struct kbase_mem_pool *pool, const struct kbase_mem_pool unsigned int order, int group_id, struct kbase_device *kbdev, struct kbase_mem_pool *next_pool) { + struct shrinker *reclaim; + if (WARN_ON(group_id < 0) || WARN_ON(group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS)) { return -EINVAL; } @@ -544,18 +546,17 @@ int kbase_mem_pool_init(struct kbase_mem_pool *pool, const struct kbase_mem_pool spin_lock_init(&pool->pool_lock); INIT_LIST_HEAD(&pool->page_list); - pool->reclaim.count_objects = kbase_mem_pool_reclaim_count_objects; - pool->reclaim.scan_objects = kbase_mem_pool_reclaim_scan_objects; - pool->reclaim.seeks = DEFAULT_SEEKS; - /* Kernel versions prior to 3.1 : - * struct shrinker does not define batch - */ - pool->reclaim.batch = 0; -#if KERNEL_VERSION(6, 0, 0) > LINUX_VERSION_CODE - register_shrinker(&pool->reclaim); -#else - register_shrinker(&pool->reclaim, "mali-mem-pool"); -#endif + reclaim = KBASE_INIT_RECLAIM(pool, reclaim, "mali-mem-pool"); + if (!reclaim) + return -ENOMEM; + KBASE_SET_RECLAIM(pool, reclaim, reclaim); + + reclaim->count_objects = kbase_mem_pool_reclaim_count_objects; + reclaim->scan_objects = kbase_mem_pool_reclaim_scan_objects; + reclaim->seeks = DEFAULT_SEEKS; + reclaim->batch = 0; + + KBASE_REGISTER_SHRINKER(reclaim, "mali-mem-pool", pool); pool_dbg(pool, "initialized\n"); @@ -581,7 +582,7 @@ void kbase_mem_pool_term(struct kbase_mem_pool *pool) pool_dbg(pool, "terminate()\n"); - unregister_shrinker(&pool->reclaim); + KBASE_UNREGISTER_SHRINKER(pool->reclaim); kbase_mem_pool_lock(pool); pool->max_size = 0; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_native_mgm.c b/drivers/gpu/arm/bifrost/mali_kbase_native_mgm.c index 5e3d1eeb6d28..d688509cee03 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_native_mgm.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_native_mgm.c @@ -121,44 +121,20 @@ static vm_fault_t kbase_native_mgm_vmf_insert_pfn_prot(struct memory_group_manag return vmf_insert_pfn_prot(vma, addr, pfn, pgprot); } -/** - * kbase_native_mgm_update_gpu_pte - Native method to modify a GPU page table - * entry - * - * @mgm_dev: The memory group manager the request is being made through. - * @group_id: A physical memory group ID, which must be valid but is not used. - * Its valid range is 0 .. MEMORY_GROUP_MANAGER_NR_GROUPS-1. - * @mmu_level: The level of the MMU page table where the page is getting mapped. - * @pte: The prepared page table entry. - * - * This function simply returns the @pte without modification. - * - * Return: A GPU page table entry to be stored in a page table. - */ static u64 kbase_native_mgm_update_gpu_pte(struct memory_group_manager_device *mgm_dev, unsigned int group_id, int mmu_level, u64 pte) { - CSTD_UNUSED(mgm_dev); - CSTD_UNUSED(group_id); - CSTD_UNUSED(mmu_level); + if (WARN_ON(group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS)) + return pte; + + pte |= ((u64)group_id << PTE_PBHA_SHIFT) & PTE_PBHA_MASK; + + /* Address could be translated into a different bus address here */ + pte |= ((u64)1 << PTE_RES_BIT_MULTI_AS_SHIFT); return pte; } -/** - * kbase_native_mgm_pte_to_original_pte - Native method to undo changes done in - * kbase_native_mgm_update_gpu_pte() - * - * @mgm_dev: The memory group manager the request is being made through. - * @group_id: A physical memory group ID, which must be valid but is not used. - * Its valid range is 0 .. MEMORY_GROUP_MANAGER_NR_GROUPS-1. - * @mmu_level: The level of the MMU page table where the page is getting mapped. - * @pte: The prepared page table entry. - * - * This function simply returns the @pte without modification. - * - * Return: A GPU page table entry to be stored in a page table. - */ static u64 kbase_native_mgm_pte_to_original_pte(struct memory_group_manager_device *mgm_dev, unsigned int group_id, int mmu_level, u64 pte) { @@ -166,6 +142,11 @@ static u64 kbase_native_mgm_pte_to_original_pte(struct memory_group_manager_devi CSTD_UNUSED(group_id); CSTD_UNUSED(mmu_level); + /* Undo the group ID modification */ + pte &= ~PTE_PBHA_MASK; + /* Undo the bit set */ + pte &= ~((u64)1 << PTE_RES_BIT_MULTI_AS_SHIFT); + return pte; } diff --git a/drivers/gpu/arm/bifrost/mali_kbase_pbha.c b/drivers/gpu/arm/bifrost/mali_kbase_pbha.c index 341ea901e2e1..c5b6fada2451 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_pbha.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_pbha.c @@ -277,16 +277,16 @@ static int kbase_pbha_read_int_id_override_property(struct kbase_device *kbdev, static int kbase_pbha_read_propagate_bits_property(struct kbase_device *kbdev, const struct device_node *pbha_node) { - u32 bits = 0; + u8 bits = 0; int err; if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PBHA_HWU)) return 0; - err = of_property_read_u32(pbha_node, "propagate-bits", &bits); + err = of_property_read_u8(pbha_node, "propagate-bits", &bits); if (err == -EINVAL) { - err = of_property_read_u32(pbha_node, "propagate_bits", &bits); + err = of_property_read_u8(pbha_node, "propagate_bits", &bits); } if (err < 0) { diff --git a/drivers/gpu/arm/bifrost/mali_kbase_pbha_debugfs.c b/drivers/gpu/arm/bifrost/mali_kbase_pbha_debugfs.c index f1d2794dd86a..8ab0d1823165 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_pbha_debugfs.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_pbha_debugfs.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2021-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2021-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -50,8 +50,8 @@ static int int_id_overrides_show(struct seq_file *sfile, void *data) #endif /* MALI_USE_CSF */ for (j = 0; j < sizeof(u32); ++j) { - u8 r_val; - u8 w_val; + u8 r_val = 0; + u8 w_val = 0; switch (j) { case 0: diff --git a/drivers/gpu/arm/bifrost/mali_kbase_pm.c b/drivers/gpu/arm/bifrost/mali_kbase_pm.c index ff71524eeaaa..17c34f334aad 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_pm.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_pm.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -52,21 +52,19 @@ void kbase_pm_context_active(struct kbase_device *kbdev) (void)kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE); } -int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, - enum kbase_pm_suspend_handler suspend_handler) +int kbase_pm_context_active_handle_suspend_locked(struct kbase_device *kbdev, + enum kbase_pm_suspend_handler suspend_handler) { int c; KBASE_DEBUG_ASSERT(kbdev != NULL); dev_dbg(kbdev->dev, "%s - reason = %d, pid = %d\n", __func__, suspend_handler, current->pid); - kbase_pm_lock(kbdev); + lockdep_assert_held(&kbdev->pm.lock); #ifdef CONFIG_MALI_ARBITER_SUPPORT - if (kbase_arbiter_pm_ctx_active_handle_suspend(kbdev, suspend_handler)) { - kbase_pm_unlock(kbdev); + if (kbase_arbiter_pm_ctx_active_handle_suspend(kbdev, suspend_handler)) return 1; - } #endif /* CONFIG_MALI_ARBITER_SUPPORT */ if (kbase_pm_is_suspending(kbdev)) { @@ -76,7 +74,6 @@ int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, break; fallthrough; case KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE: - kbase_pm_unlock(kbdev); return 1; case KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE: @@ -100,21 +97,31 @@ int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, kbase_clk_rate_trace_manager_gpu_active(kbdev); } - kbase_pm_unlock(kbdev); dev_dbg(kbdev->dev, "%s %d\n", __func__, kbdev->pm.active_count); return 0; } +int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, + enum kbase_pm_suspend_handler suspend_handler) +{ + int ret; + + kbase_pm_lock(kbdev); + ret = kbase_pm_context_active_handle_suspend_locked(kbdev, suspend_handler); + kbase_pm_unlock(kbdev); + + return ret; +} + KBASE_EXPORT_TEST_API(kbase_pm_context_active); -void kbase_pm_context_idle(struct kbase_device *kbdev) +void kbase_pm_context_idle_locked(struct kbase_device *kbdev) { int c; KBASE_DEBUG_ASSERT(kbdev != NULL); - - kbase_pm_lock(kbdev); + lockdep_assert_held(&kbdev->pm.lock); c = --kbdev->pm.active_count; KBASE_KTRACE_ADD(kbdev, PM_CONTEXT_IDLE, NULL, (u64)c); @@ -133,10 +140,16 @@ void kbase_pm_context_idle(struct kbase_device *kbdev) wake_up(&kbdev->pm.zero_active_count_wait); } - kbase_pm_unlock(kbdev); dev_dbg(kbdev->dev, "%s %d (pid = %d)\n", __func__, kbdev->pm.active_count, current->pid); } +void kbase_pm_context_idle(struct kbase_device *kbdev) +{ + kbase_pm_lock(kbdev); + kbase_pm_context_idle_locked(kbdev); + kbase_pm_unlock(kbdev); +} + KBASE_EXPORT_TEST_API(kbase_pm_context_idle); static void reenable_hwcnt_on_resume(struct kbase_device *kbdev) @@ -155,7 +168,12 @@ static void reenable_hwcnt_on_resume(struct kbase_device *kbdev) #endif /* Resume HW counters intermediaries. */ - kbase_kinstr_prfcnt_resume(kbdev->kinstr_prfcnt_ctx); +#if MALI_USE_CSF + if (kbdev->csf.firmware_inited) +#endif + { + kbase_kinstr_prfcnt_resume(kbdev->kinstr_prfcnt_ctx); + } } static void resume_job_scheduling(struct kbase_device *kbdev) @@ -183,7 +201,12 @@ int kbase_pm_driver_suspend(struct kbase_device *kbdev) /* Suspend HW counter intermediaries. This blocks until workers and timers * are no longer running. */ - kbase_kinstr_prfcnt_suspend(kbdev->kinstr_prfcnt_ctx); +#if MALI_USE_CSF + if (kbdev->csf.firmware_inited) +#endif + { + kbase_kinstr_prfcnt_suspend(kbdev->kinstr_prfcnt_ctx); + } /* Disable GPU hardware counters. * This call will block until counters are disabled. @@ -200,19 +223,24 @@ int kbase_pm_driver_suspend(struct kbase_device *kbdev) mutex_unlock(&kbdev->pm.lock); #ifdef CONFIG_MALI_ARBITER_SUPPORT -#if !MALI_USE_CSF if (kbdev->arb.arb_if) { - unsigned int i; unsigned long flags; +#if MALI_USE_CSF + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_disjoint_state_up(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +#else + unsigned int i; + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbdev->js_data.runpool_irq.submit_allowed = 0; kbase_disjoint_state_up(kbdev); for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) kbase_job_slot_softstop(kbdev, i, NULL); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +#endif } -#endif /* !MALI_USE_CSF */ #endif /* CONFIG_MALI_ARBITER_SUPPORT */ /* From now on, the active count will drop towards zero. Sometimes, @@ -273,6 +301,10 @@ int kbase_pm_driver_suspend(struct kbase_device *kbdev) } #endif /* CONFIG_MALI_ARBITER_SUPPORT */ +#if MALI_USE_CSF + kbase_backend_invalidate_gpu_timestamp_offset(kbdev); +#endif + return 0; exit: @@ -338,6 +370,7 @@ void kbase_pm_driver_resume(struct kbase_device *kbdev, bool arb_gpu_start) int kbase_pm_suspend(struct kbase_device *kbdev) { int result = 0; + #ifdef CONFIG_MALI_ARBITER_SUPPORT if (kbdev->arb.arb_if) kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_OS_SUSPEND_EVENT); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_pm.h b/drivers/gpu/arm/bifrost/mali_kbase_pm.h index 46db4db5ffe0..0c3575bfd54a 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_pm.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_pm.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -33,13 +33,12 @@ struct kbase_device; #define PM_ENABLE_IRQS 0x01 #define PM_HW_ISSUES_DETECT 0x02 -#ifdef CONFIG_MALI_ARBITER_SUPPORT -/* In the case that the GPU was granted by the Arbiter, it will have +/* Case 1: the GPU was granted by the Arbiter, it will have * already been reset. The following flag ensures it is not reset * twice. + * Case 2: GPU already in reset state after power on, then no soft-reset is needed. */ #define PM_NO_RESET 0x04 -#endif /** * kbase_pm_init - Initialize the power management framework. @@ -148,6 +147,18 @@ enum kbase_pm_suspend_handler { int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, enum kbase_pm_suspend_handler suspend_handler); +/** + * kbase_pm_context_active_handle_suspend_locked - Same as kbase_pm_context_active_handle_suspend(), + * except that pm.lock is held by the caller. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @suspend_handler: The handler code for how to handle a suspend that might occur + * + * Return: 0 on success, non-zero othrewise. + */ +int kbase_pm_context_active_handle_suspend_locked(struct kbase_device *kbdev, + enum kbase_pm_suspend_handler suspend_handler); + /** * kbase_pm_context_idle - Decrement the reference count of active contexts. * @@ -159,6 +170,14 @@ int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, */ void kbase_pm_context_idle(struct kbase_device *kbdev); +/** + * kbase_pm_context_idle_locked - Same as kbase_pm_context_idle(), except that + * pm.lock is held by the caller. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_context_idle_locked(struct kbase_device *kbdev); + /* NOTE: kbase_pm_is_active() is in mali_kbase.h, because it is an inline * function */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c b/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c index 0cee2f0e6fd5..c5c16b497369 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2011-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -1408,7 +1408,7 @@ static void kbase_ext_res_process(struct kbase_jd_atom *katom, bool map) gpu_addr = ext_res->ext_res[i].ext_resource & ~(__u64)BASE_EXT_RES_ACCESS_EXCLUSIVE; if (map) { - if (!kbase_sticky_resource_acquire(katom->kctx, gpu_addr)) + if (!kbase_sticky_resource_acquire(katom->kctx, gpu_addr, NULL)) goto failed_loop; } else { if (!kbase_sticky_resource_release_force(katom->kctx, NULL, gpu_addr)) diff --git a/drivers/gpu/arm/bifrost/mmu/Kbuild b/drivers/gpu/arm/bifrost/mmu/Kbuild index 416432397b5c..3c3defdb88e9 100644 --- a/drivers/gpu/arm/bifrost/mmu/Kbuild +++ b/drivers/gpu/arm/bifrost/mmu/Kbuild @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2021 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2021-2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -21,10 +21,15 @@ bifrost_kbase-y += \ mmu/mali_kbase_mmu.o \ mmu/mali_kbase_mmu_hw_direct.o \ + mmu/mali_kbase_mmu_faults_decoder_luts.o \ + mmu/mali_kbase_mmu_faults_decoder.o \ mmu/mali_kbase_mmu_mode_aarch64.o ifeq ($(CONFIG_MALI_CSF_SUPPORT),y) - bifrost_kbase-y += mmu/backend/mali_kbase_mmu_csf.o + bifrost_kbase-y += mmu/backend/mali_kbase_mmu_csf.o \ + mmu/backend/mali_kbase_mmu_faults_decoder_luts_csf.o else - bifrost_kbase-y += mmu/backend/mali_kbase_mmu_jm.o + bifrost_kbase-y += mmu/backend/mali_kbase_mmu_jm.o \ + mmu/backend/mali_kbase_mmu_faults_decoder_luts_jm.o + endif diff --git a/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_csf.c b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_csf.c index df027c727a2c..bd5f3914b8c7 100644 --- a/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_csf.c +++ b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_csf.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -29,6 +29,7 @@ #include #include #include +#include void kbase_mmu_get_as_setup(struct kbase_mmu_table *mmut, struct kbase_mmu_setup *const setup) { @@ -99,15 +100,38 @@ void kbase_mmu_report_mcu_as_fault_and_reset(struct kbase_device *kbdev, struct u32 as_no; /* terminal fault, print info about the fault */ - dev_err(kbdev->dev, - "Unexpected Page fault in firmware address space at VA 0x%016llX\n" - "raw fault status: 0x%X\n" - "exception type 0x%X: %s\n" - "access type 0x%X: %s\n" - "source id 0x%X\n", - fault->addr, fault->status, exception_type, - kbase_gpu_exception_name(exception_type), access_type, - kbase_gpu_access_type_name(fault->status), source_id); + if (kbdev->gpu_props.gpu_id.product_model <= GPU_ID_MODEL_MAKE(13, 0)) { + dev_err(kbdev->dev, + "Unexpected Page fault in firmware address space at VA 0x%016llX\n" + "raw fault status: 0x%X\n" + "exception type 0x%X: %s\n" + "access type 0x%X: %s\n" + "source id 0x%X (core_id:utlb:IR 0x%X:0x%X:0x%X): %s, %s\n", + fault->addr, fault->status, exception_type, + kbase_gpu_exception_name(exception_type), access_type, + kbase_gpu_access_type_name(fault->status), source_id, + FAULT_SOURCE_ID_CORE_ID_GET(source_id), + FAULT_SOURCE_ID_UTLB_ID_GET(source_id), + fault_source_id_internal_requester_get(kbdev, source_id), + fault_source_id_core_type_description_get(kbdev, source_id), + fault_source_id_internal_requester_get_str(kbdev, source_id, access_type)); + } else { + dev_err(kbdev->dev, + "Unexpected Page fault in firmware address space at VA 0x%016llX\n" + "raw fault status: 0x%X\n" + "exception type 0x%X: %s\n" + "access type 0x%X: %s\n" + "source id 0x%X (type:idx:IR 0x%X:0x%X:0x%X): %s %u, %s\n", + fault->addr, fault->status, exception_type, + kbase_gpu_exception_name(exception_type), access_type, + kbase_gpu_access_type_name(fault->status), source_id, + FAULT_SOURCE_ID_CORE_TYPE_GET(source_id), + FAULT_SOURCE_ID_CORE_INDEX_GET(source_id), + fault_source_id_internal_requester_get(kbdev, source_id), + fault_source_id_core_type_description_get(kbdev, source_id), + FAULT_SOURCE_ID_CORE_INDEX_GET(source_id), + fault_source_id_internal_requester_get_str(kbdev, source_id, access_type)); + } kbase_debug_csf_fault_notify(kbdev, NULL, DF_GPU_PAGE_FAULT); @@ -139,17 +163,44 @@ void kbase_gpu_report_bus_fault_and_kill(struct kbase_context *kctx, struct kbas const uintptr_t fault_addr = fault->addr; /* terminal fault, print info about the fault */ - dev_err(kbdev->dev, - "GPU bus fault in AS%u at PA %pK\n" - "PA_VALID: %s\n" - "raw fault status: 0x%X\n" - "exception type 0x%X: %s\n" - "access type 0x%X: %s\n" - "source id 0x%X\n" - "pid: %d\n", - as_no, (void *)fault_addr, addr_valid, status, exception_type, - kbase_gpu_exception_name(exception_type), access_type, - kbase_gpu_access_type_name(access_type), source_id, kctx->pid); + if (kbdev->gpu_props.gpu_id.product_model <= GPU_ID_MODEL_MAKE(13, 0)) { + dev_err(kbdev->dev, + "GPU bus fault in AS%u at PA %pK\n" + "PA_VALID: %s\n" + "raw fault status: 0x%X\n" + "exception type 0x%X: %s\n" + "access type 0x%X: %s\n" + "source id 0x%X (core_id:utlb:IR 0x%X:0x%X:0x%X): %s, %s\n" + "pid: %d\n", + as_no, (void *)fault_addr, addr_valid, status, exception_type, + kbase_gpu_exception_name(exception_type), access_type, + kbase_gpu_access_type_name(access_type), source_id, + FAULT_SOURCE_ID_CORE_ID_GET(source_id), + FAULT_SOURCE_ID_UTLB_ID_GET(source_id), + fault_source_id_internal_requester_get(kbdev, source_id), + fault_source_id_core_type_description_get(kbdev, source_id), + fault_source_id_internal_requester_get_str(kbdev, source_id, access_type), + kctx->pid); + } else { + dev_err(kbdev->dev, + "GPU bus fault in AS%u at PA %pK\n" + "PA_VALID: %s\n" + "raw fault status: 0x%X\n" + "exception type 0x%X: %s\n" + "access type 0x%X: %s\n" + "source id 0x%X (type:idx:IR 0x%X:0x%X:0x%X): %s %u, %s\n" + "pid: %d\n", + as_no, (void *)fault_addr, addr_valid, status, exception_type, + kbase_gpu_exception_name(exception_type), access_type, + kbase_gpu_access_type_name(access_type), source_id, + FAULT_SOURCE_ID_CORE_TYPE_GET(source_id), + FAULT_SOURCE_ID_CORE_INDEX_GET(source_id), + fault_source_id_internal_requester_get(kbdev, source_id), + fault_source_id_core_type_description_get(kbdev, source_id), + FAULT_SOURCE_ID_CORE_INDEX_GET(source_id), + fault_source_id_internal_requester_get_str(kbdev, source_id, access_type), + kctx->pid); + } /* AS transaction begin */ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); @@ -195,17 +246,46 @@ void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, struct kbase_as unsigned int as_no = as->number; /* terminal fault, print info about the fault */ - dev_err(kbdev->dev, - "Unhandled Page fault in AS%u at VA 0x%016llX\n" - "Reason: %s\n" - "raw fault status: 0x%X\n" - "exception type 0x%X: %s\n" - "access type 0x%X: %s\n" - "source id 0x%X\n" - "pid: %d\n", - as_no, fault->addr, reason_str, status, exception_type, - kbase_gpu_exception_name(exception_type), access_type, - kbase_gpu_access_type_name(status), source_id, kctx->pid); + if (kbdev->gpu_props.gpu_id.product_model <= GPU_ID_MODEL_MAKE(13, 0)) { + dev_err(kbdev->dev, + "Unhandled Page fault in AS%u at VA 0x%016llX\n" + "Reason: %s\n" + "raw fault status: 0x%X\n" + "exception type 0x%X: %s\n" + "access type 0x%X: %s\n" + "source id 0x%X (core_id:utlb:IR 0x%X:0x%X:0x%X): %s, %s\n" + "pid: %d\n", + as_no, fault->addr, reason_str, status, exception_type, + kbase_gpu_exception_name(exception_type), access_type, + kbase_gpu_access_type_name(status), source_id, + FAULT_SOURCE_ID_CORE_ID_GET(source_id), + FAULT_SOURCE_ID_UTLB_ID_GET(source_id), + fault_source_id_internal_requester_get(kbdev, source_id), + fault_source_id_core_type_description_get(kbdev, source_id), + fault_source_id_internal_requester_get_str(kbdev, source_id, + access_type), + kctx->pid); + } else { + dev_err(kbdev->dev, + "Unhandled Page fault in AS%u at VA 0x%016llX\n" + "Reason: %s\n" + "raw fault status: 0x%X\n" + "exception type 0x%X: %s\n" + "access type 0x%X: %s\n" + "source id 0x%X (type:idx:IR 0x%X:0x%X:0x%X): %s %u, %s\n" + "pid: %d\n", + as_no, fault->addr, reason_str, status, exception_type, + kbase_gpu_exception_name(exception_type), access_type, + kbase_gpu_access_type_name(status), source_id, + FAULT_SOURCE_ID_CORE_TYPE_GET(source_id), + FAULT_SOURCE_ID_CORE_INDEX_GET(source_id), + fault_source_id_internal_requester_get(kbdev, source_id), + fault_source_id_core_type_description_get(kbdev, source_id), + FAULT_SOURCE_ID_CORE_INDEX_GET(source_id), + fault_source_id_internal_requester_get_str(kbdev, source_id, + access_type), + kctx->pid); + } } /* AS transaction begin */ diff --git a/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_faults_decoder_luts_csf.c b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_faults_decoder_luts_csf.c new file mode 100644 index 000000000000..d8eec91ba887 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_faults_decoder_luts_csf.c @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note +/* + * + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +/** + * DOC: Base kernel MMU faults decoder for CSF GPUs. + */ + +#include + +#define GPU_ID_ARCH_ID_MAJOR_GET(gpu_id) ((gpu_id >> 16) & 0xFF) +#define GPU_ID_ARCH_ID_MINOR_AND_REV_GET(gpu_id) (gpu_id & 0xFFFF) +#define NELEMS(s) (sizeof(s) / sizeof((s)[0])) + +struct decode_lut_element { + u16 arch_minor_rev; + u16 key; + const char *text; +}; + +static const char *decode_lut_element_lookup(u16 arch_minor_rev, u16 key, + struct decode_lut_element *decode_element_lut, + unsigned int lut_len) +{ + struct decode_lut_element *p; + + for (p = decode_element_lut; p < decode_element_lut + lut_len; p++) { + if (p->key == key && + (p->arch_minor_rev == 0xffff || p->arch_minor_rev == arch_minor_rev)) + break; + } + if (p < decode_element_lut + lut_len) + return p->text; + else + return "unknown"; +} + +/* Auto-generated code: DO NOT MODIFY! */ + +static struct decode_lut_element lut_fault_source_csf_r_t_major_10[] = { + { 0xFFFF, 0, "pref0" }, + { 0xFFFF, 4, "iter0" }, + { 0xFFFF, 12, "lsu" }, + { 0xFFFF, 13, "mcu" }, +}; + +static struct decode_lut_element lut_fault_source_csf_r_t_major_11[] = { + { 0xFFFF, 0, "pref0" }, + { 0xFFFF, 4, "iter0" }, + { 0xFFFF, 12, "lsu" }, + { 0xFFFF, 13, "mcu" }, +}; + +static struct decode_lut_element lut_fault_source_csf_r_t_major_12[] = { + { 0xFFFF, 0, "pref0" }, + { 0xFFFF, 4, "iter0" }, + { 0xFFFF, 12, "lsu" }, + { 0xFFFF, 13, "mcu" }, +}; + +static struct decode_lut_element lut_fault_source_csf_w_t_major_10[] = { + { 0xFFFF, 8, "pcb0" }, + { 0xFFFF, 12, "lsu" }, + { 0xFFFF, 13, "mcu" }, +}; + +static struct decode_lut_element lut_fault_source_csf_w_t_major_11[] = { + { 0xFFFF, 8, "pcb0" }, + { 0xFFFF, 12, "lsu" }, + { 0xFFFF, 13, "mcu" }, +}; + +static struct decode_lut_element lut_fault_source_csf_w_t_major_12[] = { + { 0xFFFF, 8, "pcb0" }, + { 0xFFFF, 12, "lsu" }, + { 0xFFFF, 13, "mcu" }, +}; + + +const char *decode_fault_source_csf_r_t(u16 idx, u32 gpu_id) +{ + u16 min_rev = GPU_ID_ARCH_ID_MINOR_AND_REV_GET(gpu_id); + const char *ret = "unknown"; + + switch (GPU_ID_ARCH_ID_MAJOR_GET(gpu_id)) { + case 10: + ret = decode_lut_element_lookup(min_rev, idx, lut_fault_source_csf_r_t_major_10, + NELEMS(lut_fault_source_csf_r_t_major_10)); + break; + case 11: + ret = decode_lut_element_lookup(min_rev, idx, lut_fault_source_csf_r_t_major_11, + NELEMS(lut_fault_source_csf_r_t_major_11)); + break; + case 12: + ret = decode_lut_element_lookup(min_rev, idx, lut_fault_source_csf_r_t_major_12, + NELEMS(lut_fault_source_csf_r_t_major_12)); + break; + } + return ret; +} + +const char *decode_fault_source_csf_w_t(u16 idx, u32 gpu_id) +{ + u16 min_rev = GPU_ID_ARCH_ID_MINOR_AND_REV_GET(gpu_id); + const char *ret = "unknown"; + + switch (GPU_ID_ARCH_ID_MAJOR_GET(gpu_id)) { + case 10: + ret = decode_lut_element_lookup(min_rev, idx, lut_fault_source_csf_w_t_major_10, + NELEMS(lut_fault_source_csf_w_t_major_10)); + break; + case 11: + ret = decode_lut_element_lookup(min_rev, idx, lut_fault_source_csf_w_t_major_11, + NELEMS(lut_fault_source_csf_w_t_major_11)); + break; + case 12: + ret = decode_lut_element_lookup(min_rev, idx, lut_fault_source_csf_w_t_major_12, + NELEMS(lut_fault_source_csf_w_t_major_12)); + break; + } + return ret; +} diff --git a/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_faults_decoder_luts_csf.h b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_faults_decoder_luts_csf.h new file mode 100644 index 000000000000..04f5c02ccc3d --- /dev/null +++ b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_faults_decoder_luts_csf.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * + * (C) COPYRIGHT 2024 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +#ifndef _MALI_KBASE_MMU_FAULTS_DECODER_LUTS_CSF_H_ +#define _MALI_KBASE_MMU_FAULTS_DECODER_LUTS_CSF_H_ +#include + +/** + * decode_fault_source_csf_r_t() - Get internal requester of a + * fault in a human readable format. + * + * @idx: Internal requester part of SOURCE_ID field of the fault. + * @gpu_id: GPU id composed of arch_major << 16 | arch_minor << 8 | arch_rev. + * + * Return: Internal requester of a fault in a human readable format for a read + * operation on a CSF core. + */ +const char *decode_fault_source_csf_r_t(u16 idx, u32 gpu_id); + +/** + * decode_fault_source_csf_w_t() - Get internal requester of a + * fault in a human readable format. + * + * @idx: Internal requester part of SOURCE_ID field of the fault. + * @gpu_id: GPU id composed of arch_major << 16 | arch_minor << 8 | arch_rev. + * + * Return: Internal requester of a fault in a human readable format for a write + * operation on a CSF core. + */ +const char *decode_fault_source_csf_w_t(u16 idx, u32 gpu_id); + +#endif /* _MALI_KBASE_MMU_FAULTS_DECODER_LUTS_CSF_H_ */ diff --git a/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_faults_decoder_luts_jm.c b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_faults_decoder_luts_jm.c new file mode 100644 index 000000000000..a053a93978b5 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_faults_decoder_luts_jm.c @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note +/* + * + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +/** + * DOC: Base kernel MMU faults decoder for Job Manager GPUs. + */ + +#include + +#define GPU_ID_ARCH_ID_MAJOR_GET(gpu_id) ((gpu_id >> 16) & 0xFF) +#define GPU_ID_ARCH_ID_MINOR_AND_REV_GET(gpu_id) (gpu_id & 0xFFFF) +#define NELEMS(s) (sizeof(s) / sizeof((s)[0])) + +struct decode_lut_element { + u16 arch_minor_rev; + u16 key; + const char *text; +}; + +static const char *decode_lut_element_lookup(u16 arch_minor_rev, u16 key, + struct decode_lut_element *decode_element_lut, + unsigned int lut_len) +{ + struct decode_lut_element *p; + + for (p = decode_element_lut; p < decode_element_lut + lut_len; p++) { + if (p->key == key && + (p->arch_minor_rev == 0xffff || p->arch_minor_rev == arch_minor_rev)) + break; + } + if (p < decode_element_lut + lut_len) + return p->text; + else + return "unknown"; +} + +/* Auto-generated code: DO NOT MODIFY! */ + +static struct decode_lut_element lut_fault_source_jm_t_major_9[] = { + { 0xFFFF, 0, "js" }, + { 0xFFFF, 1, "pcm" }, +}; + +const char *decode_fault_source_jm_t(u16 idx, u32 gpu_id) +{ + u16 min_rev = GPU_ID_ARCH_ID_MINOR_AND_REV_GET(gpu_id); + const char *ret = "unknown"; + + switch (GPU_ID_ARCH_ID_MAJOR_GET(gpu_id)) { + case 9: + ret = decode_lut_element_lookup(min_rev, idx, lut_fault_source_jm_t_major_9, + NELEMS(lut_fault_source_jm_t_major_9)); + break; + } + return ret; +} diff --git a/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_faults_decoder_luts_jm.h b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_faults_decoder_luts_jm.h new file mode 100644 index 000000000000..f686e555d86a --- /dev/null +++ b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_faults_decoder_luts_jm.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * + * (C) COPYRIGHT 2024 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +#ifndef _MALI_KBASE_MMU_FAULTS_DECODER_LUTS_JM_H_ +#define _MALI_KBASE_MMU_FAULTS_DECODER_LUTS_JM_H_ +#include + +/** + * decode_fault_source_jm_t() - Get internal requester of a + * fault in a human readable format. + * + * @idx: Internal requester part of SOURCE_ID field of the fault. + * @gpu_id: GPU id composed of arch_major << 16 | arch_minor << 8 | arch_rev. + * + * Return: Internal requester of a fault in a human readable format for a JM core. + */ +const char *decode_fault_source_jm_t(u16 idx, u32 gpu_id); + +#endif /* _MALI_KBASE_MMU_FAULTS_DECODER_LUTS_JM_H_ */ diff --git a/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_jm.c b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_jm.c index 1b2df11f3c3c..e8e136117fd8 100644 --- a/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_jm.c +++ b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_jm.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -29,6 +29,7 @@ #include #include #include +#include void kbase_mmu_get_as_setup(struct kbase_mmu_table *mmut, struct kbase_mmu_setup *const setup) { @@ -52,9 +53,10 @@ void kbase_gpu_report_bus_fault_and_kill(struct kbase_context *kctx, struct kbas struct kbase_fault *fault) { struct kbase_device *const kbdev = kctx->kbdev; - u32 const status = fault->status; - u32 const exception_type = (status & 0xFF); - u32 const exception_data = (status >> 8) & 0xFFFFFF; + const u32 status = fault->status; + const u32 exception_type = AS_FAULTSTATUS_EXCEPTION_TYPE_GET(status); + const u32 access_type = AS_FAULTSTATUS_ACCESS_TYPE_GET(status); + const u32 source_id = AS_FAULTSTATUS_SOURCE_ID_GET(status); unsigned int const as_no = as->number; unsigned long flags; const uintptr_t fault_addr = fault->addr; @@ -64,10 +66,17 @@ void kbase_gpu_report_bus_fault_and_kill(struct kbase_context *kctx, struct kbas "GPU bus fault in AS%u at PA %pK\n" "raw fault status: 0x%X\n" "exception type 0x%X: %s\n" - "exception data 0x%X\n" + "access type 0x%X: %s\n" + "source id 0x%X (core_id:utlb:IR 0x%X:0x%X:0x%X): %s, %s\n" "pid: %d\n", as_no, (void *)fault_addr, status, exception_type, - kbase_gpu_exception_name(exception_type), exception_data, kctx->pid); + kbase_gpu_exception_name(exception_type), access_type, + kbase_gpu_access_type_name(access_type), source_id, + FAULT_SOURCE_ID_CORE_ID_GET(source_id), FAULT_SOURCE_ID_UTLB_ID_GET(source_id), + fault_source_id_internal_requester_get(kbdev, source_id), + fault_source_id_core_type_description_get(kbdev, source_id), + fault_source_id_internal_requester_get_str(kbdev, source_id, access_type), + kctx->pid); /* switch to UNMAPPED mode, will abort all jobs and stop any hw counter * dumping AS transaction begin @@ -105,10 +114,10 @@ void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, struct kbase_as if (!kbase_ctx_flag(kctx, KCTX_PAGE_FAULT_REPORT_SKIP)) { /* decode the fault status */ - u32 exception_type = fault->status & 0xFF; - u32 access_type = (fault->status >> 8) & 0x3; - u32 source_id = (fault->status >> 16); - + const u32 status = fault->status; + const u32 exception_type = AS_FAULTSTATUS_EXCEPTION_TYPE_GET(status); + const u32 access_type = AS_FAULTSTATUS_ACCESS_TYPE_GET(status); + const u32 source_id = AS_FAULTSTATUS_SOURCE_ID_GET(status); /* terminal fault, print info about the fault */ dev_err(kbdev->dev, "Unhandled Page fault in AS%u at VA 0x%016llX\n" @@ -116,11 +125,17 @@ void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, struct kbase_as "raw fault status: 0x%X\n" "exception type 0x%X: %s\n" "access type 0x%X: %s\n" - "source id 0x%X\n" + "source id 0x%X (core_id:utlb:IR 0x%X:0x%X:0x%X): %s, %s\n" "pid: %d\n", - as_no, fault->addr, reason_str, fault->status, exception_type, + as_no, fault->addr, reason_str, status, exception_type, kbase_gpu_exception_name(exception_type), access_type, - kbase_gpu_access_type_name(fault->status), source_id, kctx->pid); + kbase_gpu_access_type_name(status), source_id, + FAULT_SOURCE_ID_CORE_ID_GET(source_id), + FAULT_SOURCE_ID_UTLB_ID_GET(source_id), + fault_source_id_internal_requester_get(kbdev, source_id), + fault_source_id_core_type_description_get(kbdev, source_id), + fault_source_id_internal_requester_get_str(kbdev, source_id, access_type), + kctx->pid); } /* hardware counters dump fault handling */ diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.c b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.c index becbb02aa15a..8d52e90f9579 100644 --- a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.c +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -151,6 +151,44 @@ static void mmu_invalidate(struct kbase_device *kbdev, struct kbase_context *kct spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } +/** + * mmu_invalidate_on_teardown() - Perform an invalidate operation on MMU caches on page + * table teardown. + * @kbdev: The Kbase device. + * @kctx: The Kbase context. + * @vpfn: The virtual page frame number at which teardown is done. + * @num_pages: The number of entries that were invalidated in top most level PGD, that + * was affected by the teardown operation. + * @level: The top most PGD level that was touched on teardown. + * @as_nr: GPU address space number for which invalidate is required. + * + * Perform an MMU invalidate operation after the teardown of top most level PGD on a + * particular address space by issuing a UNLOCK command. + */ +static inline void mmu_invalidate_on_teardown(struct kbase_device *kbdev, + struct kbase_context *kctx, u64 vpfn, + size_t num_pages, int level, int as_nr) +{ + u32 invalidate_range_num_pages = num_pages; + u64 invalidate_range_start_vpfn = vpfn; + struct kbase_mmu_hw_op_param op_param; + + if (level != MIDGARD_MMU_BOTTOMLEVEL) { + invalidate_range_num_pages = 1 << ((3 - level) * 9); + invalidate_range_start_vpfn = vpfn - (vpfn & (invalidate_range_num_pages - 1)); + } + + op_param = (struct kbase_mmu_hw_op_param){ + .vpfn = invalidate_range_start_vpfn, + .nr = invalidate_range_num_pages, + .mmu_sync_info = CALLER_MMU_ASYNC, + .kctx_id = kctx ? kctx->id : 0xFFFFFFFF, + .flush_skip_levels = (1ULL << level) - 1, + }; + + mmu_invalidate(kbdev, kctx, as_nr, &op_param); +} + /* Perform a flush/invalidate on a particular address space */ static void mmu_flush_invalidate_as(struct kbase_device *kbdev, struct kbase_as *as, @@ -318,14 +356,16 @@ static int kbase_mmu_update_pages_no_flush(struct kbase_device *kbdev, struct kb * @mmut: GPU MMU page table. * @pgds: Physical addresses of page directories to be freed. * @vpfn: The virtual page frame number. - * @level: The level of MMU page table. + * @level: The level of MMU page table that needs to be updated. * @flush_op: The type of MMU flush operation to perform. * @dirty_pgds: Flags to track every level where a PGD has been updated. + * @as_nr: GPU address space number for which invalidate is required. */ static void kbase_mmu_update_and_free_parent_pgds(struct kbase_device *kbdev, struct kbase_mmu_table *mmut, phys_addr_t *pgds, u64 vpfn, int level, - enum kbase_mmu_op_type flush_op, u64 *dirty_pgds); + enum kbase_mmu_op_type flush_op, u64 *dirty_pgds, + int as_nr); static void kbase_mmu_account_freed_pgd(struct kbase_device *kbdev, struct kbase_mmu_table *mmut) { @@ -776,7 +816,7 @@ static bool page_fault_try_alloc(struct kbase_context *kctx, struct kbase_va_reg return false; } - if (kctx->kbdev->pagesize_2mb && new_pages >= NUM_PAGES_IN_2MB_LARGE_PAGE) { + if (kbase_is_large_pages_enabled() && new_pages >= NUM_PAGES_IN_2MB_LARGE_PAGE) { root_pool = &kctx->mem_pools.large[region->gpu_alloc->group_id]; *grow_2mb_pool = true; } else { @@ -923,7 +963,7 @@ void kbase_mmu_page_fault_worker(struct work_struct *data) int err; bool grown = false; size_t pages_to_grow; - bool grow_2mb_pool; + bool grow_2mb_pool = false; struct kbase_sub_alloc *prealloc_sas[2] = { NULL, NULL }; int i; size_t current_backed_size; @@ -1093,7 +1133,7 @@ void kbase_mmu_page_fault_worker(struct work_struct *data) } page_fault_retry: - if (kbdev->pagesize_2mb) { + if (kbase_is_large_pages_enabled()) { /* Preallocate (or re-allocate) memory for the sub-allocation structs if necessary */ for (i = 0; i != ARRAY_SIZE(prealloc_sas); ++i) { if (!prealloc_sas[i]) { @@ -1180,10 +1220,14 @@ page_fault_retry: */ op_param.mmu_sync_info = mmu_sync_info; op_param.kctx_id = kctx->id; - /* Can safely skip the invalidate for all levels in case - * of duplicate page faults. + /* Usually it is safe to skip the MMU cache invalidate for all levels + * in case of duplicate page faults. But for the pathological scenario + * where the faulty VA gets mapped by the time page fault worker runs it + * becomes imperative to invalidate MMU cache for all levels, otherwise + * there is a possibility of repeated page faults on GPUs which supports + * fine grained MMU cache invalidation. */ - op_param.flush_skip_levels = 0xF; + op_param.flush_skip_levels = 0x0; op_param.vpfn = fault_pfn; op_param.nr = 1; spin_lock_irqsave(&kbdev->hwaccess_lock, hwaccess_flags); @@ -1217,10 +1261,14 @@ page_fault_retry: /* See comment [1] about UNLOCK usage */ op_param.mmu_sync_info = mmu_sync_info; op_param.kctx_id = kctx->id; - /* Can safely skip the invalidate for all levels in case - * of duplicate page faults. + /* Usually it is safe to skip the MMU cache invalidate for all levels + * in case of duplicate page faults. But for the pathological scenario + * where the faulty VA gets mapped by the time page fault worker runs it + * becomes imperative to invalidate MMU cache for all levels, otherwise + * there is a possibility of repeated page faults on GPUs which supports + * fine grained MMU cache invalidation. */ - op_param.flush_skip_levels = 0xF; + op_param.flush_skip_levels = 0x0; op_param.vpfn = fault_pfn; op_param.nr = 1; spin_lock_irqsave(&kbdev->hwaccess_lock, hwaccess_flags); @@ -1382,7 +1430,7 @@ page_fault_retry: * Otherwise fail the allocation. */ if (pages_to_grow > 0) { - if (kbdev->pagesize_2mb && grow_2mb_pool) { + if (kbase_is_large_pages_enabled() && grow_2mb_pool) { /* Round page requirement up to nearest 2 MB */ struct kbase_mem_pool *const lp_mem_pool = &kctx->mem_pools.large[group_id]; @@ -1595,6 +1643,7 @@ static int mmu_get_lowest_valid_pgd(struct kbase_device *kbdev, struct kbase_mmu return err; } +KBASE_ALLOW_ERROR_INJECTION_TEST_API(mmu_get_lowest_valid_pgd, ERRNO); /* * On success, sets out_pgd to the PGD for the specified level of translation @@ -1700,12 +1749,21 @@ static void mmu_insert_pages_failure_recovery(struct kbase_device *kbdev, mmu_mode->entries_invalidate(&page[idx], pcount); if (!num_of_valid_entries) { + mmu_mode->set_num_valid_entries(page, 0); + kbase_kunmap(p, page); + kbase_mmu_update_and_free_parent_pgds(kbdev, mmut, pgds, vpfn, level - 1, + KBASE_MMU_OP_NONE, dirty_pgds, 0); + + /* No CPU and GPU cache maintenance is done here as caller would do the + * complete flush of GPU cache and invalidation of TLB before the PGD + * page is freed. CPU cache flush would be done when the PGD page is + * returned to the memory pool. + */ + kbase_mmu_add_to_free_pgds_list(mmut, p); - kbase_mmu_update_and_free_parent_pgds(kbdev, mmut, pgds, vpfn, level, - KBASE_MMU_OP_NONE, dirty_pgds); vpfn += count; continue; } @@ -1728,7 +1786,8 @@ next: * going to happen to these pages at this stage. They might return * movable once they are returned to a memory pool. */ - if (kbase_is_page_migration_enabled() && !ignore_page_migration && phys) { + if (kbase_is_page_migration_enabled() && !ignore_page_migration && phys && + !is_huge(*phys) && !is_partial(*phys)) { const u64 num_pages = (to_vpfn - from_vpfn) / GPU_PAGES_PER_CPU_PAGE; u64 i; @@ -2315,7 +2374,15 @@ static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mm if (count > remain) count = remain; - if (!vindex && is_huge_head(*phys)) + /* There are 3 conditions to satisfy in order to create a level 2 ATE: + * + * - The GPU VA is aligned to 2 MB. + * - The physical address is tagged as the head of a 2 MB region, + * which guarantees a contiguous physical address range. + * - There are actually 2 MB of virtual and physical pages to map, + * i.e. 512 entries for the MMU page table. + */ + if (!vindex && is_huge_head(*phys) && (count == KBASE_MMU_PAGE_ENTRIES)) cur_level = MIDGARD_MMU_LEVEL(2); else cur_level = MIDGARD_MMU_BOTTOMLEVEL; @@ -2525,6 +2592,7 @@ int kbase_mmu_insert_pages(struct kbase_device *kbdev, struct kbase_mmu_table *m } KBASE_EXPORT_TEST_API(kbase_mmu_insert_pages); +KBASE_ALLOW_ERROR_INJECTION_TEST_API(kbase_mmu_insert_pages, ERRNO); int kbase_mmu_insert_pages_skip_status_update(struct kbase_device *kbdev, struct kbase_mmu_table *mmut, u64 vpfn, @@ -2582,6 +2650,7 @@ int kbase_mmu_insert_aliased_pages(struct kbase_device *kbdev, struct kbase_mmu_ return 0; } +KBASE_ALLOW_ERROR_INJECTION_TEST_API(kbase_mmu_insert_aliased_pages, ERRNO); void kbase_mmu_update(struct kbase_device *kbdev, struct kbase_mmu_table *mmut, int as_nr) { @@ -2699,50 +2768,65 @@ KBASE_EXPORT_TEST_API(kbase_mmu_disable); static void kbase_mmu_update_and_free_parent_pgds(struct kbase_device *kbdev, struct kbase_mmu_table *mmut, phys_addr_t *pgds, u64 vpfn, int level, - enum kbase_mmu_op_type flush_op, u64 *dirty_pgds) + enum kbase_mmu_op_type flush_op, u64 *dirty_pgds, + int as_nr) { - int current_level; + phys_addr_t current_pgd = pgds[level]; + struct page *p = phys_to_page(current_pgd); + u64 *current_page = kbase_kmap(p); + unsigned int current_valid_entries = kbdev->mmu_mode->get_num_valid_entries(current_page); + unsigned int index = (vpfn >> ((3 - level) * 9)) & 0x1FFU; lockdep_assert_held(&mmut->mmu_lock); - for (current_level = level - 1; current_level >= MIDGARD_MMU_LEVEL(0); current_level--) { - phys_addr_t current_pgd = pgds[current_level]; - struct page *p = phys_to_page(current_pgd); + /* We need to track every level that needs updating */ + if (dirty_pgds) + *dirty_pgds |= 1ULL << level; - u64 *current_page = kbase_kmap(p); - unsigned int current_valid_entries = - kbdev->mmu_mode->get_num_valid_entries(current_page); - unsigned int index = (vpfn >> ((3 - current_level) * 9)) & 0x1FFU; + kbdev->mmu_mode->entries_invalidate(¤t_page[index], 1); + if (current_valid_entries == 1 && level != MIDGARD_MMU_LEVEL(0)) { + kbdev->mmu_mode->set_num_valid_entries(current_page, 0); - /* We need to track every level that needs updating */ - if (dirty_pgds) - *dirty_pgds |= 1ULL << current_level; + kbase_kunmap(p, current_page); - kbdev->mmu_mode->entries_invalidate(¤t_page[index], 1); - if (current_valid_entries == 1 && current_level != MIDGARD_MMU_LEVEL(0)) { - kbase_kunmap(p, current_page); + kbase_mmu_update_and_free_parent_pgds(kbdev, mmut, pgds, vpfn, level - 1, flush_op, + dirty_pgds, as_nr); - /* Ensure the cacheline containing the last valid entry - * of PGD is invalidated from the GPU cache, before the - * PGD page is freed. + /* Check if fine grained GPU cache maintenance is being used */ + if (flush_op == KBASE_MMU_OP_FLUSH_PT) { + /* Ensure the invalidated PTE is visible in memory right away */ + kbase_mmu_sync_pgd_cpu(kbdev, kbase_dma_addr(p) + (index * sizeof(u64)), + sizeof(u64)); + /* Invalidate the GPU cache for the whole PGD page and not just for + * the cacheline containing the invalidated PTE, as the PGD page is + * going to be freed. There is an extremely remote possibility that + * other cachelines (containing all invalid PTEs) of PGD page are + * also present in the GPU cache. */ - kbase_mmu_sync_pgd_gpu(kbdev, mmut->kctx, - current_pgd + (index * sizeof(u64)), sizeof(u64), - flush_op); - - kbase_mmu_add_to_free_pgds_list(mmut, p); - } else { - current_valid_entries--; - - kbdev->mmu_mode->set_num_valid_entries(current_page, current_valid_entries); - - kbase_kunmap(p, current_page); - - kbase_mmu_sync_pgd(kbdev, mmut->kctx, current_pgd + (index * sizeof(u64)), - kbase_dma_addr(p) + (index * sizeof(u64)), sizeof(u64), - flush_op); - break; + kbase_mmu_sync_pgd_gpu(kbdev, mmut->kctx, current_pgd, 512 * sizeof(u64), + KBASE_MMU_OP_FLUSH_PT); } + + kbase_mmu_add_to_free_pgds_list(mmut, p); + } else { + current_valid_entries--; + + kbdev->mmu_mode->set_num_valid_entries(current_page, current_valid_entries); + + kbase_kunmap(p, current_page); + + kbase_mmu_sync_pgd(kbdev, mmut->kctx, current_pgd + (index * sizeof(u64)), + kbase_dma_addr(p) + (index * sizeof(u64)), sizeof(u64), + flush_op); + + /* When fine grained GPU cache maintenance is used then invalidate the MMU caches + * now as the top most level PGD entry, affected by the teardown operation, has + * been invalidated (both in memory as well as in GPU L2 cache). This is to avoid + * the possibility of invalid ATEs being reloaded into the GPU L2 cache whilst the + * teardown is happening. + */ + if (flush_op == KBASE_MMU_OP_FLUSH_PT) + mmu_invalidate_on_teardown(kbdev, mmut->kctx, vpfn, 1, level, as_nr); } } @@ -2783,13 +2867,11 @@ static void mmu_flush_invalidate_teardown_pages(struct kbase_device *kbdev, } #if MALI_USE_CSF else { - /* Partial GPU cache flush with MMU cache invalidation */ + /* Partial GPU cache flush of the pages that were unmapped */ unsigned long irq_flags; unsigned int i; bool flush_done = false; - mmu_invalidate(kbdev, kctx, as_nr, op_param); - for (i = 0; !flush_done && i < phys_page_nr; i++) { spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); if (kbdev->pm.backend.gpu_ready && (!kctx || kctx->as_nr >= 0)) @@ -2809,7 +2891,7 @@ static void mmu_flush_invalidate_teardown_pages(struct kbase_device *kbdev, static int kbase_mmu_teardown_pgd_pages(struct kbase_device *kbdev, struct kbase_mmu_table *mmut, u64 vpfn, size_t nr, u64 *dirty_pgds, struct list_head *free_pgds_list, - enum kbase_mmu_op_type flush_op) + enum kbase_mmu_op_type flush_op, int as_nr) { struct kbase_mmu_mode const *mmu_mode = kbdev->mmu_mode; @@ -2832,8 +2914,7 @@ static int kbase_mmu_teardown_pgd_pages(struct kbase_device *kbdev, struct kbase phys_addr_t pgd = mmut->pgd; struct page *p = phys_to_page(pgd); - if (count > nr) - count = nr; + count = MIN(nr, count); /* need to check if this is a 2MB page or a small page */ for (level = MIDGARD_MMU_TOPLEVEL; level <= MIDGARD_MMU_BOTTOMLEVEL; level++) { @@ -2844,23 +2925,12 @@ static int kbase_mmu_teardown_pgd_pages(struct kbase_device *kbdev, struct kbase if (mmu_mode->ate_is_valid(page[index], level)) break; /* keep the mapping */ else if (!mmu_mode->pte_is_valid(page[index], level)) { - /* nothing here, advance */ - switch (level) { - case MIDGARD_MMU_LEVEL(0): - count = 134217728; - break; - case MIDGARD_MMU_LEVEL(1): - count = 262144; - break; - case MIDGARD_MMU_LEVEL(2): - count = 512; - break; - case MIDGARD_MMU_LEVEL(3): - count = 1; - break; - } - if (count > nr) - count = nr; + dev_warn(kbdev->dev, "Invalid PTE found @ level %d for VA %llx", + level, vpfn << PAGE_SHIFT); + /* nothing here, advance to the next PTE of the current level */ + count = (1 << ((3 - level) * 9)); + count -= (vpfn & (count - 1)); + count = MIN(nr, count); goto next; } next_pgd = mmu_mode->pte_to_phy_addr( @@ -2915,20 +2985,37 @@ static int kbase_mmu_teardown_pgd_pages(struct kbase_device *kbdev, struct kbase mmu_mode->entries_invalidate(&page[index], pcount); if (!num_of_valid_entries) { + mmu_mode->set_num_valid_entries(page, 0); + kbase_kunmap(p, page); - /* Ensure the cacheline(s) containing the last valid entries - * of PGD is invalidated from the GPU cache, before the - * PGD page is freed. + /* To avoid the invalid ATEs from the PGD page (that is going to be freed) + * from getting reloaded into the GPU L2 cache whilst the teardown is + * happening, the fine grained GPU L2 cache maintenance is done in the top + * to bottom level PGD order. MMU cache invalidation is done after + * invalidating the entry of top most level PGD, affected by the teardown. */ - kbase_mmu_sync_pgd_gpu(kbdev, mmut->kctx, pgd + (index * sizeof(u64)), - pcount * sizeof(u64), flush_op); + kbase_mmu_update_and_free_parent_pgds(kbdev, mmut, pgds, vpfn, level - 1, + flush_op, dirty_pgds, as_nr); + + /* Check if fine grained GPU cache maintenance is being used */ + if (flush_op == KBASE_MMU_OP_FLUSH_PT) { + /* Ensure the invalidated ATEs are visible in memory right away */ + kbase_mmu_sync_pgd_cpu(kbdev, + kbase_dma_addr(p) + (index * sizeof(u64)), + pcount * sizeof(u64)); + /* Invalidate the GPU cache for the whole PGD page and not just for + * the cachelines containing the invalidated ATEs, as the PGD page + * is going to be freed. There is an extremely remote possibility + * that other cachelines (containing all invalid ATEs) of PGD page + * are also present in the GPU cache. + */ + kbase_mmu_sync_pgd_gpu(kbdev, mmut->kctx, pgd, 512 * sizeof(u64), + KBASE_MMU_OP_FLUSH_PT); + } kbase_mmu_add_to_free_pgds_list(mmut, p); - kbase_mmu_update_and_free_parent_pgds(kbdev, mmut, pgds, vpfn, level, - flush_op, dirty_pgds); - vpfn += count; nr -= count; continue; @@ -2939,6 +3026,12 @@ static int kbase_mmu_teardown_pgd_pages(struct kbase_device *kbdev, struct kbase kbase_mmu_sync_pgd(kbdev, mmut->kctx, pgd + (index * sizeof(u64)), kbase_dma_addr(p) + (index * sizeof(u64)), pcount * sizeof(u64), flush_op); + + /* When fine grained GPU cache maintenance is used then invalidation of MMU cache + * is done inline for every bottom level PGD touched in the teardown. + */ + if (flush_op == KBASE_MMU_OP_FLUSH_PT) + mmu_invalidate_on_teardown(kbdev, mmut->kctx, vpfn, pcount, level, as_nr); next: kbase_kunmap(p, page); vpfn += count; @@ -3032,7 +3125,7 @@ static int mmu_teardown_pages(struct kbase_device *kbdev, struct kbase_mmu_table mutex_lock(&mmut->mmu_lock); err = kbase_mmu_teardown_pgd_pages(kbdev, mmut, vpfn, nr_virt_pages, &dirty_pgds, - &free_pgds_list, flush_op); + &free_pgds_list, flush_op, as_nr); /* Set up MMU operation parameters. See above about MMU cache flush strategy. */ op_param = (struct kbase_mmu_hw_op_param){ @@ -3069,6 +3162,7 @@ int kbase_mmu_teardown_pages(struct kbase_device *kbdev, struct kbase_mmu_table return mmu_teardown_pages(kbdev, mmut, vpfn, phys, nr_phys_pages, nr_virt_pages, as_nr, false); } +KBASE_EXPORT_TEST_API(kbase_mmu_teardown_pages); int kbase_mmu_teardown_imported_pages(struct kbase_device *kbdev, struct kbase_mmu_table *mmut, u64 vpfn, struct tagged_addr *phys, size_t nr_phys_pages, @@ -3583,14 +3677,12 @@ int kbase_mmu_migrate_page(struct tagged_addr old_phys, struct tagged_addr new_p ret = kbase_mmu_hw_do_unlock_no_addr(kbdev, as, &op_param); } } - spin_unlock_irqrestore(&kbdev->hwaccess_lock, hwaccess_flags); - /* Releasing locks before checking the migration transaction error state */ - mutex_unlock(&kbdev->mmu_hw_mutex); - spin_lock_irqsave(&kbdev->hwaccess_lock, hwaccess_flags); /* Release the transition prevention in L2 by ending the transaction */ mmu_page_migration_transaction_end(kbdev); spin_unlock_irqrestore(&kbdev->hwaccess_lock, hwaccess_flags); + /* Releasing locks before checking the migration transaction error state */ + mutex_unlock(&kbdev->mmu_hw_mutex); /* Checking the final migration transaction error state */ if (ret < 0) { diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_faults_decoder.c b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_faults_decoder.c new file mode 100644 index 000000000000..e1b72ed24321 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_faults_decoder.c @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note +/* + * + * (C) COPYRIGHT 2024 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +/** + * DOC: Base kernel MMU faults decoder. + */ + +#include +#include +#if MALI_USE_CSF +#include +#else +#include +#endif + +#include +#include + +unsigned int fault_source_id_internal_requester_get(struct kbase_device *kbdev, + unsigned int source_id) +{ + if (kbdev->gpu_props.gpu_id.product_model < GPU_ID_MODEL_MAKE(14, 0)) + return ((source_id >> 4) & 0xF); + else + return (source_id & 0x3F); +} + +static inline const char *source_id_enc_core_type_get_str(struct kbase_device *kbdev, + unsigned int source_id) +{ + if (kbdev->gpu_props.gpu_id.product_model < GPU_ID_MODEL_MAKE(14, 0)) + return decode_fault_source_core_id_t_core_type( + FAULT_SOURCE_ID_CORE_ID_GET(source_id), kbdev->gpu_props.gpu_id.arch_id); + else + return decode_fault_source_core_type_t_name( + FAULT_SOURCE_ID_CORE_TYPE_GET(source_id), kbdev->gpu_props.gpu_id.arch_id); +} +const char *fault_source_id_internal_requester_get_str(struct kbase_device *kbdev, + unsigned int source_id, + unsigned int access_type) +{ + unsigned int ir = fault_source_id_internal_requester_get(kbdev, source_id); + bool older_source_id_fmt = + (kbdev->gpu_props.gpu_id.product_model < GPU_ID_MODEL_MAKE(14, 0)); + unsigned int utlb_id = 0; + + if (older_source_id_fmt) + utlb_id = FAULT_SOURCE_ID_UTLB_ID_GET(source_id); + + if (strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "shader") == 0) { + if (utlb_id == 0) { + if (access_type == AS_FAULTSTATUS_ACCESS_TYPE_READ) + return decode_fault_source_shader_r_t( + ir, kbdev->gpu_props.gpu_id.arch_id); + else + return decode_fault_source_shader_w_t( + ir, kbdev->gpu_props.gpu_id.arch_id); + } else + return "Load/store cache"; + } else if (strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "tiler")) { +#if MALI_USE_CSF + if (utlb_id == 0) { + if (access_type == AS_FAULTSTATUS_ACCESS_TYPE_READ) + return decode_fault_source_tiler_r_t( + ir, kbdev->gpu_props.gpu_id.arch_id); + else + return decode_fault_source_tiler_w_t( + ir, kbdev->gpu_props.gpu_id.arch_id); + } else + return "The polygon list writer. No further details."; +#else + return (utlb_id == 0) ? "Anything other than the polygon list writer" : + "The polygon list writer"; +#endif + } +#if MALI_USE_CSF + else if (strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "csf")) { + if (access_type == AS_FAULTSTATUS_ACCESS_TYPE_READ) + return decode_fault_source_csf_r_t(ir, kbdev->gpu_props.gpu_id.arch_id); + else + return decode_fault_source_csf_w_t(ir, kbdev->gpu_props.gpu_id.arch_id); + } +#else + else if (strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "jm")) + return decode_fault_source_jm_t(ir, kbdev->gpu_props.gpu_id.arch_id); +#endif + else if (!strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "I2c") || + !strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "memsys") || + !strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "mmu")) { + return "Not used"; + } + + return "unknown"; +} + +const char *fault_source_id_core_type_description_get(struct kbase_device *kbdev, + unsigned int source_id) +{ + if (kbdev->gpu_props.gpu_id.product_model < GPU_ID_MODEL_MAKE(14, 0)) { + return decode_fault_source_core_id_t_desc(FAULT_SOURCE_ID_CORE_ID_GET(source_id), + kbdev->gpu_props.gpu_id.arch_id); + } else { + return decode_fault_source_core_type_t_desc( + FAULT_SOURCE_ID_CORE_TYPE_GET(source_id), kbdev->gpu_props.gpu_id.arch_id); + } +} diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_faults_decoder.h b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_faults_decoder.h new file mode 100644 index 000000000000..da5610ec94b0 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_faults_decoder.h @@ -0,0 +1,143 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * + * (C) COPYRIGHT 2024 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ +#ifndef _MALI_KBASE_MMU_FAULTS_DECODER_H_ +#define _MALI_KBASE_MMU_FAULTS_DECODER_H_ + +#include +#include + +/* FAULTSTATUS.SOURCE_ID encoding */ +#define SOURCE_ID_CORE_ID_SHIFT (9) +#define SOURCE_ID_CORE_ID_MASK (0x7F << SOURCE_ID_CORE_ID_SHIFT) +#define SOURCE_ID_UTLB_ID_SHIFT (8) +#define SOURCE_ID_UTLB_ID_MASK (0x01 << SOURCE_ID_UTLB_ID_SHIFT) +#define SOURCE_ID_CORE_TYPE_SHIFT (12) +#define SOURCE_ID_CORE_TYPE_MASK (0x0F << SOURCE_ID_CORE_TYPE_SHIFT) +#define SOURCE_ID_CORE_INDEX_SHIFT (6) +#define SOURCE_ID_CORE_INDEX_MASK (0x3F << SOURCE_ID_CORE_INDEX_SHIFT) + +/** + * FAULT_SOURCE_ID_CORE_ID_GET() - Get core ID of a fault. + * + * @source_id: SOURCE_ID field of FAULTSTATUS (MMU) or GPU_FAULTSTATUS (GPU) + * registers. + * + * Get core ID part of SOURCE_ID field of FAULTSTATUS (MMU) or + * GPU_FAULTSTATUS (GPU) registers. + * + * Return: core ID of the fault. + */ +#define FAULT_SOURCE_ID_CORE_ID_GET(source_id) \ + ((source_id & SOURCE_ID_CORE_ID_MASK) >> SOURCE_ID_CORE_ID_SHIFT) + +/** + * FAULT_SOURCE_ID_UTLB_ID_GET() - Get UTLB ID of a fault. + * + * @source_id: SOURCE_ID field of FAULTSTATUS (MMU) or GPU_FAULTSTATUS (GPU) + * registers. + * + * Get UTLB(micro-TLB) ID part of SOURCE_ID field of FAULTSTATUS (MMU) or + * GPU_FAULTSTATUS (GPU) registers. + * + * Return: UTLB ID of the fault. + */ +#define FAULT_SOURCE_ID_UTLB_ID_GET(source_id) \ + ((source_id & SOURCE_ID_UTLB_ID_MASK) >> SOURCE_ID_UTLB_ID_SHIFT) + +/** + * FAULT_SOURCE_ID_CORE_TYPE_GET() - Get core type of a fault. + * + * @source_id: SOURCE_ID field of FAULTSTATUS (MMU) or GPU_FAULTSTATUS (GPU) + * registers. + * + * Get core type part of SOURCE_ID field of FAULTSTATUS (MMU) or + * GPU_FAULTSTATUS (GPU) registers. + * + * Return: core type code of the fault. + */ +#define FAULT_SOURCE_ID_CORE_TYPE_GET(source_id) \ + ((source_id & SOURCE_ID_CORE_TYPE_MASK) >> SOURCE_ID_CORE_TYPE_SHIFT) + +/** + * FAULT_SOURCE_ID_CORE_INDEX_GET() - Get core index of a fault. + * + * @source_id: SOURCE_ID field of FAULTSTATUS (MMU) or GPU_FAULTSTATUS (GPU) + * registers. + * + * Get core index part of SOURCE_ID field of FAULTSTATUS (MMU) or + * GPU_FAULTSTATUS (GPU) registers. + * + * Return: core index of the fault. + */ +#define FAULT_SOURCE_ID_CORE_INDEX_GET(source_id) \ + ((source_id & SOURCE_ID_CORE_INDEX_MASK) >> SOURCE_ID_CORE_INDEX_SHIFT) + +/** + * fault_source_id_internal_requester_get() - Get internal_requester of a fault. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer). + * @source_id: SOURCE_ID field of FAULTSTATUS (MMU) or GPU_FAULTSTATUS (GPU) + * registers. + * + * Get internal_requester part of SOURCE_ID field of FAULTSTATUS (MMU) or + * GPU_FAULTSTATUS (GPU) registers. + * + * Return: Internal requester code of the fault. + */ +unsigned int fault_source_id_internal_requester_get(struct kbase_device *kbdev, + unsigned int source_id); + +/** + * fault_source_id_internal_requester_get_str() - Get internal_requester of a + * fault in a human readable format. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer). + * @source_id: SOURCE_ID field of FAULTSTATUS (MMU) or GPU_FAULTSTATUS (GPU) + * registers. + * @access_type: the direction of data transfer that caused the fault (atomic, + * execute, read, write) + * + * Get the human readable decoding of internal_requester part of SOURCE_ID field + * of FAULTSTATUS (MMU) or GPU_FAULTSTATUS (GPU) registers. + * + * Return: Internal requester of the fault in human readable format. + */ +const char *fault_source_id_internal_requester_get_str(struct kbase_device *kbdev, + unsigned int source_id, + unsigned int access_type); + +/** + * fault_source_id_core_type_description_get() - Get the core type of + * a fault in a human readable format. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer). + * @source_id: SOURCE_ID field of FAULTSTATUS (MMU) or GPU_FAULTSTATUS (GPU) + * registers. + * + * Get the human readable decoding of core type part of SOURCE_ID field + * of FAULTSTATUS (MMU) or GPU_FAULTSTATUS (GPU) registers. + * + * Return: core type of the fault in human readable format. + */ +const char *fault_source_id_core_type_description_get(struct kbase_device *kbdev, + unsigned int source_id); + +#endif /* _MALI_KBASE_MMU_FAULTS_DECODER_H_ */ diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_faults_decoder_luts.c b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_faults_decoder_luts.c new file mode 100644 index 000000000000..8e90cacb4efa --- /dev/null +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_faults_decoder_luts.c @@ -0,0 +1,660 @@ +// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note +/* + * + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +/** + * DOC: Base kernel MMU faults decoder. + */ + +#include + +#define GPU_ID_ARCH_ID_MAJOR_GET(gpu_id) ((gpu_id >> 16) & 0xFF) +#define GPU_ID_ARCH_ID_MINOR_AND_REV_GET(gpu_id) (gpu_id & 0xFFFF) +#define NELEMS(s) (sizeof(s) / sizeof((s)[0])) + +struct decode_lut_element { + u16 arch_minor_rev; + u16 key; + const char *text; +}; + +static const char *decode_lut_element_lookup(u16 arch_minor_rev, u16 key, + struct decode_lut_element *decode_element_lut, + unsigned int lut_len) +{ + struct decode_lut_element *p; + + for (p = decode_element_lut; p < decode_element_lut + lut_len; p++) { + if (p->key == key && + (p->arch_minor_rev == 0xffff || p->arch_minor_rev == arch_minor_rev)) + break; + } + if (p < decode_element_lut + lut_len) + return p->text; + else + return "unknown"; +} + +/* Auto-generated code: DO NOT MODIFY! */ + +static struct decode_lut_element lut_fault_source_core_type_t_name_major_9[] = { + { 0xFFFF, 0, "shader" }, { 0xFFFF, 1, "l2c" }, { 0xFFFF, 2, "tiler" }, + { 0xFFFF, 3, "mmu" }, { 0xFFFF, 4, "jm" }, { 0xFFFF, 5, "pmb" }, +}; + +static struct decode_lut_element lut_fault_source_core_type_t_desc_major_9[] = { + { 0xFFFF, 0, "Shader core" }, { 0xFFFF, 1, "Level 2 cache" }, + { 0xFFFF, 2, "Tiler" }, { 0xFFFF, 3, "MMU" }, + { 0xFFFF, 4, "Job Manager" }, { 0xFFFF, 5, "Performance Monitor Block" }, +}; + +static struct decode_lut_element lut_fault_source_core_type_t_name_major_10[] = { + { 0xFFFF, 0, "shader" }, { 0xFFFF, 1, "l2c" }, { 0xFFFF, 2, "tiler" }, + { 0xFFFF, 3, "mmu" }, { 0xFFFF, 4, "csf" }, { 0xFFFF, 5, "memsys" }, +}; + +static struct decode_lut_element lut_fault_source_core_type_t_desc_major_10[] = { + { 0xFFFF, 0, "Shader core" }, { 0xFFFF, 1, "Level 2 cache" }, + { 0xFFFF, 2, "Tiler" }, { 0xFFFF, 3, "MMU" }, + { 0xFFFF, 4, "CSF" }, { 0xFFFF, 5, "Memory system" }, +}; + +static struct decode_lut_element lut_fault_source_core_type_t_name_major_11[] = { + { 0xFFFF, 0, "shader" }, { 0xFFFF, 1, "l2c" }, { 0xFFFF, 2, "tiler" }, + { 0xFFFF, 3, "mmu" }, { 0xFFFF, 4, "csf" }, { 0xFFFF, 5, "memsys" }, +}; + +static struct decode_lut_element lut_fault_source_core_type_t_desc_major_11[] = { + { 0xFFFF, 0, "Shader core" }, { 0xFFFF, 1, "Level 2 cache" }, + { 0xFFFF, 2, "Tiler" }, { 0xFFFF, 3, "MMU" }, + { 0xFFFF, 4, "CSF" }, { 0xFFFF, 5, "Memory system" }, +}; + +static struct decode_lut_element lut_fault_source_core_type_t_name_major_12[] = { + { 0xFFFF, 0, "shader" }, { 0xFFFF, 1, "l2c" }, { 0xFFFF, 2, "tiler" }, + { 0xFFFF, 3, "mmu" }, { 0xFFFF, 4, "csf" }, { 0xFFFF, 5, "memsys" }, +}; + +static struct decode_lut_element lut_fault_source_core_type_t_desc_major_12[] = { + { 0xFFFF, 0, "Shader core" }, { 0xFFFF, 1, "Level 2 cache" }, + { 0xFFFF, 2, "Tiler" }, { 0xFFFF, 3, "MMU" }, + { 0xFFFF, 4, "CSF" }, { 0xFFFF, 5, "Memory system" }, +}; + +static struct decode_lut_element lut_fault_source_core_id_t_desc_major_9[] = { + { 0xFFFF, 0, "Shader core 0" }, + { 0xFFFF, 1, "Shader core 1" }, + { 0xFFFF, 2, "Shader core 2" }, + { 0xFFFF, 3, "Shader core 3" }, + { 0xFFFF, 4, "Shader core 4" }, + { 0xFFFF, 5, "Shader core 5" }, + { 0xFFFF, 6, "Shader core 6" }, + { 0xFFFF, 7, "Shader core 7" }, + { 0xFFFF, 8, "Shader core 8" }, + { 0xFFFF, 9, "Shader core 9" }, + { 0xFFFF, 10, "Shader core 10" }, + { 0xFFFF, 11, "Shader core 11" }, + { 0xFFFF, 12, "Shader core 12" }, + { 0xFFFF, 13, "Shader core 13" }, + { 0xFFFF, 14, "Shader core 14" }, + { 0xFFFF, 15, "Shader core 15" }, + { 0xFFFF, 16, "Shader core 16" }, + { 0xFFFF, 17, "Shader core 17" }, + { 0xFFFF, 18, "Shader core 18" }, + { 0xFFFF, 19, "Shader core 19" }, + { 0xFFFF, 20, "Shader core 20" }, + { 0xFFFF, 21, "Shader core 21" }, + { 0xFFFF, 22, "Shader core 22" }, + { 0xFFFF, 23, "Shader core 23" }, + { 0xFFFF, 24, "Shader core 24" }, + { 0xFFFF, 25, "Shader core 25" }, + { 0xFFFF, 26, "Shader core 26" }, + { 0xFFFF, 27, "Shader core 27" }, + { 0xFFFF, 28, "Shader core 28" }, + { 0xFFFF, 29, "Shader core 29" }, + { 0xFFFF, 30, "Shader core 30" }, + { 0xFFFF, 31, "Shader core 31" }, + { 0xFFFF, 41, "L2 Slice 3" }, + { 0xFFFF, 43, "L2 Slice 2" }, + { 0xFFFF, 45, "L2 Slice 1" }, + { 0xFFFF, 46, "PMB" }, + { 0xFFFF, 47, "L2 Slice 0" }, + { 0xFFFF, 51, "Tiler" }, + { 0xFFFF, 55, "MMU" }, + { 0xFFFF, 62, "Job Manager" }, +}; + +static struct decode_lut_element lut_fault_source_core_id_t_core_type_major_9[] = { + { 0xFFFF, 0, "shader" }, { 0xFFFF, 1, "shader" }, { 0xFFFF, 2, "shader" }, + { 0xFFFF, 3, "shader" }, { 0xFFFF, 4, "shader" }, { 0xFFFF, 5, "shader" }, + { 0xFFFF, 6, "shader" }, { 0xFFFF, 7, "shader" }, { 0xFFFF, 8, "shader" }, + { 0xFFFF, 9, "shader" }, { 0xFFFF, 10, "shader" }, { 0xFFFF, 11, "shader" }, + { 0xFFFF, 12, "shader" }, { 0xFFFF, 13, "shader" }, { 0xFFFF, 14, "shader" }, + { 0xFFFF, 15, "shader" }, { 0xFFFF, 16, "shader" }, { 0xFFFF, 17, "shader" }, + { 0xFFFF, 18, "shader" }, { 0xFFFF, 19, "shader" }, { 0xFFFF, 20, "shader" }, + { 0xFFFF, 21, "shader" }, { 0xFFFF, 22, "shader" }, { 0xFFFF, 23, "shader" }, + { 0xFFFF, 24, "shader" }, { 0xFFFF, 25, "shader" }, { 0xFFFF, 26, "shader" }, + { 0xFFFF, 27, "shader" }, { 0xFFFF, 28, "shader" }, { 0xFFFF, 29, "shader" }, + { 0xFFFF, 30, "shader" }, { 0xFFFF, 31, "shader" }, { 0xFFFF, 41, "l2c" }, + { 0xFFFF, 43, "l2c" }, { 0xFFFF, 45, "l2c" }, { 0xFFFF, 46, "pmb" }, + { 0xFFFF, 47, "l2c" }, { 0xFFFF, 51, "tiler" }, { 0xFFFF, 55, "mmu" }, + { 0xFFFF, 62, "jm" }, +}; + +static struct decode_lut_element lut_fault_source_core_id_t_desc_major_10[] = { + { 0xFFFF, 0, "Shader core 0" }, + { 0xFFFF, 1, "Shader core 1" }, + { 0xFFFF, 2, "Shader core 2" }, + { 0xFFFF, 3, "Shader core 3" }, + { 0xFFFF, 4, "Shader core 4" }, + { 0xFFFF, 5, "Shader core 5" }, + { 0xFFFF, 6, "Shader core 6" }, + { 0xFFFF, 7, "Shader core 7" }, + { 0xFFFF, 8, "Shader core 8" }, + { 0xFFFF, 9, "Shader core 9" }, + { 0xFFFF, 10, "Shader core 10" }, + { 0xFFFF, 11, "Shader core 11" }, + { 0xFFFF, 12, "Shader core 12" }, + { 0xFFFF, 13, "Shader core 13" }, + { 0xFFFF, 14, "Shader core 14" }, + { 0xFFFF, 15, "Shader core 15" }, + { 0xFFFF, 16, "Shader core 16" }, + { 0xFFFF, 17, "Shader core 17" }, + { 0xFFFF, 18, "Shader core 18" }, + { 0xFFFF, 19, "Shader core 19" }, + { 0xFFFF, 20, "Shader core 20" }, + { 0xFFFF, 21, "Shader core 21" }, + { 0xFFFF, 22, "Shader core 22" }, + { 0xFFFF, 23, "Shader core 23" }, + { 0xFFFF, 24, "Shader core 24" }, + { 0xFFFF, 25, "Shader core 25" }, + { 0xFFFF, 26, "Shader core 26" }, + { 0xFFFF, 27, "Shader core 27" }, + { 0xFFFF, 28, "Shader core 28" }, + { 0xFFFF, 29, "Shader core 29" }, + { 0xFFFF, 30, "Shader core 30" }, + { 0xFFFF, 31, "Shader core 31" }, + { 0xFFFF, 41, "L2 Slice 3" }, + { 0xFFFF, 43, "L2 Slice 2" }, + { 0xFFFF, 45, "L2 Slice 1" }, + { 0xFFFF, 47, "L2 Slice 0" }, + { 0xFFFF, 51, "Tiler" }, + { 0xFFFF, 55, "MMU" }, + { 0xFFFF, 33, "L2 Slice 7" }, + { 0xFFFF, 35, "L2 Slice 6" }, + { 0xFFFF, 37, "L2 Slice 5" }, + { 0xFFFF, 39, "L2 Slice 4" }, + { 0xFFFF, 48, "Memory system, undefined" }, + { 0xFFFF, 62, "Command Stream Frontend" }, +}; + +static struct decode_lut_element lut_fault_source_core_id_t_core_type_major_10[] = { + { 0xFFFF, 0, "shader" }, { 0xFFFF, 1, "shader" }, { 0xFFFF, 2, "shader" }, + { 0xFFFF, 3, "shader" }, { 0xFFFF, 4, "shader" }, { 0xFFFF, 5, "shader" }, + { 0xFFFF, 6, "shader" }, { 0xFFFF, 7, "shader" }, { 0xFFFF, 8, "shader" }, + { 0xFFFF, 9, "shader" }, { 0xFFFF, 10, "shader" }, { 0xFFFF, 11, "shader" }, + { 0xFFFF, 12, "shader" }, { 0xFFFF, 13, "shader" }, { 0xFFFF, 14, "shader" }, + { 0xFFFF, 15, "shader" }, { 0xFFFF, 16, "shader" }, { 0xFFFF, 17, "shader" }, + { 0xFFFF, 18, "shader" }, { 0xFFFF, 19, "shader" }, { 0xFFFF, 20, "shader" }, + { 0xFFFF, 21, "shader" }, { 0xFFFF, 22, "shader" }, { 0xFFFF, 23, "shader" }, + { 0xFFFF, 24, "shader" }, { 0xFFFF, 25, "shader" }, { 0xFFFF, 26, "shader" }, + { 0xFFFF, 27, "shader" }, { 0xFFFF, 28, "shader" }, { 0xFFFF, 29, "shader" }, + { 0xFFFF, 30, "shader" }, { 0xFFFF, 31, "shader" }, { 0xFFFF, 41, "l2c" }, + { 0xFFFF, 43, "l2c" }, { 0xFFFF, 45, "l2c" }, { 0xFFFF, 47, "l2c" }, + { 0xFFFF, 51, "tiler" }, { 0xFFFF, 55, "mmu" }, { 0xFFFF, 33, "l2c" }, + { 0xFFFF, 35, "l2c" }, { 0xFFFF, 37, "l2c" }, { 0xFFFF, 39, "l2c" }, + { 0xFFFF, 48, "memsys" }, { 0xFFFF, 62, "csf" }, +}; + +static struct decode_lut_element lut_fault_source_core_id_t_desc_major_11[] = { + { 0xFFFF, 0, "Shader core 0" }, + { 0xFFFF, 1, "Shader core 1" }, + { 0xFFFF, 2, "Shader core 2" }, + { 0xFFFF, 3, "Shader core 3" }, + { 0xFFFF, 4, "Shader core 4" }, + { 0xFFFF, 5, "Shader core 5" }, + { 0xFFFF, 6, "Shader core 6" }, + { 0xFFFF, 7, "Shader core 7" }, + { 0xFFFF, 8, "Shader core 8" }, + { 0xFFFF, 9, "Shader core 9" }, + { 0xFFFF, 10, "Shader core 10" }, + { 0xFFFF, 11, "Shader core 11" }, + { 0xFFFF, 12, "Shader core 12" }, + { 0xFFFF, 13, "Shader core 13" }, + { 0xFFFF, 14, "Shader core 14" }, + { 0xFFFF, 15, "Shader core 15" }, + { 0xFFFF, 16, "Shader core 16" }, + { 0xFFFF, 17, "Shader core 17" }, + { 0xFFFF, 18, "Shader core 18" }, + { 0xFFFF, 19, "Shader core 19" }, + { 0xFFFF, 20, "Shader core 20" }, + { 0xFFFF, 21, "Shader core 21" }, + { 0xFFFF, 22, "Shader core 22" }, + { 0xFFFF, 23, "Shader core 23" }, + { 0xFFFF, 24, "Shader core 24" }, + { 0xFFFF, 25, "Shader core 25" }, + { 0xFFFF, 26, "Shader core 26" }, + { 0xFFFF, 27, "Shader core 27" }, + { 0xFFFF, 28, "Shader core 28" }, + { 0xFFFF, 29, "Shader core 29" }, + { 0xFFFF, 30, "Shader core 30" }, + { 0xFFFF, 31, "Shader core 31" }, + { 0xFFFF, 41, "L2 Slice 3" }, + { 0xFFFF, 43, "L2 Slice 2" }, + { 0xFFFF, 45, "L2 Slice 1" }, + { 0xFFFF, 47, "L2 Slice 0" }, + { 0xFFFF, 51, "Tiler" }, + { 0xFFFF, 55, "MMU" }, + { 0xFFFF, 33, "L2 Slice 7" }, + { 0xFFFF, 35, "L2 Slice 6" }, + { 0xFFFF, 37, "L2 Slice 5" }, + { 0xFFFF, 39, "L2 Slice 4" }, + { 0xFFFF, 48, "Memory system, undefined" }, + { 0xFFFF, 62, "Command Stream Frontend" }, +}; + +static struct decode_lut_element lut_fault_source_core_id_t_core_type_major_11[] = { + { 0xFFFF, 0, "shader" }, { 0xFFFF, 1, "shader" }, { 0xFFFF, 2, "shader" }, + { 0xFFFF, 3, "shader" }, { 0xFFFF, 4, "shader" }, { 0xFFFF, 5, "shader" }, + { 0xFFFF, 6, "shader" }, { 0xFFFF, 7, "shader" }, { 0xFFFF, 8, "shader" }, + { 0xFFFF, 9, "shader" }, { 0xFFFF, 10, "shader" }, { 0xFFFF, 11, "shader" }, + { 0xFFFF, 12, "shader" }, { 0xFFFF, 13, "shader" }, { 0xFFFF, 14, "shader" }, + { 0xFFFF, 15, "shader" }, { 0xFFFF, 16, "shader" }, { 0xFFFF, 17, "shader" }, + { 0xFFFF, 18, "shader" }, { 0xFFFF, 19, "shader" }, { 0xFFFF, 20, "shader" }, + { 0xFFFF, 21, "shader" }, { 0xFFFF, 22, "shader" }, { 0xFFFF, 23, "shader" }, + { 0xFFFF, 24, "shader" }, { 0xFFFF, 25, "shader" }, { 0xFFFF, 26, "shader" }, + { 0xFFFF, 27, "shader" }, { 0xFFFF, 28, "shader" }, { 0xFFFF, 29, "shader" }, + { 0xFFFF, 30, "shader" }, { 0xFFFF, 31, "shader" }, { 0xFFFF, 41, "l2c" }, + { 0xFFFF, 43, "l2c" }, { 0xFFFF, 45, "l2c" }, { 0xFFFF, 47, "l2c" }, + { 0xFFFF, 51, "tiler" }, { 0xFFFF, 55, "mmu" }, { 0xFFFF, 33, "l2c" }, + { 0xFFFF, 35, "l2c" }, { 0xFFFF, 37, "l2c" }, { 0xFFFF, 39, "l2c" }, + { 0xFFFF, 48, "memsys" }, { 0xFFFF, 62, "csf" }, +}; + +static struct decode_lut_element lut_fault_source_core_id_t_desc_major_12[] = { + { 0xFFFF, 0, "Shader core 0" }, + { 0xFFFF, 1, "Shader core 1" }, + { 0xFFFF, 2, "Shader core 2" }, + { 0xFFFF, 3, "Shader core 3" }, + { 0xFFFF, 4, "Shader core 4" }, + { 0xFFFF, 5, "Shader core 5" }, + { 0xFFFF, 6, "Shader core 6" }, + { 0xFFFF, 7, "Shader core 7" }, + { 0xFFFF, 8, "Shader core 8" }, + { 0xFFFF, 9, "Shader core 9" }, + { 0xFFFF, 10, "Shader core 10" }, + { 0xFFFF, 11, "Shader core 11" }, + { 0xFFFF, 12, "Shader core 12" }, + { 0xFFFF, 13, "Shader core 13" }, + { 0xFFFF, 14, "Shader core 14" }, + { 0xFFFF, 15, "Shader core 15" }, + { 0xFFFF, 16, "Shader core 16" }, + { 0xFFFF, 17, "Shader core 17" }, + { 0xFFFF, 18, "Shader core 18" }, + { 0xFFFF, 19, "Shader core 19" }, + { 0xFFFF, 20, "Shader core 20" }, + { 0xFFFF, 21, "Shader core 21" }, + { 0xFFFF, 22, "Shader core 22" }, + { 0xFFFF, 23, "Shader core 23" }, + { 0xFFFF, 24, "Shader core 24" }, + { 0xFFFF, 25, "Shader core 25" }, + { 0xFFFF, 26, "Shader core 26" }, + { 0xFFFF, 27, "Shader core 27" }, + { 0xFFFF, 28, "Shader core 28" }, + { 0xFFFF, 29, "Shader core 29" }, + { 0xFFFF, 30, "Shader core 30" }, + { 0xFFFF, 31, "Shader core 31" }, + { 0xFFFF, 41, "L2 Slice 3" }, + { 0xFFFF, 43, "L2 Slice 2" }, + { 0xFFFF, 45, "L2 Slice 1" }, + { 0xFFFF, 47, "L2 Slice 0" }, + { 0xFFFF, 51, "Tiler" }, + { 0xFFFF, 55, "MMU" }, + { 0xFFFF, 33, "L2 Slice 7" }, + { 0xFFFF, 35, "L2 Slice 6" }, + { 0xFFFF, 37, "L2 Slice 5" }, + { 0xFFFF, 39, "L2 Slice 4" }, + { 0xFFFF, 48, "Memory system, undefined" }, + { 0xFFFF, 62, "Command Stream Frontend" }, +}; + +static struct decode_lut_element lut_fault_source_core_id_t_core_type_major_12[] = { + { 0xFFFF, 0, "shader" }, { 0xFFFF, 1, "shader" }, { 0xFFFF, 2, "shader" }, + { 0xFFFF, 3, "shader" }, { 0xFFFF, 4, "shader" }, { 0xFFFF, 5, "shader" }, + { 0xFFFF, 6, "shader" }, { 0xFFFF, 7, "shader" }, { 0xFFFF, 8, "shader" }, + { 0xFFFF, 9, "shader" }, { 0xFFFF, 10, "shader" }, { 0xFFFF, 11, "shader" }, + { 0xFFFF, 12, "shader" }, { 0xFFFF, 13, "shader" }, { 0xFFFF, 14, "shader" }, + { 0xFFFF, 15, "shader" }, { 0xFFFF, 16, "shader" }, { 0xFFFF, 17, "shader" }, + { 0xFFFF, 18, "shader" }, { 0xFFFF, 19, "shader" }, { 0xFFFF, 20, "shader" }, + { 0xFFFF, 21, "shader" }, { 0xFFFF, 22, "shader" }, { 0xFFFF, 23, "shader" }, + { 0xFFFF, 24, "shader" }, { 0xFFFF, 25, "shader" }, { 0xFFFF, 26, "shader" }, + { 0xFFFF, 27, "shader" }, { 0xFFFF, 28, "shader" }, { 0xFFFF, 29, "shader" }, + { 0xFFFF, 30, "shader" }, { 0xFFFF, 31, "shader" }, { 0xFFFF, 41, "l2c" }, + { 0xFFFF, 43, "l2c" }, { 0xFFFF, 45, "l2c" }, { 0xFFFF, 47, "l2c" }, + { 0xFFFF, 51, "tiler" }, { 0xFFFF, 55, "mmu" }, { 0xFFFF, 33, "l2c" }, + { 0xFFFF, 35, "l2c" }, { 0xFFFF, 37, "l2c" }, { 0xFFFF, 39, "l2c" }, + { 0xFFFF, 48, "memsys" }, { 0xFFFF, 62, "csf" }, +}; + +static struct decode_lut_element lut_fault_source_shader_r_t_major_9[] = { + { 0xFFFF, 0, "ic" }, { 0xFFFF, 1, "adc" }, { 0xFFFF, 4, "scm" }, + { 0xFFFF, 5, "vl" }, { 0xFFFF, 6, "plr" }, { 0xFFFF, 7, "fsdc" }, + { 0xFFFF, 8, "lsc" }, { 0xFFFF, 9, "cse" }, { 0xFFFF, 10, "tb" }, + { 0xFFFF, 11, "tmdi" }, { 0xFFFF, 12, "tmu0" }, { 0xFFFF, 13, "tmu1" }, + { 0xFFFF, 14, "tma0" }, { 0xFFFF, 15, "tma1" }, +}; + +static struct decode_lut_element lut_fault_source_shader_r_t_major_10[] = { + { 0xFFFF, 4, "scm" }, { 0xFFFF, 5, "vl" }, { 0xFFFF, 6, "plr" }, + { 0xFFFF, 7, "fsdc" }, { 0xFFFF, 8, "lsc" }, { 0xFFFF, 9, "cse" }, + { 0xFFFF, 10, "tb" }, { 0xFFFF, 11, "tmdi" }, { 0xFFFF, 12, "tmu0" }, + { 0xFFFF, 13, "tmu1" }, { 0xFFFF, 14, "tma0" }, { 0xFFFF, 15, "tma1" }, + { 0xFFFF, 0, "ic0" }, { 0xFFFF, 1, "ic1" }, { 0xFFFF, 2, "adc" }, +}; + +static struct decode_lut_element lut_fault_source_shader_r_t_major_11[] = { + { 0xFFFF, 4, "scm" }, { 0xFFFF, 5, "vl" }, { 0xFFFF, 6, "plr" }, + { 0xFFFF, 7, "fsdc" }, { 0xFFFF, 8, "lsc" }, { 0xFFFF, 9, "cse" }, + { 0xFFFF, 10, "tb" }, { 0xFFFF, 11, "tmdi" }, { 0xFFFF, 12, "tmu0" }, + { 0xFFFF, 13, "tmu1" }, { 0xFFFF, 14, "tma0" }, { 0xFFFF, 15, "tma1" }, + { 0xFFFF, 0, "ic0" }, { 0xFFFF, 1, "ic1" }, { 0xFFFF, 2, "adc" }, +}; + +static struct decode_lut_element lut_fault_source_shader_r_t_major_12[] = { + { 0xFFFF, 4, "scm" }, { 0xFFFF, 6, "plr" }, { 0xFFFF, 7, "fsdc" }, + { 0xFFFF, 8, "lsc" }, { 0xFFFF, 9, "cse" }, { 0xFFFF, 10, "tb" }, + { 0xFFFF, 11, "tmdi" }, { 0xFFFF, 12, "tmu0" }, { 0xFFFF, 13, "tmu1" }, + { 0xFFFF, 14, "tma0" }, { 0xFFFF, 15, "tma1" }, { 0xFFFF, 0, "ic0" }, + { 0xFFFF, 1, "ic1" }, { 0xFFFF, 2, "adc" }, { 0xFFFF, 3, "rtas" }, +}; + +static struct decode_lut_element lut_fault_source_shader_w_t_major_9[] = { + { 0xFFFF, 0, "pcb" }, + { 0xFFFF, 8, "lsc" }, + { 0xFFFF, 10, "tb" }, +}; + +static struct decode_lut_element lut_fault_source_shader_w_t_major_10[] = { + { 0xFFFF, 0, "pcb" }, { 0xFFFF, 8, "lsc" }, { 0xFFFF, 12, "tb0" }, + { 0xFFFF, 13, "tb1" }, { 0xFFFF, 14, "tb2" }, { 0xFFFF, 15, "tb3" }, +}; + +static struct decode_lut_element lut_fault_source_shader_w_t_major_11[] = { + { 0xFFFF, 0, "pcb" }, { 0xFFFF, 8, "lsc" }, { 0xFFFF, 12, "tb0" }, + { 0xFFFF, 13, "tb1" }, { 0xFFFF, 14, "tb2" }, { 0xFFFF, 15, "tb3" }, +}; + +static struct decode_lut_element lut_fault_source_shader_w_t_major_12[] = { + { 0xFFFF, 0, "pcb" }, { 0xFFFF, 8, "lsc" }, { 0xFFFF, 12, "tb0" }, + { 0xFFFF, 13, "tb1" }, { 0xFFFF, 14, "tb2" }, { 0xFFFF, 15, "tb3" }, +}; + +static struct decode_lut_element lut_fault_source_tiler_r_t_major_10[] = { + { 0xFFFF, 0, "pf" }, + { 0xFFFF, 1, "pcache" }, + { 0xFFFF, 2, "tcu" }, + { 0xFFFF, 3, "idx" }, +}; + +static struct decode_lut_element lut_fault_source_tiler_r_t_major_11[] = { + { 0xFFFF, 0, "pf" }, + { 0xFFFF, 1, "pcache" }, + { 0xFFFF, 2, "tcu" }, + { 0xFFFF, 3, "idx" }, +}; + +static struct decode_lut_element lut_fault_source_tiler_r_t_major_12[] = { + { 0xFFFF, 0, "pf" }, + { 0xFFFF, 1, "pcache" }, + { 0xFFFF, 2, "tcu" }, + { 0xFFFF, 3, "idx" }, +}; + +static struct decode_lut_element lut_fault_source_tiler_w_t_major_10[] = { + { 0xFFFF, 1, "pcache_wb" }, + { 0xFFFF, 2, "tcu_pcb" }, +}; + +static struct decode_lut_element lut_fault_source_tiler_w_t_major_11[] = { + { 0xFFFF, 1, "pcache_wb" }, + { 0xFFFF, 2, "tcu_pcb" }, +}; + +static struct decode_lut_element lut_fault_source_tiler_w_t_major_12[] = { + { 0xFFFF, 1, "pcache_wb" }, + { 0xFFFF, 2, "tcu_pcb" }, +}; + + +const char *decode_fault_source_core_type_t_name(u16 idx, u32 gpu_id) +{ + u16 min_rev = GPU_ID_ARCH_ID_MINOR_AND_REV_GET(gpu_id); + const char *ret = "unknown"; + + switch (GPU_ID_ARCH_ID_MAJOR_GET(gpu_id)) { + case 9: + ret = decode_lut_element_lookup(min_rev, idx, + lut_fault_source_core_type_t_name_major_9, + NELEMS(lut_fault_source_core_type_t_name_major_9)); + break; + case 10: + ret = decode_lut_element_lookup(min_rev, idx, + lut_fault_source_core_type_t_name_major_10, + NELEMS(lut_fault_source_core_type_t_name_major_10)); + break; + case 11: + ret = decode_lut_element_lookup(min_rev, idx, + lut_fault_source_core_type_t_name_major_11, + NELEMS(lut_fault_source_core_type_t_name_major_11)); + break; + case 12: + ret = decode_lut_element_lookup(min_rev, idx, + lut_fault_source_core_type_t_name_major_12, + NELEMS(lut_fault_source_core_type_t_name_major_12)); + break; + } + return ret; +} + +const char *decode_fault_source_core_type_t_desc(u16 idx, u32 gpu_id) +{ + u16 min_rev = GPU_ID_ARCH_ID_MINOR_AND_REV_GET(gpu_id); + const char *ret = "unknown"; + + switch (GPU_ID_ARCH_ID_MAJOR_GET(gpu_id)) { + case 9: + ret = decode_lut_element_lookup(min_rev, idx, + lut_fault_source_core_type_t_desc_major_9, + NELEMS(lut_fault_source_core_type_t_desc_major_9)); + break; + case 10: + ret = decode_lut_element_lookup(min_rev, idx, + lut_fault_source_core_type_t_desc_major_10, + NELEMS(lut_fault_source_core_type_t_desc_major_10)); + break; + case 11: + ret = decode_lut_element_lookup(min_rev, idx, + lut_fault_source_core_type_t_desc_major_11, + NELEMS(lut_fault_source_core_type_t_desc_major_11)); + break; + case 12: + ret = decode_lut_element_lookup(min_rev, idx, + lut_fault_source_core_type_t_desc_major_12, + NELEMS(lut_fault_source_core_type_t_desc_major_12)); + break; + } + return ret; +} + +const char *decode_fault_source_core_id_t_desc(u16 idx, u32 gpu_id) +{ + u16 min_rev = GPU_ID_ARCH_ID_MINOR_AND_REV_GET(gpu_id); + const char *ret = "unknown"; + + switch (GPU_ID_ARCH_ID_MAJOR_GET(gpu_id)) { + case 9: + ret = decode_lut_element_lookup(min_rev, idx, + lut_fault_source_core_id_t_desc_major_9, + NELEMS(lut_fault_source_core_id_t_desc_major_9)); + break; + case 10: + ret = decode_lut_element_lookup(min_rev, idx, + lut_fault_source_core_id_t_desc_major_10, + NELEMS(lut_fault_source_core_id_t_desc_major_10)); + break; + case 11: + ret = decode_lut_element_lookup(min_rev, idx, + lut_fault_source_core_id_t_desc_major_11, + NELEMS(lut_fault_source_core_id_t_desc_major_11)); + break; + case 12: + ret = decode_lut_element_lookup(min_rev, idx, + lut_fault_source_core_id_t_desc_major_12, + NELEMS(lut_fault_source_core_id_t_desc_major_12)); + break; + } + return ret; +} + +const char *decode_fault_source_core_id_t_core_type(u16 idx, u32 gpu_id) +{ + u16 min_rev = GPU_ID_ARCH_ID_MINOR_AND_REV_GET(gpu_id); + const char *ret = "unknown"; + + switch (GPU_ID_ARCH_ID_MAJOR_GET(gpu_id)) { + case 9: + ret = decode_lut_element_lookup( + min_rev, idx, lut_fault_source_core_id_t_core_type_major_9, + NELEMS(lut_fault_source_core_id_t_core_type_major_9)); + break; + case 10: + ret = decode_lut_element_lookup( + min_rev, idx, lut_fault_source_core_id_t_core_type_major_10, + NELEMS(lut_fault_source_core_id_t_core_type_major_10)); + break; + case 11: + ret = decode_lut_element_lookup( + min_rev, idx, lut_fault_source_core_id_t_core_type_major_11, + NELEMS(lut_fault_source_core_id_t_core_type_major_11)); + break; + case 12: + ret = decode_lut_element_lookup( + min_rev, idx, lut_fault_source_core_id_t_core_type_major_12, + NELEMS(lut_fault_source_core_id_t_core_type_major_12)); + break; + } + return ret; +} + +const char *decode_fault_source_shader_r_t(u16 idx, u32 gpu_id) +{ + u16 min_rev = GPU_ID_ARCH_ID_MINOR_AND_REV_GET(gpu_id); + const char *ret = "unknown"; + + switch (GPU_ID_ARCH_ID_MAJOR_GET(gpu_id)) { + case 9: + ret = decode_lut_element_lookup(min_rev, idx, lut_fault_source_shader_r_t_major_9, + NELEMS(lut_fault_source_shader_r_t_major_9)); + break; + case 10: + ret = decode_lut_element_lookup(min_rev, idx, lut_fault_source_shader_r_t_major_10, + NELEMS(lut_fault_source_shader_r_t_major_10)); + break; + case 11: + ret = decode_lut_element_lookup(min_rev, idx, lut_fault_source_shader_r_t_major_11, + NELEMS(lut_fault_source_shader_r_t_major_11)); + break; + case 12: + ret = decode_lut_element_lookup(min_rev, idx, lut_fault_source_shader_r_t_major_12, + NELEMS(lut_fault_source_shader_r_t_major_12)); + break; + } + return ret; +} + +const char *decode_fault_source_shader_w_t(u16 idx, u32 gpu_id) +{ + u16 min_rev = GPU_ID_ARCH_ID_MINOR_AND_REV_GET(gpu_id); + const char *ret = "unknown"; + + switch (GPU_ID_ARCH_ID_MAJOR_GET(gpu_id)) { + case 9: + ret = decode_lut_element_lookup(min_rev, idx, lut_fault_source_shader_w_t_major_9, + NELEMS(lut_fault_source_shader_w_t_major_9)); + break; + case 10: + ret = decode_lut_element_lookup(min_rev, idx, lut_fault_source_shader_w_t_major_10, + NELEMS(lut_fault_source_shader_w_t_major_10)); + break; + case 11: + ret = decode_lut_element_lookup(min_rev, idx, lut_fault_source_shader_w_t_major_11, + NELEMS(lut_fault_source_shader_w_t_major_11)); + break; + case 12: + ret = decode_lut_element_lookup(min_rev, idx, lut_fault_source_shader_w_t_major_12, + NELEMS(lut_fault_source_shader_w_t_major_12)); + break; + } + return ret; +} + +const char *decode_fault_source_tiler_r_t(u16 idx, u32 gpu_id) +{ + u16 min_rev = GPU_ID_ARCH_ID_MINOR_AND_REV_GET(gpu_id); + const char *ret = "unknown"; + + switch (GPU_ID_ARCH_ID_MAJOR_GET(gpu_id)) { + case 10: + ret = decode_lut_element_lookup(min_rev, idx, lut_fault_source_tiler_r_t_major_10, + NELEMS(lut_fault_source_tiler_r_t_major_10)); + break; + case 11: + ret = decode_lut_element_lookup(min_rev, idx, lut_fault_source_tiler_r_t_major_11, + NELEMS(lut_fault_source_tiler_r_t_major_11)); + break; + case 12: + ret = decode_lut_element_lookup(min_rev, idx, lut_fault_source_tiler_r_t_major_12, + NELEMS(lut_fault_source_tiler_r_t_major_12)); + break; + } + return ret; +} + +const char *decode_fault_source_tiler_w_t(u16 idx, u32 gpu_id) +{ + u16 min_rev = GPU_ID_ARCH_ID_MINOR_AND_REV_GET(gpu_id); + const char *ret = "unknown"; + + switch (GPU_ID_ARCH_ID_MAJOR_GET(gpu_id)) { + case 10: + ret = decode_lut_element_lookup(min_rev, idx, lut_fault_source_tiler_w_t_major_10, + NELEMS(lut_fault_source_tiler_w_t_major_10)); + break; + case 11: + ret = decode_lut_element_lookup(min_rev, idx, lut_fault_source_tiler_w_t_major_11, + NELEMS(lut_fault_source_tiler_w_t_major_11)); + break; + case 12: + ret = decode_lut_element_lookup(min_rev, idx, lut_fault_source_tiler_w_t_major_12, + NELEMS(lut_fault_source_tiler_w_t_major_12)); + break; + } + return ret; +} diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_faults_decoder_luts.h b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_faults_decoder_luts.h new file mode 100644 index 000000000000..2b0ca5659a6c --- /dev/null +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_faults_decoder_luts.h @@ -0,0 +1,119 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * + * (C) COPYRIGHT 2024 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +#ifndef _MALI_KBASE_MMU_FAULTS_DECODER_LUTS_H_ +#define _MALI_KBASE_MMU_FAULTS_DECODER_LUTS_H_ + +#include + +/** + * decode_fault_source_core_id_t_desc() - Get core description of a + * fault in a human readable format. + * + * @idx: Core ID part of SOURCE_ID field of the fault. + * @gpu_id: GPU id composed of arch_major << 16 | arch_minor << 8 | arch_rev. + * + * Return: core ID of the fault in human readable format. + */ +const char *decode_fault_source_core_id_t_desc(u16 idx, u32 gpu_id); + +/** + * decode_fault_source_core_id_t_core_type() - Get core type of a + * fault in a human readable format. + * + * @idx: Core ID part of SOURCE_ID field of the fault. + * @gpu_id: GPU id composed of arch_major << 16 | arch_minor << 8 | arch_rev. + * + * Return: core type of the fault in human readable format. + */ +const char *decode_fault_source_core_id_t_core_type(u16 idx, u32 gpu_id); + +/** + * decode_fault_source_core_type_t_name() - Get core type name of a + * fault. + * + * @idx: Core type part of SOURCE_ID field of the fault. + * @gpu_id: GPU id composed of arch_major << 16 | arch_minor << 8 | arch_rev. + * + * Return: core type short name of the fault. + */ +const char *decode_fault_source_core_type_t_name(u16 idx, u32 gpu_id); + +/** + * decode_fault_source_core_type_t_desc() - Get core type description of a + * fault. + * + * @idx: Core type part of SOURCE_ID field of the fault. + * @gpu_id: GPU id composed of arch_major << 16 | arch_minor << 8 | arch_rev. + * + * Return: core type description of the fault. + */ +const char *decode_fault_source_core_type_t_desc(u16 idx, u32 gpu_id); + +/** + * decode_fault_source_shader_r_t() - Get internal requester of a + * fault in a human readable format. + * + * @idx: Internal requester part of SOURCE_ID field of the fault. + * @gpu_id: GPU id composed of arch_major << 16 | arch_minor << 8 | arch_rev. + * + * Return: Internal requester of a fault in a human readable format for read + * operations on a shader core. + */ +const char *decode_fault_source_shader_r_t(u16 idx, u32 gpu_id); + +/** + * decode_fault_source_shader_w_t() - Get internal requester of a + * fault in a human readable format. + * + * @idx: Internal requester part of SOURCE_ID field of the fault. + * @gpu_id: GPU id composed of arch_major << 16 | arch_minor << 8 | arch_rev. + * + * Return: Internal requester of a fault in a human readable format for write + * operations on a shader core. + */ +const char *decode_fault_source_shader_w_t(u16 idx, u32 gpu_id); + +/** + * decode_fault_source_tiler_r_t() - Get internal requester of a + * fault in a human readable format. + * + * @idx: Internal requester part of SOURCE_ID field of the fault. + * @gpu_id: GPU id composed of arch_major << 16 | arch_minor << 8 | arch_rev. + * + * Return: Internal requester of a fault in a human readable format for read + * operations on a tiler core. + */ +const char *decode_fault_source_tiler_r_t(u16 idx, u32 gpu_id); + +/** + * decode_fault_source_tiler_w_t() - Get internal requester of a + * fault in a human readable format. + * + * @idx: Internal requester part of SOURCE_ID field of the fault. + * @gpu_id: GPU id composed of arch_major << 16 | arch_minor << 8 | arch_rev. + * + * Return: Internal requester of a fault in a human readable format for write + * operations on a tiler core. + */ +const char *decode_fault_source_tiler_w_t(u16 idx, u32 gpu_id); + +#endif /* _MALI_KBASE_MMU_FAULTS_DECODER_LUTS_H_ */ diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_aarch64.c b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_aarch64.c index d19579da2f5d..e3ad78daed59 100644 --- a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_aarch64.c +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_aarch64.c @@ -32,7 +32,7 @@ */ #define ENTRY_IS_ATE_L3 3ULL #define ENTRY_IS_ATE_L02 1ULL -#define ENTRY_IS_INVAL 2ULL +#define ENTRY_IS_INVAL 0ULL #define ENTRY_IS_PTE 3ULL #define ENTRY_ACCESS_RW (1ULL << 6) /* bits 6:7 */ diff --git a/drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_runtime_pm.c b/drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_runtime_pm.c index 2a5030745586..d0342af60fb3 100644 --- a/drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_runtime_pm.c +++ b/drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_runtime_pm.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2015-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2015-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/drivers/gpu/arm/bifrost/tests/Kbuild b/drivers/gpu/arm/bifrost/tests/Kbuild index 72ca70ac8779..479b91532ed7 100644 --- a/drivers/gpu/arm/bifrost/tests/Kbuild +++ b/drivers/gpu/arm/bifrost/tests/Kbuild @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2017-2023 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2017-2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software diff --git a/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_kprobe.h b/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_kprobe.h index f75cd776c60e..d8c3ca88166b 100644 --- a/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_kprobe.h +++ b/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_kprobe.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -22,6 +22,8 @@ #ifndef _KUTF_KPROBE_H_ #define _KUTF_KPROBE_H_ +struct dentry; + int kutf_kprobe_init(struct dentry *base_dir); void kutf_kprobe_exit(void); @@ -30,4 +32,6 @@ typedef void (*kutf_kp_handler)(int argc, char **argv); void kutf_kp_sample_handler(int argc, char **argv); void kutf_kp_sample_kernel_function(void); +void kutf_kp_delay_handler(int argc, char **argv); + #endif /* _KUTF_KPROBE_H_ */ diff --git a/drivers/gpu/arm/bifrost/tests/kutf/kutf_kprobe.c b/drivers/gpu/arm/bifrost/tests/kutf/kutf_kprobe.c index f118692c43a1..232809e1ed58 100644 --- a/drivers/gpu/arm/bifrost/tests/kutf/kutf_kprobe.c +++ b/drivers/gpu/arm/bifrost/tests/kutf/kutf_kprobe.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -29,6 +29,7 @@ #include #include #include +#include #include #define KUTF_KP_REG_MIN_ARGS 3 @@ -86,6 +87,19 @@ const struct file_operations kutf_kp_unreg_debugfs_fops = { struct kprobe kutf_kallsym_kp = { .symbol_name = "kallsyms_lookup_name" }; +void kutf_kp_delay_handler(int argc, char **argv) +{ + long delay; + + if ((!argv) || (!argv[0])) + return; + + if (kstrtol(argv[0], 0, &delay)) + return; + + mdelay(delay); +} + void kutf_kp_sample_kernel_function(void) { pr_debug("%s called\n", __func__); @@ -150,11 +164,9 @@ static ssize_t kutf_kp_reg_debugfs_write(struct file *file, const char __user *u if (count >= KUTF_KP_WRITE_BUFSIZE) return -EINVAL; - kbuf = memdup_user(user_buf, count); - if (IS_ERR(kbuf)) { + kbuf = memdup_user_nul(user_buf, count); + if (IS_ERR(kbuf)) return -ENOMEM; - } - kbuf[count - 1] = '\0'; argv = argv_split(GFP_KERNEL, kbuf, &argc); if (!argv) { @@ -245,11 +257,9 @@ static ssize_t kutf_kp_unreg_debugfs_write(struct file *file, const char __user if (count >= KUTF_KP_WRITE_BUFSIZE) return -EINVAL; - kbuf = memdup_user(user_buf, count); - if (IS_ERR(kbuf)) { + kbuf = memdup_user_nul(user_buf, count); + if (IS_ERR(kbuf)) return -ENOMEM; - } - kbuf[count - 1] = '\0'; argv = argv_split(GFP_KERNEL, kbuf, &argc); if (!argv) { diff --git a/drivers/gpu/arm/bifrost/thirdparty/mali_kbase_mmap.c b/drivers/gpu/arm/bifrost/thirdparty/mali_kbase_mmap.c index 1592eab806ac..cfb347affa2e 100644 --- a/drivers/gpu/arm/bifrost/thirdparty/mali_kbase_mmap.c +++ b/drivers/gpu/arm/bifrost/thirdparty/mali_kbase_mmap.c @@ -20,18 +20,169 @@ * kbase_context_get_unmapped_area() interface. */ +#if (KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE) +/** + * move_mt_gap() - Search the maple tree for an existing gap of a particular size + * immediately before another pre-identified gap. + * @gap_start: Pre-identified gap starting address. + * @gap_end: Pre-identified gap ending address. + * @size: Size of the new gap needed before gap_start. + * + * This function will search the calling process' maple tree + * for another gap, one that is immediately preceding the pre-identified + * gap, for a specific size, and upon success it will decrement gap_end + * by the specified size, and replace gap_start with the new gap_start of + * the newly identified gap. + * + * Return: true if large enough preceding gap is found, false otherwise. + */ +static bool move_mt_gap(unsigned long *gap_start, unsigned long *gap_end, unsigned long size) +{ + unsigned long new_gap_start, new_gap_end; + + MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); + + if (*gap_end < size) + return false; + + /* Calculate the gap end for the new, resultant gap */ + new_gap_end = *gap_end - size; + + /* If the new gap_end (i.e. new VA start address) is larger than gap_start, than the + * pre-identified gap already has space to shrink to accommodate the decrease in + * gap_end. + */ + if (new_gap_end >= *gap_start) { + /* Pre-identified gap already has space - just patch gap_end to new + * lower value and exit. + */ + *gap_end = new_gap_end; + return true; + } + + /* Since the new VA start address (new_gap_end) is below the start of the pre-identified + * gap in the maple tree, see if there is a free gap directly before the existing gap, of + * the same size as the alignment shift, such that the effective gap found is "extended". + * This may be larger than needed but leaves the same distance between gap_end and gap_start + * that currently exists. + */ + new_gap_start = *gap_start - size; + if (mas_empty_area_rev(&mas, new_gap_start, *gap_start - 1, size)) { + /* There's no gap between the new start address needed and the + * current start address - so return false to find a new + * gap from the maple tree. + */ + return false; + } + /* Suitable gap found - replace gap_start and gap_end with new values. gap_start takes the + * value of the start of new gap found, which now correctly precedes gap_end, and gap_end + * takes on the new aligned value that has now been decremented by the requested size. + */ + *gap_start = mas.index; + *gap_end = new_gap_end; + return true; +} + /** * align_and_check() - Align the specified pointer to the provided alignment and - * check that it is still in range. - * @gap_end: Highest possible start address for allocation (end of gap in - * address space) - * @gap_start: Start address of current memory area / gap in address space - * @info: vm_unmapped_area_info structure passed to caller, containing - * alignment, length and limits for the allocation - * @is_shader_code: True if the allocation is for shader code (which has - * additional alignment requirements) - * @is_same_4gb_page: True if the allocation needs to reside completely within - * a 4GB chunk + * check that it is still in range. On kernel 6.1 onwards + * this function does not require that the initial requested + * gap is extended with the maximum size needed to guarantee + * an alignment. + * @gap_end: Highest possible start address for allocation (end of gap in + * address space) + * @gap_start: Start address of current memory area / gap in address space + * @info: vm_unmapped_area_info structure passed to caller, containing + * alignment, length and limits for the allocation + * @is_shader_code: True if the allocation is for shader code (which has + * additional alignment requirements) + * @is_same_4gb_page: True if the allocation needs to reside completely within + * a 4GB chunk + * + * Return: true if gap_end is now aligned correctly and is still in range, + * false otherwise + */ +static bool align_and_check(unsigned long *gap_end, unsigned long gap_start, + struct vm_unmapped_area_info *info, bool is_shader_code, + bool is_same_4gb_page) +{ + unsigned long alignment_shift; + + /* Compute highest gap address at the desired alignment */ + *gap_end -= info->length; + alignment_shift = (*gap_end - info->align_offset) & info->align_mask; + + /* Align desired start VA (gap_end) by calculated alignment shift amount */ + if (!move_mt_gap(&gap_start, gap_end, alignment_shift)) + return false; + /* Alignment is done so far - check for further alignment requirements */ + + if (is_shader_code) { + /* Shader code allocations must not start or end on a 4GB boundary */ + alignment_shift = info->align_offset ? info->align_offset : info->length; + if (0 == (*gap_end & BASE_MEM_MASK_4GB)) { + if (!move_mt_gap(&gap_start, gap_end, alignment_shift)) + return false; + } + if (0 == ((*gap_end + info->length) & BASE_MEM_MASK_4GB)) { + if (!move_mt_gap(&gap_start, gap_end, alignment_shift)) + return false; + } + + if (!(*gap_end & BASE_MEM_MASK_4GB) || + !((*gap_end + info->length) & BASE_MEM_MASK_4GB)) + return false; + } else if (is_same_4gb_page) { + unsigned long start = *gap_end; + unsigned long end = *gap_end + info->length; + unsigned long mask = ~((unsigned long)U32_MAX); + + /* Check if 4GB boundary is straddled */ + if ((start & mask) != ((end - 1) & mask)) { + unsigned long offset = end - (end & mask); + /* This is to ensure that alignment doesn't get + * disturbed in an attempt to prevent straddling at + * 4GB boundary. The GPU VA is aligned to 2MB when the + * allocation size is > 2MB and there is enough CPU & + * GPU virtual space. + */ + unsigned long rounded_offset = ALIGN(offset, info->align_mask + 1); + + if (!move_mt_gap(&gap_start, gap_end, rounded_offset)) + return false; + /* Re-calculate start and end values */ + start = *gap_end; + end = *gap_end + info->length; + + /* The preceding 4GB boundary shall not get straddled, + * even after accounting for the alignment, as the + * size of allocation is limited to 4GB and the initial + * start location was already aligned. + */ + WARN_ON((start & mask) != ((end - 1) & mask)); + } + } + + if ((*gap_end < info->low_limit) || (*gap_end < gap_start)) + return false; + + return true; +} +#else +/** + * align_and_check() - Align the specified pointer to the provided alignment and + * check that it is still in range. For Kernel versions below + * 6.1, it requires that the length of the alignment is already + * extended by a worst-case alignment mask. + * @gap_end: Highest possible start address for allocation (end of gap in + * address space) + * @gap_start: Start address of current memory area / gap in address space + * @info: vm_unmapped_area_info structure passed to caller, containing + * alignment, length and limits for the allocation + * @is_shader_code: True if the allocation is for shader code (which has + * additional alignment requirements) + * @is_same_4gb_page: True if the allocation needs to reside completely within + * a 4GB chunk * * Return: true if gap_end is now aligned correctly and is still in range, * false otherwise @@ -41,8 +192,8 @@ static bool align_and_check(unsigned long *gap_end, unsigned long gap_start, bool is_same_4gb_page) { /* Compute highest gap address at the desired alignment */ - (*gap_end) -= info->length; - (*gap_end) -= (*gap_end - info->align_offset) & info->align_mask; + *gap_end -= info->length; + *gap_end -= (*gap_end - info->align_offset) & info->align_mask; if (is_shader_code) { /* Check for 4GB boundary */ @@ -73,6 +224,7 @@ static bool align_and_check(unsigned long *gap_end, unsigned long gap_start, start -= rounded_offset; end -= rounded_offset; + /* Patch gap_end to use new starting address for VA region */ *gap_end = start; /* The preceding 4GB boundary shall not get straddled, @@ -89,6 +241,7 @@ static bool align_and_check(unsigned long *gap_end, unsigned long gap_start, return true; } +#endif /** * kbase_unmapped_area_topdown() - allocates new areas top-down from @@ -218,31 +371,27 @@ check_current: } } #else - unsigned long length, high_limit, gap_start, gap_end; + unsigned long high_limit, gap_start, gap_end; MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); - /* Adjust search length to account for worst case alignment overhead */ - length = info->length + info->align_mask; - if (length < info->length) - return -ENOMEM; /* * Adjust search limits by the desired length. * See implementation comment at top of unmapped_area(). */ gap_end = info->high_limit; - if (gap_end < length) + if (gap_end < info->length) return -ENOMEM; - high_limit = gap_end - length; + high_limit = gap_end - info->length; if (info->low_limit > high_limit) return -ENOMEM; while (true) { - if (mas_empty_area_rev(&mas, info->low_limit, info->high_limit - 1, length)) + if (mas_empty_area_rev(&mas, info->low_limit, info->high_limit - 1, info->length)) return -ENOMEM; gap_end = mas.last + 1; - gap_start = mas.min; + gap_start = mas.index; if (align_and_check(&gap_end, gap_start, info, is_shader_code, is_same_4gb_page)) return gap_end; @@ -368,7 +517,7 @@ unsigned long kbase_context_get_unmapped_area(struct kbase_context *const kctx, kbase_gpu_vm_unlock(kctx); #ifndef CONFIG_64BIT } else { - return current->mm->get_unmapped_area(kctx->kfile->filp, addr, len, pgoff, flags); + return current->mm->get_unmapped_area(kctx->filp, addr, len, pgoff, flags); #endif } diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.c b/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.c index 742735846d49..34cabbd6e535 100644 --- a/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.c +++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.h b/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.h index b2cbfe6e528d..987d4d3dbf27 100644 --- a/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.h +++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/drivers/hwtracing/coresight/mali/Makefile b/drivers/hwtracing/coresight/mali/Makefile index 923cb0c910d9..d8186bee6e64 100644 --- a/drivers/hwtracing/coresight/mali/Makefile +++ b/drivers/hwtracing/coresight/mali/Makefile @@ -79,9 +79,9 @@ ifeq ($(MALI_KCONFIG_EXT_PREFIX),) endif EXTRA_SYMBOLS += \ - $(M)/../../../base/arm/Module.symvers \ $(GPU_SYMBOLS) + # The following were added to align with W=1 in scripts/Makefile.extrawarn # from the Linux source tree CFLAGS_MODULE += -Wall -Werror @@ -99,6 +99,8 @@ CFLAGS_MODULE += $(call cc-option, -Wstringop-truncation) CFLAGS_MODULE += -Wno-missing-field-initializers CFLAGS_MODULE += -Wno-sign-compare CFLAGS_MODULE += -Wno-type-limits +# The following ensures the stack frame does not get larger than a page +CFLAGS_MODULE += -Wframe-larger-than=4096 KBUILD_CPPFLAGS += -DKBUILD_EXTRA_WARN1 diff --git a/drivers/hwtracing/coresight/mali/sources/coresight_mali_sources.c b/drivers/hwtracing/coresight/mali/sources/coresight_mali_sources.c index e6d2dc71096b..247a8b47f05b 100644 --- a/drivers/hwtracing/coresight/mali/sources/coresight_mali_sources.c +++ b/drivers/hwtracing/coresight/mali/sources/coresight_mali_sources.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2022 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2022-2023 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -50,7 +50,11 @@ static void coresight_mali_disable_source(struct coresight_device *csdev, struct } static const struct coresight_ops_source coresight_mali_source_ops = { +#if KERNEL_VERSION(6, 3, 0) <= LINUX_VERSION_CODE + .cpu_id = coresight_mali_source_trace_id, +#else .trace_id = coresight_mali_source_trace_id, +#endif .enable = coresight_mali_enable_source, .disable = coresight_mali_disable_source }; diff --git a/drivers/xen/arm/Makefile b/drivers/xen/arm/Makefile index b2ee53723428..27bee59ac787 100644 --- a/drivers/xen/arm/Makefile +++ b/drivers/xen/arm/Makefile @@ -78,6 +78,8 @@ CFLAGS_MODULE += $(call cc-option, -Wstringop-truncation) CFLAGS_MODULE += -Wno-missing-field-initializers CFLAGS_MODULE += -Wno-sign-compare CFLAGS_MODULE += -Wno-type-limits +# The following ensures the stack frame does not get larger than a page +CFLAGS_MODULE += -Wframe-larger-than=4096 KBUILD_CPPFLAGS += -DKBUILD_EXTRA_WARN1 diff --git a/include/linux/mali_arbiter_interface.h b/include/linux/mali_arbiter_interface.h index b4162f86ebb4..ae44e82ae6dd 100644 --- a/include/linux/mali_arbiter_interface.h +++ b/include/linux/mali_arbiter_interface.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -26,6 +26,8 @@ #ifndef _MALI_KBASE_ARBITER_INTERFACE_H_ #define _MALI_KBASE_ARBITER_INTERFACE_H_ +#include + /** * DOC: Mali arbiter interface version * diff --git a/include/linux/mali_hw_access.h b/include/linux/mali_hw_access.h new file mode 100644 index 000000000000..4ed9da994e57 --- /dev/null +++ b/include/linux/mali_hw_access.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * + * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +#ifndef _MALI_HW_ACCESS_H_ +#define _MALI_HW_ACCESS_H_ + +#include +#include + + +#define mali_readl(addr) readl(addr) + +#define mali_writel(val, addr) writel(val, addr) + +#define mali_readq(addr) ((u64)mali_readl(addr) | ((u64)mali_readl(addr + 4) << 32)) + +static inline u64 mali_readq_coherent(const void __iomem *addr) +{ + u32 hi1, hi2, lo; + + do { + hi1 = mali_readl(addr + 4); + lo = mali_readl(addr); + hi2 = mali_readl(addr + 4); + } while (hi1 != hi2); + + return lo | (((u64)hi1) << 32); +} + +#define mali_writeq(val, addr) \ + do { \ + mali_writel(val & 0xFFFFFFFF, addr); \ + mali_writel(val >> 32, addr + 4); \ + } while (0) + +#define mali_ioremap(addr, size) ioremap(addr, size) + +#define mali_iounmap(addr) iounmap(addr) + +#define mali_arch_timer_get_cntfrq() arch_timer_get_cntfrq() + + +#endif /* _MALI_HW_ACCESS_H_ */ diff --git a/include/linux/memory_group_manager.h b/include/linux/memory_group_manager.h index 3820f1bff86b..e92d3dea0178 100644 --- a/include/linux/memory_group_manager.h +++ b/include/linux/memory_group_manager.h @@ -32,6 +32,10 @@ typedef int vm_fault_t; #define MEMORY_GROUP_MANAGER_NR_GROUPS (16) +#define PTE_PBHA_SHIFT (59) +#define PTE_PBHA_MASK ((uint64_t)0xf << PTE_PBHA_SHIFT) +#define PTE_RES_BIT_MULTI_AS_SHIFT (63) + struct memory_group_manager_device; struct memory_group_manager_import_data; diff --git a/include/linux/version_compat_defs.h b/include/linux/version_compat_defs.h index 366b50c4e3ec..46f227e108f6 100644 --- a/include/linux/version_compat_defs.h +++ b/include/linux/version_compat_defs.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2022-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2022-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -348,4 +348,50 @@ static inline long kbase_pin_user_pages_remote(struct task_struct *tsk, struct m #endif /* (KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE) */ +#if (KERNEL_VERSION(4, 16, 0) > LINUX_VERSION_CODE) +/* Null definition */ +#define ALLOW_ERROR_INJECTION(fname, err_type) +#endif /* (KERNEL_VERSION(4, 16, 0) > LINUX_VERSION_CODE) */ + +#if KERNEL_VERSION(6, 0, 0) > LINUX_VERSION_CODE +#define KBASE_REGISTER_SHRINKER(reclaim, name, priv_data) register_shrinker(reclaim) + +#elif ((KERNEL_VERSION(6, 7, 0) > LINUX_VERSION_CODE) && \ + !(defined(__ANDROID_COMMON_KERNEL__) && (KERNEL_VERSION(6, 6, 0) == LINUX_VERSION_CODE))) +#define KBASE_REGISTER_SHRINKER(reclaim, name, priv_data) register_shrinker(reclaim, name) + +#else +#define KBASE_REGISTER_SHRINKER(reclaim, name, priv_data) \ + do { \ + reclaim->private_data = priv_data; \ + shrinker_register(reclaim); \ + } while (0) + +#endif /* KERNEL_VERSION(6, 0, 0) > LINUX_VERSION_CODE */ + +#if ((KERNEL_VERSION(6, 7, 0) > LINUX_VERSION_CODE) && \ + !(defined(__ANDROID_COMMON_KERNEL__) && (KERNEL_VERSION(6, 6, 0) == LINUX_VERSION_CODE))) +#define KBASE_UNREGISTER_SHRINKER(reclaim) unregister_shrinker(&reclaim) +#define KBASE_GET_KBASE_DATA_FROM_SHRINKER(s, type, var) container_of(s, type, var) +#define DEFINE_KBASE_SHRINKER struct shrinker +#define KBASE_INIT_RECLAIM(var, attr, name) (&((var)->attr)) +#define KBASE_SET_RECLAIM(var, attr, reclaim) ((var)->attr = (*reclaim)) + +#else +#define KBASE_UNREGISTER_SHRINKER(reclaim) shrinker_free(reclaim) +#define KBASE_GET_KBASE_DATA_FROM_SHRINKER(s, type, var) s->private_data +#define DEFINE_KBASE_SHRINKER struct shrinker * +#define KBASE_SHRINKER_ALLOC(name) shrinker_alloc(0, name) +#define KBASE_INIT_RECLAIM(var, attr, name) (KBASE_SHRINKER_ALLOC(name)) +#define KBASE_SET_RECLAIM(var, attr, reclaim) ((var)->attr = reclaim) + +#endif + +#if (KERNEL_VERSION(4, 20, 0) <= LINUX_VERSION_CODE) +#include +#endif +#ifndef __maybe_unused +#define __maybe_unused __attribute__((unused)) +#endif + #endif /* _VERSION_COMPAT_DEFS_H_ */ diff --git a/include/uapi/gpu/arm/bifrost/backend/gpu/mali_kbase_model_dummy.h b/include/uapi/gpu/arm/bifrost/backend/gpu/mali_kbase_model_dummy.h index 564f477e57d1..b80817f04255 100644 --- a/include/uapi/gpu/arm/bifrost/backend/gpu/mali_kbase_model_dummy.h +++ b/include/uapi/gpu/arm/bifrost/backend/gpu/mali_kbase_model_dummy.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2021-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2021-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/include/uapi/gpu/arm/bifrost/csf/mali_base_csf_kernel.h b/include/uapi/gpu/arm/bifrost/csf/mali_base_csf_kernel.h index 0fb824267184..b4b38f242bd2 100644 --- a/include/uapi/gpu/arm/bifrost/csf/mali_base_csf_kernel.h +++ b/include/uapi/gpu/arm/bifrost/csf/mali_base_csf_kernel.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -46,7 +46,11 @@ */ #define BASE_MEM_CSF_EVENT ((base_mem_alloc_flags)1 << 19) -#define BASE_MEM_RESERVED_BIT_20 ((base_mem_alloc_flags)1 << 20) +/* Unused bit for CSF, only used in JM for BASE_MEM_TILER_ALIGN_TOP */ +#define BASE_MEM_UNUSED_BIT_20 ((base_mem_alloc_flags)1 << 20) + +/* Unused bit for CSF, only used in JM for BASE_MEM_FLAG_MAP_FIXED */ +#define BASE_MEM_UNUSED_BIT_27 ((base_mem_alloc_flags)1 << 27) /* Must be FIXABLE memory: its GPU VA will be determined at a later point, * at which time it will be at a fixed GPU VA. @@ -62,9 +66,14 @@ */ #define BASEP_MEM_FLAGS_KERNEL_ONLY (BASEP_MEM_PERMANENT_KERNEL_MAPPING | BASEP_MEM_NO_USER_FREE) -/* A mask of all currently reserved flags - */ -#define BASE_MEM_FLAGS_RESERVED BASE_MEM_RESERVED_BIT_20 +/* A mask of all flags that should not be queried */ +#define BASE_MEM_DONT_QUERY (BASE_MEM_COHERENT_SYSTEM_REQUIRED | BASE_MEM_IMPORT_SHARED) + +/* A mask of all currently reserved flags */ +#define BASE_MEM_FLAGS_RESERVED ((base_mem_alloc_flags)0) + +/* A mask of all bits that are not used by a flag on CSF */ +#define BASE_MEM_FLAGS_UNUSED (BASE_MEM_UNUSED_BIT_20 | BASE_MEM_UNUSED_BIT_27) /* Special base mem handles specific to CSF. */ @@ -474,7 +483,26 @@ struct base_gpu_queue_error_fatal_payload { }; /** - * enum base_gpu_queue_group_error_type - GPU Fatal error type. + * struct base_gpu_queue_error_fault_payload - Recoverable fault + * error information related to GPU command queue. + * + * @sideband: Additional information about this recoverable fault. + * @status: Recoverable fault information. + * This consists of exception type (least significant byte) and + * data (remaining bytes). One example of exception type is + * INSTR_INVALID_PC (0x50). + * @csi_index: Index of the CSF interface the queue is bound to. + * @padding: Padding to make multiple of 64bits + */ +struct base_gpu_queue_error_fault_payload { + __u64 sideband; + __u32 status; + __u8 csi_index; + __u8 padding[3]; +}; + +/** + * enum base_gpu_queue_group_error_type - GPU error type. * * @BASE_GPU_QUEUE_GROUP_ERROR_FATAL: Fatal error associated with GPU * command queue group. @@ -484,7 +512,9 @@ struct base_gpu_queue_error_fatal_payload { * progress timeout. * @BASE_GPU_QUEUE_GROUP_ERROR_TILER_HEAP_OOM: Fatal error due to running out * of tiler heap memory. - * @BASE_GPU_QUEUE_GROUP_ERROR_FATAL_COUNT: The number of fatal error types + * @BASE_GPU_QUEUE_GROUP_QUEUE_ERROR_FAULT: Fault error associated with GPU + * command queue. + * @BASE_GPU_QUEUE_GROUP_ERROR_FATAL_COUNT: The number of GPU error types * * This type is used for &struct_base_gpu_queue_group_error.error_type. */ @@ -493,6 +523,7 @@ enum base_gpu_queue_group_error_type { BASE_GPU_QUEUE_GROUP_QUEUE_ERROR_FATAL, BASE_GPU_QUEUE_GROUP_ERROR_TIMEOUT, BASE_GPU_QUEUE_GROUP_ERROR_TILER_HEAP_OOM, + BASE_GPU_QUEUE_GROUP_QUEUE_ERROR_FAULT, BASE_GPU_QUEUE_GROUP_ERROR_FATAL_COUNT }; @@ -512,6 +543,7 @@ struct base_gpu_queue_group_error { union { struct base_gpu_queue_group_error_fatal_payload fatal_group; struct base_gpu_queue_error_fatal_payload fatal_queue; + struct base_gpu_queue_error_fault_payload fault_queue; } payload; }; @@ -519,8 +551,7 @@ struct base_gpu_queue_group_error { * enum base_csf_notification_type - Notification type * * @BASE_CSF_NOTIFICATION_EVENT: Notification with kernel event - * @BASE_CSF_NOTIFICATION_GPU_QUEUE_GROUP_ERROR: Notification with GPU fatal - * error + * @BASE_CSF_NOTIFICATION_GPU_QUEUE_GROUP_ERROR: Notification with GPU error * @BASE_CSF_NOTIFICATION_CPU_QUEUE_DUMP: Notification with dumping cpu * queue * @BASE_CSF_NOTIFICATION_COUNT: The number of notification type diff --git a/include/uapi/gpu/arm/bifrost/csf/mali_kbase_csf_ioctl.h b/include/uapi/gpu/arm/bifrost/csf/mali_kbase_csf_ioctl.h index 537c90d6efa5..28e7db49f676 100644 --- a/include/uapi/gpu/arm/bifrost/csf/mali_kbase_csf_ioctl.h +++ b/include/uapi/gpu/arm/bifrost/csf/mali_kbase_csf_ioctl.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -95,15 +95,31 @@ * 1.22: * - Add comp_pri_threshold and comp_pri_ratio attributes to * kbase_ioctl_cs_queue_group_create. + * - Made the BASE_MEM_DONT_NEED memory flag queryable. * 1.23: * - Disallows changing the sharability on the GPU of imported dma-bufs to * BASE_MEM_COHERENT_SYSTEM using KBASE_IOCTL_MEM_FLAGS_CHANGE. * 1.24: * - Implement full block state support for hardware counters. + * 1.25: + * - Add support for CS_FAULT reporting to userspace + * 1.26: + * - Made the BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP and BASE_MEM_KERNEL_SYNC memory + * flags queryable. + * 1.27: + * - Implement support for HWC block state availability. + * 1.28: + * - Made the SAME_VA memory flag queryable. + * 1.29: + * - Re-allow child process to do supported file operations (like mmap, ioctl + * read, poll) on the file descriptor of mali device that was inherited + * from the parent process. + * 1.30: + * - Implement support for setting GPU Timestamp Offset register. */ #define BASE_UK_VERSION_MAJOR 1 -#define BASE_UK_VERSION_MINOR 24 +#define BASE_UK_VERSION_MINOR 30 /** * struct kbase_ioctl_version_check - Check version compatibility between @@ -340,6 +356,8 @@ union kbase_ioctl_cs_queue_group_create_1_18 { * @in.csi_handlers: Flags to signal that the application intends to use CSI * exception handlers in some linear buffers to deal with * the given exception types. + * @in.cs_fault_report_enable: Flag to indicate reporting of CS_FAULTs + * to userspace. * @in.padding: Currently unused, must be zero * @out: Output parameters * @out.group_handle: Handle of a newly created queue group. @@ -360,7 +378,8 @@ union kbase_ioctl_cs_queue_group_create { /** * @in.reserved: Reserved, currently unused, must be zero. */ - __u16 reserved; + __u8 reserved; + __u8 cs_fault_report_enable; /** * @in.dvs_buf: buffer for deferred vertex shader */ @@ -480,7 +499,7 @@ union kbase_ioctl_cs_tiler_heap_init { /** * union kbase_ioctl_cs_tiler_heap_init_1_13 - Initialize chunked tiler memory heap, - * earlier version upto 1.13 + * earlier version up to 1.13 * @in: Input parameters * @in.chunk_size: Size of each chunk. * @in.initial_chunks: Initial number of chunks that heap will be created with. @@ -637,6 +656,22 @@ union kbase_ioctl_read_user_page { #define KBASE_IOCTL_READ_USER_PAGE _IOWR(KBASE_IOCTL_TYPE, 60, union kbase_ioctl_read_user_page) +/** + * struct kbase_ioctl_queue_group_clear_faults - Re-enable CS FAULT reporting for the GPU queues + * + * @addr: CPU VA to an array of GPU VAs of the buffers backing the queues + * @nr_queues: Number of queues in the array + * @padding: Padding to round up to a multiple of 8 bytes, must be zero + */ +struct kbase_ioctl_queue_group_clear_faults { + __u64 addr; + __u32 nr_queues; + __u8 padding[4]; +}; + +#define KBASE_IOCTL_QUEUE_GROUP_CLEAR_FAULTS \ + _IOW(KBASE_IOCTL_TYPE, 61, struct kbase_ioctl_queue_group_clear_faults) + /*************** * test ioctls * ***************/ diff --git a/include/uapi/gpu/arm/bifrost/gpu/mali_kbase_gpu_id.h b/include/uapi/gpu/arm/bifrost/gpu/mali_kbase_gpu_id.h index d3478546e244..d4d12aed780d 100644 --- a/include/uapi/gpu/arm/bifrost/gpu/mali_kbase_gpu_id.h +++ b/include/uapi/gpu/arm/bifrost/gpu/mali_kbase_gpu_id.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2015-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2015-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/include/uapi/gpu/arm/bifrost/jm/mali_base_jm_kernel.h b/include/uapi/gpu/arm/bifrost/jm/mali_base_jm_kernel.h index 9478334ce667..b5351c50aa30 100644 --- a/include/uapi/gpu/arm/bifrost/jm/mali_base_jm_kernel.h +++ b/include/uapi/gpu/arm/bifrost/jm/mali_base_jm_kernel.h @@ -30,15 +30,11 @@ * See base_mem_alloc_flags. */ -/* Used as BASE_MEM_FIXED in other backends */ -#define BASE_MEM_RESERVED_BIT_8 ((base_mem_alloc_flags)1 << 8) +/* Unused bit for JM, only used in CSF for BASE_MEM_FIXED */ +#define BASE_MEM_UNUSED_BIT_8 ((base_mem_alloc_flags)1 << 8) -/** - * BASE_MEM_RESERVED_BIT_19 - Bit 19 is reserved. - * - * Do not remove, use the next unreserved bit for new flags - */ -#define BASE_MEM_RESERVED_BIT_19 ((base_mem_alloc_flags)1 << 19) +/* Unused bit for JM, only used in CSF for BASE_CSF_EVENT */ +#define BASE_MEM_UNUSED_BIT_19 ((base_mem_alloc_flags)1 << 19) /** * BASE_MEM_TILER_ALIGN_TOP - Memory starting from the end of the initial commit is aligned @@ -64,9 +60,14 @@ (BASEP_MEM_PERMANENT_KERNEL_MAPPING | BASEP_MEM_NO_USER_FREE | BASE_MEM_FLAG_MAP_FIXED | \ BASEP_MEM_PERFORM_JIT_TRIM) -/* A mask of all currently reserved flags - */ -#define BASE_MEM_FLAGS_RESERVED (BASE_MEM_RESERVED_BIT_8 | BASE_MEM_RESERVED_BIT_19) +/* A mask of all flags that should not be queried */ +#define BASE_MEM_DONT_QUERY (BASE_MEM_COHERENT_SYSTEM_REQUIRED | BASE_MEM_IMPORT_SHARED) + +/* A mask of all currently reserved flags */ +#define BASE_MEM_FLAGS_RESERVED ((base_mem_alloc_flags)0) + +/* A mask of all bits that are not used by a flag on JM */ +#define BASE_MEM_FLAGS_UNUSED (BASE_MEM_UNUSED_BIT_8 | BASE_MEM_UNUSED_BIT_19) /* Similar to BASE_MEM_TILER_ALIGN_TOP, memory starting from the end of the * initial commit is aligned to 'extension' pages, where 'extension' must be a power diff --git a/include/uapi/gpu/arm/bifrost/jm/mali_kbase_jm_ioctl.h b/include/uapi/gpu/arm/bifrost/jm/mali_kbase_jm_ioctl.h index 2a7a06a995be..1c115adb8172 100644 --- a/include/uapi/gpu/arm/bifrost/jm/mali_kbase_jm_ioctl.h +++ b/include/uapi/gpu/arm/bifrost/jm/mali_kbase_jm_ioctl.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -149,15 +149,25 @@ * from the parent process. * 11.40: * - Remove KBASE_IOCTL_HWCNT_READER_SETUP and KBASE_HWCNT_READER_* ioctls. + * - Made the BASE_MEM_DONT_NEED memory flag queryable. * 11.41: * - Disallows changing the sharability on the GPU of imported dma-bufs to * BASE_MEM_COHERENT_SYSTEM using KBASE_IOCTL_MEM_FLAGS_CHANGE. * 11.42: * - Implement full block state support for hardware counters. + * 11.43: + * - Made the BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP and BASE_MEM_KERNEL_SYNC memory + * flags queryable. + * 11.44: + * - Made the SAME_VA memory flag queryable. + * 11.45: + * - Re-allow child process to do supported file operations (like mmap, ioctl + * read, poll) on the file descriptor of mali device that was inherited + * from the parent process. */ #define BASE_UK_VERSION_MAJOR 11 -#define BASE_UK_VERSION_MINOR 42 +#define BASE_UK_VERSION_MINOR 45 /** * struct kbase_ioctl_version_check - Check version compatibility between diff --git a/include/uapi/gpu/arm/bifrost/mali_base_kernel.h b/include/uapi/gpu/arm/bifrost/mali_base_kernel.h index cb1a1e8dd550..198d6b8a3942 100644 --- a/include/uapi/gpu/arm/bifrost/mali_base_kernel.h +++ b/include/uapi/gpu/arm/bifrost/mali_base_kernel.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -89,10 +89,9 @@ typedef __u32 base_mem_alloc_flags; /* A mask of all the flags that can be returned via the base_mem_get_flags() * interface. */ -#define BASE_MEM_FLAGS_QUERYABLE \ - (BASE_MEM_FLAGS_INPUT_MASK & \ - ~(BASE_MEM_SAME_VA | BASE_MEM_COHERENT_SYSTEM_REQUIRED | BASE_MEM_IMPORT_SHARED | \ - BASE_MEM_FLAGS_RESERVED | BASEP_MEM_FLAGS_KERNEL_ONLY)) +#define BASE_MEM_FLAGS_QUERYABLE \ + (BASE_MEM_FLAGS_INPUT_MASK & ~(BASE_MEM_DONT_QUERY | BASE_MEM_FLAGS_RESERVED | \ + BASE_MEM_FLAGS_UNUSED | BASEP_MEM_FLAGS_KERNEL_ONLY)) /** * enum base_mem_import_type - Memory types supported by @a base_mem_import @@ -619,15 +618,15 @@ struct base_gpu_props { #define BASE_TIMEINFO_TIMESTAMP_FLAG (1U << 1) /* For GPU cycle counter */ #define BASE_TIMEINFO_CYCLE_COUNTER_FLAG (1U << 2) -/* Specify kernel GPU register timestamp */ -#define BASE_TIMEINFO_KERNEL_SOURCE_FLAG (1U << 30) -/* Specify userspace cntvct_el0 timestamp source */ -#define BASE_TIMEINFO_USER_SOURCE_FLAG (1U << 31) -#define BASE_TIMEREQUEST_ALLOWED_FLAGS \ - (BASE_TIMEINFO_MONOTONIC_FLAG | BASE_TIMEINFO_TIMESTAMP_FLAG | \ - BASE_TIMEINFO_CYCLE_COUNTER_FLAG | BASE_TIMEINFO_KERNEL_SOURCE_FLAG | \ - BASE_TIMEINFO_USER_SOURCE_FLAG) +/* Specify TimeReques flags allowed if time source is cpu/gpu register */ +#define BASE_TIMEREQUEST_CPU_GPU_SRC_ALLOWED_FLAGS \ + (BASE_TIMEINFO_MONOTONIC_FLAG | BASE_TIMEINFO_TIMESTAMP_FLAG | \ + BASE_TIMEINFO_CYCLE_COUNTER_FLAG) + +/* Specify TimeReques flags allowed if time source is system(user) space */ +#define BASE_TIMEREQUEST_SYSTEM_SRC_ALLOWED_FLAGS \ + (BASE_TIMEINFO_MONOTONIC_FLAG | BASE_TIMEINFO_TIMESTAMP_FLAG) /* Maximum number of source allocations allowed to create an alias allocation. * This needs to be 4096 * 6 to allow cube map arrays with up to 4096 array From d1b62d2e45b49ef7792d43a660d048d8618dee46 Mon Sep 17 00:00:00 2001 From: Zhen Chen Date: Mon, 12 Aug 2024 15:30:09 +0800 Subject: [PATCH 2/2] MALI: rockchip: upgrade bifrost DDK to g25p0-00eac0, from g24p0-00eac0 mali_csffw.bin from Valhall DDK g25(r50) is included. Change-Id: Ic454428c384456a14b29d9651f537eb59c11284d Signed-off-by: Zhen Chen --- .../arm/arm,coresight-mali-source.yaml | 163 ++++ .../devicetree/bindings/arm/mali-bifrost.txt | 7 +- drivers/base/arm/Kconfig | 2 +- .../memory_group_manager.c | 20 +- drivers/gpu/arm/bifrost/Kbuild | 10 +- drivers/gpu/arm/bifrost/Kconfig | 13 +- drivers/gpu/arm/bifrost/Makefile | 2 +- .../arm/bifrost/arbiter/mali_kbase_arbif.c | 29 +- .../bifrost/arbiter/mali_kbase_arbiter_pm.c | 119 +-- .../gpu/mali_kbase_clk_rate_trace_mgr.c | 16 +- .../backend/gpu/mali_kbase_gpuprops_backend.c | 8 +- .../bifrost/backend/gpu/mali_kbase_jm_hw.c | 159 +--- .../bifrost/backend/gpu/mali_kbase_jm_rb.c | 25 +- .../backend/gpu/mali_kbase_js_backend.c | 6 +- .../backend/gpu/mali_kbase_model_dummy.c | 3 +- .../backend/gpu/mali_kbase_pm_backend.c | 41 +- .../bifrost/backend/gpu/mali_kbase_pm_ca.c | 18 +- .../bifrost/backend/gpu/mali_kbase_pm_defs.h | 32 +- .../backend/gpu/mali_kbase_pm_driver.c | 161 ++-- .../backend/gpu/mali_kbase_pm_internal.h | 41 +- .../backend/gpu/mali_kbase_pm_policy.c | 16 + .../arm/bifrost/backend/gpu/mali_kbase_time.c | 10 +- drivers/gpu/arm/bifrost/build.bp | 13 +- .../context/backend/mali_kbase_context_csf.c | 10 +- .../context/backend/mali_kbase_context_jm.c | 12 +- drivers/gpu/arm/bifrost/csf/Kbuild | 4 +- .../ipa_control/mali_kbase_csf_ipa_control.c | 5 +- drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c | 125 ++- .../bifrost/csf/mali_kbase_csf_csg_debugfs.c | 87 +- .../gpu/arm/bifrost/csf/mali_kbase_csf_defs.h | 37 +- .../arm/bifrost/csf/mali_kbase_csf_firmware.c | 478 ++++++++--- .../arm/bifrost/csf/mali_kbase_csf_firmware.h | 34 +- .../csf/mali_kbase_csf_firmware_no_mali.c | 34 +- .../arm/bifrost/csf/mali_kbase_csf_fw_io.c | 251 ++++++ .../arm/bifrost/csf/mali_kbase_csf_fw_io.h | 362 +++++++++ .../csf/mali_kbase_csf_fw_io_no_mali.c | 294 +++++++ .../csf/mali_kbase_csf_heap_context_alloc.c | 7 +- .../gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c | 14 +- .../bifrost/csf/mali_kbase_csf_registers.h | 86 ++ .../bifrost/csf/mali_kbase_csf_reset_gpu.c | 2 - .../bifrost/csf/mali_kbase_csf_scheduler.c | 156 ++-- .../bifrost/csf/mali_kbase_csf_scheduler.h | 2 + .../bifrost/csf/mali_kbase_csf_tiler_heap.c | 16 +- .../bifrost/csf/mali_kbase_csf_trace_buffer.c | 7 +- .../backend/mali_kbase_debug_coresight_csf.c | 4 +- .../mali_kbase_debug_ktrace_codes_csf.h | 5 +- .../mali_kbase_debug_linux_ktrace_csf.h | 3 +- .../debug/mali_kbase_debug_ktrace_codes.h | 12 +- .../debug/mali_kbase_debug_linux_ktrace.h | 11 +- .../device/backend/mali_kbase_device_csf.c | 12 +- .../device/backend/mali_kbase_device_hw_csf.c | 33 +- .../device/backend/mali_kbase_device_hw_jm.c | 4 +- .../device/backend/mali_kbase_device_jm.c | 4 - .../arm/bifrost/device/mali_kbase_device.c | 37 +- .../bifrost/hw_access/mali_kbase_hw_access.c | 12 +- .../hw_access/mali_kbase_hw_access_regmap.h | 2 +- .../mali_kbase_hw_access_regmap_legacy.h | 2 +- .../regmap/mali_kbase_regmap_csf_macros.h | 1 + .../hwcnt/backend/mali_kbase_hwcnt_backend.h | 20 +- .../backend/mali_kbase_hwcnt_backend_csf.c | 18 + .../backend/mali_kbase_hwcnt_backend_csf_if.h | 20 + .../mali_kbase_hwcnt_backend_csf_if_fw.c | 22 + .../backend/mali_kbase_hwcnt_backend_jm.c | 12 +- .../mali_kbase_hwcnt_backend_jm_watchdog.c | 12 +- .../gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt.c | 8 +- .../arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.h | 2 +- .../bifrost/hwcnt/mali_kbase_hwcnt_types.h | 61 +- .../arm/bifrost/ipa/mali_kbase_ipa_debugfs.c | 4 +- .../gpu/arm/bifrost/jm/mali_kbase_jm_defs.h | 81 +- drivers/gpu/arm/bifrost/jm/mali_kbase_jm_js.h | 21 +- .../arm/bifrost/mali_base_hwconfig_features.h | 163 ---- .../arm/bifrost/mali_base_hwconfig_issues.h | 621 -------------- drivers/gpu/arm/bifrost/mali_csffw.bin | Bin 278528 -> 278528 bytes drivers/gpu/arm/bifrost/mali_kbase.h | 29 +- drivers/gpu/arm/bifrost/mali_kbase_caps.h | 97 ++- .../arm/bifrost/mali_kbase_config_defaults.h | 44 +- .../gpu/arm/bifrost/mali_kbase_core_linux.c | 239 +++--- .../arm/bifrost/mali_kbase_cs_experimental.h | 5 +- .../arm/bifrost/mali_kbase_debug_mem_allocs.c | 4 +- drivers/gpu/arm/bifrost/mali_kbase_defs.h | 104 ++- .../gpu/arm/bifrost/mali_kbase_dummy_job_wa.c | 6 +- .../gpu/arm/bifrost/mali_kbase_gpu_metrics.c | 5 +- drivers/gpu/arm/bifrost/mali_kbase_gpuprops.c | 6 +- drivers/gpu/arm/bifrost/mali_kbase_gwt.c | 16 +- drivers/gpu/arm/bifrost/mali_kbase_hw.c | 14 +- drivers/gpu/arm/bifrost/mali_kbase_hw.h | 10 +- .../bifrost/mali_kbase_hwconfig_features.h | 158 ++++ .../arm/bifrost/mali_kbase_hwconfig_issues.h | 609 ++++++++++++++ .../arm/bifrost/mali_kbase_ioctl_helpers.h | 542 +++++++++++++ drivers/gpu/arm/bifrost/mali_kbase_jd.c | 78 +- drivers/gpu/arm/bifrost/mali_kbase_js.c | 762 ++---------------- .../gpu/arm/bifrost/mali_kbase_kinstr_jm.c | 10 +- drivers/gpu/arm/bifrost/mali_kbase_mem.c | 126 ++- drivers/gpu/arm/bifrost/mali_kbase_mem.h | 49 +- .../gpu/arm/bifrost/mali_kbase_mem_linux.c | 69 +- .../gpu/arm/bifrost/mali_kbase_mem_linux.h | 13 +- .../gpu/arm/bifrost/mali_kbase_mem_migrate.c | 6 + .../gpu/arm/bifrost/mali_kbase_native_mgm.c | 36 +- drivers/gpu/arm/bifrost/mali_kbase_pbha.c | 65 +- .../gpu/arm/bifrost/mali_kbase_pbha_debugfs.c | 2 +- drivers/gpu/arm/bifrost/mali_kbase_pm.c | 46 +- drivers/gpu/arm/bifrost/mali_kbase_pm.h | 8 +- .../gpu/arm/bifrost/mali_kbase_reg_track.c | 4 +- drivers/gpu/arm/bifrost/mali_kbase_softjobs.c | 19 +- .../bifrost/mmu/backend/mali_kbase_mmu_csf.c | 78 +- .../bifrost/mmu/backend/mali_kbase_mmu_jm.c | 57 +- drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.c | 637 +++++++++++---- .../mmu/mali_kbase_mmu_faults_decoder.c | 8 +- .../gpu/arm/bifrost/mmu/mali_kbase_mmu_hw.h | 4 +- .../bifrost/mmu/mali_kbase_mmu_hw_direct.c | 15 +- .../arm/bifrost/mmu/mali_kbase_mmu_internal.h | 15 +- .../bifrost/mmu/mali_kbase_mmu_mode_aarch64.c | 4 +- drivers/gpu/arm/bifrost/tests/Kconfig | 2 +- .../kernel/mali_kutf_clk_rate_trace_test.c | 4 +- .../mali_kutf_mgm_integration_test_main.c | 6 +- .../tl/backend/mali_kbase_timeline_csf.c | 4 +- .../arm/bifrost/tl/mali_kbase_timeline_io.c | 5 +- .../arm/bifrost/tl/mali_kbase_tracepoints.c | 28 + .../arm/bifrost/tl/mali_kbase_tracepoints.h | 24 + drivers/hwtracing/coresight/mali/build.bp | 2 +- .../itm/coresight_mali_source_itm_core.c | 6 +- include/linux/mali_hw_access.h | 23 +- include/linux/memory_group_manager.h | 36 +- include/linux/version_compat_defs.h | 129 ++- .../arm/bifrost/csf/mali_base_csf_kernel.h | 10 +- .../arm/bifrost/csf/mali_kbase_csf_ioctl.h | 6 +- .../gpu/arm/bifrost/jm/mali_base_jm_kernel.h | 128 +-- .../gpu/arm/bifrost/jm/mali_kbase_jm_ioctl.h | 10 +- .../gpu/arm/bifrost/mali_base_common_kernel.h | 9 +- .../uapi/gpu/arm/bifrost/mali_base_kernel.h | 9 +- .../uapi/gpu/arm/bifrost/mali_kbase_ioctl.h | 6 +- .../mali_kbase_mem_profile_debugfs_buf_size.h | 4 +- 132 files changed, 5331 insertions(+), 3263 deletions(-) create mode 100644 Documentation/devicetree/bindings/arm/arm,coresight-mali-source.yaml create mode 100644 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_fw_io.c create mode 100644 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_fw_io.h create mode 100644 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_fw_io_no_mali.c delete mode 100644 drivers/gpu/arm/bifrost/mali_base_hwconfig_features.h delete mode 100644 drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h create mode 100644 drivers/gpu/arm/bifrost/mali_kbase_hwconfig_features.h create mode 100644 drivers/gpu/arm/bifrost/mali_kbase_hwconfig_issues.h create mode 100644 drivers/gpu/arm/bifrost/mali_kbase_ioctl_helpers.h diff --git a/Documentation/devicetree/bindings/arm/arm,coresight-mali-source.yaml b/Documentation/devicetree/bindings/arm/arm,coresight-mali-source.yaml new file mode 100644 index 000000000000..d844ad10932c --- /dev/null +++ b/Documentation/devicetree/bindings/arm/arm,coresight-mali-source.yaml @@ -0,0 +1,163 @@ +# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note +# +# (C) COPYRIGHT 2022-2024 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU license. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/arm/arm,coresight-mali-source.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: ARM CoreSight Mali Source integration + +maintainers: + - ARM Ltd. + +description: | + See Documentation/trace/coresight/coresight.rst for detailed information + about Coresight. + + This documentation will cover Mali specific devicetree integration. + + References to Sink ports are given as examples. Access to Sink is specific + to an implementation and would require dedicated kernel modules. + + Arm Mali GPU are supporting 3 different sources: ITM, ETM, ELA + + ELA source configuration via SysFS entries: + + The register values used by CoreSight for ELA can be configured using SysFS + interfaces. This implicitly includes configuring the ELA for independent or + shared JCN request and response channels. + +properties: + compatible: + enum: + - arm,coresight-mali-source-itm + - arm,coresight-mali-source-etm + - arm,coresight-mali-source-ela + + gpu: + minItems: 1 + maxItems: 1 + description: + Phandle to a Mali GPU definition + + port: + description: + Output connection to CoreSight Sink Trace bus. + + Legacy binding between Coresight Sources and CoreSight Sink. + For Linux kernel < v4.20. + $ref: /schemas/graph.yaml#/properties/port + + out-ports: + description: + Binding between Coresight Sources and CoreSight Sink. + For Linux kernel >= v4.20. + $ref: /schemas/graph.yaml#/properties/ports + + properties: + port: + description: Output connection to CoreSight Sink Trace bus. + $ref: /schemas/graph.yaml#/properties/port + +required: + - compatible + - gpu + - port + - out-ports + +additionalProperties: false + +examples: + +# A Sink node without legacy CoreSight connections + - | + mali-source-itm { + compatible = "arm,coresight-mali-source-itm"; + gpu = <&gpu>; + + out-ports { + port { + mali_source_itm_out_port0: endpoint { + remote-endpoint = <&mali_sink_in_port0>; + }; + }; + }; + }; + + mali-source-ela { + compatible = "arm,coresight-mali-source-ela"; + gpu = <&gpu>; + + out-ports { + port { + mali_source_ela_out_port0: endpoint { + remote-endpoint = <&mali_sink_in_port1>; + }; + }; + }; + }; + + mali-source-etm { + compatible = "arm,coresight-mali-source-etm"; + gpu = <&gpu>; + + out-ports { + port { + mali_source_etm_out_port0: endpoint { + remote-endpoint = <&mali_sink_in_port2>; + }; + }; + }; + }; + +# A Sink node with legacy CoreSight connections + - | + mali-source-itm { + compatible = "arm,coresight-mali-source-itm"; + gpu = <&gpu>; + + port { + mali_source_itm_out_port0: endpoint { + remote-endpoint = <&mali_sink_in_port0>; + }; + }; + }; + + mali-source-etm { + compatible = "arm,coresight-mali-source-etm"; + gpu = <&gpu>; + + port { + mali_source_etm_out_port0: endpoint { + remote-endpoint = <&mali_sink_in_port1>; + }; + }; + }; + + mali-source-ela { + compatible = "arm,coresight-mali-source-ela"; + gpu = <&gpu>; + + port { + mali_source_ela_out_port0: endpoint { + remote-endpoint = <&mali_sink_in_port2>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/arm/mali-bifrost.txt b/Documentation/devicetree/bindings/arm/mali-bifrost.txt index 3f80d97b0064..8ada052ebe56 100644 --- a/Documentation/devicetree/bindings/arm/mali-bifrost.txt +++ b/Documentation/devicetree/bindings/arm/mali-bifrost.txt @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2013-2023 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2013-2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -132,6 +132,10 @@ for details. set and the setting coresponding to the SYSC_ALLOC register. - propagate-bits: Used to write to L2_CONFIG.PBHA_HWU. This bitset establishes which PBHA bits are propagated on the AXI bus. +- mma-wa-id: Sets the PBHA ID to be used for the PBHA override based MMA violation workaround. + The read and write allocation override bits for the PBHA are set to NONCACHEABLE + and the driver encodes the PBHA ID in the PTEs where this workaround is to be applied. + Valid values are from 1 to 15. Example for a Mali GPU with 1 clock and 1 regulator: @@ -241,6 +245,7 @@ gpu@0xfc010000 { pbha { int-id-override = <2 0x32>, <9 0x05>, <16 0x32>; propagate-bits = /bits/ 8 <0x03>; + mma-wa-id = <2>; }; ... }; diff --git a/drivers/base/arm/Kconfig b/drivers/base/arm/Kconfig index e8bb8a40d2c5..c24a377723ca 100644 --- a/drivers/base/arm/Kconfig +++ b/drivers/base/arm/Kconfig @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2021-2023 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2021-2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software diff --git a/drivers/base/arm/memory_group_manager/memory_group_manager.c b/drivers/base/arm/memory_group_manager/memory_group_manager.c index c5fba5f1b522..da4a0c39e63a 100644 --- a/drivers/base/arm/memory_group_manager/memory_group_manager.c +++ b/drivers/base/arm/memory_group_manager/memory_group_manager.c @@ -299,7 +299,8 @@ static int example_mgm_get_import_memory_id(struct memory_group_manager_device * } static u64 example_mgm_update_gpu_pte(struct memory_group_manager_device *const mgm_dev, - unsigned int const group_id, int const mmu_level, u64 pte) + unsigned int const group_id, unsigned int const pbha_id, + unsigned int pte_flags, int const mmu_level, u64 pte) { struct mgm_groups *const data = mgm_dev->data; @@ -309,7 +310,10 @@ static u64 example_mgm_update_gpu_pte(struct memory_group_manager_device *const if (WARN_ON(group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS)) return pte; - pte |= ((u64)group_id << PTE_PBHA_SHIFT) & PTE_PBHA_MASK; + if (pte_flags & BIT(MMA_VIOLATION)) { + pr_warn_once("MMA violation! Applying PBHA override workaround to PTE\n"); + pte |= ((u64)pbha_id << PTE_PBHA_SHIFT) & PTE_PBHA_MASK; + } /* Address could be translated into a different bus address here */ pte |= ((u64)1 << PTE_RES_BIT_MULTI_AS_SHIFT); @@ -362,6 +366,16 @@ static vm_fault_t example_mgm_vmf_insert_pfn_prot(struct memory_group_manager_de return fault; } +static bool example_mgm_get_import_memory_cached_access_permitted( + struct memory_group_manager_device *mgm_dev, + struct memory_group_manager_import_data *import_data) +{ + CSTD_UNUSED(mgm_dev); + CSTD_UNUSED(import_data); + + return true; +} + static int mgm_initialize_data(struct mgm_groups *mgm_data) { int i; @@ -408,6 +422,8 @@ static int memory_group_manager_probe(struct platform_device *pdev) mgm_dev->ops.mgm_vmf_insert_pfn_prot = example_mgm_vmf_insert_pfn_prot; mgm_dev->ops.mgm_update_gpu_pte = example_mgm_update_gpu_pte; mgm_dev->ops.mgm_pte_to_original_pte = example_mgm_pte_to_original_pte; + mgm_dev->ops.mgm_get_import_memory_cached_access_permitted = + example_mgm_get_import_memory_cached_access_permitted; mgm_data = kzalloc(sizeof(*mgm_data), GFP_KERNEL); if (!mgm_data) { diff --git a/drivers/gpu/arm/bifrost/Kbuild b/drivers/gpu/arm/bifrost/Kbuild index b19e4abf3e2b..d64c439fbabc 100644 --- a/drivers/gpu/arm/bifrost/Kbuild +++ b/drivers/gpu/arm/bifrost/Kbuild @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2012-2023 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2012-2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -69,7 +69,7 @@ endif # # Driver version string which is returned to userspace via an ioctl -MALI_RELEASE_NAME ?= '"g24p0-00eac0"' +MALI_RELEASE_NAME ?= '"g25p0-00eac0"' # Set up defaults if not defined by build system ifeq ($(CONFIG_MALI_BIFROST_DEBUG), y) MALI_UNIT_TEST = 1 @@ -104,7 +104,6 @@ endif # # Experimental features must default to disabled, e.g.: # MALI_EXPERIMENTAL_FEATURE ?= 0 -MALI_INCREMENTAL_RENDERING_JM ?= 0 # # ccflags @@ -117,7 +116,6 @@ ccflags-y = \ -DMALI_COVERAGE=$(MALI_COVERAGE) \ -DMALI_RELEASE_NAME=$(MALI_RELEASE_NAME) \ -DMALI_JIT_PRESSURE_LIMIT_BASE=$(MALI_JIT_PRESSURE_LIMIT_BASE) \ - -DMALI_INCREMENTAL_RENDERING_JM=$(MALI_INCREMENTAL_RENDERING_JM) \ -DMALI_PLATFORM_DIR=$(MALI_PLATFORM_DIR) @@ -212,6 +210,7 @@ endif INCLUDE_SUBDIR = \ + $(src)/arbiter/Kbuild \ $(src)/context/Kbuild \ $(src)/debug/Kbuild \ $(src)/device/Kbuild \ @@ -228,9 +227,6 @@ ifeq ($(CONFIG_MALI_CSF_SUPPORT),y) INCLUDE_SUBDIR += $(src)/csf/Kbuild endif -ifeq ($(CONFIG_MALI_ARBITER_SUPPORT),y) - INCLUDE_SUBDIR += $(src)/arbiter/Kbuild -endif ifeq ($(CONFIG_MALI_BIFROST_DEVFREQ),y) ifeq ($(CONFIG_DEVFREQ_THERMAL),y) diff --git a/drivers/gpu/arm/bifrost/Kconfig b/drivers/gpu/arm/bifrost/Kconfig index 685ce4f423ad..b8ceff10e250 100644 --- a/drivers/gpu/arm/bifrost/Kconfig +++ b/drivers/gpu/arm/bifrost/Kconfig @@ -63,6 +63,8 @@ config MALI_BIFROST_NO_MALI All calls to the simulated hardware will complete immediately as if the hardware completed the task. +endchoice + config MALI_NO_MALI_DEFAULT_GPU string "Default GPU for No Mali" depends on MALI_BIFROST_NO_MALI @@ -70,7 +72,12 @@ config MALI_NO_MALI_DEFAULT_GPU help This option sets the default GPU to identify as for No Mali builds. -endchoice +config MALI_IS_FPGA + bool "Enable build of Mali kernel driver for FPGA" + depends on MALI_BIFROST + default n + help + This is the default HW backend. menu "Platform specific options" source "$(MALI_KCONFIG_EXT_PREFIX)drivers/gpu/arm/bifrost/platform/Kconfig" @@ -340,7 +347,7 @@ config MALI_PWRSOFT_765 changes have been backported say Y to avoid compilation errors. config MALI_HW_ERRATA_1485982_NOT_AFFECTED - bool "Disable workaround for BASE_HW_ISSUE_GPU2017_1336" + bool "Disable workaround for KBASE_HW_ISSUE_GPU2017_1336" depends on MALI_BIFROST && MALI_BIFROST_EXPERT default n help @@ -352,7 +359,7 @@ config MALI_HW_ERRATA_1485982_NOT_AFFECTED coherency mode requires the L2 to be turned off. config MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE - bool "Use alternative workaround for BASE_HW_ISSUE_GPU2017_1336" + bool "Use alternative workaround for KBASE_HW_ISSUE_GPU2017_1336" depends on MALI_BIFROST && MALI_BIFROST_EXPERT && !MALI_HW_ERRATA_1485982_NOT_AFFECTED default n help diff --git a/drivers/gpu/arm/bifrost/Makefile b/drivers/gpu/arm/bifrost/Makefile index 9b636f58c6bc..e10033aabc57 100644 --- a/drivers/gpu/arm/bifrost/Makefile +++ b/drivers/gpu/arm/bifrost/Makefile @@ -156,7 +156,6 @@ ifeq ($(MALI_KCONFIG_EXT_PREFIX),) CONFIG_MALI_BIFROST \ CONFIG_MALI_CSF_SUPPORT \ CONFIG_MALI_BIFROST_GATOR_SUPPORT \ - CONFIG_MALI_ARBITER_SUPPORT \ CONFIG_MALI_ARBITRATION \ CONFIG_MALI_PARTITION_MANAGER \ CONFIG_MALI_REAL_HW \ @@ -170,6 +169,7 @@ ifeq ($(MALI_KCONFIG_EXT_PREFIX),) CONFIG_MALI_PWRSOFT_765 \ CONFIG_MALI_JOB_DUMP \ CONFIG_MALI_BIFROST_NO_MALI \ + CONFIG_MALI_IS_FPGA \ CONFIG_MALI_HW_ERRATA_1485982_NOT_AFFECTED \ CONFIG_MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE \ CONFIG_MALI_PRFCNT_SET_PRIMARY \ diff --git a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.c b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.c index 8cdae33cf919..49b42a6ec2c0 100644 --- a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.c +++ b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.c @@ -108,6 +108,7 @@ static void on_gpu_stop(struct device *dev) } KBASE_TLSTREAM_TL_ARBITER_STOP_REQUESTED(kbdev, kbdev); + KBASE_KTRACE_ADD(kbdev, ARB_GPU_STOP_REQUESTED, NULL, 0); kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_STOP_EVT); } @@ -133,6 +134,7 @@ static void on_gpu_granted(struct device *dev) } KBASE_TLSTREAM_TL_ARBITER_GRANTED(kbdev, kbdev); + KBASE_KTRACE_ADD(kbdev, ARB_GPU_GRANTED, NULL, 0); kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_GRANTED_EVT); } @@ -156,7 +158,8 @@ static void on_gpu_lost(struct device *dev) dev_err(dev, "%s(): kbdev is NULL", __func__); return; } - + KBASE_TLSTREAM_TL_ARBITER_LOST(kbdev, kbdev); + KBASE_KTRACE_ADD(kbdev, ARB_GPU_LOST, NULL, 0); kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_LOST_EVT); } @@ -178,7 +181,7 @@ static int kbase_arbif_of_init(struct kbase_device *kbdev) if (!arbiter_if_node) arbiter_if_node = of_parse_phandle(kbdev->dev->of_node, "arbiter_if", 0); if (!arbiter_if_node) { - dev_dbg(kbdev->dev, "No arbiter_if in Device Tree\n"); + dev_dbg(kbdev->dev, "No arbiter_if in Device Tree"); /* no arbiter interface defined in device tree */ kbdev->arb.arb_dev = NULL; kbdev->arb.arb_if = NULL; @@ -187,19 +190,19 @@ static int kbase_arbif_of_init(struct kbase_device *kbdev) pdev = of_find_device_by_node(arbiter_if_node); if (!pdev) { - dev_err(kbdev->dev, "Failed to find arbiter_if device\n"); + dev_err(kbdev->dev, "Failed to find arbiter_if device"); return -EPROBE_DEFER; } if (!pdev->dev.driver || !try_module_get(pdev->dev.driver->owner)) { - dev_err(kbdev->dev, "arbiter_if driver not available\n"); + dev_err(kbdev->dev, "arbiter_if driver not available"); put_device(&pdev->dev); return -EPROBE_DEFER; } kbdev->arb.arb_dev = &pdev->dev; arb_if = platform_get_drvdata(pdev); if (!arb_if) { - dev_err(kbdev->dev, "arbiter_if driver not ready\n"); + dev_err(kbdev->dev, "arbiter_if driver not ready"); module_put(pdev->dev.driver->owner); put_device(&pdev->dev); return -EPROBE_DEFER; @@ -243,6 +246,10 @@ int kbase_arbif_init(struct kbase_device *kbdev) /* Tries to init with 'arbiter-if' if present in devicetree */ err = kbase_arbif_of_init(kbdev); + if (err == -ENODEV) { + /* devicetree does not support arbitration */ + return -EPERM; + } if (err) return err; @@ -260,19 +267,19 @@ int kbase_arbif_init(struct kbase_device *kbdev) arb_if = kbdev->arb.arb_if; if (arb_if == NULL) { - dev_err(kbdev->dev, "No arbiter interface present\n"); + dev_err(kbdev->dev, "No arbiter interface present"); goto failure_term; } if (!arb_if->vm_ops.vm_arb_register_dev) { - dev_err(kbdev->dev, "arbiter_if registration callback not present\n"); + dev_err(kbdev->dev, "arbiter_if registration callback not present"); goto failure_term; } /* register kbase arbiter_if callbacks */ err = arb_if->vm_ops.vm_arb_register_dev(arb_if, kbdev->dev, &ops); if (err) { - dev_err(kbdev->dev, "Failed to register with arbiter. (err = %d)\n", err); + dev_err(kbdev->dev, "Failed to register with arbiter. (err = %d)", err); goto failure_term; } @@ -333,6 +340,7 @@ void kbase_arbif_gpu_request(struct kbase_device *kbdev) if (arb_if && arb_if->vm_ops.vm_arb_gpu_request) { KBASE_TLSTREAM_TL_ARBITER_REQUESTED(kbdev, kbdev); + KBASE_KTRACE_ADD(kbdev, ARB_GPU_REQUESTED, NULL, 0); arb_if->vm_ops.vm_arb_gpu_request(arb_if); } } @@ -349,8 +357,11 @@ void kbase_arbif_gpu_stopped(struct kbase_device *kbdev, u8 gpu_required) if (arb_if && arb_if->vm_ops.vm_arb_gpu_stopped) { KBASE_TLSTREAM_TL_ARBITER_STOPPED(kbdev, kbdev); - if (gpu_required) + KBASE_KTRACE_ADD(kbdev, ARB_GPU_STOPPED, NULL, 0); + if (gpu_required) { KBASE_TLSTREAM_TL_ARBITER_REQUESTED(kbdev, kbdev); + KBASE_KTRACE_ADD(kbdev, ARB_GPU_REQUESTED, NULL, 0); + } arb_if->vm_ops.vm_arb_gpu_stopped(arb_if, gpu_required); } } diff --git a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.c b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.c index a27085d0f4f4..9b8551609dc7 100644 --- a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.c +++ b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.c @@ -199,6 +199,7 @@ static void kbase_arbiter_pm_resume_wq(struct work_struct *data) arb_vm_state->vm_arb_starting = false; mutex_unlock(&arb_vm_state->vm_state_lock); KBASE_TLSTREAM_TL_ARBITER_STARTED(kbdev, kbdev); + KBASE_KTRACE_ADD(kbdev, ARB_GPU_STARTED, NULL, 0); dev_dbg(kbdev->dev, "<%s\n", __func__); } @@ -295,11 +296,13 @@ int kbase_arbiter_pm_early_init(struct kbase_device *kbdev) err = kbase_arbif_init(kbdev); if (err) { - dev_err(kbdev->dev, "Failed to initialise arbif module. (err = %d)\n", err); + if (err != -EPERM) + dev_err(kbdev->dev, "Failed to initialise arbif module. (err = %d)", err); + goto arbif_init_fail; } - if (kbdev->arb.arb_if) { + if (kbase_has_arbiter(kbdev)) { kbase_arbif_gpu_request(kbdev); dev_dbg(kbdev->dev, "Waiting for initial GPU assignment...\n"); @@ -345,6 +348,9 @@ void kbase_arbiter_pm_early_term(struct kbase_device *kbdev) if (arb_vm_state == NULL) return; + if (!kbase_has_arbiter(kbdev)) + return; + kbase_arbiter_pm_release_interrupts(kbdev); cancel_request_timer(kbdev); @@ -475,6 +481,12 @@ int kbase_arbiter_pm_gpu_assigned(struct kbase_device *kbdev) if (!kbdev) return result; + /* If there is no Arbiter, then there is no virtualization + * and current VM always has access to GPU. + */ + if (!kbase_has_arbiter(kbdev)) + return 1; + /* First check the GPU_LOST state */ kbase_pm_lock(kbdev); if (kbase_pm_is_gpu_lost(kbdev)) { @@ -688,7 +700,7 @@ static inline bool kbase_arbiter_pm_vm_os_suspend_ready_state(struct kbase_devic * @kbdev: The kbase device structure for the device * * Prepares OS to be in suspend state until it receives GRANT message - * from Arbiter asynchronously. + * from Arbiter asynchronously. This function assumes there is an active Arbiter. */ static void kbase_arbiter_pm_vm_os_prepare_suspend(struct kbase_device *kbdev) { @@ -696,10 +708,8 @@ static void kbase_arbiter_pm_vm_os_prepare_suspend(struct kbase_device *kbdev) enum kbase_vm_state prev_state; lockdep_assert_held(&arb_vm_state->vm_state_lock); - if (kbdev->arb.arb_if) { - if (kbdev->pm.arb_vm_state->vm_state == KBASE_VM_STATE_SUSPENDED) - return; - } + if (kbdev->pm.arb_vm_state->vm_state == KBASE_VM_STATE_SUSPENDED) + return; /* Block suspend OS function until we are in a stable state * with vm_state_lock */ @@ -791,7 +801,7 @@ void kbase_arbiter_pm_vm_event(struct kbase_device *kbdev, enum kbase_arbif_evt { struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; - if (!kbdev->arb.arb_if) + if (!kbase_has_arbiter(kbdev)) return; mutex_lock(&arb_vm_state->vm_state_lock); @@ -911,7 +921,8 @@ static inline bool kbase_arbiter_pm_vm_gpu_assigned_locked(struct kbase_device * * * This function handles a suspend event from the driver, * communicating with the arbiter and waiting synchronously for the GPU - * to be granted again depending on the VM state. + * to be granted again depending on the VM state. Returns immediately + * with success if there is no Arbiter. * * Return: 0 on success else 1 suspend handler isn not possible. */ @@ -921,58 +932,58 @@ int kbase_arbiter_pm_ctx_active_handle_suspend(struct kbase_device *kbdev, struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; int res = 0; - if (kbdev->arb.arb_if) { - mutex_lock(&arb_vm_state->vm_state_lock); - while (!kbase_arbiter_pm_vm_gpu_assigned_locked(kbdev)) { - /* Update VM state since we have GPU work to do */ - if (arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_IDLE) - kbase_arbiter_pm_vm_set_state(kbdev, - KBASE_VM_STATE_STOPPING_ACTIVE); - else if (arb_vm_state->vm_state == KBASE_VM_STATE_STOPPED) { - kbase_arbiter_pm_vm_set_state(kbdev, - KBASE_VM_STATE_STOPPED_GPU_REQUESTED); - kbase_arbif_gpu_request(kbdev); - start_request_timer(kbdev); - } else if (arb_vm_state->vm_state == KBASE_VM_STATE_INITIALIZING_WITH_GPU) + if (!kbase_has_arbiter(kbdev)) + return res; + + mutex_lock(&arb_vm_state->vm_state_lock); + while (!kbase_arbiter_pm_vm_gpu_assigned_locked(kbdev)) { + /* Update VM state since we have GPU work to do */ + if (arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_IDLE) + kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_STOPPING_ACTIVE); + else if (arb_vm_state->vm_state == KBASE_VM_STATE_STOPPED) { + kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_STOPPED_GPU_REQUESTED); + kbase_arbif_gpu_request(kbdev); + start_request_timer(kbdev); + } else if (arb_vm_state->vm_state == KBASE_VM_STATE_INITIALIZING_WITH_GPU) + break; + + if (suspend_handler != KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE) { + /* In case of GPU lost, even if + * active_count > 0, we no longer have GPU + * access + */ + if (kbase_pm_is_gpu_lost(kbdev)) + res = 1; + + switch (suspend_handler) { + case KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE: + res = 1; break; - - if (suspend_handler != KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE) { - /* In case of GPU lost, even if - * active_count > 0, we no longer have GPU - * access - */ - if (kbase_pm_is_gpu_lost(kbdev)) + case KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE: + if (kbdev->pm.active_count == 0) res = 1; - - switch (suspend_handler) { - case KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE: - res = 1; - break; - case KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE: - if (kbdev->pm.active_count == 0) - res = 1; - break; - case KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED: - break; - default: - WARN(1, "Unknown suspend_handler\n"); - res = 1; - break; - } + break; + case KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED: + break; + default: + WARN(1, "Unknown suspend_handler\n"); + res = 1; break; } - - /* Need to synchronously wait for GPU assignment */ - atomic_inc(&kbdev->pm.gpu_users_waiting); - mutex_unlock(&arb_vm_state->vm_state_lock); - kbase_pm_unlock(kbdev); - kbase_arbiter_pm_vm_wait_gpu_assignment(kbdev); - kbase_pm_lock(kbdev); - mutex_lock(&arb_vm_state->vm_state_lock); - atomic_dec(&kbdev->pm.gpu_users_waiting); + break; } + + /* Need to synchronously wait for GPU assignment */ + atomic_inc(&kbdev->pm.gpu_users_waiting); mutex_unlock(&arb_vm_state->vm_state_lock); + kbase_pm_unlock(kbdev); + kbase_arbiter_pm_vm_wait_gpu_assignment(kbdev); + kbase_pm_lock(kbdev); + mutex_lock(&arb_vm_state->vm_state_lock); + atomic_dec(&kbdev->pm.gpu_users_waiting); } + mutex_unlock(&arb_vm_state->vm_state_lock); + return res; } diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_clk_rate_trace_mgr.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_clk_rate_trace_mgr.c index e47dd440bff2..851e6feafd30 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_clk_rate_trace_mgr.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_clk_rate_trace_mgr.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -50,14 +50,22 @@ static struct kbase_clk_rate_trace_op_conf * get_clk_rate_trace_callbacks(__maybe_unused struct kbase_device *kbdev) { /* base case */ + const void *arbiter_if_node; struct kbase_clk_rate_trace_op_conf *callbacks = (struct kbase_clk_rate_trace_op_conf *)CLK_RATE_TRACE_OPS; -#if defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF) - const void *arbiter_if_node; + + /* Nothing left to do here if there is no Arbiter/virtualization or if + * CONFIG_OF is not enabled. + */ + if (!IS_ENABLED(CONFIG_OF)) + return callbacks; if (WARN_ON(!kbdev) || WARN_ON(!kbdev->dev)) return callbacks; + if (!kbase_has_arbiter(kbdev)) + return callbacks; + arbiter_if_node = of_get_property(kbdev->dev->of_node, "arbiter-if", NULL); if (!arbiter_if_node) arbiter_if_node = of_get_property(kbdev->dev->of_node, "arbiter_if", NULL); @@ -69,8 +77,6 @@ get_clk_rate_trace_callbacks(__maybe_unused struct kbase_device *kbdev) dev_dbg(kbdev->dev, "Arbitration supported but disabled by platform. Leaving clk rate callbacks as default.\n"); -#endif - return callbacks; } diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_gpuprops_backend.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_gpuprops_backend.c index 414ad546811a..a9b629ad7ea5 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_gpuprops_backend.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_gpuprops_backend.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -48,7 +48,7 @@ int kbase_backend_gpuprops_get(struct kbase_device *kbdev, struct kbasep_gpuprop /* Not a valid register on TMIX */ /* TGOx specific register */ - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_THREAD_TLS_ALLOC)) + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_THREAD_TLS_ALLOC)) regdump->thread_tls_alloc = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(THREAD_TLS_ALLOC)); #endif /* !MALI_USE_CSF */ @@ -64,7 +64,7 @@ int kbase_backend_gpuprops_get(struct kbase_device *kbdev, struct kbasep_gpuprop /* AMBA_FEATURES enum is mapped to COHERENCY_FEATURES enum */ regdump->coherency_features = KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(COHERENCY_FEATURES)); - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_CORE_FEATURES)) + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_CORE_FEATURES)) regdump->core_features = KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(CORE_FEATURES)); #if MALI_USE_CSF @@ -116,7 +116,7 @@ int kbase_backend_gpuprops_get_curr_config(struct kbase_device *kbdev, int kbase_backend_gpuprops_get_l2_features(struct kbase_device *kbdev, struct kbasep_gpuprops_regdump *regdump) { - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_L2_CONFIG)) { + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_L2_CONFIG)) { regdump->l2_features = KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(L2_FEATURES)); regdump->l2_config = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(L2_CONFIG)); diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c index cc8a0ff7fa42..b251de4fc23e 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c @@ -98,82 +98,6 @@ static u64 kbase_job_write_affinity(struct kbase_device *kbdev, base_jd_core_req return affinity; } -/** - * select_job_chain() - Select which job chain to submit to the GPU - * @katom: Pointer to the atom about to be submitted to the GPU - * - * Selects one of the fragment job chains attached to the special atom at the - * end of a renderpass, or returns the address of the single job chain attached - * to any other type of atom. - * - * Which job chain is selected depends upon whether the tiling phase of the - * renderpass completed normally or was soft-stopped because it used too - * much memory. It also depends upon whether one of the fragment job chains - * has already been run as part of the same renderpass. - * - * Return: GPU virtual address of the selected job chain - */ -static u64 select_job_chain(struct kbase_jd_atom *katom) -{ - struct kbase_context *const kctx = katom->kctx; - u64 jc = katom->jc; - struct kbase_jd_renderpass *rp; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - if (!(katom->core_req & BASE_JD_REQ_END_RENDERPASS)) - return jc; - - compiletime_assert((1ull << (sizeof(katom->renderpass_id) * 8)) <= - ARRAY_SIZE(kctx->jctx.renderpasses), - "Should check invalid access to renderpasses"); - - rp = &kctx->jctx.renderpasses[katom->renderpass_id]; - /* We can read a subset of renderpass state without holding - * higher-level locks (but not end_katom, for example). - * If the end-of-renderpass atom is running with as-yet indeterminate - * OOM state then assume that the start atom was not soft-stopped. - */ - switch (rp->state) { - case KBASE_JD_RP_OOM: - /* Tiling ran out of memory. - * Start of incremental rendering, used once. - */ - jc = katom->jc_fragment.norm_read_forced_write; - break; - case KBASE_JD_RP_START: - case KBASE_JD_RP_PEND_OOM: - /* Tiling completed successfully first time. - * Single-iteration rendering, used once. - */ - jc = katom->jc_fragment.norm_read_norm_write; - break; - case KBASE_JD_RP_RETRY_OOM: - /* Tiling ran out of memory again. - * Continuation of incremental rendering, used as - * many times as required. - */ - jc = katom->jc_fragment.forced_read_forced_write; - break; - case KBASE_JD_RP_RETRY: - case KBASE_JD_RP_RETRY_PEND_OOM: - /* Tiling completed successfully this time. - * End of incremental rendering, used once. - */ - jc = katom->jc_fragment.forced_read_norm_write; - break; - default: - WARN_ON(1); - break; - } - - dev_dbg(kctx->kbdev->dev, "Selected job chain 0x%llx for end atom %pK in state %d\n", jc, - (void *)katom, (int)rp->state); - - katom->jc = jc; - return jc; -} - static inline bool kbasep_jm_wait_js_free(struct kbase_device *kbdev, unsigned int js, struct kbase_context *kctx) { @@ -196,7 +120,7 @@ int kbase_job_hw_submit(struct kbase_device *kbdev, struct kbase_jd_atom *katom, { struct kbase_context *kctx; u32 cfg; - u64 const jc_head = select_job_chain(katom); + u64 jc_head = katom->jc; u64 affinity; struct slot_rb *ptr_slot_rb = &kbdev->hwaccess.backend.slot_rb[js]; @@ -220,21 +144,21 @@ int kbase_job_hw_submit(struct kbase_device *kbdev, struct kbase_jd_atom *katom, */ cfg = (u32)kctx->as_nr; - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION) && + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_FLUSH_REDUCTION) && !(kbdev->serialize_jobs & KBASE_SERIALIZE_RESET)) cfg |= JS_CONFIG_ENABLE_FLUSH_REDUCTION; if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_START)) { /* Force a cache maintenance operation if the newly submitted * katom to the slot is from a different kctx. For a JM GPU - * that has the feature BASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER, + * that has the feature KBASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER, * applies a FLUSH_INV_SHADER_OTHER. Otherwise, do a * FLUSH_CLEAN_INVALIDATE. */ u64 tagged_kctx = ptr_slot_rb->last_kctx_tagged; if (tagged_kctx != SLOT_RB_NULL_TAG_VAL && tagged_kctx != SLOT_RB_TAG_KCTX(kctx)) { - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER)) + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER)) cfg |= JS_CONFIG_START_FLUSH_INV_SHADER_OTHER; else cfg |= JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE; @@ -246,15 +170,14 @@ int kbase_job_hw_submit(struct kbase_device *kbdev, struct kbase_jd_atom *katom, if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_END) && !(kbdev->serialize_jobs & KBASE_SERIALIZE_RESET)) cfg |= JS_CONFIG_END_FLUSH_NO_ACTION; - else if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_CLEAN_ONLY_SAFE)) + else if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_CLEAN_ONLY_SAFE)) cfg |= JS_CONFIG_END_FLUSH_CLEAN; else cfg |= JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE; cfg |= JS_CONFIG_THREAD_PRI(8); - if ((katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED) || - (katom->core_req & BASE_JD_REQ_END_RENDERPASS)) + if (katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED) cfg |= JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK; if (!ptr_slot_rb->job_chain_flag) { @@ -268,7 +191,7 @@ int kbase_job_hw_submit(struct kbase_device *kbdev, struct kbase_jd_atom *katom, kbase_reg_write32(kbdev, JOB_SLOT_OFFSET(js, CONFIG_NEXT), cfg); - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION)) + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_FLUSH_REDUCTION)) kbase_reg_write32(kbdev, JOB_SLOT_OFFSET(js, FLUSH_ID_NEXT), katom->flush_id); /* Write an approximate start timestamp. @@ -440,7 +363,7 @@ void kbase_job_done(struct kbase_device *kbdev, u32 done) * jobs to hang. Reset GPU before allowing * any other jobs on the slot to continue. */ - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_3076)) { + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TTRX_3076)) { if (completion_code == BASE_JD_EVENT_JOB_BUS_FAULT) { if (kbase_prepare_to_reset_gpu_locked( kbdev, RESET_FLAGS_NONE)) @@ -740,66 +663,6 @@ void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx, } } -static int softstop_start_rp_nolock(struct kbase_context *kctx, struct kbase_va_region *reg) -{ - struct kbase_device *const kbdev = kctx->kbdev; - struct kbase_jd_atom *katom; - struct kbase_jd_renderpass *rp; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - katom = kbase_gpu_inspect(kbdev, 1, 0); - - if (!katom) { - dev_dbg(kctx->kbdev->dev, "No atom on job slot\n"); - return -ESRCH; - } - - if (!(katom->core_req & BASE_JD_REQ_START_RENDERPASS)) { - dev_dbg(kctx->kbdev->dev, "Atom %pK on job slot is not start RP\n", (void *)katom); - return -EPERM; - } - - compiletime_assert((1ull << (sizeof(katom->renderpass_id) * 8)) <= - ARRAY_SIZE(kctx->jctx.renderpasses), - "Should check invalid access to renderpasses"); - - rp = &kctx->jctx.renderpasses[katom->renderpass_id]; - if (WARN_ON(rp->state != KBASE_JD_RP_START && rp->state != KBASE_JD_RP_RETRY)) - return -EINVAL; - - dev_dbg(kctx->kbdev->dev, "OOM in state %d with region %pK\n", (int)rp->state, (void *)reg); - - if (WARN_ON(katom != rp->start_katom)) - return -EINVAL; - - dev_dbg(kctx->kbdev->dev, "Adding region %pK to list %pK\n", (void *)reg, - (void *)&rp->oom_reg_list); - list_move_tail(®->link, &rp->oom_reg_list); - dev_dbg(kctx->kbdev->dev, "Added region to list\n"); - - rp->state = (rp->state == KBASE_JD_RP_START ? KBASE_JD_RP_PEND_OOM : - KBASE_JD_RP_RETRY_PEND_OOM); - - kbase_job_slot_softstop(kbdev, 1, katom); - - return 0; -} - -int kbase_job_slot_softstop_start_rp(struct kbase_context *const kctx, - struct kbase_va_region *const reg) -{ - struct kbase_device *const kbdev = kctx->kbdev; - int err; - unsigned long flags; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - err = softstop_start_rp_nolock(kctx, reg); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - return err; -} - void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx) { struct kbase_device *kbdev = kctx->kbdev; @@ -839,7 +702,7 @@ u32 kbase_backend_get_current_flush_id(struct kbase_device *kbdev) { u32 flush_id = 0; - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION)) { + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_FLUSH_REDUCTION)) { mutex_lock(&kbdev->pm.lock); if (kbdev->pm.backend.gpu_powered) flush_id = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(LATEST_FLUSH)); @@ -1085,7 +948,7 @@ static void kbasep_reset_timeout_worker(struct work_struct *data) /* The flush has completed so reset the active indicator */ kbdev->irq_reset_flush = false; - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TMIX_8463)) { + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TMIX_8463)) { u64 val; const u32 timeout_us = kbase_get_timeout_ms(kbdev, KBASE_CLEAN_CACHE_TIMEOUT) * USEC_PER_MSEC; @@ -1268,14 +1131,12 @@ bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev, unsigned int { unsigned int i; -#ifdef CONFIG_MALI_ARBITER_SUPPORT if (kbase_pm_is_gpu_lost(kbdev)) { /* GPU access has been removed, reset will be done by * Arbiter instead */ return false; } -#endif if (flags & RESET_FLAGS_HWC_UNRECOVERABLE_ERROR) kbase_instr_hwcnt_on_unrecoverable_error(kbdev); diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c index e1105bf90899..a4a640a0fb92 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -425,7 +425,7 @@ static void kbase_gpu_release_atom(struct kbase_device *kbdev, struct kbase_jd_a } } - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TGOX_R1_1234)) { + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TGOX_R1_1234)) { if (katom->atom_flags & KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT) { kbase_pm_protected_l2_override(kbdev, false); katom->atom_flags &= ~KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT; @@ -698,7 +698,7 @@ static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, struct kbas kbase_pm_protected_entry_override_disable(kbdev); - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TGOX_R1_1234)) { + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TGOX_R1_1234)) { /* * Power on L2 caches; this will also result in the * correct value written to coherency enable register. @@ -714,13 +714,13 @@ static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, struct kbas katom[idx]->protected_state.enter = KBASE_ATOM_ENTER_PROTECTED_FINISHED; - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TGOX_R1_1234)) + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TGOX_R1_1234)) return -EAGAIN; /* ***TRANSITION TO HIGHER STATE*** */ fallthrough; case KBASE_ATOM_ENTER_PROTECTED_FINISHED: - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TGOX_R1_1234)) { + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TGOX_R1_1234)) { /* * Check that L2 caches are powered and, if so, * enter protected mode. @@ -864,11 +864,7 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) lockdep_assert_held(&kbdev->hwaccess_lock); -#ifdef CONFIG_MALI_ARBITER_SUPPORT - if (kbase_reset_gpu_is_active(kbdev) || kbase_is_gpu_removed(kbdev)) -#else - if (kbase_reset_gpu_is_active(kbdev)) -#endif + if (kbase_reset_gpu_is_active(kbdev) || (kbase_is_gpu_removed(kbdev))) return; for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { @@ -896,7 +892,7 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) break; case KBASE_ATOM_GPU_RB_WAITING_BLOCKED: - if (kbase_js_atom_blocked_on_x_dep(katom[idx])) + if (katom[idx]->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) break; katom[idx]->gpu_rb_state = @@ -1236,7 +1232,7 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, unsigned int js, u32 comp * When a hard-stop is followed close after a soft-stop, the completion * code may be set to STOPPED, even though the job is terminated */ - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TMIX_8438)) { + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TMIX_8438)) { if (completion_code == BASE_JD_EVENT_STOPPED && (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_HARD_STOPPED)) { completion_code = BASE_JD_EVENT_TERMINATED; @@ -1331,6 +1327,9 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, unsigned int js, u32 comp dev_dbg(kbdev->dev, "Update job chain address of atom %pK to resume from 0x%llx\n", (void *)katom, job_tail); + /* Some of the job has been executed, so we update the job chain address to where + * we should resume from + */ katom->jc = job_tail; KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_UPDATE_HEAD, katom->kctx, katom, job_tail, js); } @@ -1381,6 +1380,8 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, unsigned int js, u32 comp dev_dbg(kbdev->dev, "Cross-slot dependency %pK has become runnable.\n", (void *)katom); + /* Cross-slot dependency has now become runnable. Try to submit it. */ + /* Check if there are lower priority jobs to soft stop */ kbase_job_slot_ctx_priority_check_locked(kctx, katom); diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_backend.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_backend.c index 202671b323d5..99037c25bf08 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_backend.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_backend.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -46,7 +46,7 @@ static inline bool timer_callback_should_run(struct kbase_device *kbdev, int nr_ } #endif /* CONFIG_MALI_BIFROST_DEBUG */ - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_9435)) { + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_9435)) { /* Timeouts would have to be 4x longer (due to micro- * architectural design) to support OpenCL conformance tests, so * only run the timer when there's: @@ -100,7 +100,7 @@ static enum hrtimer_restart timer_callback(struct hrtimer *timer) /* The current version of the model doesn't support * Soft-Stop */ - if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_5736)) { + if (!kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_5736)) { u32 ticks = atom->ticks++; #if !defined(CONFIG_MALI_JOB_DUMP) && !defined(CONFIG_MALI_VECTOR_DUMP) diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_dummy.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_dummy.c index b034ffef0ceb..0f4a8cd096bb 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_dummy.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_dummy.c @@ -1953,7 +1953,8 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value) *value = dummy->control_reg_values->gpu_features_lo; } else if (addr == GPU_CONTROL_REG(GPU_FEATURES_HI)) { *value = dummy->control_reg_values->gpu_features_hi; - } else { + } + else { model_error_log( KBASE_CORE, "Dummy model register access: Reading unsupported register 0x%x. Returning 0\n", diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c index 6db242af0578..e1941d50133a 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c @@ -36,9 +36,7 @@ #include #include #include -#ifdef CONFIG_MALI_ARBITER_SUPPORT #include -#endif /* !CONFIG_MALI_ARBITER_SUPPORT */ #endif /* !MALI_USE_CSF */ #include #include @@ -100,10 +98,8 @@ void kbase_pm_register_access_enable(struct kbase_device *kbdev) if (callbacks) callbacks->power_on_callback(kbdev); -#ifdef CONFIG_MALI_ARBITER_SUPPORT if (WARN_ON(kbase_pm_is_gpu_lost(kbdev))) dev_err(kbdev->dev, "Attempting to power on while GPU lost\n"); -#endif kbdev->pm.backend.gpu_powered = true; } @@ -136,9 +132,7 @@ int kbase_hwaccess_pm_init(struct kbase_device *kbdev) INIT_WORK(&kbdev->pm.backend.gpu_poweroff_wait_work, kbase_pm_gpu_poweroff_wait_wq); kbdev->pm.backend.ca_cores_enabled = ~0ull; -#ifdef CONFIG_MALI_ARBITER_SUPPORT kbase_pm_set_gpu_lost(kbdev, false); -#endif init_waitqueue_head(&kbdev->pm.backend.gpu_in_desired_state_wait); #if !MALI_USE_CSF @@ -180,15 +174,18 @@ int kbase_hwaccess_pm_init(struct kbase_device *kbdev) kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); #if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) - kbdev->pm.backend.gpu_sleep_supported = - kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_GPU_SLEEP) && - !kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TURSEHW_1997) && - kbdev->pm.backend.callback_power_runtime_gpu_active && - kbdev->pm.backend.callback_power_runtime_gpu_idle; + kbdev->pm.backend.gpu_sleep_allowed = 0; + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GPU_SLEEP) && + !kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TURSEHW_1997) && + kbdev->pm.backend.callback_power_runtime_gpu_active && + kbdev->pm.backend.callback_power_runtime_gpu_idle) + set_bit(KBASE_GPU_SUPPORTS_GPU_SLEEP, &kbdev->pm.backend.gpu_sleep_allowed); kbdev->pm.backend.apply_hw_issue_TITANHW_2938_wa = - kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TITANHW_2938) && - kbdev->pm.backend.gpu_sleep_supported; + kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TITANHW_2938) && + test_bit(KBASE_GPU_SUPPORTS_GPU_SLEEP, &kbdev->pm.backend.gpu_sleep_allowed); + + /* FW Sleep-on-Idle is feature is kept disabled */ #endif if (IS_ENABLED(CONFIG_MALI_HW_ERRATA_1485982_NOT_AFFECTED)) @@ -196,14 +193,14 @@ int kbase_hwaccess_pm_init(struct kbase_device *kbdev) /* WA1: L2 always_on for GPUs being affected by GPU2017-1336 */ if (!IS_ENABLED(CONFIG_MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE)) { - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_GPU2017_1336)) + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_GPU2017_1336)) kbdev->pm.backend.l2_always_on = true; return 0; } /* WA3: Clock slow down for GPUs being affected by GPU2017-1336 */ - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_GPU2017_1336)) { + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_GPU2017_1336)) { kbdev->pm.backend.gpu_clock_slow_down_wa = true; kbdev->pm.backend.gpu_clock_slow_down_desired = true; INIT_WORK(&kbdev->pm.backend.gpu_clock_control_work, @@ -348,13 +345,11 @@ static void pm_handle_power_off(struct kbase_device *kbdev) */ wait_for_mmu_fault_handling_in_gpu_poweroff_wait_wq(kbdev); -#ifdef CONFIG_MALI_ARBITER_SUPPORT /* poweron_required may have changed while pm lock * was released. */ if (kbase_pm_is_gpu_lost(kbdev)) backend->poweron_required = false; -#endif /* Turn off clock now that fault have been handled. We * dropped locks so poweron_required may have changed - @@ -948,13 +943,11 @@ void kbase_hwaccess_pm_resume(struct kbase_device *kbdev) /* System resume callback has begun */ kbdev->pm.resuming = true; kbdev->pm.suspending = false; -#ifdef CONFIG_MALI_ARBITER_SUPPORT if (kbase_pm_is_gpu_lost(kbdev)) { dev_dbg(kbdev->dev, "%s: GPU lost in progress\n", __func__); kbase_pm_unlock(kbdev); return; } -#endif kbase_pm_do_poweron(kbdev, true); #if !MALI_USE_CSF @@ -964,7 +957,6 @@ void kbase_hwaccess_pm_resume(struct kbase_device *kbdev) kbase_pm_unlock(kbdev); } -#ifdef CONFIG_MALI_ARBITER_SUPPORT void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev) { unsigned long flags; @@ -975,8 +967,10 @@ void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev) #endif struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; - if (!kbdev->arb.arb_if) + if (!kbase_has_arbiter(kbdev)) { + dev_warn(kbdev->dev, "%s called with no active arbiter!\n", __func__); return; + } mutex_lock(&kbdev->pm.lock); mutex_lock(&arb_vm_state->vm_state_lock); @@ -991,7 +985,8 @@ void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev) #if MALI_USE_CSF /* Full GPU reset will have been done by hypervisor, so cancel */ - kbase_reset_gpu_prevent_and_wait(kbdev); + if (kbase_reset_gpu_prevent_and_wait(kbdev)) + dev_warn(kbdev->dev, "Failed to prevent GPU reset."); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbase_csf_scheduler_spin_lock(kbdev, &flags_sched); @@ -1041,7 +1036,6 @@ void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev) mutex_unlock(&arb_vm_state->vm_state_lock); mutex_unlock(&kbdev->pm.lock); } -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ #if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) int kbase_pm_force_mcu_wakeup_after_sleep(struct kbase_device *kbdev) @@ -1253,4 +1247,5 @@ out: return ret; } + #endif diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.c index 8daef13388a3..6522e5ca66e9 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2013-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2013-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -55,11 +55,18 @@ void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask) unsigned long flags; #if MALI_USE_CSF u64 old_core_mask = 0; -#endif + bool mmu_sync_needed = false; + if (!IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) && + kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_GPU2019_3901)) { + mmu_sync_needed = true; + down_write(&kbdev->csf.mmu_sync_sem); + } +#endif spin_lock_irqsave(&kbdev->hwaccess_lock, flags); #if MALI_USE_CSF + if (!(core_mask & kbdev->pm.debug_core_mask)) { dev_err(kbdev->dev, "OPP core mask 0x%llX does not intersect with debug mask 0x%llX\n", @@ -98,6 +105,9 @@ void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask) old_core_mask, core_mask); } } + + if (mmu_sync_needed) + up_write(&kbdev->csf.mmu_sync_sem); #endif dev_dbg(kbdev->dev, "Devfreq policy : new core mask=%llX\n", pm_backend->ca_cores_enabled); @@ -105,6 +115,10 @@ void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask) return; unlock: spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +#if MALI_USE_CSF + if (mmu_sync_needed) + up_write(&kbdev->csf.mmu_sync_sem); +#endif } KBASE_EXPORT_TEST_API(kbase_devfreq_set_core_mask); #endif diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h index 34c34df7f82f..a25fe6bdc912 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h @@ -114,6 +114,27 @@ enum kbase_pm_runtime_suspend_abort_reason { ABORT_REASON_NON_IDLE_CGS }; +/* The following indices point to the corresponding bits stored in + * &kbase_pm_backend_data.gpu_sleep_allowed. They denote the conditions that + * would be checked against to determine the level of support for GPU sleep + * and firmware sleep-on-idle. + */ +#define KBASE_GPU_SUPPORTS_GPU_SLEEP ((uint8_t)0) +#define KBASE_GPU_SUPPORTS_FW_SLEEP_ON_IDLE ((uint8_t)1) +#define KBASE_GPU_PERF_COUNTERS_COLLECTION_ENABLED ((uint8_t)2) +#define KBASE_GPU_IGNORE_IDLE_EVENT ((uint8_t)3) +#define KBASE_GPU_NON_IDLE_OFF_SLOT_GROUPS_AVAILABLE ((uint8_t)4) + +/* FW sleep-on-idle could be enabled if + * &kbase_pm_backend_data.gpu_sleep_allowed is equal to this value. + */ +#define KBASE_GPU_FW_SLEEP_ON_IDLE_ALLOWED \ + ((uint8_t)((1 << KBASE_GPU_SUPPORTS_GPU_SLEEP) | \ + (1 << KBASE_GPU_SUPPORTS_FW_SLEEP_ON_IDLE) | \ + (0 << KBASE_GPU_PERF_COUNTERS_COLLECTION_ENABLED) | \ + (0 << KBASE_GPU_IGNORE_IDLE_EVENT) | \ + (0 << KBASE_GPU_NON_IDLE_OFF_SLOT_GROUPS_AVAILABLE))) + /** * struct kbasep_pm_metrics - Metrics data collected for use by the power * management framework. @@ -304,7 +325,7 @@ union kbase_pm_policy_data { * called previously. * See &struct kbase_pm_callback_conf. * @ca_cores_enabled: Cores that are currently available - * @apply_hw_issue_TITANHW_2938_wa: Indicates if the workaround for BASE_HW_ISSUE_TITANHW_2938 + * @apply_hw_issue_TITANHW_2938_wa: Indicates if the workaround for KBASE_HW_ISSUE_TITANHW_2938 * needs to be applied when unmapping memory from GPU. * @mcu_state: The current state of the micro-control unit, only applicable * to GPUs that have such a component @@ -350,10 +371,9 @@ union kbase_pm_policy_data { * @core_idle_work: Work item used to wait for undesired cores to become inactive. * The work item is enqueued when Host controls the power for * shader cores and down scaling of cores is performed. - * @gpu_sleep_supported: Flag to indicate that if GPU sleep feature can be - * supported by the kernel driver or not. If this - * flag is not set, then HW state is directly saved - * when GPU idle notification is received. + * @gpu_sleep_allowed: Bitmask to indicate the conditions that would be + * used to determine what support for GPU sleep is + * available. * @gpu_sleep_mode_active: Flag to indicate that the GPU needs to be in sleep * mode. It is set when the GPU idle notification is * received and is cleared when HW state has been @@ -497,7 +517,7 @@ struct kbase_pm_backend_data { struct work_struct core_idle_work; #ifdef KBASE_PM_RUNTIME - bool gpu_sleep_supported; + unsigned long gpu_sleep_allowed; bool gpu_sleep_mode_active; bool exit_gpu_sleep_mode; bool gpu_idled; diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c index 9e85cf5589f4..c6b6f3a8668a 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c @@ -47,9 +47,7 @@ #include #include #include -#ifdef CONFIG_MALI_ARBITER_SUPPORT #include -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ #if MALI_USE_CSF #include @@ -615,11 +613,11 @@ static void kbase_pm_l2_config_override(struct kbase_device *kbdev) /* * Skip if it is not supported */ - if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_L2_CONFIG)) + if (!kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_L2_CONFIG)) return; #if MALI_USE_CSF - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PBHA_HWU)) { + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_PBHA_HWU)) { val = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(L2_CONFIG)); kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(L2_CONFIG), L2_CONFIG_PBHA_HWU_SET(val, kbdev->pbha_propagate_bits)); @@ -743,16 +741,8 @@ bool kbase_pm_is_mcu_inactive(struct kbase_device *kbdev, enum kbase_mcu_state s } #ifdef KBASE_PM_RUNTIME -/** - * kbase_pm_enable_mcu_db_notification - Enable the Doorbell notification on - * MCU side - * - * @kbdev: Pointer to the device. - * - * This function is called to re-enable the Doorbell notification on MCU side - * when MCU needs to beome active again. - */ -static void kbase_pm_enable_mcu_db_notification(struct kbase_device *kbdev) + +void kbase_pm_enable_mcu_db_notification(struct kbase_device *kbdev) { u32 val = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(MCU_CONTROL)); @@ -778,7 +768,7 @@ static void wait_mcu_as_inactive(struct kbase_device *kbdev) kbase_get_timeout_ms(kbdev, KBASE_AS_INACTIVE_TIMEOUT) * USEC_PER_MSEC; lockdep_assert_held(&kbdev->hwaccess_lock); - if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TURSEHW_2716)) + if (!kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TURSEHW_2716)) return; /* Wait for the AS_ACTIVE_INT bit to become 0 for the AS used by MCU FW */ @@ -927,6 +917,18 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) if (kbase_pm_is_mcu_desired(kbdev) && !backend->policy_change_clamp_state_to_off && backend->l2_state == KBASE_L2_ON) { + kbdev->csf.mcu_halted = false; + + /* Ensure that FW would not go to sleep immediately after + * resumption. + */ + kbase_csf_firmware_global_input_mask(&kbdev->csf.global_iface, + GLB_REQ, + GLB_REQ_REQ_IDLE_DISABLE, + GLB_REQ_IDLE_DISABLE_MASK); + atomic_set(&kbdev->csf.scheduler.gpu_idle_timer_enabled, false); + atomic_set(&kbdev->csf.scheduler.fw_soi_enabled, false); + kbase_csf_firmware_trigger_reload(kbdev); backend->mcu_state = KBASE_MCU_PEND_ON_RELOAD; } @@ -1005,7 +1007,6 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) case KBASE_MCU_ON: backend->shaders_desired_mask = kbase_pm_ca_get_core_mask(kbdev); - if (!kbase_pm_is_mcu_desired(kbdev)) backend->mcu_state = KBASE_MCU_ON_HWCNT_DISABLE; else if (kbdev->csf.firmware_hctl_core_pwr) { @@ -1185,7 +1186,7 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) break; case KBASE_MCU_POWER_DOWN: - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TITANHW_2922)) { + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TITANHW_2922)) { if (!kbdev->csf.firmware_hctl_core_pwr) kbasep_pm_toggle_power_interrupt(kbdev, true); backend->mcu_state = KBASE_MCU_OFF; @@ -1206,7 +1207,20 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) #ifdef KBASE_PM_RUNTIME case KBASE_MCU_ON_SLEEP_INITIATE: if (!kbase_pm_is_mcu_desired(kbdev)) { - kbase_csf_firmware_trigger_mcu_sleep(kbdev); + bool db_notif_disabled = false; + + if (likely(test_bit(KBASE_GPU_SUPPORTS_FW_SLEEP_ON_IDLE, + &kbdev->pm.backend.gpu_sleep_allowed))) + db_notif_disabled = + kbase_reg_read32(kbdev, + GPU_CONTROL_ENUM(MCU_CONTROL)) & + MCU_CNTRL_DOORBELL_DISABLE_MASK; + + /* If DB notification is enabled on FW side then send a sleep + * request to FW. + */ + if (!db_notif_disabled) + kbase_csf_firmware_trigger_mcu_sleep(kbdev); backend->mcu_state = KBASE_MCU_ON_PEND_SLEEP; } else backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE; @@ -1240,6 +1254,16 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) case KBASE_MCU_IN_SLEEP: if (kbase_pm_is_mcu_desired(kbdev) && backend->l2_state == KBASE_L2_ON) { wait_mcu_as_inactive(kbdev); + /* Ensure that FW would not go to sleep immediately after + * resumption. + */ + kbase_csf_firmware_global_input_mask(&kbdev->csf.global_iface, + GLB_REQ, + GLB_REQ_REQ_IDLE_DISABLE, + GLB_REQ_IDLE_DISABLE_MASK); + atomic_set(&kbdev->csf.scheduler.gpu_idle_timer_enabled, false); + atomic_set(&kbdev->csf.scheduler.fw_soi_enabled, false); + KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_REQUEST_WAKEUP( kbdev, kbase_backend_get_cycle_cnt(kbdev)); kbase_pm_enable_mcu_db_notification(kbdev); @@ -1384,20 +1408,6 @@ static bool need_tiler_control(struct kbase_device *kbdev) #endif } -/** - * hctl_l2_power_down - Initiate power down of L2 cache - * - * @kbdev: The kbase device structure for the device. - * - * This function initiates the power down of L2 cache when Host controls the power - * for Tiler block. The function expects that power down of Tiler to already have - * been initiated and it triggers the L2 power down only after the power down for - * Tiler is complete. - * The function shall be called only if L2 is in ready state. - */ -static void hctl_l2_power_down(struct kbase_device *kbdev) -{ -} /** * hctl_tiler_power_up_done - Check and/or initiate power up of Tiler @@ -1444,7 +1454,6 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) u64 l2_trans = kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2); u64 l2_ready = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2); -#ifdef CONFIG_MALI_ARBITER_SUPPORT /* * kbase_pm_get_ready_cores and kbase_pm_get_trans_cores * are vulnerable to corruption if gpu is lost @@ -1473,7 +1482,6 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) } break; } -#endif /* mask off ready from trans in case transitions finished * between the register reads @@ -1574,7 +1582,7 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) case KBASE_L2_RESTORE_CLOCKS: /* We always assume only GPUs being affected by - * BASE_HW_ISSUE_GPU2017_1336 fall into this state + * KBASE_HW_ISSUE_GPU2017_1336 fall into this state */ WARN_ON_ONCE(!kbdev->pm.backend.gpu_clock_slow_down_wa); @@ -1676,7 +1684,7 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) case KBASE_L2_SLOW_DOWN_CLOCKS: /* We always assume only GPUs being affected by - * BASE_HW_ISSUE_GPU2017_1336 fall into this state + * KBASE_HW_ISSUE_GPU2017_1336 fall into this state */ WARN_ON_ONCE(!kbdev->pm.backend.gpu_clock_slow_down_wa); @@ -1725,11 +1733,6 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) case KBASE_L2_PEND_OFF: if (likely(!backend->l2_always_on)) { - if (need_tiler_control(kbdev) && l2_ready) { - hctl_l2_power_down(kbdev); - break; - } - if (l2_trans || l2_ready) break; } else if (kbdev->cache_clean_in_progress) @@ -1744,11 +1747,10 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) } #endif /* Disabling MCU after L2 cache power down is to address - * BASE_HW_ISSUE_TITANHW_2922 hardware issue. + * KBASE_HW_ISSUE_TITANHW_2922 hardware issue. */ if (backend->l2_force_off_after_mcu_halt) { - kbase_csf_firmware_disable_mcu(kbdev); - kbase_csf_firmware_disable_mcu_wait(kbdev); + kbase_csf_stop_firmware_and_wait(kbdev); WARN_ON_ONCE(backend->mcu_state != KBASE_MCU_OFF); backend->l2_force_off_after_mcu_halt = false; } @@ -1895,12 +1897,7 @@ static int kbase_pm_shaders_update_state(struct kbase_device *kbdev) * kbase_pm_get_ready_cores and kbase_pm_get_trans_cores * are vulnerable to corruption if gpu is lost */ - if (kbase_is_gpu_removed(kbdev) -#ifdef CONFIG_MALI_ARBITER_SUPPORT - || kbase_pm_is_gpu_lost(kbdev)) { -#else - ) { -#endif + if (kbase_is_gpu_removed(kbdev) || kbase_pm_is_gpu_lost(kbdev)) { backend->shaders_state = KBASE_SHADERS_OFF_CORESTACK_OFF; dev_dbg(kbdev->dev, "GPU lost has occurred - shaders off\n"); break; @@ -2005,9 +2002,8 @@ static int kbase_pm_shaders_update_state(struct kbase_device *kbdev) kbdev, KBASE_PM_POLICY_EVENT_IDLE); if (kbdev->pm.backend.protected_transition_override || -#ifdef CONFIG_MALI_ARBITER_SUPPORT - kbase_pm_is_suspending(kbdev) || kbase_pm_is_gpu_lost(kbdev) || -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + (kbase_has_arbiter(kbdev) && (kbase_pm_is_suspending(kbdev) || + kbase_pm_is_gpu_lost(kbdev))) || !stt->configured_ticks || WARN_ON(stt->cancel_queued)) { backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; @@ -2074,10 +2070,9 @@ static int kbase_pm_shaders_update_state(struct kbase_device *kbdev) kbdev, KBASE_PM_POLICY_EVENT_TIMER_MISS); backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; -#ifdef CONFIG_MALI_ARBITER_SUPPORT - } else if (kbase_pm_is_suspending(kbdev) || kbase_pm_is_gpu_lost(kbdev)) { + } else if (kbase_has_arbiter(kbdev) && + (kbase_pm_is_suspending(kbdev) || kbase_pm_is_gpu_lost(kbdev))) { backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ } break; @@ -2096,7 +2091,7 @@ static int kbase_pm_shaders_update_state(struct kbase_device *kbdev) if (!backend->partial_shaderoff) shader_poweroff_timer_queue_cancel(kbdev); - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_921)) { + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TTRX_921)) { kbase_gpu_start_cache_clean_nolock(kbdev, GPU_COMMAND_CACHE_CLN_INV_L2); backend->shaders_state = KBASE_SHADERS_L2_FLUSHING_CORESTACK_ON; @@ -2446,6 +2441,9 @@ void kbase_pm_reset_complete(struct kbase_device *kbdev) backend->in_reset = false; #if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) backend->gpu_wakeup_override = false; + backend->db_mirror_interrupt_enabled = false; + backend->gpu_sleep_mode_active = false; + backend->exit_gpu_sleep_mode = false; #endif kbase_pm_update_state(kbdev); @@ -2670,12 +2668,9 @@ static int pm_wait_for_poweroff_work_complete(struct kbase_device *kbdev, bool k const long timeout = kbase_csf_timeout_in_jiffies( kbase_get_timeout_ms(kbdev, CSF_PM_TIMEOUT) + extra_wait_time_ms); #else -#ifdef CONFIG_MALI_ARBITER_SUPPORT /* Handling of timeout error isn't supported for arbiter builds */ - const long timeout = MAX_SCHEDULE_TIMEOUT; -#else - const long timeout = (long)msecs_to_jiffies(PM_TIMEOUT_MS); -#endif + const long timeout = kbase_has_arbiter(kbdev) ? MAX_SCHEDULE_TIMEOUT : + (long)msecs_to_jiffies(PM_TIMEOUT_MS); #endif int err = 0; @@ -2796,7 +2791,8 @@ static void update_user_reg_page_mapping(struct kbase_device *kbdev) * when the context (user process) needs to access to the page. */ unmap_mapping_range(kbdev->csf.user_reg.filp->f_inode->i_mapping, - kctx->csf.user_reg.file_offset << PAGE_SHIFT, PAGE_SIZE, 1); + (loff_t)kctx->csf.user_reg.file_offset << PAGE_SHIFT, PAGE_SIZE, + 1); list_del_init(&kctx->csf.user_reg.link); dev_dbg(kbdev->dev, "Updated USER Reg page mapping of ctx %d_%d", kctx->tgid, kctx->id); @@ -2823,12 +2819,10 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) #endif /* !MALI_USE_CSF */ lockdep_assert_held(&kbdev->pm.lock); -#ifdef CONFIG_MALI_ARBITER_SUPPORT if (WARN_ON(kbase_pm_is_gpu_lost(kbdev))) { dev_err(kbdev->dev, "%s: Cannot power up while GPU lost", __func__); return; } -#endif if (backend->gpu_powered) { #if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) @@ -2876,10 +2870,8 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) * consistent state */ kbase_pm_init_hw(kbdev, PM_ENABLE_IRQS); - } -#ifdef CONFIG_MALI_ARBITER_SUPPORT - else { - if (kbdev->arb.arb_if) { + } else { + if (kbase_has_arbiter(kbdev)) { struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; /* In the case that the GPU has just been granted by @@ -2895,8 +2887,8 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) * that a repartitioning occurred. In this case the current config * should be read again. */ - kbase_gpuprops_get_curr_config_props(kbdev, &kbdev->gpu_props.curr_config); -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + if (kbase_has_arbiter(kbdev)) + kbase_gpuprops_get_curr_config_props(kbdev, &kbdev->gpu_props.curr_config); mutex_lock(&kbdev->mmu_hw_mutex); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); @@ -2988,12 +2980,7 @@ bool kbase_pm_clock_off(struct kbase_device *kbdev) } #endif - if (kbase_is_gpu_removed(kbdev) -#ifdef CONFIG_MALI_ARBITER_SUPPORT - || kbase_pm_is_gpu_lost(kbdev)) { -#else - ) { -#endif + if (kbase_is_gpu_removed(kbdev) || kbase_pm_is_gpu_lost(kbdev)) { /* Ensure we unblock any threads that are stuck waiting * for the GPU */ @@ -3011,10 +2998,7 @@ bool kbase_pm_clock_off(struct kbase_device *kbdev) /* GPU is about to be turned off, switch to dummy page */ update_user_reg_page_mapping(kbdev); #endif - -#ifdef CONFIG_MALI_ARBITER_SUPPORT kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_IDLE_EVENT); -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ if (kbdev->pm.backend.callback_power_off) kbdev->pm.backend.callback_power_off(kbdev); @@ -3068,6 +3052,7 @@ static enum hrtimer_restart kbasep_reset_timeout(struct hrtimer *timer) return HRTIMER_NORESTART; } + static int kbase_set_gpu_quirks(struct kbase_device *kbdev) { #if MALI_USE_CSF @@ -3097,7 +3082,7 @@ static int kbase_set_gpu_quirks(struct kbase_device *kbdev) kbdev->hw_quirks_gpu = hw_quirks_gpu; #endif /* !MALI_USE_CSF */ - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_IDVS_GROUP_SIZE)) { + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_IDVS_GROUP_SIZE)) { u32 default_idvs_group_size = 0xF; u32 group_size = 0; @@ -3131,10 +3116,10 @@ static int kbase_set_sc_quirks(struct kbase_device *kbdev) if (kbase_is_gpu_removed(kbdev)) return -EIO; - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_2968_TTRX_3162)) + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TTRX_2968_TTRX_3162)) hw_quirks_sc |= SC_VAR_ALGORITHM; - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_TLS_HASHING)) + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_TLS_HASHING)) hw_quirks_sc |= SC_TLS_HASH_ENABLE; kbdev->hw_quirks_sc = hw_quirks_sc; @@ -3153,7 +3138,7 @@ static int kbase_set_tiler_quirks(struct kbase_device *kbdev) return -EIO; /* Set tiler clock gate override if required */ - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3953)) + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_T76X_3953)) hw_quirks_tiler |= TC_CLOCK_GATE_OVERRIDE; kbdev->hw_quirks_tiler = hw_quirks_tiler; @@ -3370,9 +3355,8 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev) /* The GPU doesn't seem to be responding to the reset so try a hard * reset, but only when NOT in arbitration mode. */ -#ifdef CONFIG_MALI_ARBITER_SUPPORT - if (!kbdev->arb.arb_if) { -#endif + + if (!kbase_has_arbiter(kbdev)) { dev_err(kbdev->dev, "Failed to soft-reset GPU (timed out after %d ms), now attempting a hard reset\n", RESET_TIMEOUT); @@ -3402,9 +3386,7 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev) dev_err(kbdev->dev, "Failed to hard-reset the GPU (timed out after %d ms)\n", RESET_TIMEOUT); -#ifdef CONFIG_MALI_ARBITER_SUPPORT } -#endif return -EINVAL; } @@ -3487,6 +3469,7 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) kbase_amba_set_shareable_cache_support(kbdev); #if MALI_USE_CSF kbase_backend_update_gpu_timestamp_offset(kbdev); + kbdev->csf.compute_progress_timeout_cc = 0; #endif /* Sanity check protected mode was left after reset */ diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h index 033c80a7c6b4..a7fa191b89d1 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -821,6 +821,21 @@ bool kbase_pm_is_mcu_desired(struct kbase_device *kbdev); */ bool kbase_pm_is_mcu_inactive(struct kbase_device *kbdev, enum kbase_mcu_state state); +#ifdef KBASE_PM_RUNTIME + +/** + * kbase_pm_enable_mcu_db_notification - Enable the Doorbell notification on + * MCU side + * + * @kbdev: Pointer to the device. + * + * This function is called to re-enable the Doorbell notification on MCU side + * when MCU needs to beome active again. + */ +void kbase_pm_enable_mcu_db_notification(struct kbase_device *kbdev); + +#endif /* KBASE_PM_RUNTIME */ + /** * kbase_pm_idle_groups_sched_suspendable - Check whether the scheduler can be * suspended to low power state when all @@ -963,11 +978,29 @@ static inline bool kbase_pm_gpu_sleep_allowed(struct kbase_device *kbdev) * A high positive value of autosuspend_delay can be used to keep the * GPU in sleep state for a long time. */ - if (unlikely(!kbdev->dev->power.autosuspend_delay || - (kbdev->dev->power.autosuspend_delay < 0))) + if (unlikely(kbdev->dev->power.autosuspend_delay <= 0)) return false; - return kbdev->pm.backend.gpu_sleep_supported; + return test_bit(KBASE_GPU_SUPPORTS_GPU_SLEEP, &kbdev->pm.backend.gpu_sleep_allowed); +} + +/** + * kbase_pm_fw_sleep_on_idle_allowed - Check if FW sleep-on-idle could be enabled + * + * @kbdev: Device pointer + * + * This function should be called whenever the conditions that impact + * FW sleep-on-idle support change so that it could be enabled/disabled + * accordingly. + * + * Return: true if FW sleep-on-idle is allowed + */ +static inline bool kbase_pm_fw_sleep_on_idle_allowed(struct kbase_device *kbdev) +{ + if (unlikely(kbdev->dev->power.autosuspend_delay <= 0)) + return false; + + return kbdev->pm.backend.gpu_sleep_allowed == KBASE_GPU_FW_SLEEP_ON_IDLE_ALLOWED; } /** diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c index c8e3f40335d6..457e91a0a978 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c @@ -77,7 +77,16 @@ void kbase_pm_policy_init(struct kbase_device *kbdev) spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbdev->pm.backend.pm_current_policy = default_policy; kbdev->pm.backend.csf_pm_sched_flags = default_policy->pm_sched_flags; + +#ifdef KBASE_PM_RUNTIME + if (kbase_pm_idle_groups_sched_suspendable(kbdev)) + clear_bit(KBASE_GPU_IGNORE_IDLE_EVENT, &kbdev->pm.backend.gpu_sleep_allowed); + else + set_bit(KBASE_GPU_IGNORE_IDLE_EVENT, &kbdev->pm.backend.gpu_sleep_allowed); +#endif /* KBASE_PM_RUNTIME */ + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + #else CSTD_UNUSED(flags); kbdev->pm.backend.pm_current_policy = default_policy; @@ -400,6 +409,13 @@ void kbase_pm_set_policy(struct kbase_device *kbdev, const struct kbase_pm_polic /* New policy in place, release the clamping on mcu/L2 off state */ kbdev->pm.backend.policy_change_clamp_state_to_off = false; kbase_pm_update_state(kbdev); + +#ifdef KBASE_PM_RUNTIME + if (kbase_pm_idle_groups_sched_suspendable(kbdev)) + clear_bit(KBASE_GPU_IGNORE_IDLE_EVENT, &kbdev->pm.backend.gpu_sleep_allowed); + else + set_bit(KBASE_GPU_IGNORE_IDLE_EVENT, &kbdev->pm.backend.gpu_sleep_allowed); +#endif /* KBASE_PM_RUNTIME */ #endif spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c index 331c26c6a310..d3715d97d23c 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c @@ -46,7 +46,6 @@ static struct kbase_timeout_info timeout_info[KBASE_TIMEOUT_SELECTOR_COUNT] = { CSF_FIRMWARE_PING_TIMEOUT_CYCLES) }, [CSF_PM_TIMEOUT] = { "CSF_PM_TIMEOUT", CSF_PM_TIMEOUT_CYCLES }, [CSF_GPU_RESET_TIMEOUT] = { "CSF_GPU_RESET_TIMEOUT", CSF_GPU_RESET_TIMEOUT_CYCLES }, - [CSF_CSG_SUSPEND_TIMEOUT] = { "CSF_CSG_SUSPEND_TIMEOUT", CSF_CSG_SUSPEND_TIMEOUT_CYCLES }, [CSF_CSG_TERM_TIMEOUT] = { "CSF_CSG_TERM_TIMEOUT", CSF_CSG_TERM_TIMEOUT_CYCLES }, [CSF_FIRMWARE_BOOT_TIMEOUT] = { "CSF_FIRMWARE_BOOT_TIMEOUT", CSF_FIRMWARE_BOOT_TIMEOUT_CYCLES }, @@ -236,6 +235,15 @@ void kbase_device_set_timeout_ms(struct kbase_device *kbdev, enum kbase_timeout_ } selector_str = timeout_info[selector].selector_str; +#if MALI_USE_CSF + if (IS_ENABLED(CONFIG_MALI_REAL_HW) && !IS_ENABLED(CONFIG_MALI_IS_FPGA) && + unlikely(timeout_ms >= MAX_TIMEOUT_MS)) { + dev_warn(kbdev->dev, "%s is capped from %dms to %dms\n", + timeout_info[selector].selector_str, timeout_ms, MAX_TIMEOUT_MS); + timeout_ms = MAX_TIMEOUT_MS; + } +#endif + kbdev->backend_time.device_scaled_timeouts[selector] = timeout_ms; dev_dbg(kbdev->dev, "\t%-35s: %ums\n", selector_str, timeout_ms); } diff --git a/drivers/gpu/arm/bifrost/build.bp b/drivers/gpu/arm/bifrost/build.bp index 861282bd4fab..2ae771f5b546 100644 --- a/drivers/gpu/arm/bifrost/build.bp +++ b/drivers/gpu/arm/bifrost/build.bp @@ -113,7 +113,7 @@ bob_defaults { mali_hw_errata_1485982_use_clock_alternative: { kbuild_options: ["CONFIG_MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE=y"], }, - platform_is_fpga: { + mali_is_fpga: { kbuild_options: ["CONFIG_MALI_IS_FPGA=y"], }, mali_coresight: { @@ -148,7 +148,6 @@ bob_defaults { // is an umbrella feature that would be open for inappropriate use // (catch-all for experimental CS code without separating it into // different features). - "MALI_INCREMENTAL_RENDERING_JM={{.incremental_rendering_jm}}", "MALI_BASE_CSF_PERFORMANCE_TESTS={{.base_csf_performance_tests}}", ], } @@ -162,6 +161,9 @@ bob_kernel_module { "*.c", "*.h", "Kbuild", + "arbiter/*.c", + "arbiter/*.h", + "arbiter/Kbuild", "backend/gpu/*.c", "backend/gpu/*.h", "backend/gpu/Kbuild", @@ -257,13 +259,6 @@ bob_kernel_module { "ipa/backend/*_csf.h", ], }, - mali_arbiter_support: { - srcs: [ - "arbiter/*.c", - "arbiter/*.h", - "arbiter/Kbuild", - ], - }, kbuild_options: [ "CONFIG_MALI_BIFROST=m", "CONFIG_MALI_KUTF=n", diff --git a/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_csf.c b/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_csf.c index f973d39ebb22..fe1dbfaca872 100644 --- a/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_csf.c +++ b/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_csf.c @@ -186,17 +186,15 @@ void kbase_destroy_context(struct kbase_context *kctx) * Customer side that a hang could occur if context termination is * not blocked until the resume of GPU device. */ -#ifdef CONFIG_MALI_ARBITER_SUPPORT - atomic_inc(&kbdev->pm.gpu_users_waiting); -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + if (kbase_has_arbiter(kbdev)) + atomic_inc(&kbdev->pm.gpu_users_waiting); while (kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE)) { dev_dbg(kbdev->dev, "Suspend in progress when destroying context"); wait_event(kbdev->pm.resume_wait, !kbase_pm_is_suspending(kbdev)); } -#ifdef CONFIG_MALI_ARBITER_SUPPORT - atomic_dec(&kbdev->pm.gpu_users_waiting); -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + if (kbase_has_arbiter(kbdev)) + atomic_dec(&kbdev->pm.gpu_users_waiting); /* Have synchronized against the System suspend and incremented the * pm.active_count. So any subsequent invocation of System suspend diff --git a/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_jm.c b/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_jm.c index 06c2ed813de3..ef474f625f63 100644 --- a/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_jm.c +++ b/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_jm.c @@ -231,14 +231,13 @@ void kbase_destroy_context(struct kbase_context *kctx) if (WARN_ON(!kbdev)) return; - /* Context termination could happen whilst the system suspend of + /* Context termination could happen whilst the system suspend of * the GPU device is ongoing or has completed. It has been seen on * Customer side that a hang could occur if context termination is * not blocked until the resume of GPU device. */ -#ifdef CONFIG_MALI_ARBITER_SUPPORT - atomic_inc(&kbdev->pm.gpu_users_waiting); -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + if (kbase_has_arbiter(kbdev)) + atomic_inc(&kbdev->pm.gpu_users_waiting); while (kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE)) { dev_dbg(kbdev->dev, "Suspend in progress when destroying context"); @@ -255,9 +254,8 @@ void kbase_destroy_context(struct kbase_context *kctx) */ wait_event(kbdev->pm.resume_wait, !kbase_pm_is_resuming(kbdev)); -#ifdef CONFIG_MALI_ARBITER_SUPPORT - atomic_dec(&kbdev->pm.gpu_users_waiting); -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + if (kbase_has_arbiter(kbdev)) + atomic_dec(&kbdev->pm.gpu_users_waiting); kbase_mem_pool_group_mark_dying(&kctx->mem_pools); diff --git a/drivers/gpu/arm/bifrost/csf/Kbuild b/drivers/gpu/arm/bifrost/csf/Kbuild index 5df35864efc7..8159bc9d10e8 100644 --- a/drivers/gpu/arm/bifrost/csf/Kbuild +++ b/drivers/gpu/arm/bifrost/csf/Kbuild @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -48,8 +48,10 @@ bifrost_kbase-y += \ ifeq ($(CONFIG_MALI_BIFROST_NO_MALI),y) bifrost_kbase-y += csf/mali_kbase_csf_firmware_no_mali.o +bifrost_kbase-y += csf/mali_kbase_csf_fw_io_no_mali.o else bifrost_kbase-y += csf/mali_kbase_csf_firmware.o +bifrost_kbase-y += csf/mali_kbase_csf_fw_io.o endif bifrost_kbase-$(CONFIG_DEBUG_FS) += csf/mali_kbase_debug_csf_fault.o diff --git a/drivers/gpu/arm/bifrost/csf/ipa_control/mali_kbase_csf_ipa_control.c b/drivers/gpu/arm/bifrost/csf/ipa_control/mali_kbase_csf_ipa_control.c index 61a4be9ccc94..ec47b88fac53 100644 --- a/drivers/gpu/arm/bifrost/csf/ipa_control/mali_kbase_csf_ipa_control.c +++ b/drivers/gpu/arm/bifrost/csf/ipa_control/mali_kbase_csf_ipa_control.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -943,6 +943,8 @@ void kbase_ipa_control_protm_entered(struct kbase_device *kbdev) struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control; lockdep_assert_held(&kbdev->hwaccess_lock); + + ipa_ctrl->protm_start = ktime_get_raw_ns(); } @@ -955,6 +957,7 @@ void kbase_ipa_control_protm_exited(struct kbase_device *kbdev) lockdep_assert_held(&kbdev->hwaccess_lock); + for (i = 0; i < KBASE_IPA_CONTROL_MAX_SESSIONS; i++) { struct kbase_ipa_control_session *session = &ipa_ctrl->sessions[i]; diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c index 9a33169ec554..d3300ea8dcde 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c @@ -45,6 +45,9 @@ #define CS_RING_BUFFER_MAX_SIZE ((uint32_t)(1 << 31)) /* 2GiB */ #define CS_RING_BUFFER_MIN_SIZE ((uint32_t)4096) +/* 0.2 second assuming 600 MHz GPU clock, which is double of iterator disabling timeout */ +#define MAX_PROGRESS_TIMEOUT_EVENT_DELAY ((u32)120000000) + #define PROTM_ALLOC_MAX_RETRIES ((u8)5) const u8 kbasep_csf_queue_group_priority_to_relative[BASE_QUEUE_GROUP_PRIORITY_COUNT] = { @@ -1261,6 +1264,7 @@ static int create_queue_group(struct kbase_context *const kctx, INIT_LIST_HEAD(&group->error_fatal.link); INIT_WORK(&group->timer_event_work, timer_event_worker); INIT_LIST_HEAD(&group->protm_event_work); + group->progress_timer_state = 0; atomic_set(&group->pending_protm_event_work, 0); bitmap_zero(group->protm_pending_bitmap, MAX_SUPPORTED_STREAMS_PER_GROUP); @@ -1306,12 +1310,6 @@ int kbase_csf_queue_group_create(struct kbase_context *const kctx, const u32 tiler_count = hweight64(create->in.tiler_mask); const u32 fragment_count = hweight64(create->in.fragment_mask); const u32 compute_count = hweight64(create->in.compute_mask); - size_t i; - - for (i = 0; i < ARRAY_SIZE(create->in.padding); i++) { - if (create->in.padding[i] != 0) - return -EINVAL; - } mutex_lock(&kctx->csf.lock); @@ -2178,19 +2176,74 @@ static void timer_event_worker(struct work_struct *data) } /** - * handle_progress_timer_event() - Progress timer timeout event handler. + * handle_progress_timer_events() - Progress timer timeout events handler. * - * @group: Pointer to GPU queue group for which the timeout event is received. + * @kbdev: Instance of a GPU platform device that implements a CSF interface. + * @slot_mask: Bitmap reflecting the slots on which progress timer timeouts happen. * * Notify a waiting user space client of the timeout. * Enqueue a work item to terminate the group and notify the event notification * thread of progress timeout fault for the GPU command queue group. + * Ignore fragment timeout if it is following a compute timeout. */ -static void handle_progress_timer_event(struct kbase_queue_group *const group) +static void handle_progress_timer_events(struct kbase_device *const kbdev, unsigned long *slot_mask) { - kbase_debug_csf_fault_notify(group->kctx->kbdev, group->kctx, DF_PROGRESS_TIMER_TIMEOUT); + u32 max_csg_slots = kbdev->csf.global_iface.group_num; + u32 csg_nr; + struct kbase_queue_group *group = NULL; + struct kbase_csf_cmd_stream_group_info *ginfo; - queue_work(group->kctx->csf.wq, &group->timer_event_work); + kbase_csf_scheduler_spin_lock_assert_held(kbdev); + if (likely(bitmap_empty(slot_mask, MAX_SUPPORTED_CSGS))) + return; + + /* Log each timeout and Update timestamp of compute progress timeout */ + for_each_set_bit(csg_nr, slot_mask, max_csg_slots) { + group = kbdev->csf.scheduler.csg_slots[csg_nr].resident_group; + ginfo = &kbdev->csf.global_iface.groups[csg_nr]; + group->progress_timer_state = + kbase_csf_firmware_csg_output(ginfo, CSG_PROGRESS_TIMER_STATE); + + dev_info( + kbdev->dev, + "[%llu] Iterator PROGRESS_TIMER timeout notification received for group %u of ctx %d_%d on slot %u with state %x", + kbase_backend_get_cycle_cnt(kbdev), group->handle, group->kctx->tgid, + group->kctx->id, csg_nr, group->progress_timer_state); + + if (CSG_PROGRESS_TIMER_STATE_GET(group->progress_timer_state) == + CSG_PROGRESS_TIMER_STATE_COMPUTE) + kbdev->csf.compute_progress_timeout_cc = kbase_backend_get_cycle_cnt(kbdev); + } + + /* Ignore fragment timeout if it is following a compute timeout. + * Otherwise, terminate the command stream group. + */ + for_each_set_bit(csg_nr, slot_mask, max_csg_slots) { + group = kbdev->csf.scheduler.csg_slots[csg_nr].resident_group; + + /* Check if it is a fragment timeout right after another compute timeout. + * In such case, kill compute CSG and give fragment CSG a second chance + */ + if (CSG_PROGRESS_TIMER_STATE_GET(group->progress_timer_state) == + CSG_PROGRESS_TIMER_STATE_FRAGMENT) { + u64 cycle_counter = kbase_backend_get_cycle_cnt(kbdev); + u64 compute_progress_timeout_cc = kbdev->csf.compute_progress_timeout_cc; + + if (compute_progress_timeout_cc <= cycle_counter && + cycle_counter <= compute_progress_timeout_cc + + MAX_PROGRESS_TIMEOUT_EVENT_DELAY) { + dev_info( + kbdev->dev, + "Ignored Fragment iterator timeout for group %d on slot %d", + group->handle, group->csg_nr); + continue; + } + } + + kbase_debug_csf_fault_notify(group->kctx->kbdev, group->kctx, + DF_PROGRESS_TIMER_TIMEOUT); + queue_work(group->kctx->csf.wq, &group->timer_event_work); + } } /** @@ -2291,14 +2344,14 @@ static void handle_fault_event(struct kbase_queue *const queue, const u32 cs_ack const u8 cs_fault_exception_type = CS_FAULT_EXCEPTION_TYPE_GET(cs_fault); const u32 cs_fault_exception_data = CS_FAULT_EXCEPTION_DATA_GET(cs_fault); const u64 cs_fault_info_exception_data = CS_FAULT_INFO_EXCEPTION_DATA_GET(cs_fault_info); - bool use_old_log_format = true; + bool has_trace_info = false; bool skip_fault_report = kbase_ctx_flag(queue->kctx, KCTX_PAGE_FAULT_REPORT_SKIP); kbase_csf_scheduler_spin_lock_assert_held(kbdev); - if (use_old_log_format && !skip_fault_report) + if (!has_trace_info && !skip_fault_report) dev_warn(kbdev->dev, "Ctx %d_%d Group %d CSG %d CSI: %d\n" "CS_FAULT.EXCEPTION_TYPE: 0x%x (%s)\n" @@ -2478,13 +2531,13 @@ static void handle_fatal_event(struct kbase_queue *const queue, const u32 cs_fatal_exception_type = CS_FATAL_EXCEPTION_TYPE_GET(cs_fatal); const u32 cs_fatal_exception_data = CS_FATAL_EXCEPTION_DATA_GET(cs_fatal); const u64 cs_fatal_info_exception_data = CS_FATAL_INFO_EXCEPTION_DATA_GET(cs_fatal_info); - bool use_old_log_format = true; + bool has_trace_info = false; bool skip_fault_report = kbase_ctx_flag(queue->kctx, KCTX_PAGE_FAULT_REPORT_SKIP); kbase_csf_scheduler_spin_lock_assert_held(kbdev); - if (use_old_log_format && !skip_fault_report) + if (!has_trace_info && !skip_fault_report) dev_warn(kbdev->dev, "Ctx %d_%d Group %d CSG %d CSI: %d\n" "CS_FATAL.EXCEPTION_TYPE: 0x%x (%s)\n" @@ -2649,6 +2702,8 @@ static void process_cs_interrupts(struct kbase_queue_group *const group, * @csg_nr: CSG number. * @track: Pointer that tracks the highest idle CSG and the newly possible viable * protected mode requesting group, in current IRQ context. + * @progress_timeout_slot_mask: slot mask to indicate on which slot progress timeout + * happens. * * Handles interrupts for a CSG and for CSs within it. * @@ -2660,7 +2715,8 @@ static void process_cs_interrupts(struct kbase_queue_group *const group, * See process_cs_interrupts() for details of per-stream interrupt handling. */ static void process_csg_interrupts(struct kbase_device *const kbdev, u32 const csg_nr, - struct irq_idle_and_protm_track *track) + struct irq_idle_and_protm_track *track, + unsigned long *progress_timeout_slot_mask) { struct kbase_csf_cmd_stream_group_info *ginfo; struct kbase_queue_group *group = NULL; @@ -2747,13 +2803,9 @@ static void process_csg_interrupts(struct kbase_device *const kbdev, u32 const c KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_INTERRUPT_PROGRESS_TIMER_EVENT, group, req ^ ack); - dev_info( - kbdev->dev, - "[%llu] Iterator PROGRESS_TIMER timeout notification received for group %u of ctx %d_%d on slot %u\n", - kbase_backend_get_cycle_cnt(kbdev), group->handle, group->kctx->tgid, - group->kctx->id, csg_nr); - handle_progress_timer_event(group); + set_bit(csg_nr, progress_timeout_slot_mask); + } process_cs_interrupts(group, ginfo, irqreq, irqack, track); @@ -2863,6 +2915,7 @@ static inline void check_protm_enter_req_complete(struct kbase_device *kbdev, u3 dev_dbg(kbdev->dev, "Protected mode entry interrupt received"); kbdev->protected_mode = true; + kbase_ipa_protection_mode_switch_event(kbdev); kbase_ipa_control_protm_entered(kbdev); kbase_hwcnt_backend_csf_protm_entered(&kbdev->hwcnt_gpu_iface); @@ -3047,18 +3100,25 @@ void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val) struct irq_idle_and_protm_track track = { .protm_grp = NULL, .idle_seq = U32_MAX, .idle_slot = S8_MAX }; + DECLARE_BITMAP(progress_timeout_csgs, MAX_SUPPORTED_CSGS) = { 0 }; kbase_csf_scheduler_spin_lock(kbdev, &flags); - /* Looping through and track the highest idle and protm groups */ + /* Looping through and track the highest idle and protm groups. + * Also track the groups for which progress timer timeout happened. + */ while (csg_interrupts != 0) { u32 const csg_nr = (u32)ffs((int)csg_interrupts) - 1; - process_csg_interrupts(kbdev, csg_nr, &track); + process_csg_interrupts(kbdev, csg_nr, &track, + progress_timeout_csgs); csg_interrupts &= ~(1U << csg_nr); } /* Handle protm from the tracked information */ process_tracked_info_for_protm(kbdev, &track); + /* Handle pending progress timeout(s) */ + handle_progress_timer_events(kbdev, progress_timeout_csgs); + kbase_csf_scheduler_spin_unlock(kbdev, flags); } @@ -3087,11 +3147,28 @@ void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val) /* Handle IDLE Hysteresis notification event */ if ((glb_req ^ glb_ack) & GLB_REQ_IDLE_EVENT_MASK) { + u32 const glb_idle_timer_cfg = + kbase_csf_firmware_global_input_read( + global_iface, GLB_IDLE_TIMER_CONFIG); + dev_dbg(kbdev->dev, "Idle-hysteresis event flagged"); kbase_csf_firmware_global_input_mask( global_iface, GLB_REQ, glb_ack, GLB_REQ_IDLE_EVENT_MASK); + if (glb_idle_timer_cfg & + GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_MASK) { + /* The FW is going to sleep, we shall: + * - Enable fast GPU idle handling to avoid + * confirming CSGs status in gpu_idle_worker(). + * - Enable doorbell mirroring to minimise the + * chance of KBase raising kernel doorbells which + * would cause the FW to be woken up. + */ + kbdev->csf.scheduler.fast_gpu_idle_handling = true; + kbase_pm_enable_db_mirror_interrupt(kbdev); + } + glb_idle_irq_received = true; /* Defer handling this IRQ to account for a race condition * where the idle worker could be executed before we have diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_csg_debugfs.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_csg_debugfs.c index c885845bc62e..32f33a58a6f7 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_csg_debugfs.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_csg_debugfs.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -28,6 +28,8 @@ #include #include #include +#include +#include #define MAX_SCHED_STATE_STRING_LEN (16) /** @@ -268,6 +270,87 @@ static const struct file_operations kbasep_csf_debugfs_scheduler_state_fops = { .open = simple_open, .llseek = default_llseek, }; +static int kbasep_csf_debugfs_eviction_timeout_get(void *data, u64 *val) +{ + struct kbase_device *const kbdev = data; + unsigned long flags; + + kbase_csf_scheduler_spin_lock(kbdev, &flags); + *val = kbdev->csf.csg_suspend_timeout_ms - CSG_SUSPEND_TIMEOUT_HOST_ADDED_MS; + kbase_csf_scheduler_spin_unlock(kbdev, flags); + + return 0; +} + +static int kbasep_csf_debugfs_eviction_timeout_set(void *data, u64 val) +{ + struct kbase_device *const kbdev = data; + unsigned long flags_schd, flags_hw; + u64 dur_ms = val; + int ret = 0; + + if (unlikely(dur_ms < CSG_SUSPEND_TIMEOUT_FIRMWARE_MS_MIN || + dur_ms > CSG_SUSPEND_TIMEOUT_FIRMWARE_MS_MAX)) { + dev_err(kbdev->dev, "Invalid CSG suspend timeout input (%llu)", dur_ms); + return -EFAULT; + } + dur_ms = dur_ms + CSG_SUSPEND_TIMEOUT_HOST_ADDED_MS; + + /* The 'fw_load_lock' is taken to synchronize against the deferred + * loading of FW, update will take effect after firmware gets loaded. + */ + mutex_lock(&kbdev->fw_load_lock); + if (unlikely(!kbdev->csf.firmware_inited)) { + kbase_csf_scheduler_spin_lock(kbdev, &flags_schd); + kbdev->csf.csg_suspend_timeout_ms = (unsigned int)dur_ms; + kbase_csf_scheduler_spin_unlock(kbdev, flags_schd); + mutex_unlock(&kbdev->fw_load_lock); + dev_info(kbdev->dev, "CSF set csg suspend timeout deferred till fw is loaded"); + goto end; + } + mutex_unlock(&kbdev->fw_load_lock); + + /* Firmware reloading is triggered by silent reset, and then update will take effect. + */ + kbase_csf_scheduler_pm_active(kbdev); + if (kbase_csf_scheduler_killable_wait_mcu_active(kbdev)) { + dev_err(kbdev->dev, + "Unable to activate the MCU, the csg suspend timeout value shall remain unchanged"); + kbase_csf_scheduler_pm_idle(kbdev); + ret = -EFAULT; + goto exit; + } + spin_lock_irqsave(&kbdev->hwaccess_lock, flags_hw); + if (kbase_reset_gpu_silent(kbdev)) { + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags_hw); + dev_err(kbdev->dev, "CSF set csg suspend timeout pending reset, try again"); + kbase_csf_scheduler_pm_idle(kbdev); + ret = -EFAULT; + goto exit; + } + /* GPU reset is placed and it will take place only after hwaccess_lock is released, + * update on host side should be done after GPU reset is placed and before it takes place. + */ + kbase_csf_scheduler_spin_lock(kbdev, &flags_schd); + kbdev->csf.csg_suspend_timeout_ms = (unsigned int)dur_ms; + kbase_csf_scheduler_spin_unlock(kbdev, flags_schd); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags_hw); + /* Keep PM active until reset finished to allow FW reloading to take place, + * and then update request will be sent to FW during initialization. + */ + kbase_reset_gpu_wait(kbdev); + kbase_csf_scheduler_pm_idle(kbdev); + +end: + dev_info(kbdev->dev, "CSF set csg suspend timeout: %u ms", (unsigned int)dur_ms); + +exit: + return ret; +} + +DEFINE_DEBUGFS_ATTRIBUTE(kbasep_csf_debugfs_eviction_timeout_fops, + &kbasep_csf_debugfs_eviction_timeout_get, + &kbasep_csf_debugfs_eviction_timeout_set, "%llu\n"); void kbase_csf_debugfs_init(struct kbase_device *kbdev) { @@ -280,6 +363,8 @@ void kbase_csf_debugfs_init(struct kbase_device *kbdev) &kbasep_csf_debugfs_scheduling_timer_kick_fops); debugfs_create_file("scheduler_state", 0644, kbdev->mali_debugfs_directory, kbdev, &kbasep_csf_debugfs_scheduler_state_fops); + debugfs_create_file("eviction_timeout_ms", 0644, kbdev->mali_debugfs_directory, kbdev, + &kbasep_csf_debugfs_eviction_timeout_fops); kbase_csf_tl_reader_debugfs_init(kbdev); } diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h index 155c20aaa356..38e7cb940d97 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h @@ -30,6 +30,7 @@ #include "mali_kbase_csf_firmware.h" #include "mali_kbase_csf_event.h" #include +#include "mali_kbase_csf_fw_io.h" #include @@ -267,7 +268,6 @@ enum kbase_queue_group_priority { * @CSF_PM_TIMEOUT: Timeout for GPU Power Management to reach the desired * Shader, L2 and MCU state. * @CSF_GPU_RESET_TIMEOUT: Waiting timeout for GPU reset to complete. - * @CSF_CSG_SUSPEND_TIMEOUT: Timeout given for a CSG to be suspended. * @CSF_CSG_TERM_TIMEOUT: Timeout given for a CSG to be terminated. * @CSF_FIRMWARE_BOOT_TIMEOUT: Maximum time to wait for firmware to boot. * @CSF_FIRMWARE_PING_TIMEOUT: Maximum time to wait for firmware to respond @@ -290,7 +290,6 @@ enum kbase_timeout_selector { CSF_FIRMWARE_TIMEOUT, CSF_PM_TIMEOUT, CSF_GPU_RESET_TIMEOUT, - CSF_CSG_SUSPEND_TIMEOUT, CSF_CSG_TERM_TIMEOUT, CSF_FIRMWARE_BOOT_TIMEOUT, CSF_FIRMWARE_PING_TIMEOUT, @@ -554,6 +553,8 @@ struct kbase_protected_suspend_buffer { * returned to userspace if such an error has occurred. * @timer_event_work: Work item to handle the progress timeout fatal event * for the group. + * @progress_timer_state: Value of CSG_PROGRESS_TIMER_STATE register when progress + * timer timeout is reported for the group. * @deschedule_deferred_cnt: Counter keeping a track of the number of threads * that tried to deschedule the group and had to defer * the descheduling due to the dump on fault. @@ -610,6 +611,7 @@ struct kbase_queue_group { struct kbase_csf_notification error_fatal; struct work_struct timer_event_work; + u32 progress_timer_state; /** * @dvs_buf: Address and size of scratch memory. @@ -931,13 +933,11 @@ struct kbase_csf_reset_gpu { * of CSG slots. * @resident_group: pointer to the queue group that is resident on the CSG slot. * @state: state of the slot as per enum @kbase_csf_csg_slot_state. - * @trigger_jiffies: value of jiffies when change in slot state is recorded. * @priority: dynamic priority assigned to CSG slot. */ struct kbase_csf_csg_slot { struct kbase_queue_group *resident_group; atomic_t state; - unsigned long trigger_jiffies; u8 priority; }; @@ -1137,6 +1137,8 @@ struct kbase_csf_mcu_shared_regions { * @gpuq_kthread: Dedicated thread primarily used to handle * latency-sensitive tasks such as GPU queue * submissions. + * @gpu_idle_timer_enabled: Tracks whether the GPU idle timer is enabled or disabled. + * @fw_soi_enabled: True if FW Sleep-on-Idle is currently enabled. */ struct kbase_csf_scheduler { struct mutex lock; @@ -1214,6 +1216,8 @@ struct kbase_csf_scheduler { */ spinlock_t gpu_metrics_lock; #endif /* CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD */ + atomic_t gpu_idle_timer_enabled; + atomic_t fw_soi_enabled; }; /* @@ -1677,6 +1681,7 @@ struct kbase_csf_user_reg { * @gpu_idle_dur_count_no_modifier: Update csffw_glb_req_idle_enable to make the shr(10) * modifier conditional on the new flag * in GLB_IDLE_TIMER_CONFIG. + * @csg_suspend_timeout_ms: Timeout given for a CSG to be suspended. * for any request sent to the firmware. * @hwcnt: Contain members required for handling the dump of * HW counters. @@ -1695,8 +1700,21 @@ struct kbase_csf_user_reg { * kbase_queue.pending_kick_link. * @quirks_ext: Pointer to an allocated buffer containing the firmware * workarounds configuration. + * @mmu_sync_sem: RW Semaphore to defer MMU operations till the P.Mode entrance + * or DCS request has been completed. * @pmode_sync_sem: RW Semaphore to prevent MMU operations during P.Mode entrance. - * @gpu_idle_timer_enabled: Tracks whether the GPU idle timer is enabled or disabled. + * @page_fault_cnt_ptr_address: GPU VA of the location in FW data memory, extracted from the + * FW image header, that will store the GPU VA of FW visible + * memory location where the @page_fault_cnt value will be written to. + * @page_fault_cnt_ptr: CPU VA of the FW visible memory location where the @page_fault_cnt + * value will be written to. + * @page_fault_cnt: Counter that is incremented on every GPU page fault, just before the + * MMU is unblocked to retry the memory transaction that caused the GPU + * page fault. The access to counter is serialized appropriately. + * @mcu_halted: Flag to inform MCU FSM that the MCU has already halted. + * @fw_io: Firmware I/O interface. + * @compute_progress_timeout_cc: Value of GPU cycle count register when progress + * timer timeout is reported for the compute iterator. */ struct kbase_csf_device { struct kbase_mmu_table mcu_mmu; @@ -1734,6 +1752,7 @@ struct kbase_csf_device { u64 gpu_idle_hysteresis_ns; u32 gpu_idle_dur_count; u32 gpu_idle_dur_count_no_modifier; + u32 csg_suspend_timeout_ms; struct kbase_csf_hwcnt hwcnt; struct kbase_csf_mcu_fw fw; struct kbase_csf_firmware_log fw_log; @@ -1752,8 +1771,14 @@ struct kbase_csf_device { struct list_head pending_gpuq_kick_queues[KBASE_QUEUE_GROUP_PRIORITY_COUNT]; spinlock_t pending_gpuq_kick_queues_lock; u32 *quirks_ext; + struct rw_semaphore mmu_sync_sem; struct rw_semaphore pmode_sync_sem; - bool gpu_idle_timer_enabled; + u32 page_fault_cnt_ptr_address; + u32 *page_fault_cnt_ptr; + u32 page_fault_cnt; + bool mcu_halted; + struct kbase_csf_fw_io fw_io; + u64 compute_progress_timeout_cc; }; /** diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c index 35f09028098c..2d8f96641181 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c @@ -40,6 +40,7 @@ #include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h" #include #include +#include #include #include #include @@ -55,6 +56,7 @@ #include #include +#include #define MALI_MAX_DEFAULT_FIRMWARE_NAME_LEN ((size_t)64) #define DEFAULT_FW_NAME MALI_RELEASE_NAME".mali_csffw.bin" @@ -68,6 +70,7 @@ static unsigned int csf_firmware_boot_timeout_ms; module_param(csf_firmware_boot_timeout_ms, uint, 0444); MODULE_PARM_DESC(csf_firmware_boot_timeout_ms, "Maximum time to wait for firmware to boot."); +static bool kbase_iter_trace_enable; #ifdef CONFIG_MALI_BIFROST_DEBUG /* Makes Driver wait indefinitely for an acknowledgment for the different @@ -97,6 +100,7 @@ MODULE_PARM_DESC(fw_debug, "Enables effective use of a debugger for debugging fi #define CSF_FIRMWARE_ENTRY_TYPE_TIMELINE_METADATA (4) #define CSF_FIRMWARE_ENTRY_TYPE_BUILD_INFO_METADATA (6) #define CSF_FIRMWARE_ENTRY_TYPE_FUNC_CALL_LIST (7) +#define CSF_FIRMWARE_ENTRY_TYPE_PAGE_FAULT_CNT (8) #define CSF_FIRMWARE_ENTRY_TYPE_CORE_DUMP (9) #define CSF_FIRMWARE_CACHE_MODE_NONE (0ul << 3) @@ -115,7 +119,8 @@ MODULE_PARM_DESC(fw_debug, "Enables effective use of a debugger for debugging fi #define CSF_GLB_REQ_CFG_MASK \ (GLB_REQ_CFG_ALLOC_EN_MASK | GLB_REQ_CFG_PROGRESS_TIMER_MASK | \ - GLB_REQ_CFG_PWROFF_TIMER_MASK | GLB_REQ_IDLE_ENABLE_MASK) + GLB_REQ_CFG_PWROFF_TIMER_MASK | GLB_REQ_IDLE_ENABLE_MASK | \ + GLB_REQ_CFG_EVICTION_TIMER_MASK | GLB_REQ_ITER_TRACE_ENABLE_MASK) static inline u32 input_page_read(const u32 *const input, const u32 offset) { @@ -179,6 +184,92 @@ struct firmware_timeline_metadata { size_t size; }; +static void reinit_page_fault_cnt_firmware_memory(struct kbase_device *kbdev) +{ + if (!kbdev->csf.page_fault_cnt_ptr) + return; + + /* Store the GPU address of shared memory location, where the page fault counter + * value will be written, inside the FW data memory. + */ + kbase_csf_update_firmware_memory( + kbdev, kbdev->csf.page_fault_cnt_ptr_address, + (u32)((kbdev->csf.firmware_trace_buffers.mcu_rw.va_reg->start_pfn << PAGE_SHIFT) + + PAGE_SIZE - sizeof(u32))); + + *kbdev->csf.page_fault_cnt_ptr = kbdev->csf.page_fault_cnt = 0; +} + +static void init_page_fault_cnt_firmware_memory(struct kbase_device *kbdev) +{ + if (!kbdev->csf.page_fault_cnt_ptr_address) + return; + + if (WARN_ON_ONCE(!kbdev->csf.firmware_trace_buffers.mcu_rw.va_reg)) + return; + + /* Save the CPU address of shared memory location where the page fault counter + * value will be written. + * The shared memory location comes from the last 4 bytes of the page that + * is allocated to maintain the extract offset value for different trace + * buffers. Only the first 4 bytes of every cacheline is used for the extract offset + * value. + */ + kbdev->csf.page_fault_cnt_ptr = + (u32 *)((u8 *)kbdev->csf.firmware_trace_buffers.mcu_rw.cpu_addr + PAGE_SIZE - + sizeof(u32)); + reinit_page_fault_cnt_firmware_memory(kbdev); +} + +/** + * set_iterator_trace_enable - Set the value for 'kbase_iter_trace_enable' global variable + * according to the value of GLB_FEATURES.ITER_TRACE_SUPPORTED bit, + * and the corresponding device tree entry. + * @kbdev: Kernel base device pointer + */ +static void set_iterator_trace_enable(struct kbase_device *kbdev) +{ + const struct kbase_csf_global_iface *iface = &kbdev->csf.global_iface; + bool dev_support_iter_trace = iface->features & GLB_FEATURES_ITER_TRACE_SUPPORTED_MASK; + const void *dt_iter_trace_param; + unsigned int val; + + if (!dev_support_iter_trace) { + kbase_iter_trace_enable = false; + return; + } + + + /* check device tree for iterator trace enable property and + * fallback to "iter_trace_enable" if not found and try again + */ + dt_iter_trace_param = of_get_property(kbdev->dev->of_node, "iter-trace-enable", NULL); + + if (!dt_iter_trace_param) + dt_iter_trace_param = + of_get_property(kbdev->dev->of_node, "iter_trace_enable", NULL); + + val = (dt_iter_trace_param) ? be32_to_cpup(dt_iter_trace_param) : 0; + dev_dbg(kbdev->dev, "Iterator trace enable device-tree config value: %u", val); + + kbase_iter_trace_enable = val ? true : false; +} + +static void iterator_trace_reinit(struct kbase_device *kbdev) +{ + if (kbase_iter_trace_enable) { + kbase_csf_firmware_global_input_mask(&kbdev->csf.global_iface, GLB_REQ, + GLB_REQ_ITER_TRACE_ENABLE_MASK, + GLB_REQ_ITER_TRACE_ENABLE_MASK); + } +} + +static void iterator_trace_init(struct kbase_device *kbdev) +{ + set_iterator_trace_enable(kbdev); + iterator_trace_reinit(kbdev); +} + /* The shared interface area, used for communicating with firmware, is managed * like a virtual memory zone. Reserve the virtual space from that zone * corresponding to shared interface entry parsed from the firmware image. @@ -217,7 +308,7 @@ void kbase_csf_firmware_disable_mcu(struct kbase_device *kbdev) kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(MCU_CONTROL), MCU_CONTROL_REQ_DISABLE); } -static void wait_for_firmware_stop(struct kbase_device *kbdev) +void kbase_csf_firmware_disable_mcu_wait(struct kbase_device *kbdev) { u32 val; const u32 timeout_us = @@ -232,17 +323,12 @@ static void wait_for_firmware_stop(struct kbase_device *kbdev) KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_OFF(kbdev, kbase_backend_get_cycle_cnt(kbdev)); } -void kbase_csf_firmware_disable_mcu_wait(struct kbase_device *kbdev) -{ - wait_for_firmware_stop(kbdev); -} - -static void stop_csf_firmware(struct kbase_device *kbdev) +void kbase_csf_stop_firmware_and_wait(struct kbase_device *kbdev) { /* Stop the MCU firmware */ kbase_csf_firmware_disable_mcu(kbdev); - wait_for_firmware_stop(kbdev); + kbase_csf_firmware_disable_mcu_wait(kbdev); } static void wait_for_firmware_boot(struct kbase_device *kbdev) @@ -261,7 +347,6 @@ static void wait_for_firmware_boot(struct kbase_device *kbdev) */ remaining = wait_event_timeout(kbdev->csf.event_wait, kbdev->csf.interrupt_received == true, wait_timeout); - if (!remaining) dev_err(kbdev->dev, "Timed out waiting for fw boot completion"); @@ -485,6 +570,8 @@ static int reload_fw_image(struct kbase_device *kbdev) kbdev->csf.firmware_full_reload_needed = false; kbase_csf_firmware_reload_trace_buffers_data(kbdev); + reinit_page_fault_cnt_firmware_memory(kbdev); + iterator_trace_reinit(kbdev); out: return ret; } @@ -1043,6 +1130,14 @@ static int load_firmware_entry(struct kbase_device *kbdev, const struct kbase_cs } kbase_csf_firmware_log_parse_logging_call_list_entry(kbdev, entry); return 0; + case CSF_FIRMWARE_ENTRY_TYPE_PAGE_FAULT_CNT: + /* Entry about the location of page fault counter */ + if (size < sizeof(*entry)) { + dev_err(kbdev->dev, "Page fault counter entry too short (size=%u)", size); + return -EINVAL; + } + kbdev->csf.page_fault_cnt_ptr_address = *entry; + return 0; case CSF_FIRMWARE_ENTRY_TYPE_CORE_DUMP: /* Core Dump section */ if (size < CORE_DUMP_ENTRY_START_ADDR_OFFSET + sizeof(*entry)) { @@ -1657,12 +1752,13 @@ static inline void set_gpu_idle_timer_glb_req(struct kbase_device *const kbdev, global_iface, GLB_REQ, GLB_REQ_REQ_IDLE_DISABLE, GLB_REQ_IDLE_DISABLE_MASK); } - kbdev->csf.gpu_idle_timer_enabled = set; + atomic_set(&kbdev->csf.scheduler.gpu_idle_timer_enabled, set); } static void enable_gpu_idle_timer(struct kbase_device *const kbdev) { struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; + bool const fw_soi_allowed = kbase_pm_fw_sleep_on_idle_allowed(kbdev); kbase_csf_scheduler_spin_lock_assert_held(kbdev); @@ -1670,14 +1766,114 @@ static void enable_gpu_idle_timer(struct kbase_device *const kbdev) kbdev->csf.gpu_idle_dur_count); kbase_csf_firmware_global_input_mask(global_iface, GLB_IDLE_TIMER_CONFIG, - kbdev->csf.gpu_idle_dur_count_no_modifier, + kbdev->csf.gpu_idle_dur_count_no_modifier + << GLB_IDLE_TIMER_CONFIG_NO_MODIFIER_SHIFT, GLB_IDLE_TIMER_CONFIG_NO_MODIFIER_MASK); + kbase_csf_firmware_global_input_mask(global_iface, GLB_IDLE_TIMER_CONFIG, + fw_soi_allowed + << GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_SHIFT, + GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_MASK); set_gpu_idle_timer_glb_req(kbdev, true); + atomic_set(&kbdev->csf.scheduler.fw_soi_enabled, fw_soi_allowed); dev_dbg(kbdev->dev, "Enabling GPU idle timer with count-value: 0x%.8x", kbdev->csf.gpu_idle_dur_count); } +/** + * convert_dur_to_suspend_count() - Convert CSG suspend timeout from ms to cycle count + * @kbdev: Instance of a GPU platform device that implements a CSF interface + * @dur_ms: Timeout value in ms + * @no_modifier: Indicate whether bit-shift is applied, 0 when applied, 1 otherwise + * + * Convert CSG suspend timeout from ms to cycle count, then generate a register value + * combining cycle count and timer source + * + * Return: Register value which will be stored into register GLB_EVICTION_TIMER. + */ +static u32 convert_dur_to_suspend_count(struct kbase_device *kbdev, const u64 dur_ms, + u32 *no_modifier) +{ + /* Get the cntfreq_el0 value, which drives the SYSTEM_TIMESTAMP */ + u64 freq = kbase_arch_timer_get_cntfrq(kbdev); + u64 dur_val = dur_ms; + u32 cnt_val_u32, reg_val_u32; + const bool src_system_timestamp = freq > 0; + const u8 SUSPEND_VAL_UNIT_SHIFT = 10; + + if (!src_system_timestamp) { + /* Get the cycle_counter source alternative */ + spin_lock(&kbdev->pm.clk_rtm.lock); + if (kbdev->pm.clk_rtm.clks[0]) + freq = kbdev->pm.clk_rtm.clks[0]->clock_val; + else + dev_err(kbdev->dev, "No GPU clock, unexpected intregration issue!"); + spin_unlock(&kbdev->pm.clk_rtm.lock); + + dev_info(kbdev->dev, + "No timestamp frequency, use cycle counter for csg suspend timeout!"); + } + + /* Formula for dur_val = (dur/1e3) * freq_HZ) */ + dur_val = dur_val * freq; + dur_val = div_u64(dur_val, MSEC_PER_SEC); + if (dur_val < S32_MAX) { + *no_modifier = 1; + } else { + dur_val = dur_val >> SUSPEND_VAL_UNIT_SHIFT; + *no_modifier = 0; + } + + /* Interface limits the value field to S32_MAX */ + cnt_val_u32 = (dur_val > S32_MAX) ? S32_MAX : (u32)dur_val; + + reg_val_u32 = GLB_EVICTION_TIMER_TIMEOUT_SET(0, cnt_val_u32); + /* add the source flag */ + reg_val_u32 = GLB_EVICTION_TIMER_TIMER_SOURCE_SET( + reg_val_u32, + (src_system_timestamp ? GLB_EVICTION_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP : + GLB_EVICTION_TIMER_TIMER_SOURCE_GPU_COUNTER)); + + return reg_val_u32; +} + +/** + * set_csg_suspend_timeout() - Update CSG suspend timeout setting on FW side + * + * @kbdev: Instance of a GPU platform device that implements a CSF interface + */ +static void set_csg_suspend_timeout(struct kbase_device *const kbdev) +{ + u32 dur_ms, dur_val; + u32 no_modifier = 0; + struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; + + kbase_csf_scheduler_spin_lock_assert_held(kbdev); + + dur_ms = kbdev->csf.csg_suspend_timeout_ms; + if (unlikely(dur_ms < CSG_SUSPEND_TIMEOUT_FIRMWARE_MS_MIN + + CSG_SUSPEND_TIMEOUT_HOST_ADDED_MS || + dur_ms > CSG_SUSPEND_TIMEOUT_FIRMWARE_MS_MAX + + CSG_SUSPEND_TIMEOUT_HOST_ADDED_MS)) { + dev_err(kbdev->dev, "Unexpected CSG suspend timeout: %ums, default to: %ums", + dur_ms, CSG_SUSPEND_TIMEOUT_MS); + kbdev->csf.csg_suspend_timeout_ms = CSG_SUSPEND_TIMEOUT_MS; + dur_ms = CSG_SUSPEND_TIMEOUT_MS; + } + dur_ms = dur_ms - CSG_SUSPEND_TIMEOUT_HOST_ADDED_MS; + + dur_val = convert_dur_to_suspend_count(kbdev, dur_ms, &no_modifier); + + kbase_csf_firmware_global_input(global_iface, GLB_EVICTION_TIMER, dur_val); + + kbase_csf_firmware_global_input_mask(global_iface, GLB_EVICTION_TIMER_CONFIG, no_modifier, + GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_MASK); + + set_global_request(global_iface, GLB_REQ_CFG_EVICTION_TIMER_MASK); + + dev_dbg(kbdev->dev, "Updating CSG suspend timeout with count-value: 0x%.8x", dur_val); +} + static bool global_debug_request_complete(struct kbase_device *const kbdev, u32 const req_mask) { struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; @@ -1766,7 +1962,8 @@ static void global_init(struct kbase_device *const kbdev, u64 core_mask) GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_MASK | GLB_ACK_IRQ_MASK_PROTM_ENTER_MASK | GLB_ACK_IRQ_MASK_PROTM_EXIT_MASK | GLB_ACK_IRQ_MASK_FIRMWARE_CONFIG_UPDATE_MASK | GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_MASK | GLB_ACK_IRQ_MASK_IDLE_EVENT_MASK | - GLB_REQ_DEBUG_CSF_REQ_MASK | GLB_ACK_IRQ_MASK_IDLE_ENABLE_MASK; + GLB_REQ_DEBUG_CSF_REQ_MASK | GLB_ACK_IRQ_MASK_IDLE_ENABLE_MASK | + GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_MASK | GLB_ACK_IRQ_MASK_ITER_TRACE_ENABLE_MASK; const struct kbase_csf_global_iface *const global_iface = &kbdev->csf.global_iface; unsigned long flags; @@ -1781,11 +1978,10 @@ static void global_init(struct kbase_device *const kbdev, u64 core_mask) set_timeout_global(global_iface, kbase_csf_timeout_get(kbdev)); - /* The GPU idle timer is always enabled for simplicity. Checks will be - * done before scheduling the GPU idle worker to see if it is - * appropriate for the current power policy. + /* The csg suspend timeout is always enabled so customer has the flexibility to update it + * at any time. */ - enable_gpu_idle_timer(kbdev); + set_csg_suspend_timeout(kbdev); /* Unmask the interrupts */ kbase_csf_firmware_global_input(global_iface, GLB_ACK_IRQ_MASK, ack_irq_mask); @@ -1963,6 +2159,7 @@ void kbase_csf_firmware_reload_completed(struct kbase_device *kbdev) KBASE_KTRACE_ADD(kbdev, CSF_FIRMWARE_REBOOT, NULL, 0u); + /* Tell MCU state machine to transit to next state */ kbdev->csf.firmware_reloaded = true; kbase_pm_update_state(kbdev); @@ -2068,23 +2265,34 @@ u32 kbase_csf_firmware_set_gpu_idle_hysteresis_time(struct kbase_device *kbdev, * gets serialized. */ kbase_csf_scheduler_lock(kbdev); + while (atomic_read(&kbdev->csf.scheduler.pending_gpu_idle_work) > 0) { + kbase_csf_scheduler_unlock(kbdev); + kbase_csf_scheduler_wait_for_kthread_pending_work( + kbdev, &kbdev->csf.scheduler.pending_gpu_idle_work); + kbase_csf_scheduler_lock(kbdev); + } kbase_csf_scheduler_spin_lock(kbdev, &flags); + kbdev->csf.gpu_idle_hysteresis_ns = dur_ns; kbdev->csf.gpu_idle_dur_count = hysteresis_val; kbdev->csf.gpu_idle_dur_count_no_modifier = no_modifier; - if (kbdev->csf.gpu_idle_timer_enabled) { + if (atomic_read(&kbdev->csf.scheduler.gpu_idle_timer_enabled)) { /* Timer is already enabled. Disable the timer as FW only reads * the new idle timer value when timer is re-enabled. */ kbase_csf_firmware_disable_gpu_idle_timer(kbdev); kbase_csf_scheduler_spin_unlock(kbdev, flags); /* Ensure that the request has taken effect */ - wait_for_global_request(kbdev, GLB_REQ_IDLE_DISABLE_MASK); + if (wait_for_global_request(kbdev, GLB_REQ_IDLE_DISABLE_MASK)) + dev_err(kbdev->dev, + "Failed to disable GLB_IDLE timer when setting a new idle hysteresis timeout"); kbase_csf_scheduler_spin_lock(kbdev, &flags); kbase_csf_firmware_enable_gpu_idle_timer(kbdev); kbase_csf_scheduler_spin_unlock(kbdev, flags); - wait_for_global_request(kbdev, GLB_REQ_IDLE_ENABLE_MASK); + if (wait_for_global_request(kbdev, GLB_REQ_IDLE_ENABLE_MASK)) + dev_err(kbdev->dev, + "Failed to re-enable GLB_IDLE timer when setting a new idle hysteresis timeout"); } else { kbase_csf_scheduler_spin_unlock(kbdev, flags); } @@ -2190,78 +2398,6 @@ u32 kbase_csf_firmware_reset_mcu_core_pwroff_time(struct kbase_device *kbdev) return kbase_csf_firmware_set_mcu_core_pwroff_time(kbdev, DEFAULT_GLB_PWROFF_TIMEOUT_NS); } -/** - * kbase_csf_get_iterator_trace_enable - Parsing the iterator_trace enable firstly from - * the module parameter, and then from device-tree. - * @kbdev: Kernel base device pointer - * - * Return: true on enabled, otherwise false. - */ -static bool kbase_csf_get_iterator_trace_enable(struct kbase_device *kbdev) -{ - const void *dt_iter_trace_param; - unsigned int val; - - - /* check device tree for iterator trace enable property and - * fallback to "iter_trace_enable" if not found and try again - */ - dt_iter_trace_param = of_get_property(kbdev->dev->of_node, "iter-trace-enable", NULL); - - if (!dt_iter_trace_param) - dt_iter_trace_param = - of_get_property(kbdev->dev->of_node, "iter_trace_enable", NULL); - - val = (dt_iter_trace_param) ? be32_to_cpup(dt_iter_trace_param) : 0; - dev_dbg(kbdev->dev, "Iterator trace enable device-tree config value: %u", val); - - return (val != 0); -} - -/** - * kbase_device_csf_iterator_trace_init - Send request to enable iterator - * trace port. - * @kbdev: Kernel base device pointer - * - * Return: 0 on success (or if enable request is not sent), or error - * code -EINVAL on failure of GPU to acknowledge enable request. - */ -static int kbase_device_csf_iterator_trace_init(struct kbase_device *kbdev) -{ - /* Enable the iterator trace port if supported by the GPU and is - * configured to do so. The FW must advertise this feature in GLB_FEATURES. - */ - if (kbdev->pm.backend.gpu_powered) { - const struct kbase_csf_global_iface *iface = &kbdev->csf.global_iface; - bool dev_support_iter_trace = iface->features & - GLB_FEATURES_ITER_TRACE_SUPPORTED_MASK; - - dev_dbg(kbdev->dev, "Device supporting iterator trace: %s\n", - dev_support_iter_trace ? "true" : "false"); - if (dev_support_iter_trace && kbase_csf_get_iterator_trace_enable(kbdev)) { - long ack_timeout = kbase_csf_timeout_in_jiffies( - kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT)); - - /* write enable request to global input */ - kbase_csf_firmware_global_input_mask(iface, GLB_REQ, - GLB_REQ_ITER_TRACE_ENABLE_MASK, - GLB_REQ_ITER_TRACE_ENABLE_MASK); - /* Ring global doorbell */ - kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); - - ack_timeout = wait_event_timeout( - kbdev->csf.event_wait, - !((kbase_csf_firmware_global_input_read(iface, GLB_REQ) ^ - kbase_csf_firmware_global_output(iface, GLB_ACK)) & - GLB_REQ_ITER_TRACE_ENABLE_MASK), - ack_timeout); - - return ack_timeout ? 0 : -EINVAL; - } - } - return 0; -} - int kbase_csf_firmware_early_init(struct kbase_device *kbdev) { init_waitqueue_head(&kbdev->csf.event_wait); @@ -2275,9 +2411,7 @@ int kbase_csf_firmware_early_init(struct kbase_device *kbdev) INIT_WORK(&kbdev->csf.firmware_reload_work, kbase_csf_firmware_reload_worker); INIT_WORK(&kbdev->csf.fw_error_work, firmware_error_worker); - kbdev->csf.glb_init_request_pending = true; - - init_rwsem(&kbdev->csf.pmode_sync_sem); + init_rwsem(&kbdev->csf.mmu_sync_sem); mutex_init(&kbdev->csf.reg_lock); kbase_csf_pending_gpuq_kick_queues_init(kbdev); @@ -2307,6 +2441,8 @@ int kbase_csf_firmware_late_init(struct kbase_device *kbdev) convert_dur_to_idle_count(kbdev, kbdev->csf.gpu_idle_hysteresis_ns, &no_modifier); kbdev->csf.gpu_idle_dur_count_no_modifier = no_modifier; + kbdev->csf.csg_suspend_timeout_ms = CSG_SUSPEND_TIMEOUT_MS; + return 0; } @@ -2495,6 +2631,8 @@ int kbase_csf_firmware_load_init(struct kbase_device *kbdev) goto err_out; } + init_page_fault_cnt_firmware_memory(kbdev); + ret = kbase_csf_firmware_cfg_fw_wa_init(kbdev); if (ret != 0) { dev_err(kbdev->dev, "Failed to initialize firmware workarounds"); @@ -2515,6 +2653,8 @@ int kbase_csf_firmware_load_init(struct kbase_device *kbdev) if (ret != 0) goto err_out; + iterator_trace_init(kbdev); + ret = kbase_csf_doorbell_mapping_init(kbdev); if (ret != 0) goto err_out; @@ -2545,10 +2685,6 @@ int kbase_csf_firmware_load_init(struct kbase_device *kbdev) if (ret != 0) goto err_out; - ret = kbase_device_csf_iterator_trace_init(kbdev); - if (ret != 0) - goto err_out; - if (kbdev->csf.fw_core_dump.available) kbase_csf_firmware_core_dump_init(kbdev); @@ -2598,7 +2734,7 @@ void kbase_csf_firmware_unload_term(struct kbase_device *kbdev) kbdev->csf.firmware_inited = false; if (WARN_ON(kbdev->pm.backend.mcu_state != KBASE_MCU_OFF)) { kbdev->pm.backend.mcu_state = KBASE_MCU_OFF; - stop_csf_firmware(kbdev); + kbase_csf_stop_firmware_and_wait(kbdev); } spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); @@ -2804,6 +2940,7 @@ void kbase_csf_firmware_disable_gpu_idle_timer(struct kbase_device *kbdev) kbase_csf_scheduler_spin_lock_assert_held(kbdev); set_gpu_idle_timer_glb_req(kbdev, false); + atomic_set(&kbdev->csf.scheduler.fw_soi_enabled, false); dev_dbg(kbdev->dev, "Sending request to disable gpu idle timer"); kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); @@ -2947,7 +3084,6 @@ void kbase_csf_firmware_trigger_mcu_sleep(struct kbase_device *kbdev) KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_REQUEST_SLEEP(kbdev, kbase_backend_get_cycle_cnt(kbdev)); kbase_csf_scheduler_spin_lock(kbdev, &flags); - set_gpu_idle_timer_glb_req(kbdev, false); set_global_request(global_iface, GLB_REQ_SLEEP_MASK); dev_dbg(kbdev->dev, "Sending sleep request to MCU"); kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); @@ -2956,11 +3092,23 @@ void kbase_csf_firmware_trigger_mcu_sleep(struct kbase_device *kbdev) bool kbase_csf_firmware_is_mcu_in_sleep(struct kbase_device *kbdev) { + bool db_notif_disabled; + lockdep_assert_held(&kbdev->hwaccess_lock); - return (global_request_complete(kbdev, GLB_REQ_SLEEP_MASK) && - kbase_csf_firmware_mcu_halted(kbdev)); + db_notif_disabled = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(MCU_CONTROL)) & + MCU_CNTRL_DOORBELL_DISABLE_MASK; + + if (!db_notif_disabled || !kbase_csf_firmware_mcu_halted(kbdev)) + return false; + + if (global_request_complete(kbdev, GLB_REQ_SLEEP_MASK)) + return true; + + kbase_pm_enable_mcu_db_notification(kbdev); + dev_dbg(kbdev->dev, "Enabled DB notification"); + return false; } #endif @@ -3222,3 +3370,127 @@ void kbase_csf_firmware_mcu_shared_mapping_term(struct kbase_device *kbdev, vunmap(csf_mapping->cpu_addr); kfree(csf_mapping->phys); } + +#ifdef KBASE_PM_RUNTIME + +void kbase_csf_firmware_soi_update(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + unsigned long flags; + + /* There are 3 possibilities: + * - Sleep-on-Idle allowed + * - Sleep-on-Idle not allowed, GLB_IDLE timer disabled + * - Sleep-on-Idle not allowed, GLB_IDLE timer enabled + */ + if (kbase_pm_fw_sleep_on_idle_allowed(kbdev)) { + if (likely(atomic_read(&kbdev->csf.scheduler.fw_soi_enabled))) + return; + } else { + if (test_bit(KBASE_GPU_NON_IDLE_OFF_SLOT_GROUPS_AVAILABLE, + &kbdev->pm.backend.gpu_sleep_allowed)) { + if (likely(!atomic_read(&kbdev->csf.scheduler.gpu_idle_timer_enabled))) + return; + } else if (likely(atomic_read(&kbdev->csf.scheduler.gpu_idle_timer_enabled))) { + return; + } + } + + if (kbase_reset_gpu_try_prevent(kbdev)) + return; + + kbase_csf_scheduler_lock(kbdev); + + if (atomic_read(&scheduler->pending_gpu_idle_work) > 0) + goto out_unlock_scheduler_lock; + + if ((scheduler->state == SCHED_SUSPENDED) || (scheduler->state == SCHED_SLEEPING)) + goto out_unlock_scheduler_lock; + + if (kbdev->pm.backend.mcu_state != KBASE_MCU_ON) + goto out_unlock_scheduler_lock; + + /* Ensure that an existing DISABLE request is completed before + * proceeding. They are made without waiting for them to complete such + * as when enabling the MCU. + */ + if (wait_for_global_request(kbdev, GLB_REQ_IDLE_DISABLE_MASK)) { + dev_err(kbdev->dev, + "Existing GLB_IDLE timer config change failed to complete in time (gpu_sleep_allowed:%lx)", + kbdev->pm.backend.gpu_sleep_allowed); + goto out_unlock_scheduler_lock; + } + + /* Disable the GLB IDLE timer if it's currently enabled */ + if (atomic_read(&kbdev->csf.scheduler.gpu_idle_timer_enabled)) { + kbase_csf_scheduler_spin_lock(kbdev, &flags); + kbase_csf_firmware_disable_gpu_idle_timer(kbdev); + kbase_csf_scheduler_spin_unlock(kbdev, flags); + if (wait_for_global_request(kbdev, GLB_REQ_IDLE_DISABLE_MASK)) { + dev_err(kbdev->dev, + "Failed to disable GLB_IDLE timer following FW Sleep-on-Idle config change (gpu_sleep_allowed:%lx)", + kbdev->pm.backend.gpu_sleep_allowed); + goto out_unlock_scheduler_lock; + } + } + + /* The GLB IDLE timer and, consequently, FW Sleep-on-Idle could remain + * disabled in certain cases. Otherwise, we shall re-enable GLB IDLE + * timer with the new FW Sleep-on-Idle configuration. + */ + if (!test_bit(KBASE_GPU_NON_IDLE_OFF_SLOT_GROUPS_AVAILABLE, + &kbdev->pm.backend.gpu_sleep_allowed)) { + kbase_csf_scheduler_spin_lock(kbdev, &flags); + kbase_csf_firmware_enable_gpu_idle_timer(kbdev); + kbase_csf_scheduler_spin_unlock(kbdev, flags); + if (wait_for_global_request(kbdev, GLB_REQ_IDLE_ENABLE_MASK)) { + dev_err(kbdev->dev, + "Failed to re-enable GLB_IDLE timer following FW Sleep-on-Idle config change (gpu_sleep_allowed:%lx)", + kbdev->pm.backend.gpu_sleep_allowed); + goto out_unlock_scheduler_lock; + } + } + + if (atomic_read(&scheduler->fw_soi_enabled)) { + dev_dbg(kbdev->dev, "FW Sleep-on-Idle was enabled"); + KBASE_KTRACE_ADD(kbdev, FIRMWARE_SLEEP_ON_IDLE_CHANGED, NULL, true); + } else { + dev_dbg(kbdev->dev, "FW Sleep-on-Idle was disabled"); + KBASE_KTRACE_ADD(kbdev, FIRMWARE_SLEEP_ON_IDLE_CHANGED, NULL, false); + } + +out_unlock_scheduler_lock: + kbase_csf_scheduler_unlock(kbdev); + kbase_reset_gpu_allow(kbdev); +} + +int kbase_csf_firmware_soi_disable_on_scheduler_suspend(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + unsigned long flags; + + lockdep_assert_held(&scheduler->lock); + + if (WARN_ON_ONCE(scheduler->state != SCHED_INACTIVE)) + return 0; + + if (!atomic_read(&kbdev->csf.scheduler.fw_soi_enabled)) + return 0; + + kbase_csf_scheduler_spin_lock(kbdev, &flags); + if (atomic_read(&scheduler->fw_soi_enabled)) { + kbase_csf_firmware_disable_gpu_idle_timer(kbdev); + kbase_csf_scheduler_spin_unlock(kbdev, flags); + if (wait_for_global_request(kbdev, GLB_REQ_IDLE_DISABLE_MASK)) { + dev_err(kbdev->dev, "Failed to disable Sleep-on-Idle config"); + return -ETIMEDOUT; + } + KBASE_KTRACE_ADD(kbdev, FIRMWARE_SLEEP_ON_IDLE_CHANGED, NULL, false); + } else { + kbase_csf_scheduler_spin_unlock(kbdev, flags); + } + + return 0; +} + +#endif /* KBASE_PM_RUNTIME */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.h index f7a9c07dd6a9..20cb03991bbe 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.h +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.h @@ -591,13 +591,20 @@ void kbase_csf_firmware_enable_mcu(struct kbase_device *kbdev); void kbase_csf_firmware_disable_mcu(struct kbase_device *kbdev); /** - * kbase_csf_firmware_disable_mcu_wait - Wait for the MCU to reach disabled - * status. + * kbase_csf_firmware_disable_mcu_wait - Wait for the MCU to reach disabled status. * * @kbdev: Instance of a GPU platform device that implements a CSF interface. */ void kbase_csf_firmware_disable_mcu_wait(struct kbase_device *kbdev); +/** + * kbase_csf_stop_firmware_and_wait - Disable firmware and wait for the MCU to reach + * disabled status. + * + * @kbdev: Instance of a GPU platform device that implements a CSF interface. + */ +void kbase_csf_stop_firmware_and_wait(struct kbase_device *kbdev); + #ifdef KBASE_PM_RUNTIME /** * kbase_csf_firmware_trigger_mcu_sleep - Send the command to put MCU in sleep @@ -927,4 +934,27 @@ int kbase_csf_trigger_firmware_config_update(struct kbase_device *kbdev); */ int kbase_csf_firmware_req_core_dump(struct kbase_device *const kbdev); +#ifdef KBASE_PM_RUNTIME + +/** + * kbase_csf_firmware_soi_update - Update FW Sleep-on-Idle config + * + * @kbdev: Device pointer + * + * This function reconfigures the FW Sleep-on-Idle configuration if necessary. + */ +void kbase_csf_firmware_soi_update(struct kbase_device *kbdev); + +/** + * kbase_csf_firmware_soi_disable_on_scheduler_suspend - Disable FW Sleep-on-Idle config + * on scheduler suspension + * + * @kbdev: Device pointer + * + * Return: 0 on success, otherwise failure + */ +int kbase_csf_firmware_soi_disable_on_scheduler_suspend(struct kbase_device *kbdev); + +#endif /* KBASE_PM_RUNTIME */ + #endif diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_no_mali.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_no_mali.c index 0af560fd4260..a206ed3da210 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_no_mali.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_no_mali.c @@ -684,7 +684,7 @@ static inline void set_gpu_idle_timer_glb_req(struct kbase_device *const kbdev, global_iface, GLB_REQ, GLB_REQ_REQ_IDLE_DISABLE, GLB_REQ_IDLE_DISABLE_MASK); } - kbdev->csf.gpu_idle_timer_enabled = set; + atomic_set(&kbdev->csf.scheduler.gpu_idle_timer_enabled, set); } static void enable_gpu_idle_timer(struct kbase_device *const kbdev) @@ -788,12 +788,6 @@ static void global_init(struct kbase_device *const kbdev, u64 core_mask) set_timeout_global(global_iface, kbase_csf_timeout_get(kbdev)); - /* The GPU idle timer is always enabled for simplicity. Checks will be - * done before scheduling the GPU idle worker to see if it is - * appropriate for the current power policy. - */ - enable_gpu_idle_timer(kbdev); - /* Unmask the interrupts */ kbase_csf_firmware_global_input(global_iface, GLB_ACK_IRQ_MASK, ack_irq_mask); @@ -901,6 +895,7 @@ void kbase_csf_firmware_trigger_reload(struct kbase_device *kbdev) kbdev->csf.firmware_reloaded = true; } } +KBASE_EXPORT_TEST_API(kbase_csf_firmware_trigger_reload); void kbase_csf_firmware_reload_completed(struct kbase_device *kbdev) { @@ -909,6 +904,7 @@ void kbase_csf_firmware_reload_completed(struct kbase_device *kbdev) if (unlikely(!kbdev->csf.firmware_inited)) return; + /* Tell MCU state machine to transit to next state */ kbdev->csf.firmware_reloaded = true; kbase_pm_update_state(kbdev); @@ -1019,7 +1015,7 @@ u32 kbase_csf_firmware_set_gpu_idle_hysteresis_time(struct kbase_device *kbdev, kbdev->csf.gpu_idle_dur_count = hysteresis_val; kbdev->csf.gpu_idle_dur_count_no_modifier = no_modifier; - if (kbdev->csf.gpu_idle_timer_enabled) { + if (atomic_read(&kbdev->csf.scheduler.gpu_idle_timer_enabled)) { /* Timer is already enabled. Disable the timer as FW only reads * the new idle timer value when timer is re-enabled. */ @@ -1142,7 +1138,7 @@ int kbase_csf_firmware_early_init(struct kbase_device *kbdev) INIT_WORK(&kbdev->csf.firmware_reload_work, kbase_csf_firmware_reload_worker); INIT_WORK(&kbdev->csf.fw_error_work, firmware_error_worker); - init_rwsem(&kbdev->csf.pmode_sync_sem); + init_rwsem(&kbdev->csf.mmu_sync_sem); mutex_init(&kbdev->csf.reg_lock); kbase_csf_pending_gpuq_kick_queues_init(kbdev); @@ -1210,6 +1206,7 @@ int kbase_csf_firmware_load_init(struct kbase_device *kbdev) /* NO_MALI: Don't load the MMU tables or boot CSF firmware */ + ret = invent_capabilities(kbdev); if (ret != 0) goto error; @@ -1540,6 +1537,12 @@ void kbase_csf_firmware_disable_mcu(struct kbase_device *kbdev) kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(MCU_CONTROL), MCU_CONTROL_REQ_DISABLE); } +void kbase_csf_stop_firmware_and_wait(struct kbase_device *kbdev) +{ + /* Stop the MCU firmware, no wait required on NO_MALI instance */ + kbase_csf_firmware_disable_mcu(kbdev); +} + void kbase_csf_firmware_disable_mcu_wait(struct kbase_device *kbdev) { /* NO_MALI: Nothing to do here */ @@ -1662,3 +1665,16 @@ void kbase_csf_firmware_mcu_shared_mapping_term(struct kbase_device *kbdev, vunmap(csf_mapping->cpu_addr); kfree(csf_mapping->phys); } + +#ifdef KBASE_PM_RUNTIME + +void kbase_csf_firmware_soi_update(struct kbase_device *kbdev) +{ +} + +int kbase_csf_firmware_soi_disable_on_scheduler_suspend(struct kbase_device *kbdev) +{ + return 0; +} + +#endif /* KBASE_PM_RUNTIME */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_fw_io.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_fw_io.c new file mode 100644 index 000000000000..c65f837a9f72 --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_fw_io.c @@ -0,0 +1,251 @@ +// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note +/* + * + * (C) COPYRIGHT 2024 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +#include "mali_kbase.h" +#include "mali_kbase_csf_fw_io.h" +#include + +#include + +static inline u32 input_page_read(const u32 *const input, const u32 offset) +{ + WARN_ON(offset % sizeof(u32)); + + return input[offset / sizeof(u32)]; +} + +static inline void input_page_write(u32 *const input, const u32 offset, const u32 value) +{ + WARN_ON(offset % sizeof(u32)); + + input[offset / sizeof(u32)] = value; +} + +static inline void input_page_partial_write(u32 *const input, const u32 offset, u32 value, u32 mask) +{ + WARN_ON(offset % sizeof(u32)); + + input[offset / sizeof(u32)] = (input_page_read(input, offset) & ~mask) | (value & mask); +} + +static inline u32 output_page_read(const u32 *const output, const u32 offset) +{ + WARN_ON(offset % sizeof(u32)); + + return output[offset / sizeof(u32)]; +} + +void kbase_csf_fw_io_init(struct kbase_csf_fw_io *fw_io) +{ + spin_lock_init(&fw_io->lock); + bitmap_zero(fw_io->status, KBASE_FW_IO_STATUS_NUM_BITS); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_init); + +void kbase_csf_fw_io_term(struct kbase_csf_fw_io *fw_io) +{ + /* Nothing to do. */ +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_term); + +void kbase_csf_fw_io_global_write(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_global_iface *iface, u32 offset, u32 value) +{ + const struct kbase_device *const kbdev = iface->kbdev; + + lockdep_assert_held(&fw_io->lock); + + dev_dbg(kbdev->dev, "glob input w: reg %08x val %08x\n", offset, value); + input_page_write(iface->input, offset, value); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_global_write); + +void kbase_csf_fw_io_global_write_mask(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_global_iface *iface, u32 offset, + u32 value, u32 mask) +{ + const struct kbase_device *const kbdev = iface->kbdev; + + lockdep_assert_held(&fw_io->lock); + + dev_dbg(kbdev->dev, "glob input w: reg %08x val %08x mask %08x\n", offset, value, mask); + input_page_partial_write(iface->input, offset, value, mask); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_global_write_mask); + +u32 kbase_csf_fw_io_global_input_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_global_iface *iface, u32 offset) +{ + const struct kbase_device *const kbdev = iface->kbdev; + u32 val; + + lockdep_assert_held(&fw_io->lock); + + val = input_page_read(iface->input, offset); + dev_dbg(kbdev->dev, "glob input r: reg %08x val %08x\n", offset, val); + + return val; +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_global_input_read); + +u32 kbase_csf_fw_io_global_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_global_iface *iface, u32 offset) +{ + const struct kbase_device *const kbdev = iface->kbdev; + u32 val; + + lockdep_assert_held(&fw_io->lock); + + val = output_page_read(iface->output, offset); + dev_dbg(kbdev->dev, "glob output r: reg %08x val %08x\n", offset, val); + + return val; +} + +void kbase_csf_fw_io_group_write(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_group_info *info, u32 offset, + u32 value) +{ + const struct kbase_device *const kbdev = info->kbdev; + + lockdep_assert_held(&fw_io->lock); + + dev_dbg(kbdev->dev, "csg input w: reg %08x val %08x\n", offset, value); + input_page_write(info->input, offset, value); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_group_write); + +void kbase_csf_fw_io_group_write_mask(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_group_info *info, + u32 offset, u32 value, u32 mask) +{ + const struct kbase_device *const kbdev = info->kbdev; + + lockdep_assert_held(&fw_io->lock); + + dev_dbg(kbdev->dev, "csg input w: reg %08x val %08x mask %08x\n", offset, value, mask); + input_page_partial_write(info->input, offset, value, mask); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_group_write_mask); + +u32 kbase_csf_fw_io_group_input_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_group_info *info, u32 offset) +{ + const struct kbase_device *const kbdev = info->kbdev; + u32 val; + + lockdep_assert_held(&fw_io->lock); + + val = input_page_read(info->input, offset); + dev_dbg(kbdev->dev, "csg input r: reg %08x val %08x\n", offset, val); + + return val; +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_group_input_read); + +u32 kbase_csf_fw_io_group_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_group_info *info, u32 offset) +{ + const struct kbase_device *const kbdev = info->kbdev; + u32 val; + + lockdep_assert_held(&fw_io->lock); + + val = output_page_read(info->output, offset); + dev_dbg(kbdev->dev, "csg output r: reg %08x val %08x\n", offset, val); + + return val; +} + +void kbase_csf_fw_io_stream_write(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_info *info, u32 offset, + u32 value) +{ + const struct kbase_device *const kbdev = info->kbdev; + + lockdep_assert_held(&fw_io->lock); + + dev_dbg(kbdev->dev, "cs input w: reg %08x val %08x\n", offset, value); + input_page_write(info->input, offset, value); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_stream_write); + +void kbase_csf_fw_io_stream_write_mask(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_info *info, u32 offset, + u32 value, u32 mask) +{ + const struct kbase_device *const kbdev = info->kbdev; + + lockdep_assert_held(&fw_io->lock); + + dev_dbg(kbdev->dev, "cs input w: reg %08x val %08x mask %08x\n", offset, value, mask); + input_page_partial_write(info->input, offset, value, mask); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_stream_write_mask); + +u32 kbase_csf_fw_io_stream_input_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_info *info, u32 offset) +{ + const struct kbase_device *const kbdev = info->kbdev; + u32 val; + + lockdep_assert_held(&fw_io->lock); + + val = input_page_read(info->input, offset); + dev_dbg(kbdev->dev, "cs input r: reg %08x val %08x\n", offset, val); + + return val; +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_stream_input_read); + +u32 kbase_csf_fw_io_stream_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_info *info, u32 offset) +{ + const struct kbase_device *const kbdev = info->kbdev; + u32 val; + + lockdep_assert_held(&fw_io->lock); + + val = output_page_read(info->output, offset); + dev_dbg(kbdev->dev, "cs output r: reg %08x val %08x\n", offset, val); + + return val; +} + +void kbase_csf_fw_io_set_status(struct kbase_csf_fw_io *fw_io, + enum kbase_csf_fw_io_status_bits status_bit) +{ + set_bit(status_bit, fw_io->status); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_set_status); + +void kbase_csf_fw_io_clear_status(struct kbase_csf_fw_io *fw_io, + enum kbase_csf_fw_io_status_bits status_bit) +{ + clear_bit(status_bit, fw_io->status); +} + +bool kbase_csf_fw_io_test_status(struct kbase_csf_fw_io *fw_io, + enum kbase_csf_fw_io_status_bits status_bit) +{ + return test_bit(status_bit, fw_io->status); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_test_status); diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_fw_io.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_fw_io.h new file mode 100644 index 000000000000..a8eb1ab51fbc --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_fw_io.h @@ -0,0 +1,362 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * + * (C) COPYRIGHT 2024 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +#ifndef _KBASE_CSF_FW_IO_H_ +#define _KBASE_CSF_FW_IO_H_ + +#include +#include +#include +#include + +/** The wait completed because the GPU was lost. */ +#define KBASE_CSF_FW_IO_WAIT_GPU_LOST 1 + +/** The wait was aborted because of an unexpected event. */ +#define KBASE_CSF_FW_IO_WAIT_UNSUPPORTED 255 + +/** + * enum kbase_csf_fw_io_status_bits - Status bits for firmware I/O interface. + * + * @KBASE_FW_IO_STATUS_GPU_SUSPENDED: The GPU is suspended. + * @KBASE_FW_IO_STATUS_NUM_BITS: Number of bits used to encode the state. + */ +enum kbase_csf_fw_io_status_bits { + KBASE_FW_IO_STATUS_GPU_SUSPENDED = 0, + KBASE_FW_IO_STATUS_NUM_BITS, +}; + +/** + * struct kbase_csf_fw_io - Manager of firmware input/output interface. + * + * @lock: Mutex to serialize access to the interface. + * @status: Internal status of the MCU interface. + */ +struct kbase_csf_fw_io { + spinlock_t lock; + DECLARE_BITMAP(status, KBASE_FW_IO_STATUS_NUM_BITS); +}; + +struct kbase_csf_global_iface; +struct kbase_csf_cmd_stream_group_info; +struct kbase_csf_cmd_stream_info; + +/** + * kbase_csf_fw_io_init() - Initialize manager of firmware input/output interface. + * + * @fw_io: Firmware I/O interface to initialize. + */ +void kbase_csf_fw_io_init(struct kbase_csf_fw_io *fw_io); + +/** + * kbase_csf_fw_io_term() - Terminate manager of firmware input/output interface. + * + * @fw_io: Firmware I/O interface to terminate. + */ +void kbase_csf_fw_io_term(struct kbase_csf_fw_io *fw_io); + +/** + * kbase_csf_fw_io_open() - Start a transaction with the firmware input/output interface. + * + * @fw_io: Firmware I/O interface to open. + * + * Return: 0 on success, otherwise an error code reflecting the status of the + * interface. + */ +static inline int kbase_csf_fw_io_open(struct kbase_csf_fw_io *fw_io) +{ + if (test_bit(KBASE_FW_IO_STATUS_GPU_SUSPENDED, fw_io->status)) + return -KBASE_CSF_FW_IO_WAIT_GPU_LOST; + + spin_lock(&fw_io->lock); + + return 0; +} + +/** + * kbase_csf_fw_io_open_force() - Force a transaction with the firmware input/output interface. + * + * @fw_io: Firmware I/O interface to open. + * + * This function forces the start of a transaction regardless of the status + * of the interface. + */ +static inline void kbase_csf_fw_io_open_force(struct kbase_csf_fw_io *fw_io) +{ + spin_lock(&fw_io->lock); +} + +/** + * kbase_csf_fw_io_close() - End a transaction with the firmware input/output interface. + * + * @fw_io: Firmware I/O interface to close. + */ +static inline void kbase_csf_fw_io_close(struct kbase_csf_fw_io *fw_io) +{ + spin_unlock(&fw_io->lock); +} + +/** + * kbase_csf_fw_io_assert_opened() - Assert if a transaction with the firmware input/output + * interface has started. + * + * @fw_io: Firmware I/O interface. + */ +static inline void kbase_csf_fw_io_assert_opened(struct kbase_csf_fw_io *fw_io) +{ + lockdep_assert_held(&fw_io->lock); +} + +/** + * kbase_csf_fw_io_global_write() - Write a word in the global input page. + * + * @fw_io: Firmware I/O manager. + * @iface: CSF interface provided by the firmware. + * @offset: Offset of the word to write, in bytes. + * @value: Value to be written. + */ +void kbase_csf_fw_io_global_write(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_global_iface *iface, u32 offset, + u32 value); + +/** + * kbase_csf_fw_io_global_write_mask() - Write part of a word in the global input page. + * + * @fw_io: Firmware I/O manager. + * @iface: CSF interface provided by the firmware. + * @offset: Offset of the word to write, in bytes. + * @value: Value to be written. + * @mask: Bitmask with the bits to be modified set. + */ +void kbase_csf_fw_io_global_write_mask(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_global_iface *iface, u32 offset, + u32 value, u32 mask); + +/** + * kbase_csf_fw_io_global_input_read() - Read a word in the global input page. + * + * @fw_io: Firmware I/O manager. + * @iface: CSF interface provided by the firmware. + * @offset: Offset of the word to be read, in bytes. + * + * Return: Value of the word read from the global input page. + */ +u32 kbase_csf_fw_io_global_input_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_global_iface *iface, u32 offset); + +/** + * kbase_csf_fw_io_global_read() - Read a word in the global output page. + * + * @fw_io: Firmware I/O manager. + * @iface: CSF interface provided by the firmware. + * @offset: Offset of the word to be read, in bytes. + * + * Return: Value of the word read from the global output page. + */ +u32 kbase_csf_fw_io_global_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_global_iface *iface, u32 offset); + +/** + * kbase_csf_fw_io_group_write() - Write a word in a CSG's input page. + * + * @fw_io: Firmware I/O manager. + * @info: CSG interface provided by the firmware. + * @offset: Offset of the word to write, in bytes. + * @value: Value to be written. + */ +void kbase_csf_fw_io_group_write(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_group_info *info, u32 offset, + u32 value); + +/** + * kbase_csf_fw_io_group_write_mask() - Write part of a word in a CSG's input page. + * + * @fw_io: Firmware I/O manager. + * @info: CSG interface provided by the firmware. + * @offset: Offset of the word to write, in bytes. + * @value: Value to be written. + * @mask: Bitmask with the bits to be modified set. + */ +void kbase_csf_fw_io_group_write_mask(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_group_info *info, + u32 offset, u32 value, u32 mask); + +/** + * kbase_csf_fw_io_group_input_read() - Read a word in a CSG's input page. + * + * @fw_io: Firmware I/O manager. + * @info: CSG interface provided by the firmware. + * @offset: Offset of the word to be read, in bytes. + * + * Return: Value of the word read from a CSG's input page. + */ +u32 kbase_csf_fw_io_group_input_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_group_info *info, + u32 offset); + +/** + * kbase_csf_fw_io_group_read() - Read a word in a CSG's output page. + * + * @fw_io: Firmware I/O manager. + * @info: CSG interface provided by the firmware. + * @offset: Offset of the word to be read, in bytes. + * + * Return: Value of the word read from the CSG's output page. + */ +u32 kbase_csf_fw_io_group_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_group_info *info, u32 offset); + +/** + * kbase_csf_fw_io_stream_write() - Write a word in a CS's input page. + * + * @fw_io: Firmware I/O manager. + * @info: CSI interface provided by the firmware. + * @offset: Offset of the word to write, in bytes. + * @value: Value to be written. + */ +void kbase_csf_fw_io_stream_write(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_info *info, u32 offset, + u32 value); + +/** + * kbase_csf_fw_io_stream_write_mask() - Write part of a word in a CS's input page. + * + * @fw_io: Firmware I/O manager. + * @info: CSI interface provided by the firmware. + * @offset: Offset of the word to write, in bytes. + * @value: Value to be written. + * @mask: Bitmask with the bits to be modified set. + */ +void kbase_csf_fw_io_stream_write_mask(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_info *info, u32 offset, + u32 value, u32 mask); + +/** + * kbase_csf_fw_io_stream_input_read() - Read a word in a CS's input page. + * + * @fw_io: Firmware I/O manager. + * @info: CSI interface provided by the firmware. + * @offset: Offset of the word to be read, in bytes. + * + * Return: Value of the word read from a CS's input page. + */ +u32 kbase_csf_fw_io_stream_input_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_info *info, u32 offset); + +/** + * kbase_csf_fw_io_stream_read() - Read a word in a CS's output page. + * + * @fw_io: Firmware I/O manager. + * @info: CSI interface provided by the firmware. + * @offset: Offset of the word to be read, in bytes. + * + * Return: Value of the word read from the CS's output page. + */ +u32 kbase_csf_fw_io_stream_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_info *info, u32 offset); + +/** + * kbase_csf_fw_io_set_status() - Set a FW I/O status bit. + * + * @fw_io: Firmware I/O manager. + * @status_bit: Status bit to set. + */ +void kbase_csf_fw_io_set_status(struct kbase_csf_fw_io *fw_io, + enum kbase_csf_fw_io_status_bits status_bit); + +/** + * kbase_csf_fw_io_clear_status() - Clear a FW I/O status bit. + * + * @fw_io: Firmware I/O manager. + * @status_bit: Status bit to clear. + */ +void kbase_csf_fw_io_clear_status(struct kbase_csf_fw_io *fw_io, + enum kbase_csf_fw_io_status_bits status_bit); + +/** + * kbase_csf_fw_io_test_status() - Test a FW I/O status bit. + * + * @fw_io: Firmware I/O manager. + * @status_bit: Status bit to test. + * + * Return: Value of the tested status bit. + */ +bool kbase_csf_fw_io_test_status(struct kbase_csf_fw_io *fw_io, + enum kbase_csf_fw_io_status_bits status_bit); + +/** + * kbase_csf_fw_io_wait_event_timeout() - Wait until condition gets true, timeout + * occurs or a FW I/O status bit is set. The rest of the functionalities is equal + * to wait_event_timeout(). + * + * @fw_io: Firmware I/O manager. + * @wq_head: The waitqueue to wait on. + * @condition: C expression for the event to wait for + * @timeout: Timeout, in jiffies + * + * Return: Remaining jiffies (at least 1) on success, + * 0 on timeout, + * negative KBASE_CSF_FW_IO_WAIT_* error codes otherwise. + */ +#define kbase_csf_fw_io_wait_event_timeout(fw_io, wq_head, condition, timeout) \ + ({ \ + int __ret; \ + int __wait_remaining = wait_event_timeout( \ + wq_head, condition || kbasep_csf_fw_io_check_status(fw_io), timeout); \ + __ret = kbasep_csf_fw_io_handle_wait_result(fw_io, __wait_remaining); \ + __ret; \ + }) + +/** + * kbasep_csf_fw_io_check_status() - Private function to check if any FW I/O status bit is set. + * + * @fw_io: Firmware I/O manager. + * + * Return: True if any FW I/O status bit is set, false otherwise. + */ +static inline bool kbasep_csf_fw_io_check_status(struct kbase_csf_fw_io *fw_io) +{ + return !bitmap_empty(fw_io->status, KBASE_FW_IO_STATUS_NUM_BITS); +} + +/** + * kbasep_csf_fw_io_handle_wait_result() - Private function to handle the wait_event_timeout() + * result. + * + * @fw_io: Firmware I/O manager + * @wait_remaining: Remaining jiffies returned by wait_event_timeout() + * + * Return: Remaining jiffies (at least 1) on success, + * 0 on timeout, + * negative KBASE_CSF_FW_IO_WAIT_* error codes otherwise. + */ +static inline int kbasep_csf_fw_io_handle_wait_result(struct kbase_csf_fw_io *fw_io, + int wait_remaining) +{ + /* Check for any FW IO status bit set */ + if (!bitmap_empty(fw_io->status, KBASE_FW_IO_STATUS_NUM_BITS)) + return (test_bit(KBASE_FW_IO_STATUS_GPU_SUSPENDED, fw_io->status)) ? + -KBASE_CSF_FW_IO_WAIT_GPU_LOST : + -KBASE_CSF_FW_IO_WAIT_UNSUPPORTED; + + return wait_remaining; +} +#endif diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_fw_io_no_mali.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_fw_io_no_mali.c new file mode 100644 index 000000000000..0cffc8475654 --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_fw_io_no_mali.c @@ -0,0 +1,294 @@ +// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note +/* + * + * (C) COPYRIGHT 2024 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +#include "mali_kbase.h" +#include "mali_kbase_csf_fw_io.h" +#include + +#include + +static inline u32 input_page_read(const u32 *const input, const u32 offset) +{ + WARN_ON(offset % sizeof(u32)); + + return input[offset / sizeof(u32)]; +} + +static inline void input_page_write(u32 *const input, const u32 offset, const u32 value) +{ + WARN_ON(offset % sizeof(u32)); + + input[offset / sizeof(u32)] = value; +} + +static inline void input_page_partial_write(u32 *const input, const u32 offset, u32 value, u32 mask) +{ + WARN_ON(offset % sizeof(u32)); + + input[offset / sizeof(u32)] = (input_page_read(input, offset) & ~mask) | (value & mask); +} + +static inline u32 output_page_read(const u32 *const output, const u32 offset) +{ + WARN_ON(offset % sizeof(u32)); + + return output[offset / sizeof(u32)]; +} + +static inline void output_page_write(u32 *const output, const u32 offset, const u32 value) +{ + WARN_ON(offset % sizeof(u32)); + + output[offset / sizeof(u32)] = value; +} + +void kbase_csf_fw_io_init(struct kbase_csf_fw_io *fw_io) +{ + spin_lock_init(&fw_io->lock); + bitmap_zero(fw_io->status, KBASE_FW_IO_STATUS_NUM_BITS); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_init); + +void kbase_csf_fw_io_term(struct kbase_csf_fw_io *fw_io) +{ + /* Nothing to do. */ +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_term); + +void kbase_csf_fw_io_global_write(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_global_iface *iface, u32 offset, u32 value) +{ + const struct kbase_device *const kbdev = iface->kbdev; + + lockdep_assert_held(&fw_io->lock); + + dev_dbg(kbdev->dev, "glob input w: reg %08x val %08x\n", offset, value); + input_page_write(iface->input, offset, value); + + if (offset == GLB_REQ) { + /* NO_MALI: Immediately acknowledge requests - except for PRFCNT_ENABLE + * and PRFCNT_SAMPLE. These will be processed along with the + * corresponding performance counter registers when the global doorbell + * is rung in order to emulate the performance counter sampling behavior + * of the real firmware. + */ + const u32 ack = output_page_read(iface->output, GLB_ACK); + const u32 req_mask = ~(GLB_REQ_PRFCNT_ENABLE_MASK | GLB_REQ_PRFCNT_SAMPLE_MASK); + const u32 toggled = (value ^ ack) & req_mask; + + output_page_write(iface->output, GLB_ACK, ack ^ toggled); + } +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_global_write); + +void kbase_csf_fw_io_global_write_mask(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_global_iface *iface, u32 offset, + u32 value, u32 mask) +{ + const struct kbase_device *const kbdev = iface->kbdev; + + lockdep_assert_held(&fw_io->lock); + + dev_dbg(kbdev->dev, "glob input w: reg %08x val %08x mask %08x\n", offset, value, mask); + + /* NO_MALI: Go through existing function to capture writes */ + kbase_csf_fw_io_global_write(fw_io, iface, offset, + (input_page_read(iface->input, offset) & ~mask) | + (value & mask)); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_global_write_mask); + +u32 kbase_csf_fw_io_global_input_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_global_iface *iface, u32 offset) +{ + const struct kbase_device *const kbdev = iface->kbdev; + u32 val; + + lockdep_assert_held(&fw_io->lock); + + val = input_page_read(iface->input, offset); + dev_dbg(kbdev->dev, "glob input r: reg %08x val %08x\n", offset, val); + + return val; +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_global_input_read); + +u32 kbase_csf_fw_io_global_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_global_iface *iface, u32 offset) +{ + const struct kbase_device *const kbdev = iface->kbdev; + u32 val; + + lockdep_assert_held(&fw_io->lock); + + val = output_page_read(iface->output, offset); + dev_dbg(kbdev->dev, "glob output r: reg %08x val %08x\n", offset, val); + + return val; +} + +void kbase_csf_fw_io_group_write(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_group_info *info, u32 offset, + u32 value) +{ + const struct kbase_device *const kbdev = info->kbdev; + + lockdep_assert_held(&fw_io->lock); + + dev_dbg(kbdev->dev, "csg input w: reg %08x val %08x\n", offset, value); + input_page_write(info->input, offset, value); + + if (offset == CSG_REQ) { + /* NO_MALI: Immediately acknowledge requests */ + output_page_write(info->output, CSG_ACK, value); + } +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_group_write); + +void kbase_csf_fw_io_group_write_mask(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_group_info *info, + u32 offset, u32 value, u32 mask) +{ + const struct kbase_device *const kbdev = info->kbdev; + + lockdep_assert_held(&fw_io->lock); + + dev_dbg(kbdev->dev, "csg input w: reg %08x val %08x mask %08x\n", offset, value, mask); + + /* NO_MALI: Go through existing function to capture writes */ + kbase_csf_fw_io_group_write(fw_io, info, offset, + (input_page_read(info->input, offset) & ~mask) | + (value & mask)); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_group_write_mask); + +u32 kbase_csf_fw_io_group_input_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_group_info *info, u32 offset) +{ + const struct kbase_device *const kbdev = info->kbdev; + u32 val; + + lockdep_assert_held(&fw_io->lock); + + val = input_page_read(info->input, offset); + dev_dbg(kbdev->dev, "csg input r: reg %08x val %08x\n", offset, val); + + return val; +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_group_input_read); + +u32 kbase_csf_fw_io_group_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_group_info *info, u32 offset) +{ + const struct kbase_device *const kbdev = info->kbdev; + u32 val; + + lockdep_assert_held(&fw_io->lock); + + val = output_page_read(info->output, offset); + dev_dbg(kbdev->dev, "csg output r: reg %08x val %08x\n", offset, val); + + return val; +} + +void kbase_csf_fw_io_stream_write(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_info *info, u32 offset, + u32 value) +{ + const struct kbase_device *const kbdev = info->kbdev; + + lockdep_assert_held(&fw_io->lock); + + dev_dbg(kbdev->dev, "cs input w: reg %08x val %08x\n", offset, value); + input_page_write(info->input, offset, value); + + if (offset == CS_REQ) { + /* NO_MALI: Immediately acknowledge requests */ + output_page_write(info->output, CS_ACK, value); + } +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_stream_write); + +void kbase_csf_fw_io_stream_write_mask(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_info *info, u32 offset, + u32 value, u32 mask) +{ + const struct kbase_device *const kbdev = info->kbdev; + + lockdep_assert_held(&fw_io->lock); + + dev_dbg(kbdev->dev, "cs input w: reg %08x val %08x mask %08x\n", offset, value, mask); + + /* NO_MALI: Go through existing function to capture writes */ + kbase_csf_fw_io_stream_write(fw_io, info, offset, + (input_page_read(info->input, offset) & ~mask) | + (value & mask)); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_stream_write_mask); + +u32 kbase_csf_fw_io_stream_input_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_info *info, u32 offset) +{ + const struct kbase_device *const kbdev = info->kbdev; + u32 val; + + lockdep_assert_held(&fw_io->lock); + + val = input_page_read(info->input, offset); + dev_dbg(kbdev->dev, "cs input r: reg %08x val %08x\n", offset, val); + + return val; +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_stream_input_read); + +u32 kbase_csf_fw_io_stream_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_info *info, u32 offset) +{ + const struct kbase_device *const kbdev = info->kbdev; + u32 val; + + lockdep_assert_held(&fw_io->lock); + + val = output_page_read(info->output, offset); + dev_dbg(kbdev->dev, "cs output r: reg %08x val %08x\n", offset, val); + + return val; +} + +void kbase_csf_fw_io_set_status(struct kbase_csf_fw_io *fw_io, + enum kbase_csf_fw_io_status_bits status_bit) +{ + set_bit(status_bit, fw_io->status); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_set_status); + +void kbase_csf_fw_io_clear_status(struct kbase_csf_fw_io *fw_io, + enum kbase_csf_fw_io_status_bits status_bit) +{ + clear_bit(status_bit, fw_io->status); +} + +bool kbase_csf_fw_io_test_status(struct kbase_csf_fw_io *fw_io, + enum kbase_csf_fw_io_status_bits status_bit) +{ + return test_bit(status_bit, fw_io->status); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_test_status); diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_heap_context_alloc.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_heap_context_alloc.c index 12a79b4852fb..fb181026719f 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_heap_context_alloc.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_heap_context_alloc.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -180,8 +180,9 @@ void kbase_csf_heap_context_allocator_term(struct kbase_csf_heap_context_allocat u64 kbase_csf_heap_context_allocator_alloc(struct kbase_csf_heap_context_allocator *const ctx_alloc) { struct kbase_context *const kctx = ctx_alloc->kctx; - u64 flags = BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR | BASE_MEM_PROT_CPU_WR | - BASEP_MEM_NO_USER_FREE | BASE_MEM_PROT_CPU_RD; + base_mem_alloc_flags flags = BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR | + BASE_MEM_PROT_CPU_WR | BASEP_MEM_NO_USER_FREE | + BASE_MEM_PROT_CPU_RD; u64 nr_pages = PFN_UP(MAX_TILER_HEAPS * ctx_alloc->heap_context_size_aligned); u64 heap_gpu_va = 0; diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c index ba47b7190395..09c92f0bed4e 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c @@ -371,7 +371,7 @@ static int kbase_kcpu_jit_allocate_prepare(struct kbase_kcpu_command_queue *kcpu goto out; } - if (copy_from_user(info, data, sizeof(*info) * count) != 0) { + if (copy_from_user(info, data, size_mul(sizeof(*info), count)) != 0) { ret = -EINVAL; goto out_free; } @@ -563,7 +563,7 @@ static int kbase_kcpu_jit_free_prepare(struct kbase_kcpu_command_queue *kcpu_que goto out_free; } - if (copy_from_user(ids, data, sizeof(*ids) * count)) { + if (copy_from_user(ids, data, size_mul(sizeof(*ids), count))) { ret = -EINVAL; goto out_free; } @@ -852,7 +852,8 @@ static int kbase_kcpu_cqs_wait_prepare(struct kbase_kcpu_command_queue *queue, if (!objs) return -ENOMEM; - if (copy_from_user(objs, u64_to_user_ptr(cqs_wait_info->objs), nr_objs * sizeof(*objs))) { + if (copy_from_user(objs, u64_to_user_ptr(cqs_wait_info->objs), + size_mul(nr_objs, sizeof(*objs)))) { kfree(objs); return -ENOMEM; } @@ -957,7 +958,8 @@ static int kbase_kcpu_cqs_set_prepare(struct kbase_kcpu_command_queue *kcpu_queu if (!objs) return -ENOMEM; - if (copy_from_user(objs, u64_to_user_ptr(cqs_set_info->objs), nr_objs * sizeof(*objs))) { + if (copy_from_user(objs, u64_to_user_ptr(cqs_set_info->objs), + size_mul(nr_objs, sizeof(*objs)))) { kfree(objs); return -ENOMEM; } @@ -1115,7 +1117,7 @@ static int kbase_kcpu_cqs_wait_operation_prepare( return -ENOMEM; if (copy_from_user(objs, u64_to_user_ptr(cqs_wait_operation_info->objs), - nr_objs * sizeof(*objs))) { + size_mul(nr_objs, sizeof(*objs)))) { kfree(objs); return -ENOMEM; } @@ -1280,7 +1282,7 @@ static int kbase_kcpu_cqs_set_operation_prepare( return -ENOMEM; if (copy_from_user(objs, u64_to_user_ptr(cqs_set_operation_info->objs), - nr_objs * sizeof(*objs))) { + size_mul(nr_objs, sizeof(*objs)))) { kfree(objs); return -ENOMEM; } diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_registers.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_registers.h index 9a7c6e451f66..7e96a9d01fc7 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_registers.h +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_registers.h @@ -172,6 +172,11 @@ #define CSG_STATUS_EP_CURRENT 0x0010 /* () Endpoint allocation status register */ #define CSG_STATUS_EP_REQ 0x0014 /* () Endpoint request status register */ #define CSG_RESOURCE_DEP 0x001C /* () Current resource dependencies */ +/* TODO: GPUCORE-xxxx: Remove after spec alignment, use 0x1C as CSG_RESOURCE_DEP is deprecated*/ +/* CSG_OUTPUT_BLOCK register offsets */ +#ifndef CSG_PROGRESS_TIMER_STATE +#define CSG_PROGRESS_TIMER_STATE 0x001C /* () Current resource status */ +#endif /* GLB_CONTROL_BLOCK register offsets */ #define GLB_VERSION 0x0000 /* () Global interface version */ @@ -1243,6 +1248,21 @@ CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_MASK)) +/* CSG_PROGRESS_TIMER_STATE register */ +#ifndef CSG_PROGRESS_TIMER_STATE_GET +#define CSG_PROGRESS_TIMER_STATE_SHIFT 0 +#define CSG_PROGRESS_TIMER_STATE_MASK ((u32)0xFFFFFFFF << CSG_PROGRESS_TIMER_STATE_SHIFT) +#define CSG_PROGRESS_TIMER_STATE_GET(reg_val) \ + (((reg_val)&CSG_PROGRESS_TIMER_STATE_MASK) >> CSG_PROGRESS_TIMER_STATE_SHIFT) +#define CSG_PROGRESS_TIMER_STATE_SET(reg_val, value) \ + (((reg_val) & ~CSG_PROGRESS_TIMER_STATE_MASK) | \ + (((value) << CSG_PROGRESS_TIMER_STATE_SHIFT) & CSG_PROGRESS_TIMER_STATE_MASK)) +/* CSG_PROGRESS_TIMER_STATE values */ +#define CSG_PROGRESS_TIMER_STATE_COMPUTE 0x0 +#define CSG_PROGRESS_TIMER_STATE_FRAGMENT 0x1 +#define CSG_PROGRESS_TIMER_STATE_TILER 0x2 +#define CSG_PROGRESS_TIMER_STATE_NEURAL 0x3 +#endif /* End of CSG_OUTPUT_BLOCK register set definitions */ /* STREAM_CONTROL_BLOCK register set definitions */ @@ -1380,6 +1400,13 @@ #define GLB_REQ_SLEEP_SET(reg_val, value) \ (((reg_val) & ~GLB_REQ_SLEEP_MASK) | \ (((value) << GLB_REQ_SLEEP_SHIFT) & GLB_REQ_SLEEP_MASK)) +#define GLB_REQ_CFG_EVICTION_TIMER_SHIFT 16 +#define GLB_REQ_CFG_EVICTION_TIMER_MASK (0x1 << GLB_REQ_CFG_EVICTION_TIMER_SHIFT) +#define GLB_REQ_CFG_EVICTION_TIMER_GET(reg_val) \ + (((reg_val)&GLB_REQ_CFG_EVICTION_TIMER_MASK) >> GLB_REQ_CFG_EVICTION_TIMER_SHIFT) +#define GLB_REQ_CFG_EVICTION_TIMER_SET(reg_val, value) \ + (((reg_val) & ~GLB_REQ_CFG_EVICTION_TIMER_MASK) | \ + (((value) << GLB_REQ_CFG_EVICTION_TIMER_SHIFT) & GLB_REQ_CFG_EVICTION_TIMER_MASK)) #define GLB_REQ_INACTIVE_COMPUTE_SHIFT 20 #define GLB_REQ_INACTIVE_COMPUTE_MASK (0x1 << GLB_REQ_INACTIVE_COMPUTE_SHIFT) #define GLB_REQ_INACTIVE_COMPUTE_GET(reg_val) \ @@ -1524,6 +1551,17 @@ (((reg_val) & ~GLB_ACK_IRQ_MASK_FIRMWARE_CONFIG_UPDATE_MASK) | \ (((value) << GLB_ACK_IRQ_MASK_FIRMWARE_CONFIG_UPDATE_SHIFT) & \ GLB_ACK_IRQ_MASK_FIRMWARE_CONFIG_UPDATE_MASK)) +#define GLB_ACK_IRQ_MASK_ITER_TRACE_ENABLE_SHIFT 11 +#define GLB_ACK_IRQ_MASK_ITER_TRACE_ENABLE_MASK (0x1 << GLB_ACK_IRQ_MASK_ITER_TRACE_ENABLE_SHIFT) +#define GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_SHIFT 16 +#define GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_MASK (0x1 << GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_SHIFT) +#define GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_GET(reg_val) \ + (((reg_val)&GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_MASK) >> \ + GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_SHIFT) +#define GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_SET(reg_val, value) \ + (((reg_val) & ~GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_MASK) | \ + (((value) << GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_SHIFT) & \ + GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_MASK)) #define GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_SHIFT 20 #define GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_MASK (0x1 << GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_SHIFT) #define GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_GET(reg_val) \ @@ -1635,6 +1673,45 @@ GLB_PWROFF_TIMER_CONFIG_NO_MODIFIER_MASK)) #endif /* End of GLB_PWROFF_TIMER_CONFIG values */ +/* GLB_EVICTION_TIMER register */ +#ifndef GLB_EVICTION_TIMER +#define GLB_EVICTION_TIMER 0x0090 +#define GLB_EVICTION_TIMER_TIMEOUT_SHIFT (0) +#define GLB_EVICTION_TIMER_TIMEOUT_MASK ((0x7FFFFFFF) << GLB_EVICTION_TIMER_TIMEOUT_SHIFT) +#define GLB_EVICTION_TIMER_TIMEOUT_GET(reg_val) \ + (((reg_val)&GLB_EVICTION_TIMER_TIMEOUT_MASK) >> GLB_EVICTION_TIMER_TIMEOUT_SHIFT) +#define GLB_EVICTION_TIMER_TIMEOUT_SET(reg_val, value) \ + (((reg_val) & ~GLB_EVICTION_TIMER_TIMEOUT_MASK) | \ + (((value) << GLB_EVICTION_TIMER_TIMEOUT_SHIFT) & GLB_EVICTION_TIMER_TIMEOUT_MASK)) +#define GLB_EVICTION_TIMER_TIMER_SOURCE_SHIFT (31) +#define GLB_EVICTION_TIMER_TIMER_SOURCE_MASK ((0x1) << GLB_EVICTION_TIMER_TIMER_SOURCE_SHIFT) +#define GLB_EVICTION_TIMER_TIMER_SOURCE_GET(reg_val) \ + (((reg_val)&GLB_EVICTION_TIMER_TIMER_SOURCE_MASK) >> GLB_EVICTION_TIMER_TIMER_SOURCE_SHIFT) +#define GLB_EVICTION_TIMER_TIMER_SOURCE_SET(reg_val, value) \ + (((reg_val) & ~GLB_EVICTION_TIMER_TIMER_SOURCE_MASK) | \ + (((value) << GLB_EVICTION_TIMER_TIMER_SOURCE_SHIFT) & \ + GLB_EVICTION_TIMER_TIMER_SOURCE_MASK)) +/* GLB_EVICTION_TIMER_TIMER_SOURCE values */ +#define GLB_EVICTION_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP 0x0U +#define GLB_EVICTION_TIMER_TIMER_SOURCE_GPU_COUNTER 0x1U +/* End of GLB_EVICTION_TIMER_TIMER_SOURCE values */ +#endif /* End of GLB_EVICTION_TIMER */ + +/* GLB_EVICTION_TIMER_CONFIG register */ +#ifndef GLB_EVICTION_TIMER_CONFIG +#define GLB_EVICTION_TIMER_CONFIG 0x0094 /* () Configuration fields for GLB_EVICTION_TIMER */ +#define GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_SHIFT 0 +#define GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_MASK \ + (0x1 << GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_SHIFT) +#define GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_GET(reg_val) \ + (((reg_val)&GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_MASK) >> \ + GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_SHIFT) +#define GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_SET(reg_val, value) \ + (((reg_val) & ~GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_MASK) | \ + (((value) << GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_SHIFT) & \ + GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_MASK)) +#endif /* End of GLB_EVICTION_TIMER_CONFIG values */ + /* GLB_ALLOC_EN register */ #define GLB_ALLOC_EN_MASK_SHIFT 0 #define GLB_ALLOC_EN_MASK_MASK (GPU_ULL(0xFFFFFFFFFFFFFFFF) << GLB_ALLOC_EN_MASK_SHIFT) @@ -1717,6 +1794,15 @@ (((reg_val) & ~GLB_IDLE_TIMER_CONFIG_NO_MODIFIER_MASK) | \ (((value) << GLB_IDLE_TIMER_CONFIG_NO_MODIFIER_SHIFT) & \ GLB_IDLE_TIMER_CONFIG_NO_MODIFIER_MASK)) +#define GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_SHIFT 9 +#define GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_MASK (0x1 << GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_SHIFT) +#define GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_GET(reg_val) \ + (((reg_val)&GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_MASK) >> \ + GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_SHIFT) +#define GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_SET(reg_val, value) \ + (((reg_val) & ~GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_MASK) | \ + (((value) << GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_SHIFT) & \ + GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_MASK)) #endif /* End of GLB_IDLE_TIMER_CONFIG values */ /* GLB_INSTR_FEATURES register */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_reset_gpu.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_reset_gpu.c index b07cc9600a04..ffd27318cba3 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_reset_gpu.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_reset_gpu.c @@ -496,12 +496,10 @@ static void kbase_csf_reset_gpu_worker(struct work_struct *data) bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev, unsigned int flags) { -#ifdef CONFIG_MALI_ARBITER_SUPPORT if (kbase_pm_is_gpu_lost(kbdev)) { /* GPU access has been removed, reset will be done by Arbiter instead */ return false; } -#endif if (flags & RESET_FLAGS_HWC_UNRECOVERABLE_ERROR) kbase_hwcnt_backend_csf_on_unrecoverable_error(&kbdev->hwcnt_gpu_iface); diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.c index 642531c1033c..cd6abd62f6c5 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.c @@ -42,6 +42,7 @@ #include #endif /* CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD */ + /* Value to indicate that a queue group is not groups_to_schedule list */ #define KBASEP_GROUP_PREPARED_SEQ_NUM_INVALID (U32_MAX) @@ -93,6 +94,11 @@ static bool queue_group_scheduled_locked(struct kbase_queue_group *group); #define kctx_as_enabled(kctx) (!kbase_ctx_flag(kctx, KCTX_AS_DISABLED_ON_FAULT)) +bool is_gpu_level_suspend_supported(struct kbase_device *const kbdev) +{ + return false; +} + #if IS_ENABLED(CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD) /** * gpu_metrics_ctx_init() - Take a reference on GPU metrics context if it exists, @@ -1196,13 +1202,20 @@ static void scheduler_wakeup(struct kbase_device *kbdev, bool kick) scheduler_enable_tick_timer_nolock(kbdev); } -static void scheduler_suspend(struct kbase_device *kbdev) +static int scheduler_suspend(struct kbase_device *kbdev) { struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; lockdep_assert_held(&scheduler->lock); if (!WARN_ON(scheduler->state == SCHED_SUSPENDED)) { +#if KBASE_PM_RUNTIME + int ret; + + ret = kbase_csf_firmware_soi_disable_on_scheduler_suspend(kbdev); + if (ret) + return ret; +#endif /* KBASE_PM_RUNTIME */ dev_dbg(kbdev->dev, "Suspending the Scheduler"); scheduler_pm_idle(kbdev); scheduler->state = SCHED_SUSPENDED; @@ -1211,6 +1224,8 @@ static void scheduler_suspend(struct kbase_device *kbdev) #endif KBASE_KTRACE_ADD(kbdev, SCHED_SUSPENDED, NULL, scheduler->state); } + + return 0; } /** @@ -1462,7 +1477,7 @@ static int sched_halt_stream(struct kbase_queue *queue) long remaining; int slot; int err = 0; - const u32 group_schedule_timeout = kbase_get_timeout_ms(kbdev, CSF_CSG_SUSPEND_TIMEOUT); + const u32 group_schedule_timeout = kbdev->csf.csg_suspend_timeout_ms; const u32 fw_timeout_ms = kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT); if (WARN_ON(!group)) @@ -1942,7 +1957,6 @@ static enum kbase_csf_csg_slot_state update_csg_slot_status(struct kbase_device if ((state == CSG_ACK_STATE_START) || (state == CSG_ACK_STATE_RESUME)) { slot_state = CSG_SLOT_RUNNING; atomic_set(&csg_slot->state, slot_state); - csg_slot->trigger_jiffies = jiffies; KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_RUNNING, csg_slot->resident_group, state); dev_dbg(kbdev->dev, "Group %u running on slot %d\n", @@ -1953,7 +1967,6 @@ static enum kbase_csf_csg_slot_state update_csg_slot_status(struct kbase_device if ((state == CSG_ACK_STATE_SUSPEND) || (state == CSG_ACK_STATE_TERMINATE)) { slot_state = CSG_SLOT_STOPPED; atomic_set(&csg_slot->state, slot_state); - csg_slot->trigger_jiffies = jiffies; KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_STOPPED, csg_slot->resident_group, state); dev_dbg(kbdev->dev, "Group %u stopped on slot %d\n", @@ -2052,7 +2065,6 @@ static void halt_csg_slot(struct kbase_queue_group *group, bool suspend) kbase_csf_ring_csg_doorbell(kbdev, slot); spin_unlock_irqrestore(&kbdev->csf.scheduler.interrupt_lock, flags); atomic_set(&csg_slot[slot].state, CSG_SLOT_DOWN2STOP); - csg_slot[slot].trigger_jiffies = jiffies; KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_STOP_REQ, group, halt_cmd); KBASE_TLSTREAM_TL_KBASE_DEVICE_HALTING_CSG(kbdev, kbdev->id, (u32)slot, suspend); @@ -2367,11 +2379,6 @@ static void cancel_tock_work(struct kbase_csf_scheduler *const scheduler) atomic_set(&scheduler->pending_tock_work, false); } -static void cancel_gpu_idle_work(struct kbase_csf_scheduler *const scheduler) -{ - atomic_set(&scheduler->pending_gpu_idle_work, false); -} - static void remove_group_from_runnable(struct kbase_csf_scheduler *const scheduler, struct kbase_queue_group *group, enum kbase_csf_group_state run_state) @@ -2776,7 +2783,6 @@ static bool cleanup_csg_slot(struct kbase_queue_group *group) spin_unlock_bh(&kbdev->csf.scheduler.gpu_metrics_lock); #endif - csg_slot->trigger_jiffies = jiffies; atomic_set(&csg_slot->state, CSG_SLOT_READY); KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_CLEANED, group, (u64)slot); @@ -2990,7 +2996,6 @@ static void program_csg_slot(struct kbase_queue_group *group, s8 slot, u8 prio) /* Update status before rings the door-bell, marking ready => run */ atomic_set(&csg_slot->state, CSG_SLOT_READY2RUN); - csg_slot->trigger_jiffies = jiffies; csg_slot->priority = prio; /* Trace the programming of the CSG on the slot */ @@ -3556,8 +3561,7 @@ static void program_suspending_csg_slots(struct kbase_device *kbdev) while (!bitmap_empty(slot_mask, MAX_SUPPORTED_CSGS)) { DECLARE_BITMAP(changed, MAX_SUPPORTED_CSGS); - long remaining = kbase_csf_timeout_in_jiffies( - kbase_get_timeout_ms(kbdev, CSF_CSG_SUSPEND_TIMEOUT)); + long remaining = kbase_csf_timeout_in_jiffies(kbdev->csf.csg_suspend_timeout_ms); bitmap_copy(changed, slot_mask, MAX_SUPPORTED_CSGS); @@ -4101,7 +4105,7 @@ static void scheduler_group_check_protm_enter(struct kbase_device *const kbdev, * entry to protected mode happens with a memory region being locked and * the same region is then accessed by the GPU in protected mode. */ - down_write(&kbdev->csf.pmode_sync_sem); + down_write(&kbdev->csf.mmu_sync_sem); spin_lock_irqsave(&scheduler->interrupt_lock, flags); /* Check if the previous transition to enter & exit the protected @@ -4167,7 +4171,7 @@ static void scheduler_group_check_protm_enter(struct kbase_device *const kbdev, spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); err = kbase_csf_wait_protected_mode_enter(kbdev); - up_write(&kbdev->csf.pmode_sync_sem); + up_write(&kbdev->csf.mmu_sync_sem); if (err) schedule_actions_trigger_df( @@ -4182,7 +4186,7 @@ static void scheduler_group_check_protm_enter(struct kbase_device *const kbdev, } spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); - up_write(&kbdev->csf.pmode_sync_sem); + up_write(&kbdev->csf.mmu_sync_sem); } /** @@ -4760,27 +4764,28 @@ static int suspend_active_groups_on_powerdown(struct kbase_device *kbdev, bool s { struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; DECLARE_BITMAP(slot_mask, MAX_SUPPORTED_CSGS) = { 0 }; - int ret; - ret = suspend_active_queue_groups(kbdev, slot_mask, false); + if (unlikely(suspend_active_queue_groups(kbdev, slot_mask, false))) { + if (!is_gpu_level_suspend_supported(kbdev)) { + const int csg_nr = ffs(slot_mask[0]) - 1; + struct kbase_queue_group *group; + enum dumpfault_error_type error_type = DF_CSG_SUSPEND_TIMEOUT; - if (unlikely(ret)) { - const int csg_nr = ffs(slot_mask[0]) - 1; - struct kbase_queue_group *group = scheduler->csg_slots[csg_nr].resident_group; - enum dumpfault_error_type error_type = DF_CSG_SUSPEND_TIMEOUT; + group = scheduler->csg_slots[csg_nr].resident_group; - /* The suspend of CSGs failed, - * trigger the GPU reset to be in a deterministic state. - */ - dev_warn( - kbdev->dev, - "[%llu] Timeout (%d ms) waiting for CSG slots to suspend on power down, slot_mask: 0x%*pb\n", - kbase_backend_get_cycle_cnt(kbdev), - kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT), - kbdev->csf.global_iface.group_num, slot_mask); - if (kbase_csf_firmware_ping_wait(kbdev, FW_PING_AFTER_ERROR_TIMEOUT_MS)) - error_type = DF_PING_REQUEST_TIMEOUT; - schedule_actions_trigger_df(kbdev, group->kctx, error_type); + /* The suspend of CSGs failed, + * trigger the GPU reset to be in a deterministic state. + */ + dev_warn( + kbdev->dev, + "[%llu] Timeout (%d ms) waiting for CSG slots to suspend on power down, slot_mask: 0x%*pb\n", + kbase_backend_get_cycle_cnt(kbdev), + kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT), + kbdev->csf.global_iface.group_num, slot_mask); + if (kbase_csf_firmware_ping_wait(kbdev, FW_PING_AFTER_ERROR_TIMEOUT_MS)) + error_type = DF_PING_REQUEST_TIMEOUT; + schedule_actions_trigger_df(kbdev, group->kctx, error_type); + } if (kbase_prepare_to_reset_gpu(kbdev, RESET_FLAGS_NONE)) kbase_reset_gpu(kbdev); @@ -4788,6 +4793,8 @@ static int suspend_active_groups_on_powerdown(struct kbase_device *kbdev, bool s return -1; } + kbdev->csf.mcu_halted = false; + /* Check if the groups became active whilst the suspend was ongoing, * but only for the case where the system suspend is not in progress */ @@ -4947,9 +4954,13 @@ static bool scheduler_suspend_on_idle(struct kbase_device *kbdev) } dev_dbg(kbdev->dev, "Scheduler to be suspended on GPU becoming idle"); - scheduler_suspend(kbdev); - cancel_tick_work(scheduler); - return true; + ret = scheduler_suspend(kbdev); + if (!ret) { + cancel_tick_work(scheduler); + return true; + } + + return false; } static void gpu_idle_worker(struct kbase_device *kbdev) @@ -5193,8 +5204,7 @@ static int wait_csg_slots_suspend(struct kbase_device *kbdev, unsigned long *slo bitmap_copy(slot_mask_local, slot_mask, MAX_SUPPORTED_CSGS); while (!bitmap_empty(slot_mask_local, MAX_SUPPORTED_CSGS)) { - long remaining = kbase_csf_timeout_in_jiffies( - kbase_get_timeout_ms(kbdev, CSF_CSG_SUSPEND_TIMEOUT)); + long remaining = kbase_csf_timeout_in_jiffies(kbdev->csf.csg_suspend_timeout_ms); DECLARE_BITMAP(changed, MAX_SUPPORTED_CSGS); bitmap_copy(changed, slot_mask_local, MAX_SUPPORTED_CSGS); @@ -5343,20 +5353,6 @@ static void evict_lru_or_blocked_csg(struct kbase_device *kbdev) } } -static void scheduler_enable_gpu_idle_timer(struct kbase_device *kbdev) -{ - struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; - unsigned long flags; - - lockdep_assert_held(&scheduler->lock); - - if (!kbdev->csf.gpu_idle_timer_enabled) { - spin_lock_irqsave(&scheduler->interrupt_lock, flags); - kbase_csf_firmware_enable_gpu_idle_timer(kbdev); - spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); - } -} - static void schedule_actions(struct kbase_device *kbdev, bool is_tick) { struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; @@ -5399,7 +5395,6 @@ static void schedule_actions(struct kbase_device *kbdev, bool is_tick) * in particular, no alterations to on-slot CSGs. */ if (keep_lru_on_slots(kbdev)) { - scheduler_enable_gpu_idle_timer(kbdev); return; } } @@ -5472,7 +5467,6 @@ redo_local_tock: wait_csg_slots_start(kbdev); wait_csg_slots_finish_prio_update(kbdev); - scheduler_enable_gpu_idle_timer(kbdev); if (new_protm_top_grp) { scheduler_group_check_protm_enter(kbdev, scheduler->top_grp); @@ -5495,6 +5489,15 @@ redo_local_tock: } evict_lru_or_blocked_csg(kbdev); + +#ifdef KBASE_PM_RUNTIME + if (atomic_read(&scheduler->non_idle_offslot_grps)) + set_bit(KBASE_GPU_NON_IDLE_OFF_SLOT_GROUPS_AVAILABLE, + &kbdev->pm.backend.gpu_sleep_allowed); + else + clear_bit(KBASE_GPU_NON_IDLE_OFF_SLOT_GROUPS_AVAILABLE, + &kbdev->pm.backend.gpu_sleep_allowed); +#endif /* KBASE_PM_RUNTIME */ } /** @@ -5653,19 +5656,20 @@ static int suspend_active_queue_groups(struct kbase_device *kbdev, unsigned long { struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; u32 num_groups = kbdev->csf.global_iface.group_num; + struct kbase_queue_group *group; u32 slot_num; int ret; lockdep_assert_held(&scheduler->lock); - for (slot_num = 0; slot_num < num_groups; slot_num++) { - struct kbase_queue_group *group = scheduler->csg_slots[slot_num].resident_group; + for (slot_num = 0; slot_num < num_groups; slot_num++) { + group = scheduler->csg_slots[slot_num].resident_group; - if (group) { + if (group) { suspend_queue_group(group); - set_bit(slot_num, slot_mask); + set_bit(slot_num, slot_mask); + } } - } ret = wait_csg_slots_suspend(kbdev, slot_mask); return ret; @@ -5819,7 +5823,10 @@ static void scheduler_inner_reset(struct kbase_device *kbdev) /* Cancel any potential queued delayed work(s) */ cancel_tick_work(scheduler); cancel_tock_work(scheduler); - cancel_gpu_idle_work(scheduler); + /* gpu_idle_worker() might already be running at this point, which + * could decrement the pending_gpu_idle_worker counter to below 0. + * It'd be safer to let it run if one has already been scheduled. + */ cancel_delayed_work_sync(&scheduler->ping_work); mutex_lock(&scheduler->lock); @@ -5837,10 +5844,22 @@ static void scheduler_inner_reset(struct kbase_device *kbdev) scheduler->top_kctx = NULL; scheduler->top_grp = NULL; + atomic_set(&scheduler->gpu_idle_timer_enabled, false); + atomic_set(&scheduler->fw_soi_enabled, false); + KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_TOP_GRP, scheduler->top_grp, scheduler->num_active_address_spaces | (((u64)scheduler->total_runnable_grps) << 32)); +#ifdef KBASE_PM_RUNTIME + if (scheduler->state == SCHED_SLEEPING) { +#if IS_ENABLED(CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD) + hrtimer_cancel(&scheduler->gpu_metrics_timer); +#endif + scheduler->state = SCHED_SUSPENDED; + KBASE_KTRACE_ADD(kbdev, SCHED_SUSPENDED, NULL, scheduler->state); + } +#endif mutex_unlock(&scheduler->lock); } @@ -6726,6 +6745,11 @@ static int kbase_csf_scheduler_kthread(void *data) while (atomic_read(&scheduler->pending_gpu_idle_work) > 0) gpu_idle_worker(kbdev); + /* Update GLB_IDLE timer/FW Sleep-on-Idle config (which might + * have been disabled during FW boot et. al.). + */ + kbase_csf_firmware_soi_update(kbdev); + dev_dbg(kbdev->dev, "Waking up for event after a scheduling iteration."); wake_up_all(&kbdev->csf.event_wait); } @@ -6786,6 +6810,9 @@ int kbase_csf_scheduler_init(struct kbase_device *kbdev) scheduler->gpu_metrics_timer.function = gpu_metrics_timer_callback; #endif /* CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD */ + atomic_set(&scheduler->gpu_idle_timer_enabled, false); + atomic_set(&scheduler->fw_soi_enabled, false); + return kbase_csf_mcu_shared_regs_data_init(kbdev); } @@ -6988,8 +7015,9 @@ int kbase_csf_scheduler_pm_suspend_no_lock(struct kbase_device *kbdev) goto exit; } else { dev_dbg(kbdev->dev, "Scheduler PM suspend"); - scheduler_suspend(kbdev); - cancel_tick_work(scheduler); + result = scheduler_suspend(kbdev); + if (!result) + cancel_tick_work(scheduler); } } diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.h index e84994600809..915945bb495e 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.h +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.h @@ -693,4 +693,6 @@ void kbase_csf_scheduler_force_wakeup(struct kbase_device *kbdev); void kbase_csf_scheduler_force_sleep(struct kbase_device *kbdev); #endif +bool is_gpu_level_suspend_supported(struct kbase_device *const kbdev); + #endif /* _KBASE_CSF_SCHEDULER_H_ */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.c index 51d665f23970..5a5a4c315396 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -260,8 +260,9 @@ static struct kbase_csf_tiler_heap_chunk *alloc_new_chunk(struct kbase_context * u64 chunk_size) { u64 nr_pages = PFN_UP(chunk_size); - u64 flags = BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR | BASE_MEM_PROT_CPU_WR | - BASEP_MEM_NO_USER_FREE | BASE_MEM_COHERENT_LOCAL | BASE_MEM_PROT_CPU_RD; + base_mem_alloc_flags flags = BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR | + BASE_MEM_PROT_CPU_WR | BASEP_MEM_NO_USER_FREE | + BASE_MEM_COHERENT_LOCAL | BASE_MEM_PROT_CPU_RD; struct kbase_csf_tiler_heap_chunk *chunk = NULL; /* The chunk kernel mapping needs to be large enough to: * - initially zero the CHUNK_HDR_SIZE area @@ -350,13 +351,14 @@ static struct kbase_csf_tiler_heap_chunk *alloc_new_chunk(struct kbase_context * } remove_external_chunk_mappings(kctx, chunk); - kbase_gpu_vm_unlock(kctx); /* If page migration is enabled, we don't want to migrate tiler heap pages. * This does not change if the constituent pages are already marked as isolated. */ if (kbase_is_page_migration_enabled()) - kbase_set_phy_alloc_page_status(chunk->region->gpu_alloc, NOT_MOVABLE); + kbase_set_phy_alloc_page_status(kctx, chunk->region->gpu_alloc, NOT_MOVABLE); + + kbase_gpu_vm_unlock(kctx); return chunk; @@ -640,7 +642,7 @@ static bool kbasep_is_buffer_descriptor_region_suitable(struct kbase_context *co if (!(reg->flags & KBASE_REG_CPU_RD) || kbase_is_region_shrinkable(reg) || (reg->flags & KBASE_REG_PF_GROW)) { - dev_err(kctx->kbdev->dev, "Region has invalid flags: 0x%lX!\n", reg->flags); + dev_err(kctx->kbdev->dev, "Region has invalid flags: 0x%llX!\n", reg->flags); return false; } @@ -737,7 +739,7 @@ int kbase_csf_tiler_heap_init(struct kbase_context *const kctx, u32 const chunk_ KBASE_VMAP_FLAG_PERMANENT_MAP_ACCOUNTING); if (kbase_is_page_migration_enabled()) - kbase_set_phy_alloc_page_status(buf_desc_reg->gpu_alloc, NOT_MOVABLE); + kbase_set_phy_alloc_page_status(kctx, buf_desc_reg->gpu_alloc, NOT_MOVABLE); kbase_gpu_vm_unlock(kctx); diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.c index cdab5a17f70c..a9469c5949b4 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -469,14 +469,15 @@ unsigned int kbase_csf_firmware_trace_buffer_read_data(struct firmware_trace_buf } else { unsigned int bytes_copied_head, bytes_copied_tail; - bytes_copied_tail = min_t(unsigned int, num_bytes, (buffer_size - extract_offset)); + bytes_copied_tail = + min_t(unsigned int, num_bytes, size_sub(buffer_size, extract_offset)); memcpy(data, &data_cpu_va[extract_offset], bytes_copied_tail); bytes_copied_head = min_t(unsigned int, (num_bytes - bytes_copied_tail), insert_offset); memcpy(&data[bytes_copied_tail], data_cpu_va, bytes_copied_head); - bytes_copied = bytes_copied_head + bytes_copied_tail; + bytes_copied = size_add(bytes_copied_head, bytes_copied_tail); extract_offset += bytes_copied; if (extract_offset >= buffer_size) extract_offset = bytes_copied_head; diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_coresight_csf.c b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_coresight_csf.c index fe8201f7f7e6..da56d71f473f 100644 --- a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_coresight_csf.c +++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_coresight_csf.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2022-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2022-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -443,7 +443,7 @@ kbase_debug_coresight_csf_config_create(void *client_data, } config = kzalloc(sizeof(struct kbase_debug_coresight_csf_config), GFP_KERNEL); - if (WARN_ON(!client)) + if (WARN_ON(!config)) return NULL; config->client = client; diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_codes_csf.h b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_codes_csf.h index 18520db15502..04da9c8b9057 100644 --- a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_codes_csf.h +++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_codes_csf.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -212,6 +212,9 @@ KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_EVICT_CTX_SLOTS_START), KBASE_KTRACE_CODE_MAKE_CODE(SCHED_BUSY), KBASE_KTRACE_CODE_MAKE_CODE(SCHED_INACTIVE), KBASE_KTRACE_CODE_MAKE_CODE(SCHED_SUSPENDED), KBASE_KTRACE_CODE_MAKE_CODE(SCHED_SLEEPING), + /* info_val == true if FW Sleep-on-Idle is enabled, false otherwise */ + KBASE_KTRACE_CODE_MAKE_CODE(FIRMWARE_SLEEP_ON_IDLE_CHANGED), + /* info_val = mcu state */ #define KBASEP_MCU_STATE(n) KBASE_KTRACE_CODE_MAKE_CODE(PM_MCU_##n), #include "backend/gpu/mali_kbase_pm_mcu_states.h" diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_linux_ktrace_csf.h b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_linux_ktrace_csf.h index 0b0de2385f85..7c40f472a78b 100644 --- a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_linux_ktrace_csf.h +++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_linux_ktrace_csf.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -69,6 +69,7 @@ DEFINE_MALI_ADD_EVENT(SCHED_BUSY); DEFINE_MALI_ADD_EVENT(SCHED_INACTIVE); DEFINE_MALI_ADD_EVENT(SCHED_SUSPENDED); DEFINE_MALI_ADD_EVENT(SCHED_SLEEPING); +DEFINE_MALI_ADD_EVENT(FIRMWARE_SLEEP_ON_IDLE_CHANGED); #define KBASEP_MCU_STATE(n) DEFINE_MALI_ADD_EVENT(PM_MCU_##n); #include "backend/gpu/mali_kbase_pm_mcu_states.h" #undef KBASEP_MCU_STATE diff --git a/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_codes.h b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_codes.h index 991f70fe8540..d40eec013cb5 100644 --- a/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_codes.h +++ b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_codes.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2011-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -149,13 +149,17 @@ KBASE_KTRACE_CODE_MAKE_CODE(CORE_CTX_DESTROY), KBASE_KTRACE_CODE_MAKE_CODE(SCHED_RETAIN_CTX_NOLOCK), /* info_val == kctx->refcount */ KBASE_KTRACE_CODE_MAKE_CODE(SCHED_RELEASE_CTX), -#ifdef CONFIG_MALI_ARBITER_SUPPORT /* * Arbitration events */ - KBASE_KTRACE_CODE_MAKE_CODE(ARB_GPU_LOST), KBASE_KTRACE_CODE_MAKE_CODE(ARB_VM_STATE), + KBASE_KTRACE_CODE_MAKE_CODE(ARB_VM_STATE), KBASE_KTRACE_CODE_MAKE_CODE(ARB_VM_EVT), -#endif + KBASE_KTRACE_CODE_MAKE_CODE(ARB_GPU_GRANTED), + KBASE_KTRACE_CODE_MAKE_CODE(ARB_GPU_LOST), + KBASE_KTRACE_CODE_MAKE_CODE(ARB_GPU_STARTED), + KBASE_KTRACE_CODE_MAKE_CODE(ARB_GPU_STOP_REQUESTED), + KBASE_KTRACE_CODE_MAKE_CODE(ARB_GPU_STOPPED), + KBASE_KTRACE_CODE_MAKE_CODE(ARB_GPU_REQUESTED), #if MALI_USE_CSF #include "debug/backend/mali_kbase_debug_ktrace_codes_csf.h" diff --git a/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_linux_ktrace.h b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_linux_ktrace.h index 1ebddfa3f44f..acc78eb5b0b2 100644 --- a/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_linux_ktrace.h +++ b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_linux_ktrace.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -95,13 +95,16 @@ DEFINE_MALI_ADD_EVENT(PM_RUNTIME_RESUME_CALLBACK); #undef KBASEP_L2_STATE DEFINE_MALI_ADD_EVENT(SCHED_RETAIN_CTX_NOLOCK); DEFINE_MALI_ADD_EVENT(SCHED_RELEASE_CTX); -#ifdef CONFIG_MALI_ARBITER_SUPPORT -DEFINE_MALI_ADD_EVENT(ARB_GPU_LOST); DEFINE_MALI_ADD_EVENT(ARB_VM_STATE); DEFINE_MALI_ADD_EVENT(ARB_VM_EVT); +DEFINE_MALI_ADD_EVENT(ARB_GPU_GRANTED); +DEFINE_MALI_ADD_EVENT(ARB_GPU_LOST); +DEFINE_MALI_ADD_EVENT(ARB_GPU_STARTED); +DEFINE_MALI_ADD_EVENT(ARB_GPU_STOP_REQUESTED); +DEFINE_MALI_ADD_EVENT(ARB_GPU_STOPPED); +DEFINE_MALI_ADD_EVENT(ARB_GPU_REQUESTED); -#endif #if MALI_USE_CSF #include "backend/mali_kbase_debug_linux_ktrace_csf.h" #else diff --git a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_csf.c b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_csf.c index 6fae88665d43..218022ac3186 100644 --- a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_csf.c +++ b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_csf.c @@ -132,11 +132,15 @@ static int kbase_backend_late_init(struct kbase_device *kbdev) fail_update_l2_features: kbase_backend_devfreq_term(kbdev); -fail_devfreq_init: - kbasep_pm_metrics_term(kbdev); -fail_pm_metrics_init: - kbase_ipa_control_term(kbdev); +fail_devfreq_init: + { + kbasep_pm_metrics_term(kbdev); + } +fail_pm_metrics_init: + { + kbase_ipa_control_term(kbdev); + } #ifdef CONFIG_MALI_BIFROST_DEBUG #if IS_ENABLED(CONFIG_MALI_REAL_HW) fail_interrupt_test: diff --git a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_csf.c b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_csf.c index 4706a4f15b65..3b27b87657a5 100644 --- a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_csf.c +++ b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_csf.c @@ -32,7 +32,7 @@ bool kbase_is_gpu_removed(struct kbase_device *kbdev) { - if (!IS_ENABLED(CONFIG_MALI_ARBITER_SUPPORT)) + if (!kbase_has_arbiter(kbdev)) return false; @@ -88,6 +88,7 @@ static void kbase_gpu_fault_interrupt(struct kbase_device *kbdev) void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) { u32 power_changed_mask = (POWER_CHANGED_ALL | MCU_STATUS_GPU_IRQ); + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ, NULL, val); @@ -95,7 +96,6 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) kbase_gpu_fault_interrupt(kbdev); if (val & GPU_PROTECTED_FAULT) { - struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; unsigned long flags; dev_err_ratelimited(kbdev->dev, "GPU fault in protected mode"); @@ -149,10 +149,33 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) unsigned long flags; dev_dbg(kbdev->dev, "Doorbell mirror interrupt received"); + + /* Assume that the doorbell comes from userspace which + * presents new works in order to invalidate a possible GPU + * idle event. + * If the doorbell was raised by KBase then the FW would handle + * the pending doorbell then raise a 2nd GBL_IDLE IRQ which + * would allow us to put the GPU to sleep. + */ + atomic_set(&scheduler->gpu_no_longer_idle, true); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbase_pm_disable_db_mirror_interrupt(kbdev); - kbdev->pm.backend.exit_gpu_sleep_mode = true; - kbase_csf_scheduler_invoke_tick(kbdev); + + if (likely(kbdev->pm.backend.mcu_state == KBASE_MCU_IN_SLEEP)) { + kbdev->pm.backend.exit_gpu_sleep_mode = true; + kbase_csf_scheduler_invoke_tick(kbdev); + } else if (likely(test_bit(KBASE_GPU_SUPPORTS_FW_SLEEP_ON_IDLE, + &kbdev->pm.backend.gpu_sleep_allowed)) && + (kbdev->pm.backend.mcu_state != KBASE_MCU_ON_PEND_SLEEP)) { + /* The firmware is going to sleep on its own but new + * doorbells were rung before we manage to handle + * the GLB_IDLE IRQ in the bottom half. We shall enable + * DB notification to allow the DB to be handled by FW. + */ + dev_dbg(kbdev->dev, "Re-enabling MCU immediately following DB_MIRROR IRQ"); + kbase_pm_enable_mcu_db_notification(kbdev); + } spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } #endif @@ -179,7 +202,7 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) * cores. */ if (kbdev->pm.backend.l2_always_on || - kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_921)) + kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TTRX_921)) kbase_pm_power_changed(kbdev); } diff --git a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_jm.c b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_jm.c index c0cb835de69c..4dd9a228aa11 100644 --- a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_jm.c +++ b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_jm.c @@ -30,7 +30,7 @@ bool kbase_is_gpu_removed(struct kbase_device *kbdev) { - if (!IS_ENABLED(CONFIG_MALI_ARBITER_SUPPORT)) + if (!kbase_has_arbiter(kbdev)) return false; return (KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(GPU_ID)) == 0); @@ -103,7 +103,7 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) * cores. */ if (kbdev->pm.backend.l2_always_on || - kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_921)) + kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TTRX_921)) kbase_pm_power_changed(kbdev); } diff --git a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_jm.c b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_jm.c index 0fe76918296a..8cdf26e28ac6 100644 --- a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_jm.c +++ b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_jm.c @@ -32,10 +32,6 @@ #include #include -#ifdef CONFIG_MALI_ARBITER_SUPPORT -#include -#endif - #include #include #include diff --git a/drivers/gpu/arm/bifrost/device/mali_kbase_device.c b/drivers/gpu/arm/bifrost/device/mali_kbase_device.c index ccb62c2a5cb5..e5bed33d1129 100644 --- a/drivers/gpu/arm/bifrost/device/mali_kbase_device.c +++ b/drivers/gpu/arm/bifrost/device/mali_kbase_device.c @@ -51,10 +51,7 @@ #include "backend/gpu/mali_kbase_irq_internal.h" #include "mali_kbase_regs_history_debugfs.h" #include "mali_kbase_pbha.h" - -#ifdef CONFIG_MALI_ARBITER_SUPPORT #include "arbiter/mali_kbase_arbiter_pm.h" -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ #if defined(CONFIG_DEBUG_FS) && !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) @@ -69,6 +66,22 @@ static DEFINE_MUTEX(kbase_dev_list_lock); static LIST_HEAD(kbase_dev_list); static unsigned int kbase_dev_nr; +static unsigned int mma_wa_id; + +static int set_mma_wa_id(const char *val, const struct kernel_param *kp) +{ + return kbase_param_set_uint_minmax(val, kp, 1, 15); +} + +static const struct kernel_param_ops mma_wa_id_ops = { + .set = set_mma_wa_id, + .get = param_get_uint, +}; + +module_param_cb(mma_wa_id, &mma_wa_id_ops, &mma_wa_id, 0444); +__MODULE_PARM_TYPE(mma_wa_id, "uint"); +MODULE_PARM_DESC(mma_wa_id, "PBHA ID for MMA workaround. Valid range is from 1 to 15."); + struct kbase_device *kbase_device_alloc(void) { return vzalloc(sizeof(struct kbase_device)); @@ -320,6 +333,10 @@ int kbase_device_misc_init(struct kbase_device *const kbdev) if (err) goto dma_set_mask_failed; + /* Set mma_wa_id if it has been passed in as a module parameter */ + if ((kbdev->gpu_props.gpu_id.arch_id >= GPU_ID_ARCH_MAKE(14, 8, 0)) && mma_wa_id != 0) + kbdev->mma_wa_id = mma_wa_id; + err = kbase_pbha_read_dtb(kbdev); if (err) goto term_as; @@ -597,17 +614,12 @@ int kbase_device_early_init(struct kbase_device *kbdev) /* We're done accessing the GPU registers for now. */ kbase_pm_register_access_disable(kbdev); -#ifdef CONFIG_MALI_ARBITER_SUPPORT - if (kbdev->arb.arb_if) { + if (kbase_has_arbiter(kbdev)) { if (kbdev->pm.arb_vm_state) err = kbase_arbiter_pm_install_interrupts(kbdev); } else { err = kbase_install_interrupts(kbdev); } -#else - err = kbase_install_interrupts(kbdev); -#endif - if (err) goto gpuprops_term; @@ -633,15 +645,10 @@ ktrace_term: void kbase_device_early_term(struct kbase_device *kbdev) { -#ifdef CONFIG_MALI_ARBITER_SUPPORT - if (kbdev->arb.arb_if) + if (kbase_has_arbiter(kbdev)) kbase_arbiter_pm_release_interrupts(kbdev); else kbase_release_interrupts(kbdev); -#else - kbase_release_interrupts(kbdev); -#endif - kbase_gpuprops_term(kbdev); kbase_device_backend_term(kbdev); kbase_regmap_term(kbdev); diff --git a/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access.c b/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access.c index 2cce391606c3..d7dd6200d497 100644 --- a/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access.c +++ b/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access.c @@ -31,8 +31,8 @@ #define KBASE_REGMAP_ACCESS_ALWAYS_POWERED (1U << 16) static u32 always_powered_regs[] = { -#if MALI_USE_CSF -#else /* MALI_USE_CSF */ + +#if !MALI_USE_CSF PTM_AW_IRQ_CLEAR, PTM_AW_IRQ_INJECTION, PTM_AW_IRQ_MASK, @@ -44,13 +44,19 @@ static u32 always_powered_regs[] = { PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE1, PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE_STATUS, PTM_ID, -#endif /* MALI_USE_CSF */ +#endif /* !MALI_USE_CSF */ }; static void kbasep_reg_setup_always_powered_registers(struct kbase_device *kbdev) { u32 i; + +#if !MALI_USE_CSF + if (kbdev->gpu_props.gpu_id.arch_id < GPU_ID_ARCH_MAKE(9, 14, 0)) + return; +#endif /* MALI_USE_CSF */ + for (i = 0; i < ARRAY_SIZE(always_powered_regs); i++) { u32 reg_enum = always_powered_regs[i]; diff --git a/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access_regmap.h b/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access_regmap.h index a8708fafc638..591391c6a8a1 100644 --- a/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access_regmap.h +++ b/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access_regmap.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access_regmap_legacy.h b/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access_regmap_legacy.h index a62d1707ebb7..9392d44f684b 100644 --- a/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access_regmap_legacy.h +++ b/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access_regmap_legacy.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_csf_macros.h b/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_csf_macros.h index 56ec5e015d11..e0568d8f8c6d 100644 --- a/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_csf_macros.h +++ b/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_csf_macros.h @@ -185,6 +185,7 @@ */ #define AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_SHARED 0x0 + /* CSF_CONFIG register */ #define CSF_CONFIG_FORCE_COHERENCY_FEATURES_SHIFT 2 diff --git a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend.h b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend.h index cc3ba98ab6fe..a6d418b8e82c 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend.h +++ b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -78,6 +78,18 @@ typedef int kbase_hwcnt_backend_init_fn(const struct kbase_hwcnt_backend_info *i */ typedef void kbase_hwcnt_backend_term_fn(struct kbase_hwcnt_backend *backend); +/** + * typedef kbase_hwcnt_backend_acquire_fn - Enable counter collection. + * @backend: Non-NULL pointer to backend interface. + */ +typedef void kbase_hwcnt_backend_acquire_fn(const struct kbase_hwcnt_backend *backend); + +/** + * typedef kbase_hwcnt_backend_release_fn - Disable counter collection. + * @backend: Non-NULL pointer to backend interface. + */ +typedef void kbase_hwcnt_backend_release_fn(const struct kbase_hwcnt_backend *backend); + /** * typedef kbase_hwcnt_backend_timestamp_ns_fn - Get the current backend * timestamp. @@ -206,6 +218,10 @@ typedef int kbase_hwcnt_backend_dump_get_fn(struct kbase_hwcnt_backend *backend, * metadata. * @init: Function ptr to initialise an instance of the backend. * @term: Function ptr to terminate an instance of the backend. + * @acquire: Callback to indicate that counter collection has + * been enabled. + * @release: Callback to indicate that counter collection has + * been disabled. * @timestamp_ns: Function ptr to get the current backend timestamp. * @dump_enable: Function ptr to enable dumping. * @dump_enable_nolock: Function ptr to enable dumping while the @@ -222,6 +238,8 @@ struct kbase_hwcnt_backend_interface { kbase_hwcnt_backend_metadata_fn *metadata; kbase_hwcnt_backend_init_fn *init; kbase_hwcnt_backend_term_fn *term; + kbase_hwcnt_backend_acquire_fn *acquire; + kbase_hwcnt_backend_release_fn *release; kbase_hwcnt_backend_timestamp_ns_fn *timestamp_ns; kbase_hwcnt_backend_dump_enable_fn *dump_enable; kbase_hwcnt_backend_dump_enable_nolock_fn *dump_enable_nolock; diff --git a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf.c b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf.c index 8bdfc9a0bfda..b937c047a94a 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf.c +++ b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf.c @@ -1710,6 +1710,22 @@ static void kbasep_hwcnt_backend_csf_term(struct kbase_hwcnt_backend *backend) kbasep_hwcnt_backend_csf_destroy(backend_csf); } +static void kbasep_hwcnt_backend_csf_acquire(const struct kbase_hwcnt_backend *backend) +{ + struct kbase_hwcnt_backend_csf *backend_csf = (struct kbase_hwcnt_backend_csf *)backend; + struct kbase_hwcnt_backend_csf_info *csf_info = backend_csf->info; + + csf_info->csf_if->acquire(csf_info->csf_if->ctx); +} + +static void kbasep_hwcnt_backend_csf_release(const struct kbase_hwcnt_backend *backend) +{ + struct kbase_hwcnt_backend_csf *backend_csf = (struct kbase_hwcnt_backend_csf *)backend; + struct kbase_hwcnt_backend_csf_info *csf_info = backend_csf->info; + + csf_info->csf_if->release(csf_info->csf_if->ctx); +} + /** * kbasep_hwcnt_backend_csf_info_destroy() - Destroy a CSF backend info. * @info: Pointer to info to destroy. @@ -2168,6 +2184,8 @@ int kbase_hwcnt_backend_csf_create(struct kbase_hwcnt_backend_csf_if *csf_if, u3 iface->metadata = kbasep_hwcnt_backend_csf_metadata; iface->init = kbasep_hwcnt_backend_csf_init; iface->term = kbasep_hwcnt_backend_csf_term; + iface->acquire = kbasep_hwcnt_backend_csf_acquire; + iface->release = kbasep_hwcnt_backend_csf_release; iface->timestamp_ns = kbasep_hwcnt_backend_csf_timestamp_ns; iface->dump_enable = kbasep_hwcnt_backend_csf_dump_enable; iface->dump_enable_nolock = kbasep_hwcnt_backend_csf_dump_enable_nolock; diff --git a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if.h b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if.h index c982900cf755..81f809fdc83a 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if.h +++ b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if.h @@ -114,6 +114,20 @@ typedef void (*kbase_hwcnt_backend_csf_if_lock_fn)(struct kbase_hwcnt_backend_cs typedef void (*kbase_hwcnt_backend_csf_if_unlock_fn)(struct kbase_hwcnt_backend_csf_if_ctx *ctx, unsigned long flags); +/** + * typedef kbase_hwcnt_backend_csf_if_acquire_fn - Enable counter collection. + * + * @ctx: Non-NULL pointer to a CSF context. + */ +typedef void (*kbase_hwcnt_backend_csf_if_acquire_fn)(struct kbase_hwcnt_backend_csf_if_ctx *ctx); + +/** + * typedef kbase_hwcnt_backend_csf_if_release_fn - Disable counter collection. + * + * @ctx: Non-NULL pointer to a CSF context. + */ +typedef void (*kbase_hwcnt_backend_csf_if_release_fn)(struct kbase_hwcnt_backend_csf_if_ctx *ctx); + /** * typedef kbase_hwcnt_backend_csf_if_get_prfcnt_info_fn - Get performance * counter information. @@ -272,6 +286,10 @@ typedef void (*kbase_hwcnt_backend_csf_if_get_gpu_cycle_count_fn)( * @assert_lock_held: Function ptr to assert backend spinlock is held. * @lock: Function ptr to acquire backend spinlock. * @unlock: Function ptr to release backend spinlock. + * @acquire: Callback to indicate that counter collection has + * been enabled. + * @release: Callback to indicate that counter collection has + * been disabled. * @get_prfcnt_info: Function ptr to get performance counter related * information. * @ring_buf_alloc: Function ptr to allocate ring buffer for CSF HWC. @@ -292,6 +310,8 @@ struct kbase_hwcnt_backend_csf_if { kbase_hwcnt_backend_csf_if_assert_lock_held_fn assert_lock_held; kbase_hwcnt_backend_csf_if_lock_fn lock; kbase_hwcnt_backend_csf_if_unlock_fn unlock; + kbase_hwcnt_backend_csf_if_acquire_fn acquire; + kbase_hwcnt_backend_csf_if_release_fn release; kbase_hwcnt_backend_csf_if_get_prfcnt_info_fn get_prfcnt_info; kbase_hwcnt_backend_csf_if_ring_buf_alloc_fn ring_buf_alloc; kbase_hwcnt_backend_csf_if_ring_buf_sync_fn ring_buf_sync; diff --git a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.c b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.c index 3cf039f70056..29f8a2a8838d 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.c +++ b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.c @@ -131,6 +131,26 @@ static void kbasep_hwcnt_backend_csf_if_fw_unlock(struct kbase_hwcnt_backend_csf kbase_csf_scheduler_spin_unlock(kbdev, flags); } +static void kbasep_hwcnt_backend_csf_if_fw_acquire(struct kbase_hwcnt_backend_csf_if_ctx *ctx) +{ + struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx = + (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx; + + /* Mark performance counters collection as enabled */ + set_bit(KBASE_GPU_PERF_COUNTERS_COLLECTION_ENABLED, + &fw_ctx->kbdev->pm.backend.gpu_sleep_allowed); +} + +static void kbasep_hwcnt_backend_csf_if_fw_release(struct kbase_hwcnt_backend_csf_if_ctx *ctx) +{ + struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx = + (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx; + + /* Mark performance counters collection as disabled */ + clear_bit(KBASE_GPU_PERF_COUNTERS_COLLECTION_ENABLED, + &fw_ctx->kbdev->pm.backend.gpu_sleep_allowed); +} + /** * kbasep_hwcnt_backend_csf_if_fw_on_freq_change() - On freq change callback * @@ -813,6 +833,8 @@ int kbase_hwcnt_backend_csf_if_fw_create(struct kbase_device *kbdev, if_fw->assert_lock_held = kbasep_hwcnt_backend_csf_if_fw_assert_lock_held; if_fw->lock = kbasep_hwcnt_backend_csf_if_fw_lock; if_fw->unlock = kbasep_hwcnt_backend_csf_if_fw_unlock; + if_fw->acquire = kbasep_hwcnt_backend_csf_if_fw_acquire; + if_fw->release = kbasep_hwcnt_backend_csf_if_fw_release; if_fw->get_prfcnt_info = kbasep_hwcnt_backend_csf_if_fw_get_prfcnt_info; if_fw->ring_buf_alloc = kbasep_hwcnt_backend_csf_if_fw_ring_buf_alloc; if_fw->ring_buf_sync = kbasep_hwcnt_backend_csf_if_fw_ring_buf_sync; diff --git a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_jm.c b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_jm.c index 1ee0e3b823ad..c3f2bcdbf256 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_jm.c +++ b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_jm.c @@ -683,7 +683,7 @@ static int kbasep_hwcnt_backend_jm_dump_alloc(const struct kbase_hwcnt_backend_j struct kbase_context *kctx, u64 *gpu_dump_va) { struct kbase_va_region *reg; - u64 flags; + base_mem_alloc_flags flags; u64 nr_pages; /* Calls to this function are inherently asynchronous, with respect to @@ -851,6 +851,14 @@ static void kbasep_hwcnt_backend_jm_term(struct kbase_hwcnt_backend *backend) kbasep_hwcnt_backend_jm_destroy((struct kbase_hwcnt_backend_jm *)backend); } +static void kbasep_hwcnt_backend_jm_acquire(const struct kbase_hwcnt_backend *backend) +{ +} + +static void kbasep_hwcnt_backend_jm_release(const struct kbase_hwcnt_backend *backend) +{ +} + /** * kbasep_hwcnt_backend_jm_info_destroy() - Destroy a JM backend info. * @info: Pointer to info to destroy. @@ -932,6 +940,8 @@ int kbase_hwcnt_backend_jm_create(struct kbase_device *kbdev, iface->metadata = kbasep_hwcnt_backend_jm_metadata; iface->init = kbasep_hwcnt_backend_jm_init; iface->term = kbasep_hwcnt_backend_jm_term; + iface->acquire = kbasep_hwcnt_backend_jm_acquire; + iface->release = kbasep_hwcnt_backend_jm_release; iface->timestamp_ns = kbasep_hwcnt_backend_jm_timestamp_ns; iface->dump_enable = kbasep_hwcnt_backend_jm_dump_enable; iface->dump_enable_nolock = kbasep_hwcnt_backend_jm_dump_enable_nolock; diff --git a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_jm_watchdog.c b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_jm_watchdog.c index cf2a2e65bc25..88917e72ac58 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_jm_watchdog.c +++ b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_jm_watchdog.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2021-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2021-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -317,6 +317,14 @@ kbasep_hwcnt_backend_jm_watchdog_term_partial(struct kbase_hwcnt_backend_jm_watc kfree(wd_backend); } +static void kbasep_hwcnt_backend_jm_watchdog_acquire(const struct kbase_hwcnt_backend *backend) +{ +} + +static void kbasep_hwcnt_backend_jm_watchdog_release(const struct kbase_hwcnt_backend *backend) +{ +} + /* Job manager watchdog backend, implementation of kbase_hwcnt_backend_term_fn * Calling term does *not* destroy the interface */ @@ -807,6 +815,8 @@ int kbase_hwcnt_backend_jm_watchdog_create(struct kbase_hwcnt_backend_interface .metadata = kbasep_hwcnt_backend_jm_watchdog_metadata, .init = kbasep_hwcnt_backend_jm_watchdog_init, .term = kbasep_hwcnt_backend_jm_watchdog_term, + .acquire = kbasep_hwcnt_backend_jm_watchdog_acquire, + .release = kbasep_hwcnt_backend_jm_watchdog_release, .timestamp_ns = kbasep_hwcnt_backend_jm_watchdog_timestamp_ns, .dump_enable = kbasep_hwcnt_backend_jm_watchdog_dump_enable, .dump_enable_nolock = kbasep_hwcnt_backend_jm_watchdog_dump_enable_nolock, diff --git a/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt.c b/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt.c index 8b1de2e1cdaf..8d308f1138a7 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt.c +++ b/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -599,6 +599,9 @@ int kbase_hwcnt_accumulator_acquire(struct kbase_hwcnt_context *hctx, return errcode; } + /* Inform the backend that counter collection has been enabled. */ + hctx->iface->acquire(hctx->accum.backend); + spin_lock_irqsave(&hctx->state_lock, flags); WARN_ON(hctx->disable_count == 0); @@ -646,6 +649,9 @@ void kbase_hwcnt_accumulator_release(struct kbase_hwcnt_accumulator *accum) mutex_unlock(&hctx->accum_lock); + /* Inform the backend that counter collection has been disabled. */ + hctx->iface->release(hctx->accum.backend); + kbasep_hwcnt_accumulator_term(hctx); mutex_lock(&hctx->accum_lock); diff --git a/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.h b/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.h index 1f25282d378a..896f1389eb37 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.h +++ b/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.h @@ -169,7 +169,7 @@ enum kbase_hwcnt_physical_set { /** * struct kbase_hwcnt_gpu_info - Information about hwcnt blocks on the GPUs. * @l2_count: L2 cache count. - * @sc_core_mask: Shader core mask. May be sparse. + * @sc_core_mask: Shader core mask. May be sparse. * @clk_cnt: Number of clock domains available. * @csg_cnt: Number of CSGs available. * @prfcnt_values_per_block: Total entries (header + counters) of performance diff --git a/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_types.h b/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_types.h index c7afe173d426..45f67f7c9a1b 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_types.h +++ b/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_types.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -19,65 +19,6 @@ * */ -/* - * Hardware counter types. - * Contains structures for describing the physical layout of hardware counter - * dump buffers and enable maps within a system. - * - * Also contains helper functions for manipulation of these dump buffers and - * enable maps. - * - * Through use of these structures and functions, hardware counters can be - * enabled, copied, accumulated, and generally manipulated in a generic way, - * regardless of the physical counter dump layout. - * - * Terminology: - * - * Hardware Counter System: - * A collection of hardware counter blocks, making a full hardware counter - * system. - * Hardware Counter Block: - * A block of hardware counters (e.g. shader block, tiler block). - * Hardware Counter Block Instance: - * An instance of a Hardware Counter Block (e.g. an MP4 GPU might have - * 4 shader block instances). - * - * Block Header: - * A header value inside a counter block. Headers don't count anything, - * so it is only valid to copy or zero them. Headers are always the first - * values in the block. - * Block Counter: - * A counter value inside a counter block. Counters can be zeroed, copied, - * or accumulated. Counters are always immediately after the headers in the - * block. - * Block Value: - * A catch-all term for block headers and block counters. - * - * Enable Map: - * An array of u64 bitfields, where each bit either enables exactly one - * block value, or is unused (padding). Note that this is derived from - * the client configuration, and is not obtained from the hardware. - * Dump Buffer: - * An array of u64 values, where each u64 corresponds either to one block - * value, or is unused (padding). - * Block State Buffer: - * An array of blk_stt_t values, where each blk_stt_t corresponds to one block - * instance and is used to track the on/off power state transitions, as well has - * hardware resource availability, and whether the block was operating - * in normal or protected mode. - * Availability Mask: - * A bitfield, where each bit corresponds to whether a block instance is - * physically available (e.g. an MP3 GPU may have a sparse core mask of - * 0b1011, meaning it only has 3 cores but for hardware counter dumps has the - * same dump buffer layout as an MP4 GPU with a core mask of 0b1111. In this - * case, the availability mask might be 0b1011111 (the exact layout will - * depend on the specific hardware architecture), with the 3 extra early bits - * corresponding to other block instances in the hardware counter system). - * Metadata: - * Structure describing the physical layout of the enable map and dump buffers - * for a specific hardware counter system. - */ - #ifndef _KBASE_HWCNT_TYPES_H_ #define _KBASE_HWCNT_TYPES_H_ diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_debugfs.c b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_debugfs.c index e4138580de20..9305747ff472 100644 --- a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_debugfs.c +++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_debugfs.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2017-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2017-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -115,7 +115,7 @@ static ssize_t param_string_set(struct file *file, const char __user *user_buf, goto end; } - buf_size = min(param->size - 1, count); + buf_size = min(size_sub(param->size, 1), count); if (copy_from_user(param->addr.str, user_buf, buf_size)) { ret = -EFAULT; goto end; diff --git a/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_defs.h b/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_defs.h index 373b9b1b73b3..23e919314333 100644 --- a/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_defs.h +++ b/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_defs.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -308,11 +308,11 @@ enum kbase_atom_gpu_rb_state { * powered down and GPU shall come out of fully * coherent mode before entering protected mode. * @KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY: Prepare coherency change; - * for BASE_HW_ISSUE_TGOX_R1_1234 also request L2 power on + * for KBASE_HW_ISSUE_TGOX_R1_1234 also request L2 power on * so that coherency register contains correct value when * GPU enters protected mode. * @KBASE_ATOM_ENTER_PROTECTED_FINISHED: End state; for - * BASE_HW_ISSUE_TGOX_R1_1234 check + * KBASE_HW_ISSUE_TGOX_R1_1234 check * that L2 is powered up and switch GPU to protected mode. */ enum kbase_atom_enter_protected_state { @@ -500,10 +500,6 @@ enum kbase_atom_exit_protected_state { * is snapshot of the age_count counter in kbase * context. * @jobslot: Job slot to use when BASE_JD_REQ_JOB_SLOT is specified. - * @renderpass_id:Renderpass identifier used to associate an atom that has - * BASE_JD_REQ_START_RENDERPASS set in its core requirements - * with an atom that has BASE_JD_REQ_END_RENDERPASS set. - * @jc_fragment: Set of GPU fragment job chains */ struct kbase_jd_atom { struct work_struct work; @@ -564,8 +560,6 @@ struct kbase_jd_atom { enum base_jd_event_code event_code; base_jd_core_req core_req; u8 jobslot; - u8 renderpass_id; - struct base_jd_fragment jc_fragment; u32 ticks; int sched_priority; @@ -676,71 +670,6 @@ static inline bool kbase_jd_atom_is_earlier(const struct kbase_jd_atom *katom_a, #define KBASE_JD_DEP_QUEUE_SIZE 256 -/** - * enum kbase_jd_renderpass_state - State of a renderpass - * @KBASE_JD_RP_COMPLETE: Unused or completed renderpass. Can only transition to - * START. - * @KBASE_JD_RP_START: Renderpass making a first attempt at tiling. - * Can transition to PEND_OOM or COMPLETE. - * @KBASE_JD_RP_PEND_OOM: Renderpass whose first attempt at tiling used too much - * memory and has a soft-stop pending. Can transition to - * OOM or COMPLETE. - * @KBASE_JD_RP_OOM: Renderpass whose first attempt at tiling used too much - * memory and therefore switched to incremental - * rendering. The fragment job chain is forced to run. - * Can only transition to RETRY. - * @KBASE_JD_RP_RETRY: Renderpass making a second or subsequent attempt at - * tiling. Can transition to RETRY_PEND_OOM or COMPLETE. - * @KBASE_JD_RP_RETRY_PEND_OOM: Renderpass whose second or subsequent attempt at - * tiling used too much memory again and has a - * soft-stop pending. Can transition to RETRY_OOM - * or COMPLETE. - * @KBASE_JD_RP_RETRY_OOM: Renderpass whose second or subsequent attempt at - * tiling used too much memory again. The fragment job - * chain is forced to run. Can only transition to RETRY. - * - * A state machine is used to control incremental rendering. - */ -enum kbase_jd_renderpass_state { - KBASE_JD_RP_COMPLETE, /* COMPLETE => START */ - KBASE_JD_RP_START, /* START => PEND_OOM or COMPLETE */ - KBASE_JD_RP_PEND_OOM, /* PEND_OOM => OOM or COMPLETE */ - KBASE_JD_RP_OOM, /* OOM => RETRY */ - KBASE_JD_RP_RETRY, /* RETRY => RETRY_PEND_OOM or COMPLETE */ - KBASE_JD_RP_RETRY_PEND_OOM, /* RETRY_PEND_OOM => RETRY_OOM or COMPLETE */ - KBASE_JD_RP_RETRY_OOM /* RETRY_OOM => RETRY */ -}; - -/** - * struct kbase_jd_renderpass - Data for a renderpass - * @state: Current state of the renderpass. If KBASE_JD_RP_COMPLETE then - * all other members are invalid. - * Both the job dispatcher context and hwaccess_lock must be - * locked to modify this so that it can be read with either - * (or both) locked. - * @start_katom: Address of the atom that is the start of a renderpass. - * Both the job dispatcher context and hwaccess_lock must be - * locked to modify this so that it can be read with either - * (or both) locked. - * @end_katom: Address of the atom that is the end of a renderpass, or NULL - * if that atom hasn't been added to the job scheduler yet. - * The job dispatcher context and hwaccess_lock must be - * locked to modify this so that it can be read with either - * (or both) locked. - * @oom_reg_list: A list of region structures which triggered out-of-memory. - * The hwaccess_lock must be locked to access this. - * - * Atoms tagged with BASE_JD_REQ_START_RENDERPASS or BASE_JD_REQ_END_RENDERPASS - * are associated with an object of this type, which is created and maintained - * by kbase to keep track of each renderpass. - */ -struct kbase_jd_renderpass { - enum kbase_jd_renderpass_state state; - struct kbase_jd_atom *start_katom; - struct kbase_jd_atom *end_katom; - struct list_head oom_reg_list; -}; - /** * struct kbase_jd_context - per context object encapsulating all the * Job dispatcher related state. @@ -751,9 +680,6 @@ struct kbase_jd_renderpass { * @atoms: Array of the objects representing atoms, * containing the complete state and attributes * of an atom. - * @renderpasses: Array of renderpass state for incremental - * rendering, indexed by user-specified renderpass - * ID. * @job_nr: Tracks the number of atoms being processed by the * kbase. This includes atoms that are not tracked by * scheduler: 'not ready to run' & 'dependency-only' @@ -803,7 +729,6 @@ struct kbase_jd_context { struct mutex lock; struct kbasep_js_kctx_info sched_info; struct kbase_jd_atom atoms[BASE_JD_ATOM_COUNT]; - struct kbase_jd_renderpass renderpasses[BASE_JD_RP_COUNT]; struct workqueue_struct *job_done_wq; wait_queue_head_t zero_jobs_wait; diff --git a/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_js.h b/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_js.h index 333ad2d2b150..65b54c68d8c7 100644 --- a/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_js.h +++ b/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_js.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -567,22 +567,6 @@ bool kbase_js_complete_atom_wq(struct kbase_context *kctx, struct kbase_jd_atom */ struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, ktime_t *end_timestamp); -/** - * kbase_js_atom_blocked_on_x_dep - Decide whether to ignore a cross-slot - * dependency - * @katom: Pointer to an atom in the slot ringbuffer - * - * A cross-slot dependency is ignored if necessary to unblock incremental - * rendering. If the atom at the start of a renderpass used too much memory - * and was soft-stopped then the atom at the end of a renderpass is submitted - * to hardware regardless of its dependency on the start-of-renderpass atom. - * This can happen multiple times for the same pair of atoms. - * - * Return: true to block the atom or false to allow it to be submitted to - * hardware. - */ -bool kbase_js_atom_blocked_on_x_dep(struct kbase_jd_atom *katom); - /** * kbase_js_sched - Submit atoms from all available contexts. * @@ -809,8 +793,7 @@ static inline bool kbasep_js_has_atom_finished(const struct kbasep_js_atom_retained_state *katom_retained_state) { return (bool)(katom_retained_state->event_code != BASE_JD_EVENT_STOPPED && - katom_retained_state->event_code != BASE_JD_EVENT_REMOVED_FROM_NEXT && - katom_retained_state->event_code != BASE_JD_EVENT_END_RP_DONE); + katom_retained_state->event_code != BASE_JD_EVENT_REMOVED_FROM_NEXT); } /** diff --git a/drivers/gpu/arm/bifrost/mali_base_hwconfig_features.h b/drivers/gpu/arm/bifrost/mali_base_hwconfig_features.h deleted file mode 100644 index 843094076d69..000000000000 --- a/drivers/gpu/arm/bifrost/mali_base_hwconfig_features.h +++ /dev/null @@ -1,163 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* AUTOMATICALLY GENERATED FILE. If you want to amend the issues/features, - * please update base/tools/hwconfig_generator/hwc_{issues,features}.py - * For more information see base/tools/docs/hwconfig_generator.md - */ - -#ifndef _BASE_HWCONFIG_FEATURES_H_ -#define _BASE_HWCONFIG_FEATURES_H_ - -#include - -enum base_hw_feature { - BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_TLS_HASHING, - BASE_HW_FEATURE_THREAD_GROUP_SPLIT, - BASE_HW_FEATURE_CLEAN_ONLY_SAFE, - BASE_HW_FEATURE_IDVS_GROUP_SIZE, - BASE_HW_FEATURE_L2_CONFIG, - BASE_HW_FEATURE_L2_SLICE_HASH, - BASE_HW_FEATURE_GPU_SLEEP, - BASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER, - BASE_HW_FEATURE_CORE_FEATURES, - BASE_HW_FEATURE_PBHA_HWU, - BASE_HW_FEATURE_LARGE_PAGE_ALLOC, - BASE_HW_FEATURE_THREAD_TLS_ALLOC, - BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_generic[] = { - BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tMIx[] = { - BASE_HW_FEATURE_THREAD_GROUP_SPLIT, BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tHEx[] = { - BASE_HW_FEATURE_THREAD_GROUP_SPLIT, BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tSIx[] = { - BASE_HW_FEATURE_THREAD_GROUP_SPLIT, BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tDVx[] = { - BASE_HW_FEATURE_THREAD_GROUP_SPLIT, BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tNOx[] = { - BASE_HW_FEATURE_THREAD_GROUP_SPLIT, BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, BASE_HW_FEATURE_TLS_HASHING, - BASE_HW_FEATURE_IDVS_GROUP_SIZE, BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tGOx[] = { - BASE_HW_FEATURE_THREAD_GROUP_SPLIT, BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, BASE_HW_FEATURE_TLS_HASHING, - BASE_HW_FEATURE_IDVS_GROUP_SIZE, BASE_HW_FEATURE_CORE_FEATURES, - BASE_HW_FEATURE_THREAD_TLS_ALLOC, BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tTRx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_IDVS_GROUP_SIZE, BASE_HW_FEATURE_CLEAN_ONLY_SAFE, - BASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER, BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tNAx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_IDVS_GROUP_SIZE, BASE_HW_FEATURE_CLEAN_ONLY_SAFE, - BASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER, BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tBEx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_IDVS_GROUP_SIZE, - BASE_HW_FEATURE_L2_CONFIG, - BASE_HW_FEATURE_CLEAN_ONLY_SAFE, - BASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER, - BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tBAx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_IDVS_GROUP_SIZE, - BASE_HW_FEATURE_L2_CONFIG, - BASE_HW_FEATURE_CLEAN_ONLY_SAFE, - BASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER, - BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tODx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_L2_CONFIG, BASE_HW_FEATURE_CLEAN_ONLY_SAFE, BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tGRx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_L2_CONFIG, BASE_HW_FEATURE_CLEAN_ONLY_SAFE, - BASE_HW_FEATURE_CORE_FEATURES, BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tVAx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_L2_CONFIG, BASE_HW_FEATURE_CLEAN_ONLY_SAFE, - BASE_HW_FEATURE_CORE_FEATURES, BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tTUx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_L2_CONFIG, BASE_HW_FEATURE_CLEAN_ONLY_SAFE, - BASE_HW_FEATURE_L2_SLICE_HASH, BASE_HW_FEATURE_GPU_SLEEP, - BASE_HW_FEATURE_CORE_FEATURES, BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tTIx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_L2_CONFIG, - BASE_HW_FEATURE_CLEAN_ONLY_SAFE, - BASE_HW_FEATURE_L2_SLICE_HASH, - BASE_HW_FEATURE_GPU_SLEEP, - BASE_HW_FEATURE_CORE_FEATURES, - BASE_HW_FEATURE_PBHA_HWU, - BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tKRx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_L2_CONFIG, BASE_HW_FEATURE_CLEAN_ONLY_SAFE, - BASE_HW_FEATURE_L2_SLICE_HASH, BASE_HW_FEATURE_GPU_SLEEP, - BASE_HW_FEATURE_CORE_FEATURES, BASE_HW_FEATURE_PBHA_HWU, - BASE_HW_FEATURE_LARGE_PAGE_ALLOC, BASE_HW_FEATURE_END -}; - - -#endif /* _BASE_HWCONFIG_FEATURES_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h b/drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h deleted file mode 100644 index 409e2e8bedcf..000000000000 --- a/drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h +++ /dev/null @@ -1,621 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* AUTOMATICALLY GENERATED FILE. If you want to amend the issues/features, - * please update base/tools/hwconfig_generator/hwc_{issues,features}.py - * For more information see base/tools/docs/hwconfig_generator.md - */ - -#ifndef _BASE_HWCONFIG_ISSUES_H_ -#define _BASE_HWCONFIG_ISSUES_H_ - -#include - -enum base_hw_issue { - BASE_HW_ISSUE_5736, - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_10682, - BASE_HW_ISSUE_11054, - BASE_HW_ISSUE_T76X_3953, - BASE_HW_ISSUE_TMIX_7891, - BASE_HW_ISSUE_TMIX_7940, - BASE_HW_ISSUE_TMIX_8042, - BASE_HW_ISSUE_TMIX_8133, - BASE_HW_ISSUE_TMIX_8138, - BASE_HW_ISSUE_TMIX_8206, - BASE_HW_ISSUE_TMIX_8343, - BASE_HW_ISSUE_TMIX_8463, - BASE_HW_ISSUE_TMIX_8456, - BASE_HW_ISSUE_TSIX_1116, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TMIX_8438, - BASE_HW_ISSUE_TNOX_1194, - BASE_HW_ISSUE_TGOX_R1_1234, - BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_TSIX_1792, - BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_3076, - BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_GPU2017_1336, - BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TTRX_3485, - BASE_HW_ISSUE_GPU2019_3212, - BASE_HW_ISSUE_TURSEHW_1997, - BASE_HW_ISSUE_GPU2019_3878, - BASE_HW_ISSUE_TURSEHW_2716, - BASE_HW_ISSUE_GPU2019_3901, - BASE_HW_ISSUE_GPU2021PRO_290, - BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_TITANHW_2679, - BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2922, - BASE_HW_ISSUE_TITANHW_2952, - BASE_HW_ISSUE_KRAKEHW_2151, - BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_KRAKEHW_2269, - BASE_HW_ISSUE_TURSEHW_2934, - BASE_HW_ISSUE_END -}; - -__attribute__(( - unused)) static const enum base_hw_issue base_hw_issues_generic[] = { BASE_HW_ISSUE_END }; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tMIx_r0p0_05dev0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_10682, BASE_HW_ISSUE_11054, - BASE_HW_ISSUE_T76X_3953, BASE_HW_ISSUE_TMIX_7891, BASE_HW_ISSUE_TMIX_8042, - BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TMIX_8138, BASE_HW_ISSUE_TMIX_8206, - BASE_HW_ISSUE_TMIX_8343, BASE_HW_ISSUE_TMIX_8463, BASE_HW_ISSUE_TMIX_8456, - BASE_HW_ISSUE_TMIX_8438, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tMIx_r0p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_10682, BASE_HW_ISSUE_11054, - BASE_HW_ISSUE_TMIX_7891, BASE_HW_ISSUE_TMIX_7940, BASE_HW_ISSUE_TMIX_8042, - BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TMIX_8138, BASE_HW_ISSUE_TMIX_8206, - BASE_HW_ISSUE_TMIX_8343, BASE_HW_ISSUE_TMIX_8463, BASE_HW_ISSUE_TMIX_8456, - BASE_HW_ISSUE_TMIX_8438, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tMIx_r0p1[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_10682, BASE_HW_ISSUE_11054, - BASE_HW_ISSUE_TMIX_7891, BASE_HW_ISSUE_TMIX_7940, BASE_HW_ISSUE_TMIX_8042, - BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TMIX_8138, BASE_HW_ISSUE_TMIX_8206, - BASE_HW_ISSUE_TMIX_8343, BASE_HW_ISSUE_TMIX_8463, BASE_HW_ISSUE_TMIX_8456, - BASE_HW_ISSUE_TMIX_8438, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tMIx[] = { - BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_7891, - BASE_HW_ISSUE_TMIX_7940, BASE_HW_ISSUE_TMIX_8042, BASE_HW_ISSUE_TMIX_8133, - BASE_HW_ISSUE_TMIX_8138, BASE_HW_ISSUE_TMIX_8206, BASE_HW_ISSUE_TMIX_8343, - BASE_HW_ISSUE_TMIX_8456, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tHEx_r0p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_10682, BASE_HW_ISSUE_11054, - BASE_HW_ISSUE_TMIX_7891, BASE_HW_ISSUE_TMIX_8042, BASE_HW_ISSUE_TMIX_8133, - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tHEx_r0p1[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_10682, BASE_HW_ISSUE_11054, - BASE_HW_ISSUE_TMIX_7891, BASE_HW_ISSUE_TMIX_8042, BASE_HW_ISSUE_TMIX_8133, - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tHEx_r0p2[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_10682, BASE_HW_ISSUE_11054, - BASE_HW_ISSUE_TMIX_7891, BASE_HW_ISSUE_TMIX_8042, BASE_HW_ISSUE_TMIX_8133, - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tHEx_r0p3[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_10682, BASE_HW_ISSUE_TMIX_7891, - BASE_HW_ISSUE_TMIX_8042, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tHEx[] = { - BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_7891, - BASE_HW_ISSUE_TMIX_8042, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tSIx_r0p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_11054, BASE_HW_ISSUE_TMIX_8133, - BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TSIX_1792, - BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tSIx_r0p1[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_11054, BASE_HW_ISSUE_TMIX_8133, - BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TSIX_1792, - BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tSIx_r1p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_11054, BASE_HW_ISSUE_TMIX_8133, - BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tSIx_r1p1[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, - BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tSIx[] = { - BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, - BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tDVx_r0p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, - BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tDVx[] = { - BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, - BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tNOx_r0p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TNOX_1194, BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tNOx[] = { - BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, - BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tGOx_r0p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TNOX_1194, BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tGOx_r1p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TGOX_R1_1234, BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tGOx[] = { - BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, - BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tTRx_r0p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_3076, BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, BASE_HW_ISSUE_GPU2017_1336, - BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TTRX_3485, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tTRx_r0p1[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_3076, BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, BASE_HW_ISSUE_GPU2017_1336, - BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TTRX_3485, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tTRx_r0p2[] = { - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_3076, - BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_GPU2017_1336, - BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tTRx[] = { - BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_3414, BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tNAx_r0p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_3076, BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, BASE_HW_ISSUE_GPU2017_1336, - BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TTRX_3485, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tNAx_r0p1[] = { - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_3076, - BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_GPU2017_1336, - BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tNAx[] = { - BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_3414, BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tBEx_r0p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TTRX_3485, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tBEx_r0p1[] = { - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tBEx_r1p0[] = { - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tBEx_r1p1[] = { - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tBEx[] = { - BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_3414, BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_lBEx_r1p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TTRX_3485, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_lBEx_r1p1[] = { - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tBAx_r0p0[] = { - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tBAx_r0p1[] = { - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tBAx_r0p2[] = { - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tBAx[] = { - BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_3414, BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tODx_r0p0[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3212, - BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tODx[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3212, - BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tGRx_r0p0[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, - BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tGRx[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, - BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tVAx_r0p0[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, - BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tVAx_r0p1[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, - BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tVAx[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, - BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r0p0[] = { - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_TURSEHW_1997, - BASE_HW_ISSUE_GPU2019_3878, - BASE_HW_ISSUE_TURSEHW_2716, - BASE_HW_ISSUE_GPU2019_3901, - BASE_HW_ISSUE_GPU2021PRO_290, - BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_TITANHW_2679, - BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_TURSEHW_2934, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r0p1[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_1997, - BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2019_3901, - BASE_HW_ISSUE_GPU2021PRO_290, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_TURSEHW_2934, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tTUx[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, - BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_GPU2021PRO_290, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_TURSEHW_2934, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r1p0[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, - BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_GPU2021PRO_290, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_TURSEHW_2934, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r1p1[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, - BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_GPU2021PRO_290, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_TURSEHW_2934, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r1p2[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, - BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_GPU2021PRO_290, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_TURSEHW_2934, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r1p3[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, - BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_GPU2021PRO_290, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_TURSEHW_2934, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tTIx[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_2716, - BASE_HW_ISSUE_GPU2021PRO_290, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2952, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_TURSEHW_2934, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tTIx_r0p0[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_2716, - BASE_HW_ISSUE_GPU2021PRO_290, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2952, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_TURSEHW_2934, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tTIx_r0p1[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2021PRO_290, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_TURSEHW_2934, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tKRx_r0p0[] = { - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_KRAKEHW_2151, BASE_HW_ISSUE_KRAKEHW_2269, BASE_HW_ISSUE_TITANHW_2922, - BASE_HW_ISSUE_TURSEHW_2934, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tKRx_r0p1[] = { - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_KRAKEHW_2269, BASE_HW_ISSUE_TURSEHW_2934, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tKRx[] = { - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_KRAKEHW_2151, BASE_HW_ISSUE_KRAKEHW_2269, BASE_HW_ISSUE_TURSEHW_2934, - BASE_HW_ISSUE_END -}; - - -#endif /* _BASE_HWCONFIG_ISSUES_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_csffw.bin b/drivers/gpu/arm/bifrost/mali_csffw.bin index a1168ffdc7436dd69b31f643f2a77b9fc993d6e0..4319d6b74e3bdea3eec1abd666bcf9c06ddbdcbe 100644 GIT binary patch delta 35016 zcmb5W33yY*+BiOQk|w7cO&1_tC@1NbmO@k3vWV$IC=_TBgsP}1AO-{six3oVQm9fC zwJ=BuqE6hzc;uL=d1(@>I=q-}ohoRflne)s;rf1f9vGy6L; z@4WMFGj@Bt-5%dijx)DT8L!C9LP%Y+^|2I%x&|Xbz4f*0U!U~ieDX1V3G2x+W)>bv z-erd1BBGJ?CyXo|Pk^rxgA9qq2qT2y#&1#5^YUR6+|v!hpKQi11l?PwkRR>=SS`vB6C z^X%xpJiB8>1M}=)Qe{W|$Jx=XciLeHfBt*-PAGH*50&k-iXVl_Uvp~FwWW4a9G`sa zPIWChkX(zBm)gZ@&nDT?O#`Iglw>=yEw`g(OYGQ+AXCzb8^*L=Z06 zNYC z9Y%~JP8@+8aZ-6fEk}^Y=yIUc2$&KBj)uHwM`fez=-cP*Xi+%K8!g3`KuA0I z8-%o-2aBZmlMvGQ{AfFx^{gGuyBQ|>IXl{AumLXZv!j?|DPa+WwBaAaVQ)CxQX-|# zD3!u(Wm3bZAfyeKOp@Xa5YqT7;m~T7+C2~9CIC!6S<3G%m*PpbDN;gRINUN-inl{Z zA!I6~_+#PF84kVSaLzO-|H*LZgpk(jHyy&)?C5p~X~*Xvq<}4)0ZRb`oLDKP+m^xy z&9H~V)^PZFIP`?WOEaa0Kfgr^2h5Vf@^EMlhmS!>XK>masok!*uxnqkqm)>w!!;1n z{xbWCp$%=hP0Dx?LYmUVk#5_BC z|7AN;-7WQ#P$PxIAauQAv!e#_!;biHMsGMie7;oB03mH~^dc!NPm5$UjgOGMq(irae8L6DJO{%~5J!wW-A*4%Ebli@Lq22dB zlG5)w0Thj9ya68+^2a`r!c!2^hS^W0g3H6fBZs<2Oy;N-i8?{ zd>gPmO$uj!BZbZ3(0WdaPyIH$|L0wh8oC84(EXkA?{I@ZN$FLCr1)JB(hg4j0<`u3 z(2ZZE@MJjrA4~OSj zDSc|B6gGy#_rhUJl$2f+4i|>Q1>jFaKF z1W`GfJai;1BarZPULL1zvrPW|Dr+Cw(OD@gZx@g=fxo{evKbvkZIsL#E@RmT=8qVw^alY!C3aVb_jZOr;!3C z*D1b?(#f5OPwRy`d!(cmyOfC-D0=cigH;a+B(^3^stc?p86pqpj(0) z$UV7(2b?g`){#bP|7GLROZ-!)jWp(37`BQ;<>~3nO~_kbG@-l9U1jXYHwx9pSpH0) z+K4q+i+O!FO?^VB@Je7iI}rY?BL-J^C&gEI%T?rGdBZcV^u)QN_Ei}p_@VwNv6;KX zW}Xem!s%1Vm?60gtn&OJlT*D_2>_4wJ!pzHF(nKgG^rxThv*X6D#bnuFuD@0GAxeZ zj|rrANOmGy)t6YiS_F?@L&oHfpaWT&Uo!9!|8Z!LW*~rjM9@G-z|SNf_Q`rQ#7gYuvOCZ)6v+4 z1iMO)`8WM^mW`qOvRM95cO`GHI!^izOD$OHrx0Db4{16U)Gm1PXFBUr?e-2N&i9+p z6J42wM$FGAcMls%NA={eyn-!>Tg_jZKj?Te@s!Y@Iwfq^f6&p8INjT|z;!>&TgJ=6 zw1iWFUG;%bo)FR5OfC+~P)_dU0F1>XzMz6x>L+&9UZe`Z;Q8OCu}N8Th?+x{*;1>=-#Yr%%sV<%;0P2Cz=%WOU11 zY$s;%XgnsutM1Ph{QM=|{dFX@aQuj})67t%6Aw)`MmW*Xn6?$L7RX&=+$tCnKUR)% zZ5KGz$BM$!cnni;_M38;ukmD4;h^zfbT?1Zxr(eR9>ASs<5R0@KOD% zDLQL#QJqkTrdU5#%AC=Be{%Qm0>*WSGz?cK7KC7tO`N!lM=x0na=xA%9{vn-wwFA) zX+T`a_Yy3Nyczl|mc!yaJVKN3OSc%W1sYHc^9VGNG>^!`zmac8+^nfZvBq|#$_bVD z-Myxk?LsZuF6>d!VWHeQp_YsvSr9Yw7$qQ&H&=LL5Lq)aN9IjKq-kVoGK2syY_`Wy zL<6ja9##SaYjE* zi)oQg+Pv5tKacDmRf!)Vp-~%Sq^>FCWh&Az`bbi1@4aqk$BB#vK_^2Ev~hPYF^$P) z0zpzgCR1MFu_ut`F&wTTt})Y@e*_3uG(_etizhRR9vzy|EgLFt)nG360v`AC*Cjy0BuJC(E6 zSR=LZ{3}6nZ0udC-ucjqHo;0tw5MPf%_sTew0JtH9H$%Pt%^68Ay@Nm2jc!h!M2a9 zj;`vFxtYBhW3R7@_{L4fIppT?1x%NTSjIoe^qRI!xZ}SP!8H?#56!u_ORVt@P*nKW^*x+WO%F+s%rdJ(nJMS7&U77i8wbp6D;WiU?i{lv#B zfnLvp({KtM^Z!b<>3m~rF^@5}L~s!lb)TZ73jmL=(9&EwTQHPXk;b>)JpM1kV+^;; z1pi+l zM-88)#PidHr54=GAEl$&)Vu+R4qH{><=2uu#T-6PJ}4d)wXX;1E&--OqP%1xK0!)L zZoYeb=n){jPYZtoLA4h>0-?_Mv~W-jdn%-}8ocJ2svtJO*1JvzRW2F-c_7aH85}!R zunxb#A<3_Wk>;>&oiQ3Hq}<)>yVzR=--pSClEK`{UY&6!pj#aOw_c>Pb?w;yGdL-#Kt-QULdFZc$HFms134 zyL$gqm3#!org-x1@xMJu>JI^e-PJ$BPbBOdErlZWF1SKAWl6 zkaN@Zcp8bGk;?q|0MX5uMo;`TGcpw`xM~b6e%a47a7&Q#V1vpW+zCTIRFZZvCzvs)->2MSnsz3fHZ3UcOp-zB5}G;|$QdnJ93*$m9FR6D zXj&5INm`-e+`_hDW1J@j#-$6AhM5zX%^~vn%mJxAfr!q*=#3JNA1seNW$@lv z1kJDpql)KAXl82lzXGq62wzgL=7y9o4t-S^pb+QIHy$Y&1aglFvwznlY36`?DeNyY3O{oDRqVR@=yd0PB<^gAV&U39UtdV#A|Ksg2LHh9)u zul>BAyf}M{t=bQgipDt^8x5WbVNhp4H^VGr!`)1V5^T7{Jz?N)f;5%p>}1Ah@M3** zxa4q1Rfkh2Tu;q`R8FezrR)|(jzN&tMAJq>1NGs=L`d8dE+`MDMM7G8IBje=EkrKd znt0PSL2o=M<#^sPNzGn?G-tTl!-u+J1~78^YOye}M`DqMi5*(7yWDlAx{U1IRohPxUM-Ma@EJW1jF z5fC>IGZnt{Vl{-3H|uakinxiO!v0K8C;AgPeEmFHBi9m-s2!!-liRvE+8HWGdwB;XUlw~ zk!5`DLA0nAn`%w*CViXCH2-aS#;tt&+C{?Lu{2+9M7A0m9$T~a<2P|G^Nmd8Ul~dYFi!|Ml6t_K?y_PeI#VSW%V8{!1B-%CxD?I!I+4T6&(pihs zkS^Df;5yxbafL@%N&81;m5DWj3ay zAW!bXTm-)(NXyY$Ooo)d;3~sJnQE2FuaJ%JK}bUjb0Ds(fNxR&Pc$~Dv*IEZABxji zY+aA+?*qEWx8;y53*~U2!59~2ekl*72~yD_XDvtpqbqrqt>mMkpewH3gR(*W(!gxr zE}#ZjU6dR|_dG4=n<+m-#a!FrP82u52}uSW91~KbwBTYZ(n4WgKQE4dHK3z1nid!> z3&S(O(nayp{Er9+t-x+s9ueBD3a@&=(?ToD>eAX+?h*KE1^Mq0p?>8f!fO_0cbiMz zDMVCu^;?5-bjEnitG7<-kgtv7f3o$cM(akseQlWSsta+)h3eSj!o1iz!5*_m&_lR4 zrf-t_{f9|r#7X`sfE!y|P7C==gHV`?`Ce>?u)-rLWUv*FjfdI9Fq=QTAt330=lvsuN2pwHA&zyakj3+{mG*4XJa$d(5DTfx-i@j z>0)+n6>4G)JSMl@6>HrXNiL?b5BGqan>U;PGX^rdIbdLa739~VEDAHz zh_1z~fP-Kw;x`8X8Ad9+MWg9zf|MLZ5d#n+li3-WwN*gPsX;W^A?O>5Y+-%aK@k-o zsj3K4ZV~Z=R$d3F0EKxDP~mxhk%$VL1bq|c1HDDd*_J4NncxuiA*^FvMSvP{epSmR zVP6yLrm4CXW#?Y3H!}P;-Prx8fSZbpA9WmmEsEC(MMg%mNw6KR+rnxJhW=>$r|jo8W7c@LChaDaAGABDq(cL; zM)Sd1fo)K9s!+TJSp1!F{IYI5Mm175nl1SHOQtd8d!h%3NO%$ZFpY7B=~1K9dXH~g zE0t|FHsglA(#To`00|bd*EDSaj0nrDRFH>N;u3>cRAZ8;nrw!hONQ5!<1OT&ns1rj z8d5!f9HrHW?3kY+^QzMh9GPE&K_j?rVVZtn;6r0>vOXERb3x-7yEV$#Z>P-o<|WJE zm*B^Z^}7mMVs~1~#u-Q;4*&!&hM_nH*rsqs?~2-~Doc$oTM3nTe{7?2cfV&Fpj$lV5%@1F zc%AgWr#Sfozri@dm{Z0=r{4)h?r}ye4(SDQ|2+elfA~rLJtJ+FwamL zQA#PP^Qt*a6+B$6h3alt{>(vGg*A|pJ=|KAhVt~5DA@eMO4!b;AeBnL842c8b0Q@J z`H-^;jsQw5C=t&}FsGQ#3rIhN9tR=`8DcZtZa{f)-`1z2+~$Ctue@BPV%r$*1@nm- z&ImYJjQAKqA13U23+UjFa-i6?7`lDCkE%e|#;=E_^LGn~n*{)Ef##zrJ)+=bq`?U! zh+kg{$ejdA7(9}G^`*WqB+y9Ls!U+qp*V$W6`8sy+a}ddHX*0#=+R3m=dUeWQ1df9`KRoa)XK=Xp2MqZo3TBY%2u^ zU{&9Wa5k))3e^O0-zd$L*gj%*39OwnK4C)qBcM#{_956=v|;h^4L8i11_HUQaz1jI zjbL&-_Ut#Ay2Q8++=6G&|PXI4htjZogWhVgk4m*c}Go)7~O*`i#pMmw>jB z($KnI&c^5_yC`@nU>+*3gNCdIWt5YGXE#4)t~YJCj4-Vc*R6M(Y)qCc2N#)uuefm} z%b@$Ro}9IV+*vl6?N6sQ%^Cw)So?nit2p>agaMU8Ekzl=xZAh-2n7NQzln#;X3&^` z_|g(7@<1@FRv4u2ENFiU5log$H{XeMybERGtSUI?^Tbu(1=$z?`n~%xAq|u*F$_%h z>Rn{=;tU&9RdKxD0G5Pewv;oS6XXal!+acmOwg$CQv1?RIv!I|^~vYGVp%ihRhX8f zX*+w){b{~v+TtK*j?mn5l|h_|_S}!(12&3oAMWPBLMe zgk3Hh9dFdNm^C-X93hRlikgo?zQX`J;%ACuFA9&nNE~}wPl-paj_SOmM45n?kzfyF z^$#|;d<`1GU4jNteGKN;moC2ge9Z|#u0J8L+zA*?k&)BVyr9qD!`Tkvk-9Oe(QN*O zFDX~|nGn+i2RIz!aI^tecK~!;W233DETLA`7~w4{Y!Hy0o(bw8dIwHign3y=mRSVz zNH@4c*n=o@y9tC@rRxkV4l;ivG$PjgQAZA<=QM3=j~(mgxPSvN%rpYU!Yr*`A~w)$ z#F@1Dbyln$0L(zDxPlJ`fe|oNdVgP~%%otR>Bvra-xC_q3PcEHrx;WjC%vdN*$`j) z5qaUhL|d!5K~F6rX9PC#j1ZIj5!hvJfLMsIeiLZl`_5ipxTfY#du7zASQsunfariFSYx%3?#LUdnc?I}EzDAQ2nj)^k%CSK zLV-R@hAvIE6&dCGFhATs0lJzk?2fAew8a|E1YPq#2x|A4jvS1&plwf_4>`~oqYg6j z>q|ZY!%1XNre!*J%Ojk0Ia|$VgxxA^;$Xg;V5avViL-~5Kr-N$sIlv9fr4uQvlZwo zZ;|DwZ2%qsqoK2`3NlGl+bterVXG=V-k7!~8`d|iHNk(=+K6k)dk%HtsA#V`H4^Nl zl+mTbVsq%AWR3DnK#*qI4X|?edzhzV2|3z+A}H&c<|eV42e_y}*2 z8hN_rqHFo)HzLoIy0lm-eu{wtp^z$Sc7z9sWnI!(>6u41{G6lkgOeSFtWy6 zP{ZEVi$}?ZenU^02fq_y8f5e+&pQXRLl@=~2*3Fgy(x#iInWQLU=_^4C^j2FrMz#} zi8ob2wyHtUbHfif1=ZM8+kpmQ4>uQ%Tf4Bw!Z@8B*^^m%xUQdIZWkI2aK>BOJ95_E zYZz}xZ*vMwmWK@OLX%!B&w+raLHx-O?Fdh#rmoLjC*F(zk~%0*4y5ftKLzoI=(rQ) zymlo$Ec;AQGbJ08oSzb)nII7$YR#gRH~Ud02ijy^kdh}qRJu;kMA5JugcNei10r z^%k0TgZ*D@m1&`LaDWgEH2V(!zD^8kc70Q{uHx(umk_Urno9qsPfeUU668dl^p@B_ zkB$JZuZDEM;j{c2|JZQK<#5U}|I@J1Er+S#Nkzgw(UW#UMbCbGj$dp;6gm;Vp;zrT z1L>&%69X)L2BrD*IEV5aJ=$egHZV7ibQ`ClkZkQiyG5A+P|?=45yWUwD__~`+Y&Z2 z)EiMQY5-EO;RZKDjhl%Z=+-rOP()*3FAp;o2^(_`&IG>BNAalU!Z{$G^kl_Q`ukcq z3t+gH{Iy2JeIlq*BCCK-9o!P`mqquR1aK@n@q8ci0j9{C{!7leUA&sfcVqo zaZ75d;sUMno3INSHYlmyVwdomNR^);?^=?X2km6o$^r6z{JjG4TGE$%040$~GXNls zDj1>a_GJmTUN{FN_#BYMnTA?mIkBMyH0a9nQP<*YI4};ga71BV=kIIVQ2UI4ObjLM z0COMl+$CTYZJmL2$Pmc-6 z=+VydwP@EV&{muh<_*I7;!cF$gV=izLqM>wmgS!aT;GCYL;@5QA+iH74LPWhp4RGM z)B}A)7xqE(VIaJZ2>Y6mXPy0rOVRv;32WD8Wfj?qvXpC!vWlWIb;jGmryEq|7h9vW z1+cAdGfoxhwv@(bi{{1hx?TpfFnUCH!F|Em?ymqLV6<|2PNtm#t|W@qllHhe2-a#V zqfK=)f9Zj?W{c%ZZ zHLE~y-nO;~KS{=}U18fV&;|J%IDCe`wO8~F*aULWWU59SWCTi}D+1eMpNqqP6Ce4> zN`7l=D{O6b8_JV84aRtw=P4newqq(hhIP_%(}n)R8(KHLMKp$zTx%>lA|wruOtf+d zgZnBxSiy-^B8kPCZIgw|$lFBUflVEh!3Sv#*Q(5 zv&uBF{EBX>W~G8{$oKf627w=SLW^?2bir`WQeF<4Ilg~99N=-deaTj^%&IZv)1$44 zplQHur975JTaYH+WFX(JOD*F2HwgAL%=bV&d)m88D!VLPgDHZG;`bbay1A!aX9r2|L)Faj0I6QD zwH14}4yWEecBZ{UrlCVu8G^ovVEJYF>;P&}9p#oI-4$0ST}dz+OKOV#V0Dlb6&5GB zeV`Ix&+C90kvX--Tj0ouq>2XynUBMPpfbxlcQeO^L}AO5>4G18#8<202v4}zr}Tv-10w|V$Y^6lR$nK6En zZ_C8blbN>u1CRA=7uJIZ%RzN*i5#{Wod%X)K_0e^85GMWOKw{xCI2~mRSD&TYfI9= z7lh@%B`#YsK0~h9#!|1gY`d9_t))p}GQ-`mq5b&ZtD)aIAp(B~+4A1siP z?{~djIoK>DEl=`M%Te$$sfzp)mQsh2Sbl=k*qYj5>Kl#GY6||);pVr1Kb%62)LI!; zG8w&LBr`BXmTp)bU#l{=fPNW)e-7yEgT7Bm=Z5~^>v_5(8?V_u9ZvRcSi zwU!OPQcSSFPb@innA`-D^)|AyAk5v1INbqggR6|931k{i{=I~oNorq6%|tC z_PSx%PBzySl#Ozp7TDa=Ld;mYii-uial6pY>cHdZU6!((br71&XelT);9Cw`0f94_ z!boSev!~xdI7eLMx4Hodk|b6P-cjjowe`5L)_y?y$Z*W4a><-WrzQ1yxUjJLA>UT8 zEx~%e{b)8e!4mgZHyZ-JNk^%gF8(^bfeDKwQJXksR?l@8662<9ZB$Pm32`0%>*B@m z+8%JaQmcRMuCa1foPqhl%)3`h;$;EGBcAogHceozbd&B)Y9{|68Bm|ie9}!O)~90~ zfRdOV@NMcAm2x#$OPBnKA7KH&i)N$br1fO|UCi4d@>~6AW><)0Z?-eHg~;B`BbkXI z^8Mxk%IRXT zp3t=wUdwz^(r^d3V&r{S%nblXkIM})mf*;ha^H3|^S`|~lFk|_n>Om=P?l>CPO zM($q!jocT>7u)(XUH6iox1ApLlOW?#+$<9Vmz-$p|J@6VTq5;pU8X_C_piz(zkgLp z70gtDUDQ~j{bsti*F?5#zxBWRopo=w{lzisv8T0oG5N>SwQ|32v_OXMSO86^Xw!YJ zY2{Ri=BRHJ`0qCF&_z}U8J+6Oc&Cii>=^l9OY0rO$F+20qfA$R7I8mZ!u(`Lk&d|% zH{uL%Zczu}KRXLo{IRp$jdESxFfJaJtV=KO!oN2V-m&n|mHU``=&+Z(_9=jK9 zjpT3cAxXP*cq6%KR~A#=Ll*8TN&!{}DYwRhUmO(=SOP{Ag&sCXSvPrm*Ms?x4Z2^H zlNe-Hvd>fH%|KAq;GA2m3_n=*-qn%>WkA+F#wrq!dC#P)qxt8-X&~gAB*cV|ZoA}V zx9ype*iG)7@9N06XxY%EPSXdJmpEG z=;fc8>IH;*Wv0?w4Gxx2()!A67^Z_TTDNY^tusqszY#4YEg#Y7aw4Te&*UPTmbdX$H07iauJ|SBM<&#vJD0a z!_Pbo%$ACm#YH#`JPQ*5%$G`a)W9=TgioNXSPA$$fR6~pP=i`er)rit!JG_k6KItR zRFh1``;Zp$?eMcLh*KSQq8pNA4{qY22W_!aSxWM4J_@nMzoDlb|1Vk*ft7$^IOn8 zfUuB)KyxyYS!1p(Np%wK{t}t_J~`RCs6Sc1pPgj_Y(#WAwb&d_$5Cvkrj1I?K{MCC zZis@2-(eSg`vOpIF%02egXec@U&)1E4gf~Z>@UD7qHJEFURr~ONiC&|+Fw}QNSN9C zNJDcP{+j%wc_0Ik51TU;3jthH4dI)UBKO*5oQ$6xJWpa<3N;^!8v`22Jl&Bgs$Nfb zq-5N3S%wr&1m!l9yITehH+BOKbqoL&v`W}3veI%;`m_8vF+CgD(M${&!fAOK!ve?H z#F1B8vT+CbtYv7%4{%CaP0$4=2XSy&u+*bR3P2l#s6#xXk@4@6L2njDZ0|;yHU-$arf_pu z7w@8FXg)IQT8cgT@HcW(e|CEqbEkWch-*uY1hH^Ot94PaXC!&+tt|CFdSW}-7;4yw z>5SxLN8U_^>!M)RJHzXZCYGlx_5UC)q)-~zD^}oO}%*vOubnA z(A)KyCb16#7)N7I_mLlLoC)jrSPA1~^(;TvPZqtC4fL`K?hd@8wKWM$d$BdQg8DQ0$fVD6~F&i$sR)NN|n91GVHc#LYI+MC;4}7Rp45##B{{q-7G>EPCCiHg7&0qLYYhs+Gv+OB}D4e{sgRNbuGXz`0!WE>h!wgbk_WmDH-52ff}q z1V}b`-k4`X0;d2k9Bjk62HtLO0HZRsrOXCB(N$1JajMDg?ZR1>8cvtkw+r8}bh_@b zCuM<|3`A(j@q{WL=(nV!rFT`L!B;6VKqeeDW*h~LMYr!!f%cDcbkH2AY!A@1dqWW* zearbdWZ%)Twypc!OE#IPDTBH`xm=Xb#iKWrvBPX#%l1<&TgiW0U#&VVovf(3i(W}L0?mkoX-LuO?nNnwHmNZR^d^vg^5s9c+543 zEI2k-F5YoGc5LLtvn$@~G%%H21`#1ZX30>uC6=Kp;Nk&JC*3O$2jJZUAb{bN{c!Qvoiv&#~W0U|=9};L0$e-l6gDnQnJPcHc z{Lq$~Kc>fekP^DSJRos;Aay_jZ`3t-w%}==^@w)HCiUnJPV+P(!n8jskK{RlY-!Kr z1|E~VwEVt%yCqp0#>oP1Dkw!=znD1Nhs^p5{!i!z-gO{=m~*BhMm(C++GHHVzbc#& zk8E`t!~Y1B`$y1Y(zU>_{kp{siL57Dpstq8cz3vQb59>hz-9)n=XuNtaQPt!^Ynt~ zd75B7r5xcLzBGZ4iz|TC6L91lUrUc2IDQ_0vub69_r(?D!*_4j^wG~^m?XP+zS5JN z1*J3~v4-I{lH&IUF?JI%y_XqXM^^`=Xv~+Ar`}7={xzTgCoK&qaU=O{0X|uzaR$t< z5VcZu%?eP8Kvm~^x|q6q>8Pm{JhGR3c>I>sl0a)Rr>R|8=HcRxdrJ%31%)bFuYm%$ zlNIj|?AMRBF}bc;DYdcVeO+{&NKOq<Bgt&OB76)JAnGVBz89<4^fX z(eZKIKmAc0D=3jQP1GGs#(ab5yYxqxqHn;YBOe@}WvKB>AWiacnh!ja;%|Vou4k;; zP!^@tmlc+MvNA?5Hd>7Np`tnPk-mXee~|ZwMvs#xKDaF<0It*XW_|`X^uqvX^mRag z6SjSr5BT)8K=dEZ&W+?>6QVe3LKSa4RbqbA5vp0*7;ZZztn@XI(22o$H)djRH+uy| z*BYq;N2#qgasrbIPUb*_=xnS4li!EqQX=`;0L9Xo(V*8+Y89Q4;mMjJkf|R{0J4$` zeX~LYGN(zq8}kuj@7G7R_oF|KY&ZGtqd^QGBHbTNOm>F;JV2>&mOoBHA7sU-5M1cQ z!$Dj7dWh7Wn4Ml5`tvaFIxGz{yv~MCVVxHNu4L#bBi0Kz1?H{M$ zEb{Hgsr}9z`Y+Y)Cu2UjgW1zdwtq5uKz%PAx-|LJ*|r$7a2t!bKPgF{*DJx&H|Vi{ zX*A{rEG|FE`}lT7(JKPe(LCCMNSp2Q=b231BBJ^tow>D}=)ai2jP54OzsRI?puw5V)S1bV zFF4#wzWd^4=KCN?cN&>PL9)oH`$LIVCkI+&TEZlj7%~CwME!A3B3|8AR>GCen!?@6 zAHlvJ_YpcFIO{Xz}j%y$Q;0fo?NlkDbrVGmC9r)| z_AAC8Cc{-1-s&fpPY+8o0SPhPYE5SK(n=tHf~bNPN!lx4X_>lvNcC6KC~V=_Ctp?3 zSlZVjq~%|y%UQlvAWObBz{d`Oyz#Xmh2>X^TagWK#l$d1E*1!ThL)QwkZEV~Xgew2 zzyKDLSI%f5St*d^-;^;whR8p^DPi6V5#bwC!dBRNOJ@p$=?IFkl82jMOU{jEmW8%` zqy9q+<=Y3t-Z8*>TcK-HxO{$zxvk2V70@#3}RqS z&YeqRR`e3#oQM+X-|Fa0mw%hruO2kU<=_r54j7A}y~O^lIFi@D6_?`Nx9ONC{%=Dzg zQBb^Zb$)iEwpk_ItVy6c5?84Fd%)p^B--kmNE#Q<}p9Fl+0H~P>oT`FadA=6pny&>^_`uS<(s;x0r75Lg z=PG5LQ~;mhHGx8iS`;Gq_YPAL9Yk>uzUjdp;gkb!+U>ikm!<&UnFYc*;0=>S)jx~D zs8Y!ztaK&tawgr@W&i<)W?&7Y#HSA))VVo63Q=tk)fz#yj6SUqaH?UA(MmG_WL6od zrYPupAY?g6wM42vbRO!VDI9c6Yny>FWWf)a4vQ_7w^6|F{lSC zrL^uctHhU;3UtLjO&N7XnLOQit5Dr$yd13 zqte=x@B&E$pAaOo|Ghfr6(O*c`h7M`zU{IcDV-odXb}I5Fm72n@J$Vq5x-mr`eu{g z|E*D19YWkkpqT-(KRM`|aEyOToU?e6{Uet=3MK{ygYXqJV2p9DV2)d7&bo|HCVBeD zdvG-g{&-tTpZcI3ynsA*&?-C#I){fqJ+Y1~|H+`!9;xu2zwX}4fk(iL7PJ+7PANl5 zSwAHP4Yt!lzWzz4>hi@2)C*YB5IEP9em`5&+t+~FOHX;pY7^AS2F{Z5@;M{Hc6ekB z+4J+9l(P>&Et;7%)I6+gTQc=EXmt=By@;et&ewty0`&sX#M zg&Vf5mMRw^UEhnjE{hH1!A;h_N$~yJgKw6Dx%&dRF;m&7hfKPp$$JBCdE+7-+@lS}(J5!!l;IBh6jPy#)VaPo zlQdjP%70jBvX*-4SHNb*RJyR5ux91kbmlOqm3|0O<>fWsyYT);wT3t^scjpZK}yua z&ZH6|w@=SB06n-aAVvl;+bE=g8`wx!BRKAz6?7$V!#7IqV&N(dXY~2L=w_amdy7Vb z=osAyD*?h0h8yV*^cU7GUDI-_FeQcsd-B(TL!cFjc0~wyF%=j6zG)%38`5lX@-}do zwjA!n(ei7`h15CbTNm~4JlM&`9R#VsEOGNzeM{e!F}AJSw;@0UaLg&-^`E|bZU28o zX#{H)74p5-10$BR*(A`RwH*}jM7ju9Tq=>6{mW0MU%IMB`z->2-)drDPGQ=LkQRm; zc3P(77wN#<)Q2NR19;~5qH@qrfRiPx0O-8or53D09CS!sVP`=N4Y<=L4zl5)c`jM! z30t(RvpIszE;SNxR8$tExd`|@3Z~=`5qu`MwM9j507g5j=lH6|PjO1N}jFx9U=qfU+nE<*X7x_z;L7KFyS|h3CiqjrqKo=uZER}7m3`3 zdX<3JM_7?ptr}K_3g8apL_DF(g@+fGv3Wm%lV`n3nu=^jRSHTQ6Dhl*O?f7y8jWx2 za_tg8ahgd_hR5KGs)UIBN{SrrR)xrOSJH-{Y?!q>K*1M@vH*mfh+s540ETsXQ1r7= z6M%^ne61m>W>Arzu4F_|&!lJ)`%7{)cDt86YN8&*e^4=uQkJnpVNygEHt>_g9Mknf z8ZL+GLg;W&HU$`Z8(0gHI=09;Fw4Mki|9Z!#Quxg25OYZ^6Z5<{@R=SKwb7O)2ObwUM@KF_v7|icbY-qV!BeVYF<=3?t4OAf#1N8lERAtad^t z80@}^03GEh3*JG;hK-;~)w33w8v(gKf4LE`dko5^vpOIX3F&tO*f~rClmNHB=<9BM z2JdOq?jeu;`lQVduCh!;PG3--u_?PBdBjb`1|{P>$^e9wAbM2)ye)N>e?M(eB65xNMFi89w1l}@&FW5VPx&2s~{x9+k zimKPqR3yc$1rAcWX`hm0o{+CO{;>V6F5e#E=q0;>8U*bc65#VvhaRKuf1s!BQi$9S>ej*7-`~s+>&z z9MDpq1@o;8%^;upD!|KVpiodiFEX|(ZC#7^!{r}fH}vLW6WkzOWV%@FIW3l!>!TPZ zVX?xN1hf3>XSjS6mDi{G(6|UIG+X6 zbAnr?QGpd5ffUr``z2t|M1ofP0+_|AN1&AYV}Qg2=ioH5Ah>x@E0|;9xhC_}gOu$F z`DVeu3yo0}d5f2k$nHtZ?t4j9H+}5S@!U+D5>W(h0w0pi-EB;34Vl_grP@txzL`yn zk=YGuR;gH8o{Z1_gP3;|Sin9NEVy2`Bd+<%yuqa+e2F?YD8nNM{kfy`-VZwz7`8P$4{+6Y(&3QdT<#rEI)f`J z3qC-U2cUc%ygjG@pCIY&L5CX}8o0uL@}hhnWswhbe{_*HfuV^Fjv;cKn*5CafS4)x z-WT4u2tysKGu{D9Fw}9k96yCWbNnvH3v%BH>gdG_Ezb1&E{E2w16?=Z@K!&{hqH9H z!ujAjF`vKScqjr-#t%8(kHCwEY)LGhIm>hgUfkPi`qK2C&;Y;f&F=~8o4}>AA>K(} z4MSJzIAv$EW3B?<{D<`{UXOEGaXqo-s&BqyzXH!>*3Wl<%ym1%uW;O@#KZF9k6I>& ziIs<)3lorIwQ?=3qB*$LV!wcaLAnI;+%M}KZ!2*L!_RjFlsJX4bB+NlE>}s^+yEz< zQoQ`;SPqt@feP_TkH^I=SG8RArF6?-<$8T3j(@PAAYkV@oGi98yvcD(BzDqxW)v=` z;qoZ_ifYec%UiX}Q0_Qqj1_QYjw3S~k7Qzlj@zU0>-YzU5*~(3X+CU$Ysi%66j8}~ zsin-*1dFzQfn#Y59>=h^I`+n34u9hKI0mQU{f?hvpzCK3IoMd7NxK>ri{~@!Y{#}( zJcnU#dFOI0mdo%u$L}g!I?o_1=43mVr|+W|ytMS~IIr()AOc?el6S6FVz<0=nlgG_ zDbFYr!>&4ILn2-BYx%3_D@%dsUM zXW}+TOFX_A-{$Cy$Ajf^zD95)CE($Bm}7DR9s&t^!LcR*CVYLz#pJ#T^Ukw_t_xEAgpF z{(xgd5*`wh2oH#Ak^10vAo&h+5*|J1fM35w#jD~XogA-1`Z}5eWU#H=*y&>&2a|A3 z!nt1h{KjK~yvo2|u+y1$ZV#NO_Y_s8+Xg#7)q|zI=cJgZ05&R zj?4jgDW2%qIsgyE!yWqu;Jfp0V2--2$9DwY@|0NjEpC*5&Z)Q(h3fA8`eHfkBfH-* z5~}sH_fR~dew+5pT;{J_W_rhPoJKFH%;AO6131S8GN;h)CL#_ zH;CfqrV_+2sHL6(j&?OppL!asp|7bxBQO^6pV<-Q`&}Qy0<#eynMCE95wO1R*jqc9 zIHePensJdJ_WbX?IN${VD|c2rBW@9N+^!Zt$NBhL)GB}k9{EFg6`t;uj`n0+Ka(D| zo6Yo~0n?He@dg;0_yw26o(xD(J_eBOL!V4)>(cMnje_?OqbO50VZZL8!R!RKO6dmo zCR8le-{^3pz_T*m8pnGn_-~A%)KQX(rzWg^18xe#9Zt9>j6}WrQ;sG`&oLZG0I>^% zb@1#qB!1O0cFTKL4TvoTJ3cC^%Ud@qT|NYPA{}vQIC%(_!u0MUb1Tf8*S-$yoJGJI zeAI!~{^q7v#th&dKs6y?K61=T!T@z^pM+TV1Z>KpGy3?{r*9!&!KyLpcbKiYL8AfQd)^eTB|D zrVqje+0Vj#E5WA>QZU8={Sj3|l!OESP64v@>=O70N7EoYfms`Hd_M?YG;unV=|Bj_ z1{}Ai<6@atg&fNV<7#Ykd^8xB!mSXxosxm8kkm&YN%KvPK^!i~ zf#FjoS}U1F#{a9zD0i253?jo-rO#yy)x~Mp^_6242e-SQb6ntXF;k!G$kbr7?Iegy zsDOe`pSob3-@zLTOhK#@7xafWP35wFZkkf+nLt@8CUH=MF@m-&W3Sn(Aw`CFl?bfL zrN*5cv&y&?9=o8=ZfpyOhN5e(=Kye205Uv&5)eh@!q4=zmQr@gF;I&WZ3Y>$6V@#} zfL`BW@r}pL&2h$J&jgXrV+slZq&u9np*o-(Cl^5@qMs54qR=e60(T^{m4w}zY%CSz z_sJcZg%RXCfpaUiMm!HUaaSmw>1V`STL7?4@T1{I_#Nvs^y@OZlso0m1_9>SCt0kP7q6zqWQQeRZuRfn@yGLTc&xe$iZde zblaER@NOGC2O~G?U=zO4Lvtv$fQCS)ba7UrM=mbViF@A#deY}N?=~ThtKne3&Fcjh ztbWg3imBqPL%M93DJZ^-3=y!H3g-_8*`2IO?fx8SP=B}Dl;|GQ!W4kyRM5|*GA08S zD+(Ifo!D(Mt^gpA-9^pOaKCfMcl5bkpSBZpM}S2j`rrPpnMl2a@*H@|2)WWbl(8#H0Xi3c+K&27Je8{awKNM4w>HT zn3Ih&mCf*$8}A$DsLjSx@CnDK**G_Hm7f8^6Y_oK=*h-IBdf#FLyi$SxKKG>tZ~qB zUk=W|uR5N}!DGOk#L7B8$$?7<--Z0X1V<~GBP>vVs_d8 z>+5X5qPosJe$SZ?CP-v76CIVr0YOJZGbkpYL`J0p%7+t6gseozNVBtsJ`YKF?6f6y z=u%5k+L8_ROe(u!v$jh&wxx*;lZpvVOxA8qIy9!Gt=orW-IhdF0SPn8*Z$u-XPTQd zw?5C$_s%+yj(Y{Q0tt*R?J1SZ&Z8szWWj`fykm2SpW+#w`NCP!X@ zBl)@H_I&g^md%i^75m6j>FAl&y>*Ws`|4Kgj|EL>d|Yfs6KkYpt1M_%l!%M@)A8eC zDY*9#rEgR?H-hJ44JBZs&};m7l|w4jE{sAPK{VdpSbNA&Y1MQ&Y<2SB(^v zH+bTvYD7zT}x%pOH#7&7Ttf6Y(l0zKGHb44`DQ8H$uug}tMxv4L3MCp%Ru;$|&ZwS%}#l<}s*i5Q@ zx%L=C?@LKtg_F$hQ?Nn=+As9Z=nzmrl(7ru2ETPQhy#`5|1Vkz1k<+#h&|q`W;ft* z`f2<#c_Zj>`Pf~~jph%7hr^$_G(d+{8TB;3Ga34pD-{hgv zXJAPuDm~sM0I>+32sA3U#Lws~lUk@2KRuX4DdUx{N6q&1jYL5&xcD={bmohh+++7) zH+gEVdOq1BWs5#meFDubeq$Cg6@R#cEh%!!T#V|}b&EfqN4zu%ZDlONFeXtJA}y*@ zmL{!ezJsmZuxG}A>O31gI2Xor_Wf%1%hc?1K2dqokULA{ej6$$%i=O;gk_0b7rQk( zvEy_s5(QeL{yd*7knp(J?fYVw{2h&^V>)m)`rlKSQy)F8mSDMBf_duC?n1vh-cwKV za1Z^wN$QSM9^@hA>X@a7jydkm;e1j2OV!;KeJ+-NL>Uht5FN{TY|1*kP2tXlfl{1k zo6yd$xp7EMN2euUnK)P`oU$JstZqNYVQs&jsM1E)+kY}TE3xEBqQiy?pmsCuC@oKo z|8~pqh_g{yFl2TXER)&IMpw;k7-~PQJVn`W#t#26CTen7>FSy2ALmR%bZTPAwqiE= z>MWgYMH7`LNw8?57?#0q62Hi0Z?X;{*0WXYN%6{hwomP1IySIcwqHE9fjz_w;{6S5 z`Av(lib*GDjN9 z#s?4g)?!IzW*+=f*?1%%>hf8iIr1>RcvP$`U>)p=I9W|JV{X`pcPju)axpG0V5OyM34%#IX5= z=kN0}@y2F$2w`VpeIfheg1?Fb5`Bftn_)!5XhY{ZzOeVs3_AWGJ=upA@hF9ViFJ2l z0%GDpvGZ;gTKDvgkHXjDp=+Ov=GsB?WR7^~n6PePJ6W69y9HPG`QpeHwu`0;B3oF{ z{GK{9CH54t(u}F7m^-e9ogSBAr{933c8Cv9Okww&=r3Y61Z0|IT<${f=h>v#dwg1wvWGn@4lh=&Js+)cPz;8-M`_x zXR>?OZDSs0n>`ST`ID$PHw9it@GVU$iq`GyK^(X)Q}6h8wvZckhqsF-cd%8WeFw8G zqWk|9lkIhQjM>B-6_cR@6_Xu*FEBQ5%WhZOW9mC5<3at9`iUW&sw3{l8kF?#GroC1 z%r9Y!Y#r)I*y{Vm&D?X~+HZFwtZQ&2#%=;VNfq5n8qo+7tk4Mcn4c-Y6mh)^X5|~!jpu??SaP1Ey{!B}ss~)^Q;`Y@H&zdxpU=>>&)c$%Au4FEA0ai*6 z9}rh6S>dA2I#c_u$vK*?&`wdicd^^~oi`jVipg6n>lXabSqMtrv3i~(s-o9n} zqvwX{g_Rcz*QvDR|dM~uEO^G%yW|G366W{kT4{2Ya&++N;+ZTOr z>LcyGKll9f37n!xTX@f3r{#Y)C0@RVW!h%dlW$?*>@~yor?xri?Ft9Qf=<&JU?rPR= z;6Iz`J+zno3+GkSy~+DnCCk}$MnO?Hx8hzI8yb(6%-CXh9Ak&#jA024t2@~-9=%(p z>22N5`X6HM-mw??kNH#H1(sf`pAVX_b(8qr+pMIZt5M-Zw?xtW>w}LtD>)BKP?ArV zbiBok;{PtP?FHe-6i)O@G_uMC_=t;=OR$8#?)CpgQ6oq31Qi&RXeCGBBPJx%u!O#( zkBWNpRTOpj+X^S}QX>_uuCZO=M2f@)OX%w^iJ_><(?|j=60Kw!d_<-s2bR#63|fsw zv6k`e1@>nZPB0E>OiPTHQLli$WOz3u zN3Wquf|x`rnKFTD32c%ySVCVCt&>T-xfds=Wd?NI^#rxL@U_>AJHc1fF<-L?TsrbuuC+GN)CKPo}>_#(3fmI2T8obkP|M6R0HN`m%!`qNH5{l@fGIw32=B5oaam zVF`W7;aNzAyHF`XzeFo}0Y2iQo)C0fZ5_=pL~G%TSnX;}ct z;pb5y!Kg$lX?y{d5u`|Lu!O#(eJLcBJ|qDaiB>WVJ|a_+154;jI#)o_{yHioa7eV0 zdGHa1l44jwUot-jlCHNmB1^}>ej(W z)Jqy+34Ps~EhwsQ2(=RUC0fZQ_=pxs8!VwO=`Vp~;C<9e5R_;oJK!TaB`07BeaV0q zlI`QDksu_|N}hy|I3?+WCG;g*?}KFb6lx^sk!U5)!bhB!T!1C?B|{HDvfs!#aZaL@ zya*q0NiqUU=u37#1j)ghIVXlBTFD9ch-ry&KIeqKWVjKMql-8vViK)n%3{t5n4GrgG2J|=`9Ipkys9mDf&4iE0k>tS=`nrQhQB=ojjvJjE%d5rFN*2OL6idos z34O`oZ$mPFJ?Dg5qLr+IkEoT@!4mqCu@EG^n>i(I+_z zOXy4HpMhl0e$I)oL@Rk7KH`GpA}paV={rL$=No7_&-qa;K}4d}y#yaIBAI|C^mS`~ zg`y5UjA{vnC0fa8_z2@8aZ+FjeM$cyBx7GfwFJtSB^pf|d_|TYYS>5 zuuC+GN)CKPo}>_#(3foe0FuscqBa7TL@QYgA5ku;f+h4NL)RebKE^pwD$z>T!bj9e z>R}0e$?j=L);xn6349W*WFve;lcWWf(3cFGjOcQ9qeg;&L@U__AJHM{ge7z(SGa$I zeFmixg~j!1@R=xT4T@B&@it2k9|-aAWmZ7*Pf3ecuWVLc0?&cFKr{HymCf`L`*|@W zC6=EK(qF8d=XxH;3jFYN5D9K21?xN?9K`Q_t~xD2Y!gUfqZBrQxda==Y&L zAC^jY3x0)BY!l=kPWbUx_7`y;1M*v8)%3W3?#4Dj{`|P#?!Xw16N$_M*d{h2Q3Gru zvo35CsxHe;(d|lDWeD+#GoWT2mWF&@8bCHRo08gbfU}u EANQR~asU7T delta 35200 zcmb5X34Bw<_BcLsv)!gGO$(H6l$&%REd^56f(SQl355a;i&zmgfJ;EsuqeT&<`z&9 zQ40g42r4d5af4P|SZ!5QRP-f+PXv61_Y^2q-)$(#O_MhNGdC&V*Y|zD@1M^nojY@9 z&YYP!bLMPw5rkAhNUg8JeKtw<>||AoI}3AFJq3} z!_30@Uf7FA~uG@OxDG z$KOX#*0)$dcNGin`74>{2CSMPpogy$(BxYM7{VX_y?3iXp$}Hvvr2v_)hsHkLtoq@ z5OZqg4SoC7p`jUd=(i<;+^ngOfM%q}ez#-_Xx4oKGT$wr_JTUJ3Vzkbx+~ysSuCLC z4+!W}r+_{vsk_>4Z)xvA_`2(%83UE+tSaM;20tyJH`fWM8q!tpTLZtd&j{$>@Ov>^ zK*Q?=G-#7Ns%gwnJvrCs>GHO*)?W;uajxi2I=pks2czDmj@OH-XXBYu3eN#Z^ApPYP0X?J=&=+qB zXo*H}p+{d7&{#J1Vh*H3UlY&?NNKq)A(o#9Da~hT1@y#A0@@0L7_(DAhvMnl(Xsq7 zNNEQbAf<48V`BMENNN6oL;*EDC!h~`0j+#qK#!PR0{Z%O0d<&T1!)yAK#Sw)+IX5# z87u#LJndtN0j`CV0zMGWclYEc#o!fJ#p=z3R36ymSpFiUG(X2RCH644I+i{SDaFha z&(E0}E8h`MKaQt{Y4LXPbO)rg-6=@#e?vfp(_jp{?*hU!02-piwu+I+z^yJatiBz3?3-aoQl8bwJ?^?fs_K&-W5yd^@)``A*JOz;;AV+ zRz4n5T0ZX)0WI1spna=jY5JO2+7M6stQSybqkv{ZdXEdX!AW>{5(-Xk5D6QUNo+~B9ud%U?+Iwb=dtpf zqp|WOkh|Ipd^{+|O4=)NBNuNc6rbFm7u`^WMNAf+9MzX~Y4S3s4& z#nOxM^q>EZ<rs zQelZ^PAJKjAkHEa3i~GY5ACue<5};tz&@VX3JoKw1Gmq|9LjI;Gx{TbM#(y=150?w zafNo|#6~1W1T+MR{}QyjECGIGO~=M+@_yk0W(iNS24)qXwbHseDGMQ}`>;?QIBS)m zaj0zyLhU$Vq7QMbsELqU2aaPlOe8x7W-z;X^8P?>ewWmIspgqo_CU>=|(W(q%^;T&uT0OD9_f3n*f|^k?E-+b&Yzw|0M{pwJ32%)k*sz`z4|aZYesHT;qu5i|xWsbbB}VyI%e26+GBUJ2Ye5?) zc`SZ5q1rDbk`Ciy{0zCRXaFP6b_RL2XfcCKL|ts6^H)}UZ`p`QrO#^VB|b0JT9U;d zJ8Laij}2HfMrbLKs{Oo-yr&i6uUnIFwSVN&YJZu7{G)hS-t6uapZ0aDMI}BL(#l|N zm%+T;sfd^FCc_66(cW(zG%4H9FM~lEJ_b+}#AJPZ4*Vk>q-BsX4Vl!h(<0QCsN)x@ z#33QlIjFEVGW8VJt&q_Z?i^S-SS6IuGETzK_#A) zYJIm|JZeMjQQ@|WRSwtX?{-)FuR!i&Ta+vokQWE{pQaM0NJ{R5bF~%(c$^LGgC{f0 zrBeFfT#bm>+buLt35vEy@+8m^3sRb(Ghy+K5M7C>(UQt!v4tcK(GT4zQ5-JZjr6Sx z>lQwJj;=|pT}vxUrBnG1^g-9{b_*8glADJNrjxUNNb%5;-W%=5?Vq*|>HV!#ulrWo zZ2Gj-)O%Xs=L+wAFspfgm8PbBD+#(!rK&Vl`%&_*A$gi<-5elrA4wis&FqxO+@U$+ zNq4fyMgV*Mh_nO!@=FZ!d6u4q;THjIq<-D1O@-ZShEjck6BjAG+Sirhj3{|`=(wb3 zLOSkc%k#8pDTxd%!OKaJsUXW1s_EEi8)Bhw`|7<08)0~VLD?qRb zc=0uXG5|CXc3LI*yD2wsnHLDJ6B`vC#<;}Gwqq8L!lPA}Q}1Aho;yyt#v0PE^g36? z31+ayg9m3?R30=qsd*`^4D!vlBsjm&jP!1nINcZO(i3akeUp zLQL!A>>1jFm^*wyoWzRvlOId7v+E*d&K6Cw7n;h<3)^uiwoxWEm-HSsvac1>3B+Y< zrBXB*P*ivl#ZlyzVMCeg50a;cW%Rzi8|cBt$t!;Nf`eCz9c2Ho?E{nLa^8g zpn3|PkCJ(#3z@kQvUYU7a!Np0M&26D;nn2p(bJgzVUkleNZ}u`luRjmY;bl&F<9Bm zS)#!=BLjvin}IbTl`+fXxTSGF?Lr=rQ`k@EWIO39yPpvdSuuu7{IxU2Bb4Ij*WLVIxvB2?KIE}+gHz(n8`wme%(Ooq z$K#uc;(AxA02JNSR_{viW>~htQB^|7_4k;63`_gzk+{gBC`td;2>t&M?(1u?*ded@ zwg~dO!;EhXq<;-t_x}idH@gibPHDs1)91^e^e{1xACPf091Y?y;(j|HTct-E>)@0E z?49FpQ^4TKsqs(h;V_s72kvA#5&!u}fI(zYRtK0btv1m4r*j5l*$0P^gS)K=a$HCr;0qZhM4so!x zst)*WB#mYcBjPoW)cz7d#tR;~P**-4|DDv7PheJdllt=XNh)CGMkj2t8M-h%!>_VD zomU~6!zsS6VQQ=}t-neX0^mK6^%*UR;=j9WOWFx|-&NuuVE%md_HG?=zGT#<*B zSe)KT3=?P0Y~2rM49DV&FOs5l#`Y!hT_f8Y6+4o`kcYSz(j}P2yO7ROzX)ikQbNTHfSIXzb z%MZ%shr+$%^~>Vrd*t$fE3EOQ_;Z$0U@<^1cWODEm=)b}p%=PfNA-N*RXAg47h5~? z`uF;ZTSUvgkC$!eRJQj~KKHqzUb~N4oES1_`qGJ0&#$v!YhR{@cK?w`vA1NoD&Bq z6TamGx0JyfmWMZ{PUWcG<)P!RmTRG$jD&L497&mNgm;F{SIed2bUf~E4NQQ^8pCgE zH|Hy{94hY*E$%9VR--61_>tRYYB)8FiBt7y*QcmY&IU*(fD{S66^FF6n>;pip6h7n zjhlv4@_~GAIWQDDJy^`(Lz~L4cukUk^3GyUB5M zUJA8C?z|)RZhNToZr(pV-fnYfDdfK6WAB~{4O_B&2_IPOig)s8=$$2(px}vk{^8KD zdzRnB2X@8t_lMrO=Mv;UkLT|R4ZC;wy?lV@VgT<5ExnKTmpfv)TSHBd>$JvlH-_3F z_j5dF4gFne(D8x$5Pj>?iB(XrQ^)(+cm*EvuR#7%9P={B?}EHB{@w)nw;+FuBL?0G z`9{b;63=sx-^cx3I%MJlFUQeKhgo}upB$gHl#r4;;s-v`_5`_Y_P~sQbOS6;Ei6yF zghzc)e$h*p`i?I0{Ook*CyBg2d&Bew3G^bpXA-vX{&8{4o|bt35XYD})W@L)8?SLk z9M^}TOs79NiLvkjY)XvRTqcqF8~V8Jf|p#ZL4e9%bU|-{vYqj=b@8%UQ1(i^Oc^hm z3}r3xvN7?p2~c(HnpyU0_pDPkyUvRshI%2^VDx3ouLtN3r>i!8(Bai@qEqIgu( zL|4c4d#K2}uU(Pi>PXoGr;!XnC)T-N}2gol(U$jG+ zJX z4taY&-oG}UzdQQk65hWm-qG!Vn2o29bmBk4^oU4_fVemXUV|2C7xvb7?i3q41(7ut zd#LgNFK2@=3KNM6piVvtt7jtpCdp*Fdp|B^E~hTg6a{}U=deX-ApCX*OG#qw07f4s zCACM>JTh3NUA42o{`#U6@q=!nzNw7ySP6eqAA=Gj=OAN| zqYpr>_FtGOBWL05PuP)NXJhT4oXwzPV{M?U#h;jldQg6-N~ESLkzA^^Q- z2F&D9U1~MxT0M=l_Jx-iMr)(`RRl8)0=z?ds`)YvYCUoSy@rfVT&5GFDMp8@qizq= z5F18Uux&p)7odT$fsDOoAn$~zE;Dc?toA6-$ygU)%E59Mf-$L{X@JEq*XAQ9s6}As zVb-Lfa8 zHyj_p1lM%*dH@YX?UIUVNYy`ngV8dvRk<=nyf2~~WgPkbbC)hJ>sgx3A>PV%OaP=v^`UX+M z!1~u|CS2C-Ntf+bk2Wgn>H5$`q1HX7KEbQpwo#g&%!?S+L5eRYI`VvgfV|nTi~s`& zDF4r-sNBJU;N2l9FKZoYyGz-CE+;KDS@1?A)^}1SpbnIcLiv<9C>v00r8F_b0`(|W zfJDzcL8(I3mY_)}04ms77PtBx0J`W$>B0(awJ4vK&x}(=!WF70lFE9hg2&=PSNCbj zhDKW0rUdcx=yvIKq<9`1S#KE-i(S$+Jt@7;vp!mGY|^yv!X^tN)<=+Ug`>>!RqK(; z1aWn=%);n_toX;J{Z$EKSYjH0g37J?fyAoZSE%c8cmk^8f#~e&fL%=~&DC*t{T|rw z4W$;wkRW~+!J`yIf-sqsvhj`7 zNZC;CS7z0S42uSW$ySTnFQ~Z((OxBKzID)@uXWn@tJ`%bRlinZ>p^YVmLf`>c(iVW zaTGiB`{T-nXi(_7y1+dY$av%)o>4Z$dWr;F!9+$Cv^9;mzNbENS^>KYR${$vYC1J5 zD4f*501L_W4rAGTOWwJykU^u#soSc+F`Z{S&HQfOG=HoTYV&0L{5-|w)FpdwpI?rt z33S(j9MkX8XO^N&QzrHmfu%QjW1^+kHpa5|f@8o-@Z-jWyhEFkw>c`uT9#IV#4nMK z1;br=u!tTbsKAOmrSHd9RUY;s1Io*g%|fF+l{^2@KeK<|{$IBy53@=GQK6_%ugKcj zrbI{_&}(N~FQm47-I|EB9;3xzS!a0wPI@AlE$>?jD%mzAPTH>B0{q1C4q&j8VsTx_ z`ytSu+Vf+pmeE!AORZc6O&*kz8#Ft5Jqvqjw9cUzx|@8o&^&C0#9M}222w!iN2$!m zSjJeI;=}o{o$DoeH9H2ukg_anSIyeT1v=y_g{=wmBV52?g|&h#4xa2+M}b zI>+G)jXWz_!43j6WTv_3c^#PTeDk1P$qor|vvN@J1__xG;UqXC^Nbwe6fy2`TtdD+ zV5Pw$=u=PWC-gbR|MRST5Z;-*9Li2+5$7F+`dIr+8}jH5AHJaDBf;MfQJSc0q=p~g z%^Ydo-Pe`oJ<*E6rW`w~{`XVrTGo>W_O~-uXhXRboDlS^POw+;Iceb*>ap5!yK2vsr3nKs!T|JwHmQ z9<*$BTj%RkqPfgbx-X@ytHtk%nfa*iLKVi^8J1M(m6UQg!?g~^bE>s4Pvbk)iu2O! zUrG&Jw*AZ2fy3umka3PAxLC)Uiwquk5!824xJ%)+%Et4Houivfb`G2lCL5H9Z+5Yr zDzWGyN}%1-^O2%(*jwOY3V}vkS40U$-%D%@7p}wx&IyE$TP~IBm`>ada+iEO%#=7& z9&}a?XA|YpeY-gqV;t#4+i33nu*qk0F$Ic3TxJ8I<*JCH0JzNnx4*BzrJ!A2>5;y1 zz4QO0%X6LE!R!oMzda@=V;UQWFV&aF8N{rHk^B^+e+u3+Pu(39_^W>n>Y! z%IpZ1OkuQnu(7?+i}G`y3O_03z*T149e57Jhq|Gr9g8|l>(P?yx=#OrzGzuWn6s<& zcU)o+XQRFL5_7|~KxLDl{HmC?>HHP^=KcXe_6mZuF;KIiXVK_FspN$_BTE9A6?Dm{ z?Vi$nl@pt)97g-KgE5k??E_qNK#7-=9=JLvkHib|umlshJ*ah+B935q#`y4z^6;7? zaH{l8Y`>tPXET!kCyUDTU_;a2!FIV_(j#hI!s0^}i!Qw||8CnGHlwY~!Wn33I5?(@ z+mA;WN9$ap*pi=*vw^aa^njzs^w&~SqaEA_NDK)uNIt%6s^P@Y_piD z_FV{zjopfYJa9|8itz3(Oz?`oz{5i8X)^uQUhY`U$qW|WB+y8^Z2-c zjnlj?DmCIl9!Mc3UH6k>k^wgK5<5d6DcWiR|6Z=iKiyiG_K80_{ItZPEMAv)EPyI9 z`P7Q9q#Ymxw}3~cMOv#{XQB=sTBp9}DdDt~l=)RF3&y~hw7MO<1laT>{Do6{I$cDL z2lv7R&y_>mBCNxvb;2kgFK_PT2t|^qPre4tGeQnlP1E+$*u|J5XQW>DGa1n6q7CFfbvTHjEq_%jKcKTeXKS5)48I z5IC)k&fZR#w^y!g!dH;O8^i5MiQ>V>Js65YSS1&AVf5*7+EEVvBP=N$rYt}N2==w#o}7{mays57T-2hviwUqFdjnrKIjF~wRx6jOC0kM>>ND&m zc1uRtXNbigB#Nr~a_XJ$12PA_iGi|=M1RMPiI+=OCAP27CbmyiC${G&q5`sHNv`XO z{frUZRK_vXO=V1Ko-Dhm%oY}wQ{hoG6oJjy(}hPW2A`78F2xRod?u?b$WE8VX#3*< zM7h{MB+jVY2E+d!h?0@6;8088kG|TE)!JxOmC0@e*CA0rtc zG-zir$-bvEWJngnP#fSFpu3$&e5yOs2WTK{s0K#^&|@Cuo%FDSHXJ?p6m6>+%16ns z_a?cvcO_mnw2M*CVmt>Z?Kub$X_(rn?7rgsP!)mk8(m zWH9v1!TE3i2Z59eZaI4jco98}Fw{O(2TlhVsuZfT(1ZK>&>RaruID64J5p{38Zm(& z@)LRTzTqj!;;N|C!n_V=AdkfJygN2<=!MmY2=faF1>Ss^?ty&F?;LOcuvSNtq+pepGrEwj1CJPE5MdRJw5RXmmCJwo1YP zPpdj_F27nt$!R7XqT@>W7DnhSS{h;tI=UTI;5l_8D1}lxZuP3Y+04=cvsIT>cbQk z$z5*1=ZJZEf%;4>6-LRD<+rnI8tJxFTFA-e#V)xjhHyff z*B_hA?Fb)%+!4r8X$l)yacn1LEwNpSHy6r!6?9dlqPvt+S<=I76SEPu;e@ITNqVe4BZ zgGmG1pUStqWEiBYg2kQNd3D?02;06pxW1Dz(nN80_fhDiaiJS9eH5;^k^4|T+ zsTy1SKp7rH7Cx{vxk{o7^o^GT6Twt+_JK^Qz&dVEpL4BpCQj0bRr~hAp3i7T#R?B^ zNd+`2qcDQe5EHbDErd0l0+T%OZ-MBlX-ym$Jre5nmn%KGu+{{kHJ{90+cPzeU4medNwBjvlqV!B_H%HM*2GN7N4E5Sz1e)ADq$WmM~}04rRQoSRZA4j9w!i z>LmGpsb!uDkw^Yw$ekLP4<_OkzreJ#D)e+fIzAly_5f#L#g9Vd!@s1vmhgSSnjI5{ z6kfy~flad12f-SMDu8RHS2)IV0;Dw!)kYa)KO!|Co&5+%uE@Yqt+NPPK&u^SoM)}{ z2)9<+53O2^ydSY}(pU%5C=z48FnVlr+|oqJPba}KNe5O8R-QND*qS2IPL6e| zN#c)DIS|995pM%0xmBtfP*lH#jmtrlN{CDe#7fOr+_PzinmN_VWd z=aA!Yye*2g-6RajxZ0N1Qj3cZHooVP1fHA}oQ$9&GwVh$r@P3yx)sA7)!CcCVhf{x z6hmc_G2(TUbVT*X5;n~n@wU7RGKTB%UCuYytnxqbC`?XFIx zDcf7cDeH;vc_TToIxTIDZnC-xA~~^MklxV2pv^>D&5xf>BYI#P3wEH=8vt$|kXAvS zIJE*ePI(r9+`%COXeS(KC!DOE7c-{DMW+MMdZc1_-G+s0_&aG=j#D_fiSmDf9BR#C*$AbxfFfPn@CK!@Lf0z;iY>596R@E7NA~N5}IdLAa ze+G_$R7EG_myc}!W8;{EVPbkbgZXA3neup{`dJwW@zrO+?U{{__sz+JUSSi>#|BCW z+kU22tw+JuAP?f>$8TjG>n4VEqnO3rWbQhFDe5L?){V&O)$Nvi=0kfmmphgwUhZ%h zFjK;uOcI%HQv8IG`K^o0eqtDN90=%%$LR5(TW@9V>>>-+moPP5q<+1P$?YQk^}{lh zU0|@(W5@TBjs0G_J>x4#zk-xhK55`zhW-1WM?4mSonmZt>3+Z%qEx6I&Sax3NLzyK z!$C0^v4Fh#BsX=%WGF1Ef-tNJAtz_bmOG=6I(nY)4GswKT;t!<5kyHg_$3{4Gu5B6UH zebuVa)&j)uL*&^F#rRe7>4v+(^h<_4H3FCHz3Zu2VDL4QLr*)HbDdV_B#5TjW+Qx|1NLegsuC6*@D<_XUb2CJ4EmMI~`o9D$1)eFy znWTSx9$rVx^>;DHCGtjnZboGo7Z@#$!#nReA$c;uN?eGX;ov?Ah@rp2U$$)1#(vrU zk;|0SBWdXEWXtA*bfyry6}=r0{d2ZuZeF#FbCh(YUqUB!kAS z9VIvX*SNK^&90RH8MF4;W&>tP>*hM86g(Cs_ikATARK@k*Av!O1xKVDOijbWe%)eB zkUANo?s%$4L7v+(;=k5jvUM1qL0;IJ`#;gDz82=mtz-TxloEFdnPuz?P-P)`w$l)TTU`w{ENb0wUlgqakg6Is{>W#M0_!w zzFR@GFO?@MU@?YS20`p@4Y~29)lhf&-ZL-VilGJsE{=a?2n!F+DS@gLWW_7zn5qgg z@2@j7{;FlwwC1~iy$M4T$6noQNARd;!tYByr0b}2_3`LL4yAfcpaSK|3{)w8dT1-W z<<;sF*Oq>8Xs`D5UYPlH<-pLI9+Phgg3nrg0fH8b|9>(wxdH=0R zuE6Hmuz&J7C)H zFNClKakxo$m~&ug-*er;>@00U^iBs(u%|X*xY6L4)N`*PB;RWY$oCptSRXxK9jKiO zAphV$%J%Bs10;G?2dtHV#B_?}Kfte)XRGNDDyjpsD{qw)!1|f|JdmpAK^^);Wy05+ z;B7@sAi5OF^1ceB;B5PRaLHuCf?F$SS+>2Q0_7d?i_81Y^2I7siv_J5UHa$B89eWg zE{EArWbm?ci-UVE2-=yU4Fp24&B*7N)c78t0bR_lB#U>?WNNF)hr4rUS?M08^JT#H zREoNpuf5t5TEt<{gCA*GxcjSPwF^P0y@FGS#_(x!M`Nk_Q%T+)I~ud921~Ykgh^~g7(8z2U%$d(6H)`&fRDxo;j4OuxRtDIDph4iQ2uq~?It~A)e=vW!Iim36IR!UgY6+@ zr$&6VVoT>=MKqY&#eVZ@&r|C+1PjC1>KqF3pPe`#0mWM>LFlJASC=ENV^Wnv>qNyY z$ia-@4r*rs@wX3D!_^$x9yz)FFaSRTw~?Jqm^%t|%vw-+o>gLp0w75> zuBTDp0FXFTSO*S=zsban^>x1pl#0PRkLJe-L66|RQ6xBzy!g(bbVfW8#hh?hcuyfd z2(u_0hDG7YKi@GKwpio+gP{MIj~>tfNKxs}(AJ13sXDOrMlxdWFjtH0V};E)krh)E z*Ey;K4YQ_;SOfY21^{%MPOJ{>npGWmeRg%=UFy%Fa(zDh3bGs@U&48L4)w)NdDVeq z@P6MMg@_gKj>En`gyMoz$qeLS#tB)W;CUd3 z`uNDPG1W`!(D%r0Y%=?~_%rfB=zPFIsTXK}THr02gs4j(87`OYvY7q(r0d;+jK?F% z?Q9bDoF=s=^-5|72Ymgsz&W0beQzfVy*R2W6u0_ax0YL8N z$&vRAIU7UM0vpWK3j3kuEvjql*V!k-0?JUB_t)iH<*s=ruSq`9!~C#i22jWPa>m1& zSa1Z8-R~CyU$#2P$@dK=L1J3W&Y~vj(O1efEE3jZA7zqPAi(TVssP#3Om=**fU)tL z_LZx?>6U`K$)tTV(rsWuVT>oi+^B1cxiDn`dH23M@g~xDe`&uzYf{|3Qt&Bq%l?gE zHh{()^a<1w+kME+fRn5Z+*n9FBT&`-#r!oDej~Vfkcsyz zlVB32)C5`{;QhM9X@L`!ux~My83B56i+_R|@`>=A53ypf>eQA&1+_DI5GHU&+J(+Y zNe%@7`CUebA=s^OP!VQYz*j-)4~)|tDpyi3yP>REAzySOXAYQ{qvfR6LC4S~6Kgsg z+%f4G;7Fs^r~!OL9o=VeUCsezdbXCeg@b8PG!%(Ns7gON0XNc}ly6gYS-7#zeT*!!X zA~1Cbj=bPE;F^iKp*e`Wjs}e(5Cg&akRbL6#jJJK#2QhP089fckIW$8>hiot*ew0P(s{9yuMhQ4>Ia@g>ZD2# zvqPlgkY(U@FqA}sC%~GcNx+50K1~~|1m6}i?xp1~M^&JmE5$N$%i%Gur}y|4KW3xR zMKrSLZ(izT(_G#-=|DRR0jP`jP_w2+{Qjzc2zDyNnd2d5W{1)%fOjaPSt&L}APfeeEQ_~@<0{TxbDe^L!dsm`1h}!^z-mh@x)O0*Iky62IiQ~)?XT>Ikv2SG%G*rn- zQB&C(VDnH)e@o!~BXD(Kg_TTg&MNEIwQK)Xu2Us*ofxm7T#BBQl$`6Wv5*+-;QjM3 z@CcqZ9`t*}xu3G5o|nLgOu&$UQC)4!`!_&uOGtHdHv3h?xW78E&PLvCeoV=VL!)GT zOFnn(aLgPW3XY1zWP2F69q3X`+1`@LWOd7+S%0GR)e$(TX$OeVN24)l$4=Rx)08UcWd~A*Qo@`bKICzrwaem?ox{9DdW02#`A7>XH>Qq~3MCLF{ zg7{)c%#<5JNQFbTUaf6ZgAE0mRvs}Xt(Zh7qYRp|-NbohCO;#zFO$>PEvxi%sYe18 zr7eXm8dMO{Bl{6DRx~Bi^&w~q_vHj7}TmyjEO5`&dWoizmmUv za#MPn#0QqlI8doEXxr65A#jd~XC-p+ld;?b@G&zkhdPn+9?EkFKEvjV?*1gnaVU7umi3 ze&D}=x|g(maWk{Gi&(xKmG17MgN{wKCO+h+y2z$4%X9DUis9EY+_FCbO}dJoCq&L1 zoy+v?B4uBcF;ava`YMZwhRD-jWc6yI);6#Y#%O!fv1_NPii|vV3$r0YHXj?6?ugKp zZ-iB6+k23?GeUgFEKGTX7{4yh}kT{Z~LwewBAbAeSPhmJxUIKeKYfA zm}GoYHudeWR-QCdC760f zzhIyE-kL^LY$HAYO}?RL>|o=Msw*`XY9iDaOcqXJvirWsig$#`&)Dy7v%OPU__Eu(jh#dX4 zikTiFX~*S$#vL!Am@GcNf;kVCyyF(;6Nwm4%r=1M(udl08aPdON*5Y%Em8%`j6chS zn=Rrxi9CN|2;Jk~oXDUE%>B;5te8lWzMFbY?vd}TH0SwFMlSMQu2Lf&jRJB!JiHnu z3;xb$X~ebic4On)t+z~j8u06rLvp zl;dy7n?KA?eX*NjDh;5xnz2UQM*5!`o%1kkuB(JHmb%)18S(JqpEYE+-MU_V?1x$2)Xqqd)t*W>9^IZRy_dFxcZ&cWwcVs0%e zGO*3W>dB(B9XVCptF~KXse+(@u^?<*-$h28mPcqiEicE@r*q-@=G&)hn3ECGecHyn z2B!WqwakMNa^Q@uaB^fiptP%eQ6qj3Cj37P$qrs-opxP^U}MSBQ`waC zHvA)pSsf;C{No*F#cXoxPeTkVtbh?!W`I%{y-IB9q6)`O`+QI|bf7hXWU~9G2GAhZ zImr}X8NQ#a@m0IFAB>)Fd=CmA;{AMYXjuZssNxL7mK~RlSdU9Be68JBq2sgchKf{( z)?htJpp{P#*uW%2BX?Eeg9kxBzX9bRYb~Usk)>#bR|&#uBzQ+R)kLYOqJ&D*GG;L& z0S|t;NHDFaJEJL9)!N+?ZAN3ya+)ddC;ruVn4aW--UH;@vj+SQxqNo0YfTK}rF|_7 z&XLurCu<=DPxofUlh$(kOBLldZ)<*DRt4MKc)0w*Z#60}dN5`*YvdDr>=I!Tx~O9C zMWq74t$G@LOzBeY=0UiPQ7B*RDb}`O&A12GmgDA!9=z@1kri_#%okSRcAX0=n_c$n zEBK<{yatH2%>!UFE;5RI8Yo*)BSy>CUSu`+qDGC{1P$=J%o@v$Qf;&4VxoGyZOuhi z-J54Fs#TBJl@~Ds76UvHau3@wK!U#`dh8P54e|@1{Lcl`z|CMQUS zau5cv)rvMkRA)a*Js|vPu(LYQVkM9LV$@v<{uHIUYfO)CnMXeS#hKeUr=~+NQE9T) z25kyKwxp8!z!Bh&d}|JweQr+H(Ho!@eOWNrKBRI}CXHlK`M~5cgB&?$!Y*?8T*2%o z!P*kje&L2oDF%bhHTn3B7qz2YM##F992;2$V za3RB0*9b0xiOA5S8lp{9Ib^~ZLUM8-AZ)z10mAoAO2%>sB2Q3yS-1(q!GR4Qe+ZO~ zpvop_K#erGg|^EM18)Q_2;Z}D>}fLH%Yyg$!%iAL*4qnpO5y#^$E4SR7a%=biQ0F$ zFlPw`*GH8;T1g4F&20|t<lUGygHecB?9zo`@I8He>=}yncXYoVc{Wey&F}56+)zab71$rqDV!9-$f!p6`1Sf z9bR)eNaRmo)HP1`3lJP_O;UnO42|W<`HQoFlsD2SV=!N!Axu~ypnQu9QZa*CY{H2? zO{Vg4>g{fbNODW1j`5XfFkBQI|1Qj1yB7~Dtz?V;D(N z8hhIM-E|Jac$nfWNol)D%6H6x>qD%{XwPT@ZN>mLJjpl`j}Lg=(%Xne8u21VkHhmh zx|XA1?GwS6Fc%DC36x`^fvT_;zVL;OCNC;Hlt~wMGI$kDD1!|CHP5y076=a#bP_kf zEGiks0Tu=$Q><(@gAWor7@u!(i^N_Db^V(ckEEsncJL8zY^wNkluJJj@o83;YOqyi z-p_#zM)=c){FTUa`VXet0AV4Y@ygf_{DVaP9DdQe!!G-;#oZGIw zmO%ZkcS2MtLgO7OJ#s?;Hav4qwRm$#y;&*c-bYse72?f6CY9=j^nvwo!J}smq zU?-}20%3Xb{cpr&sM`{^>-2}GA2PuFBWBbw#vyb?!5W7cSW&PABRUWRwg411RnqF% zwnUB7nHpBWm0mmJl}gee+*VgFjmd_PQ!yx@Kv}5@!rwzd1$>J_2_K+xOY01%_%_e? zQiI_hAKm+^%SkTqU2Fs=4bmrCc9p6&0{GOMD!pLE@=E#ZAXkdp5|u~~PHrQ1c#vXK zO~=V)M<7)FNUPGhbG`z;JD{x33@0I8O4QX>nr*wlEnr9mh&m&z)h^U1uRu|Ts=X1` zYE}5t))J?xonsUY{n}cPs|r~Ck=7v)g`orc z4hrve1Yed8Ul2f)tM)r#5rL`O*OgPhjP5D8W=Jscz*2s0FCcZ)JDO|7`O{T;pL7>r9P@o6i$Rd%khQp`G_2W0i*n?m>7JgyCS3 z>tYJQBm`?&Dvt)Z_$YFyox`t@AKUZrE`t4oQkNc5wpW2IJQh`k!7O^czX;cmyZi$Z zFTv$5dOTzCEwa)7*L3Q)s%oZwkBRaf#=6GpfafN{U)qg7CI7xO4GZM@j$0Cb=>!WY z#M7N8FLz8}JTu789hRi5u2>wK6ucbSG^S6QgTEAfOSbvcHTDAK*;7mAU49ymAZ?cm z@Z%&ka5&+!kb;{GaK0wr2OjR7(~01&aXvbEp594P7#OiPxhq(aQsZGtZh+JEB=~g| zXd2lcti}V0N*X#C0gX}{E?l+&$-`tf+dgYWcg{a=_MedJDot93$y;Nkv40(8p;Q5} z$FE8K@AQK_1h;0Q`twF7QT+CnRrX|2I?jqfGV;m#n%p)4jNOe~?P8QuFB7*81d2Ww%3I@y>e zPstmhN<5tWJJgSX`JtIrWLoEB`~z`!4jmKl&+ z^CG-H0gUEn!8cCB3S*_90Y*CKKp@4e@Du%afiMDJbFxnXhk6mjTug_7ms+%xl+E{& zP_*cqFesn#JhH8w~L$akD56eWrdVo)~DRmPc!h_d=zNIrU z`i|f&Tb{!6H1I*V6RoDaEh664XG7<=bh03>l3L3<1lS%X_gff$n09;TB41+hU0!A2 zs4NVoMcofEIKx=-1aRuR=tRPokPuT(TR#I~U_E`R(&3u~aIesPh{0J|4UrK(#FhF^ zNJrr|j>mXHI#I>}x4NO*4TO_1WOFDB`j>)9(R5fQPg$(qF(Z{y{@R^H+d-RS&<30f zQTHeX9?}1gJxzrVD5){9Z@902DfprLaRn~oUXM~LYGEN9CN5>sZyc(@xfA=wq1yw1 z`ke?JJFk45ip~AIK0i-EU;wegovp;#ncbni@=GbG5A8k=V;y6(+ze|lz+I!n&)}o( zpOkoEQEJ2}-yFgDoxw_J)haN40}($LLM3n_&sKXLTqVC3e{|oi!jtel?!79!Xi!OS z^Nd-x@8R2h8*RsJA4~P{+uZoE^w7?aC80jm1CFwC)V7w>v>$a(QsW8NEU5V*TtxLi z_@1EtQgF8W1vQ?*EGc*U)p#!Bo8g|W!9$7_haHosR0h1sk496@?{soa`FN7n(w2Tb+i7VU>GK8Xlxnh_^-E_B2?B8zeX18>hQJPQy9)wEHJ` z3saNW8&|t)d&9RokGlWT8xP03-Fte&o7dc@d*cCkt~=Zt_fN@+m^NUw<~#Wllwy;6 zU>`h4n+BgSH6Y!7&gs6j4<4264VgCRL|qE&;Y1xWt>s{emCaR__8@XM^}+K?lDZFq z!na;hT6yuTKvx0|Pc2;yVBlKdTt4dAt^le4J~6&8eyPu|W_YWIa0~e8+{^(2mU5NYGMc-Br`#$1 za5j_G=`QJqhxPhL7o{@FZMuTzyWC6qVUD@wL-*7D@Cm%dy$~ej5}QmctPebjR*I+s zo*HEa2;zhfba?yDa`^UO9rfzF_hsPRDaXNi`i2h72V;$<@mC8IbZ^x2t>uz{$FRh z_hsUBHOTpdot{GAU4qNFGhkry6I?EP3ce*taa<30y^g+|*xXUN$2gLDVX>Ni45;Wl zZ}&)Or^W|CQphZ~U+dnO1)qvJXLcXT!jCYBcaO`)Q~KP$d!zIg2xWEfS2oT+gPPq9 z@Fod0rh!HXsysvnxW#OoF_EpqhMz7dM8#YE`Y8}9DfhCP>=Rbl9ibq6=g3!PhtJrE zmxWc}6fFb6;qz9s@;BKaz>60JYC*x`0{65Wte+rAWgC#56IoQY_WM@G))l8f7 z5<4XixN9AHDwklJilg^XM5t}n^gl7wuD4JRp3?nx4jzmTxliZd0(^rT_s1hs8@kJY zEDwfyDt+Ld)E^JckHBRtDR^g?+9k$#CI-*~iU`X4%I|@1#NOQXEZnPi@92-mF>i+4 zr}|?f{>}~mkHHM)o{)P^F24SHxc@<^1UpmVlQxSzo1{j=OwFAq=m%7=<0hb;Yv0fH zZjvgwG>zfJ4fM?twga4(Is-McmS@&Yu%p= zz!klzIF<>5NX$G0W&!bGcQ%KI4h+J_Mf5=EF^|ghe_Aa{U%8)`Wi+d)h~aC^IoNyJ zy_ACs*sr*M;;@+!65Iwow!03&#aWJ_xwxQYZ{nUH~3!6{p ztMHFg0FsJaOaAbY{TnSB+J}n0EYw1&3N$hWe%>On-0;d}naIcGZGvx0ya+!MPx-RT z-sgb^(NH)_@Y4{_t@C1cmmUv*7%F$J0jC?x{&5eb~hP+! zq01%r8QDSCOY?Xg0yHu;y970K`v&U~EZoX`TfYx$i(395gS`)T2TQ3f!ezZnCR4@_?SIJs&GKYPS6- zW%%&r=7q{_^aCOS{AgN7fywA;8dTdTa>(Ct19E2Cr{^H?V*zb}km~J#t{E;ziKrJq z*Q2d7q&%H6Pt$KBr`Nnd!90BpIDUfrp*&p4t2R>oTm#Cs94t(AL7l;(uK&7K@y*w* zxZ+0NSC=%rdK>7qwgcGt0z(dC_x5*+^$}xR2%IQK?G!*0vrFP$mx?-a>cqOLy-AJVyUh z*TAJzT4Q7({cdxcu-Wi4oAz@-LV#GK}AP^;HVrV5XnBk8Dnm>|A6k1ZTqK+}^Qe{_cY%?(& ziyC*?6E>ID*u`$DboD5E$`Nd$j!Jv5C!P+9+f;3L>|rfN-Ao9Z8N$E)eusNE+E;rv z=X~;g{O0?<@6F7c`~J+kcd}LO3UR&YQ~$a`teo@uxI~LI6Ms{^w?eF%gYNOkKfBc2 zJh3MA(c~P@s9DAF)^2F7+OC=aoKgmNkhPP+N<4e_V^RNq^-y6>pcZk&Q9JQ8C z$M03Qtw#xU19Ime3);_{TXv@U$* z>UMRxeZ2VKXe%3aCO`Tod=h=ILTcaT;^MusBQ?kyp7!i-PAwj4cKJ{D>@&tlqO)@` zS~M&654cgPyAY!`AH}H6ME;bf-2KvxpfM~Z1 zm;aKz=X_|PPfxRD#Z!)N!6lrbsxQqougUA+n>0xDA&=v$jjJS zX5fD-gZG%BUvZ-=hYMbhTcde7k1IPL4Qah!@fV8w3ZHFlwA*Z@=H^Glw9gbX$`1(1)|~#bNiS3XzJ-E zXDa&Rr=HGnTe-QoBZXV9Sc&h+Bi73F*ZgX4fw*MveKYKXIXsx*-4<~DTiSDJZAF2x z_}@=4cM%3@@r3xU@pSp~ zFz-EV5mk=l7QyfRxJB^n$XYfX@7X)e=j2cBLc@t6PZ756ogu#5i;^$(TD@8IvAeoI z1!J<*qMPvS_Kb>kCjK;TFm_tS*t|Sd_|kqwN-jQP$wSwcpfA~}Si9LdYpyw33UZ%#geE0a&)Lj444l25?-r6+(DS68nV3o`J*$aQiLlw4q~x@9$v?D4_)QgQ!k zv1-+=Qw}s$)A2iJ`fz>x8_5N&`qF}CLVTq-dQGjmVzsy!ySd~iFZsFS^GUvd>YI27 zix9a|FV_cuhEH6mw)(`98YeCs$KrX3N-XQYCKC95@TugovXjf=$H2;1HON=~7QE%# z13V7Lv%rN5=}iqt+^sv?klm7SZ{+)G1s5mZA@ysYC~DY=ksSoCJT9!uw`Bg!gO%7E z)}uYHI9C;QCo=lmPTz7G}9CN=LW@wVt!A6zB! z#B<8AR(#8W13uNTR@908>h-na9a9ItIfpURXgyfXi$GwDE5h36)zO` zi{9RQz9x!=2&tcaUF;N}sMM>)L07nH)9Ap>y`302S-1wiv1v41qq>U3F;{pS9=J*E zS|{2?K+UWZ-}6MLe(|YGIwD;cW2Ixin^FhXi@w#KGwpiRFhbz>tip7HqY`u9@)wQDP+5t2@ znRshvN9Dd@8Q(y#G_4+;u^!>L+^mS^*`wVQZ*u3mkGK8cYDGZ(;absgbTn#cZboT20VG*Y$cY};+>-V)KVo>L`fyLftt?>Ec& z;|=F{Zj1UI{?V|T&kW?E9dq*+34--j><2hj9ri1KsW_WWpIGDgpU~}e?F#-=Y3sWs zqpg0NQU=uaGI7FC>fXMvsl1L($%9Whc{7d{q2`?C`?CZpxc&pp)=zZimaedVhg-WRHYFL^%esefyp4BU3+x7$%EQO7Gq`TW}_&c@$F#}D=| zsVnJSa=lpWki(^F{bsRd&0p<$#x**>@q}JUzPN`si?XGdtu7(GUp;RAp)Dd?4wtDV zTg1-Gusvnfe06U$^`k8q@dEo(KX##ass1fu6*`icP0;h_o!3G|I;<<-F&@>vRa8mewEB3fcmd+18I++OXw<7Y1>3)780LJeUo|2J)fy|Z4*nfKAt>(50m@+yrrsh zc3xg;wXe^i?P7_1a#DG>i!9l7om#(L1hNm5&pVx!eqbcS`U?xgh4dwhwtQh>$8d|$ zV`y+NcjW&yx0I7=JAT$M)BI>d{~1Tw!P0#Ga6_4T>?U#RU+=W2T-Bml?ip8g)uPoo z7QZ>6MykaTaa=vILsVSwdHmJn4-MsCI`{0P_&nZ=UxL|zOJAB9@o0wKNYVXFAZ#hXUBL-^i;fG z{}mp02Ik}FJAc!uc*!}(34ZG(aPn>~%iY9pD^A4^B~Hb&&v|m@M6H6f57KbG>{R@l z3CEMZs}}c`)QY1H`Q7Q>={oUaDKn>fe|)RhDtvoh#YLZ&_}hWbfQXL9@1M?0h`G+| zt^C9ww}&+EX#8&N!70Dhn{}HA--A0zEX&`!^jZ0$JXp2B*&F^y{4Pa(=RHwT+|gu7 z(y5`<{OduD9MkkeiFw=}*)}Mg>WM#yvf{4qS(0>X%skY7)W~to5R@2?J@@H9VO9Gc zu_Os=jGa1!8u?H&3?;^6kBt2WtNKa|t2*$HmLw64nTI-z8u?5!1|`O0&mA1cs-Ebu zB#CN_ojQgZnbbIcXh~u`_Q_+yq|1t8 z4D%$3^_<4|p_!4jKTcaJ5BM`Pxp=AcFjG=)%NJoa4Q z5?NWC^@=5lS7YqdLe$6xO*xbpk3F(750Ux5!a*$XX^fp(jvCplsfH5cu}9YX5Lxsa zOA^1v*s0a1kvdHSlo*davUwdMD^FOGRB4Qz+JG9_t!aW1ak zP$Lg%TA;*u?2%oY5E;anJfu-$?9>+2$N^0|lo*daGF*+w<};QgA&s$9+fgHjHAkSt zc@o%P+~mx+}NF1)s7Sln+`fP#!l@)jU3bT zLy7U&Bi)UN>_XRb(ycLeYCme^xMm1SjK?17eGrj-7YRwi8e^vpp+-K`3`2?W*dvP` zM&y83ND|Q)J9QW}@|k7~N{q)IS@k#~PplM@L^Z}v9Yc*wYMhq~NsPxH8Ei-72u7+Q zF^#cPoof*R(lwb-Vm$W9&{K#^Tq7j0HfW3=nu!|8*5p8mvDhPVRX#;M^$c6i>x3j8 zjhTm}seI1cS^+FQA#@MOVsF6BN1C$t#Ju>lYL{{D{B&pIEJGB8dvRl&xCB|cq z^b8@g<~|`wy~fz7O{kHFG%ZkKJod=^-y<@(M@Z7BF?MPTYUF^X9ZHPH9_hzO7{$#k zLXwci*s1NPk;9rJP+~mxNE|GssGX-|`GzNiYz1u^GYjJ+?7{HSIWr zh7)vZjGfwp8abxvhZ5tlN45$F8qR+~!wI@I#!l@=jU3kuL5cC$BRkU&+1G`J6NEL! zP8~vxe5e_Q663K)_ANx@z`vv61QCs~Q-@I_pJ~RR#CYtHk!(bs_yrnH5Y-qvbqqB! zsc{}fs|Sq79yyYW$dNbEaDte|*s0EbM1XWnCX^VDE%MN6Bqold-2~R#8e@lMqDHbc zIZ$FOwnP91nnm9X^fp(jvCplsfH5cu}9Y5gvg@PXg7ghW9-yw)JUDC0ZNR= z9@$)n$jWnQH$j!g*r^Suk=>dmC@~&;WZNBxtZ_(5>NUnrZ9_NBAYLil7uwIPHjhx9M&9x663K);>-<4oFQzStxKdN zZ5lHV^$2R@c})+L7>_+S_9#}hBTq`wsWEnH4{GF?rXNa-#~$fEh{!IVl%!i@?9_hL z$Z^dOlo*da(t8+@ee0woVU4j1oQbsh!P8~*#e5M(L z663K)R=t486Pu(YQH`-v$511a8s}yyiSgJYgT06xsg{z&G{#PK?mz@c*JMJ8@z^6n zVMHc&;@2U8^(~F@Lo-n$*_s?EF&2AdCpXTg-eSvnmz2b#G4oJ!P$LDJLMSmFdv5nT zSkLu$|a;!<}Ae-L%s19BZNKc!yu)0}Cw5C6l#aV8H#JiNB5(B#if;Nz8?==EnhhlShubA051N$@bi!vqf#JWQzacEdr; zD-*^vx!<|Z>Vt9(7vi-T!8Bn#YU-HF(}PbjVBHN(`W=>4glWRMZ?>N0t;Ym03v0wQ z`5TymP_wWOOcU1IOdWHDF-=(SGj%L%1k;4|C{)!CiYza>kKnrX`K5U=O}3%KYZlZj ztrF9Ob-$@&X^ogBtOrdUOY6WiVZ9BiLRgw7j43jpnZ5em>w8vq$}j$Z=mW8=caGN? HzwCbj6G%pZ diff --git a/drivers/gpu/arm/bifrost/mali_kbase.h b/drivers/gpu/arm/bifrost/mali_kbase.h index ee78a1237d56..4d845ea08adb 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase.h +++ b/drivers/gpu/arm/bifrost/mali_kbase.h @@ -345,21 +345,6 @@ void kbase_job_done(struct kbase_device *kbdev, u32 done); void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx, struct kbase_jd_atom *katom); -/** - * kbase_job_slot_softstop_start_rp() - Soft-stop the atom at the start - * of a renderpass. - * @kctx: Pointer to a kernel base context. - * @reg: Reference of a growable GPU memory region in the same context. - * Takes ownership of the reference if successful. - * - * Used to switch to incremental rendering if we have nearly run out of - * virtual address space in a growable memory region and the atom currently - * executing on a job slot is the tiler job chain at the start of a renderpass. - * - * Return: 0 if successful, otherwise a negative error code. - */ -int kbase_job_slot_softstop_start_rp(struct kbase_context *kctx, struct kbase_va_region *reg); - /** * kbase_job_slot_softstop - Soft-stop the specified job slot * @@ -496,9 +481,7 @@ void kbasep_as_do_poke(struct work_struct *work); * or a dmb was executed recently (to ensure the value is most up-to-date). * However, without a lock the value could change afterwards. * - * Return: - * * false if a suspend is not in progress - * * !=false otherwise + * Return: False if a suspend is not in progress, true otherwise, */ static inline bool kbase_pm_is_suspending(struct kbase_device *kbdev) { @@ -521,21 +504,20 @@ static inline bool kbase_pm_is_resuming(struct kbase_device *kbdev) return kbdev->pm.resuming; } -#ifdef CONFIG_MALI_ARBITER_SUPPORT /* * Check whether a gpu lost is in progress * * @kbdev: The kbase device structure for the device (must be a valid pointer) * * Indicates whether a gpu lost has been received and jobs are no longer - * being scheduled + * being scheduled. * - * Return: false if gpu is lost - * Return: != false otherwise + * Return: false if GPU is already lost or if no Arbiter is present (as GPU will + * always be present in this case), true otherwise. */ static inline bool kbase_pm_is_gpu_lost(struct kbase_device *kbdev) { - return (atomic_read(&kbdev->pm.gpu_lost) == 0 ? false : true); + return (kbdev->arb.arb_if && ((bool)atomic_read(&kbdev->pm.gpu_lost))); } /* @@ -556,7 +538,6 @@ static inline void kbase_pm_set_gpu_lost(struct kbase_device *kbdev, bool gpu_lo if (new_val != cur_val) KBASE_KTRACE_ADD(kbdev, ARB_GPU_LOST, NULL, (u64)new_val); } -#endif /** * kbase_pm_is_active - Determine whether the GPU is active diff --git a/drivers/gpu/arm/bifrost/mali_kbase_caps.h b/drivers/gpu/arm/bifrost/mali_kbase_caps.h index f6bcdd06e1aa..000e30e1ed84 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_caps.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_caps.h @@ -33,13 +33,21 @@ * * @MALI_KBASE_CAP_SYSTEM_MONITOR: System Monitor * @MALI_KBASE_CAP_JIT_PRESSURE_LIMIT: JIT Pressure limit - * @MALI_KBASE_CAP_MEM_DONT_NEED: Not needed physical memory - * @MALI_KBASE_CAP_MEM_GROW_ON_GPF: Memory grow on page fault - * @MALI_KBASE_CAP_MEM_PROTECTED: Protected memory - * @MALI_KBASE_CAP_MEM_IMPORT_SYNC_ON_MAP_UNMAP: CPU cache maintenance required when - * imported GPU memory is mapped/unmapped - * @MALI_KBASE_CAP_MEM_KERNEL_SYNC: Kernel side cache sync ops required - * @MALI_KBASE_CAP_MEM_SAME_VA: Same VA on CPU and GPU + * @MALI_KBASE_CAP_QUERY_MEM_DONT_NEED: BASE_MEM_DONT_NEED is queryable + * @MALI_KBASE_CAP_QUERY_MEM_GROW_ON_GPF: BASE_MEM_GROW_ON_GPF is queryable + * @MALI_KBASE_CAP_QUERY_MEM_PROTECTED: BASE_MEM_PROTECTED is queryable + * @MALI_KBASE_CAP_QUERY_MEM_IMPORT_SYNC_ON_MAP_UNMAP: BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP is + * queryable + * @MALI_KBASE_CAP_QUERY_MEM_KERNEL_SYNC: BASE_MEM_KERNEL_SYNC is queryable + * @MALI_KBASE_CAP_QUERY_MEM_SAME_VA: BASE_MEM_SAME_VA is queryable + * @MALI_KBASE_CAP_REJECT_ALLOC_MEM_DONT_NEED: BASE_MEM_DONT_NEED is not allocatable + * @MALI_KBASE_CAP_REJECT_ALLOC_MEM_PROTECTED_IN_UNPROTECTED_ALLOCS: BASE_MEM_PROTECTED is not + * allocatable in functions other + * than base_mem_protected + * @MALI_KBASE_CAP_REJECT_ALLOC_MEM_UNUSED_BIT_8: BASE_MEM_UNUSED_BIT_8 is not allocatable + * @MALI_KBASE_CAP_REJECT_ALLOC_MEM_UNUSED_BIT_19: BASE_MEM_UNUSED_BIT_19 is not allocatable + * @MALI_KBASE_CAP_REJECT_ALLOC_MEM_UNUSED_BIT_20: BASE_MEM_UNUSED_BIT_20 is not allocatable + * @MALI_KBASE_CAP_REJECT_ALLOC_MEM_UNUSED_BIT_27: BASE_MEM_UNUSED_BIT_27 is not allocatable * @MALI_KBASE_NUM_CAPS: Delimiter * * New enumerator must not be negative and smaller than @MALI_KBASE_NUM_CAPS. @@ -47,12 +55,18 @@ enum mali_kbase_cap { MALI_KBASE_CAP_SYSTEM_MONITOR = 0, MALI_KBASE_CAP_JIT_PRESSURE_LIMIT, - MALI_KBASE_CAP_MEM_DONT_NEED, - MALI_KBASE_CAP_MEM_GROW_ON_GPF, - MALI_KBASE_CAP_MEM_PROTECTED, - MALI_KBASE_CAP_MEM_IMPORT_SYNC_ON_MAP_UNMAP, - MALI_KBASE_CAP_MEM_KERNEL_SYNC, - MALI_KBASE_CAP_MEM_SAME_VA, + MALI_KBASE_CAP_QUERY_MEM_DONT_NEED, + MALI_KBASE_CAP_QUERY_MEM_GROW_ON_GPF, + MALI_KBASE_CAP_QUERY_MEM_PROTECTED, + MALI_KBASE_CAP_QUERY_MEM_IMPORT_SYNC_ON_MAP_UNMAP, + MALI_KBASE_CAP_QUERY_MEM_KERNEL_SYNC, + MALI_KBASE_CAP_QUERY_MEM_SAME_VA, + MALI_KBASE_CAP_REJECT_ALLOC_MEM_DONT_NEED, + MALI_KBASE_CAP_REJECT_ALLOC_MEM_PROTECTED_IN_UNPROTECTED_ALLOCS, + MALI_KBASE_CAP_REJECT_ALLOC_MEM_UNUSED_BIT_8, + MALI_KBASE_CAP_REJECT_ALLOC_MEM_UNUSED_BIT_19, + MALI_KBASE_CAP_REJECT_ALLOC_MEM_UNUSED_BIT_20, + MALI_KBASE_CAP_REJECT_ALLOC_MEM_UNUSED_BIT_27, MALI_KBASE_NUM_CAPS }; @@ -68,34 +82,67 @@ static inline bool mali_kbase_supports_jit_pressure_limit(unsigned long api_vers return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_JIT_PRESSURE_LIMIT); } -static inline bool mali_kbase_supports_mem_dont_need(unsigned long api_version) +static inline bool mali_kbase_supports_query_mem_dont_need(unsigned long api_version) { - return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_MEM_DONT_NEED); + return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_QUERY_MEM_DONT_NEED); } -static inline bool mali_kbase_supports_mem_grow_on_gpf(unsigned long api_version) +static inline bool mali_kbase_supports_query_mem_grow_on_gpf(unsigned long api_version) { - return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_MEM_GROW_ON_GPF); + return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_QUERY_MEM_GROW_ON_GPF); } -static inline bool mali_kbase_supports_mem_protected(unsigned long api_version) +static inline bool mali_kbase_supports_query_mem_protected(unsigned long api_version) { - return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_MEM_PROTECTED); + return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_QUERY_MEM_PROTECTED); } -static inline bool mali_kbase_supports_mem_import_sync_on_map_unmap(unsigned long api_version) +static inline bool mali_kbase_supports_query_mem_import_sync_on_map_unmap(unsigned long api_version) { - return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_MEM_IMPORT_SYNC_ON_MAP_UNMAP); + return mali_kbase_supports_cap(api_version, + MALI_KBASE_CAP_QUERY_MEM_IMPORT_SYNC_ON_MAP_UNMAP); } -static inline bool mali_kbase_supports_mem_kernel_sync(unsigned long api_version) +static inline bool mali_kbase_supports_query_mem_kernel_sync(unsigned long api_version) { - return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_MEM_KERNEL_SYNC); + return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_QUERY_MEM_KERNEL_SYNC); } -static inline bool mali_kbase_supports_mem_same_va(unsigned long api_version) +static inline bool mali_kbase_supports_query_mem_same_va(unsigned long api_version) { - return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_MEM_SAME_VA); + return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_QUERY_MEM_SAME_VA); +} + +static inline bool mali_kbase_supports_reject_alloc_mem_dont_need(unsigned long api_version) +{ + return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_REJECT_ALLOC_MEM_DONT_NEED); +} + +static inline bool +mali_kbase_supports_reject_alloc_mem_protected_in_unprotected_allocs(unsigned long api_version) +{ + return mali_kbase_supports_cap( + api_version, MALI_KBASE_CAP_REJECT_ALLOC_MEM_PROTECTED_IN_UNPROTECTED_ALLOCS); +} + +static inline bool mali_kbase_supports_reject_alloc_mem_unused_bit_8(unsigned long api_version) +{ + return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_REJECT_ALLOC_MEM_UNUSED_BIT_8); +} + +static inline bool mali_kbase_supports_reject_alloc_mem_unused_bit_19(unsigned long api_version) +{ + return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_REJECT_ALLOC_MEM_UNUSED_BIT_19); +} + +static inline bool mali_kbase_supports_reject_alloc_mem_unused_bit_20(unsigned long api_version) +{ + return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_REJECT_ALLOC_MEM_UNUSED_BIT_20); +} + +static inline bool mali_kbase_supports_reject_alloc_mem_unused_bit_27(unsigned long api_version) +{ + return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_REJECT_ALLOC_MEM_UNUSED_BIT_27); } #endif /* __KBASE_CAPS_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h b/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h index baca78679f0b..7657c25d565c 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h @@ -255,6 +255,42 @@ enum { */ #define DEFAULT_PROGRESS_TIMEOUT_CYCLES (2500000000ull) +/* MIN value of iterators' suspend timeout*/ +#define CSG_SUSPEND_TIMEOUT_FIRMWARE_MS_MIN (200) +#if CSG_SUSPEND_TIMEOUT_FIRMWARE_MS_MIN <= 0 +#error "CSG_SUSPEND_TIMEOUT_FIRMWARE_MS_MIN should be larger than 0" +#endif + +/* MAX value of iterators' suspend timeout*/ +#define CSG_SUSPEND_TIMEOUT_FIRMWARE_MS_MAX (60000) +#if CSG_SUSPEND_TIMEOUT_FIRMWARE_MS_MAX >= (0xFFFFFFFF) +#error "CSG_SUSPEND_TIMEOUT_FIRMWARE_MS_MAX should be less than U32_MAX" +#endif + +/* Firmware iterators' suspend timeout, default 4000ms. Customer can update this by + * using debugfs -- csg_suspend_timeout + */ +#if IS_ENABLED(CONFIG_MALI_REAL_HW) && !IS_ENABLED(CONFIG_MALI_IS_FPGA) +#define CSG_SUSPEND_TIMEOUT_FIRMWARE_MS (4000) +#else +#define CSG_SUSPEND_TIMEOUT_FIRMWARE_MS (31000) +#endif +#if (CSG_SUSPEND_TIMEOUT_FIRMWARE_MS < CSG_SUSPEND_TIMEOUT_FIRMWARE_MS_MIN) || \ + (CSG_SUSPEND_TIMEOUT_FIRMWARE_MS > CSG_SUSPEND_TIMEOUT_FIRMWARE_MS_MAX) +#error "CSG_SUSPEND_TIMEOUT_FIRMWARE_MS is out of range" +#endif + +/* Additional time in milliseconds added to the firmware iterators' suspend timeout, + * default 100ms + */ +#define CSG_SUSPEND_TIMEOUT_HOST_ADDED_MS (100) + +/* Host side CSG suspend timeout */ +#define CSG_SUSPEND_TIMEOUT_MS (CSG_SUSPEND_TIMEOUT_FIRMWARE_MS + CSG_SUSPEND_TIMEOUT_HOST_ADDED_MS) + +/* MAX allowed timeout value(ms) on host side, should be less than ANR timeout */ +#define MAX_TIMEOUT_MS (4500) + #else /* MALI_USE_CSF */ /* A default timeout in clock cycles to be used when an invalid timeout @@ -327,14 +363,6 @@ enum { */ #define DEFAULT_PROGRESS_TIMEOUT ((u64)5 * 500 * 1024 * 1024) -/* Default threshold at which to switch to incremental rendering - * - * Fraction of the maximum size of an allocation that grows on GPU page fault - * that can be used up before the driver switches to incremental rendering, - * in 256ths. 0 means disable incremental rendering. - */ -#define DEFAULT_IR_THRESHOLD (192) - /* Waiting time in clock cycles for the completion of a MMU operation. * * Ideally 1.6M GPU cycles required for the L2 cache (512KiB slice) flush. diff --git a/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c b/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c index 1e7e823f44c4..9f88d4f3fbc6 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c @@ -62,9 +62,7 @@ #include "csf/mali_kbase_csf_cpu_queue.h" #include "csf/mali_kbase_csf_event.h" #endif -#ifdef CONFIG_MALI_ARBITER_SUPPORT #include "arbiter/mali_kbase_arbiter_pm.h" -#endif #include "mali_kbase_cs_experimental.h" @@ -76,6 +74,7 @@ #if IS_ENABLED(CONFIG_DEBUG_FS) #include "mali_kbase_pbha_debugfs.h" #endif +#include "mali_kbase_ioctl_helpers.h" #include #include @@ -153,21 +152,29 @@ static const struct mali_kbase_capability_def kbase_caps_table[MALI_KBASE_NUM_CA #if MALI_USE_CSF { 1, 0 }, /* SYSTEM_MONITOR */ { 1, 0 }, /* JIT_PRESSURE_LIMIT */ - { 1, 22 }, /* MEM_DONT_NEED */ - { 1, 0 }, /* MEM_GROW_ON_GPF */ - { 1, 0 }, /* MEM_PROTECTED */ - { 1, 26 }, /* MEM_IMPORT_SYNC_ON_MAP_UNMAP */ - { 1, 26 }, /* MEM_KERNEL_SYNC */ - { 1, 28 } /* MEM_SAME_VA */ + { 1, 22 }, /* QUERY_MEM_DONT_NEED */ + { 1, 0 }, /* QUERY_MEM_GROW_ON_GPF */ + { 1, 0 }, /* QUERY_MEM_PROTECTED */ + { 1, 26 }, /* QUERY_MEM_IMPORT_SYNC_ON_MAP_UNMAP */ + { 1, 26 }, /* QUERY_MEM_KERNEL_SYNC */ + { 1, 28 }, /* QUERY_MEM_SAME_VA */ + { 1, 31 }, /* REJECT_ALLOC_MEM_DONT_NEED */ + { 1, 31 }, /* REJECT_ALLOC_MEM_PROTECTED_IN_UNPROTECTED_ALLOCS */ + { 1, 31 }, /* REJECT_ALLOC_MEM_UNUSED_BIT_20 */ + { 1, 31 } /* REJECT_ALLOC_MEM_UNUSED_BIT_27 */ #else { 11, 15 }, /* SYSTEM_MONITOR */ { 11, 25 }, /* JIT_PRESSURE_LIMIT */ - { 11, 40 }, /* MEM_DONT_NEED */ - { 11, 2 }, /* MEM_GROW_ON_GPF */ - { 11, 2 }, /* MEM_PROTECTED */ - { 11, 43 }, /* MEM_IMPORT_SYNC_ON_MAP_UNMAP */ - { 11, 43 }, /* MEM_KERNEL_SYNC */ - { 11, 44 } /* MEM_SAME_VA */ + { 11, 40 }, /* QUERY_MEM_DONT_NEED */ + { 11, 2 }, /* QUERY_MEM_GROW_ON_GPF */ + { 11, 2 }, /* QUERY_MEM_PROTECTED */ + { 11, 43 }, /* QUERY_MEM_IMPORT_SYNC_ON_MAP_UNMAP */ + { 11, 43 }, /* QUERY_MEM_KERNEL_SYNC */ + { 11, 44 }, /* QUERY_MEM_SAME_VA */ + { 11, 46 }, /* REJECT_ALLOC_MEM_DONT_NEED */ + { 11, 46 }, /* REJECT_ALLOC_MEM_PROTECTED_IN_UNPROTECTED_ALLOCS */ + { 11, 46 }, /* REJECT_ALLOC_MEM_UNUSED_BIT_8 */ + { 11, 46 } /* REJECT_ALLOC_MEM_UNUSED_BIT_19 */ #endif }; @@ -819,7 +826,7 @@ static int kbase_api_mem_alloc_ex(struct kbase_context *kctx, union kbase_ioctl_mem_alloc_ex *alloc_ex) { struct kbase_va_region *reg; - u64 flags = alloc_ex->in.flags; + base_mem_alloc_flags flags = alloc_ex->in.flags; u64 gpu_va; /* Calls to this function are inherently asynchronous, with respect to @@ -929,7 +936,7 @@ static int kbase_api_mem_alloc(struct kbase_context *kctx, union kbase_ioctl_mem static int kbase_api_mem_alloc(struct kbase_context *kctx, union kbase_ioctl_mem_alloc *alloc) { struct kbase_va_region *reg; - u64 flags = alloc->in.flags; + base_mem_alloc_flags flags = alloc->in.flags; u64 gpu_va; /* Calls to this function are inherently asynchronous, with respect to @@ -1055,16 +1062,6 @@ static int kbase_api_get_ddk_version(struct kbase_context *kctx, static int kbase_api_mem_jit_init(struct kbase_context *kctx, struct kbase_ioctl_mem_jit_init *jit_init) { - size_t i; - - for (i = 0; i < sizeof(jit_init->padding); i++) { - /* Ensure all padding bytes are 0 for potential future - * extension - */ - if (jit_init->padding[i]) - return -EINVAL; - } - return kbase_region_tracker_init_jit(kctx, jit_init->va_pages, jit_init->max_allocations, jit_init->trim_level, jit_init->group_id, jit_init->phys_pages); @@ -1130,7 +1127,7 @@ static int kbase_api_mem_commit(struct kbase_context *kctx, struct kbase_ioctl_m static int kbase_api_mem_alias(struct kbase_context *kctx, union kbase_ioctl_mem_alias *alias) { struct base_mem_aliasing_info *ai; - u64 flags; + base_mem_alloc_flags flags; int err; if (alias->in.nents == 0 || alias->in.nents > BASE_MEM_ALIAS_MAX_ENTS) @@ -1141,7 +1138,7 @@ static int kbase_api_mem_alias(struct kbase_context *kctx, union kbase_ioctl_mem return -ENOMEM; err = copy_from_user(ai, u64_to_user_ptr(alias->in.aliasing_info), - sizeof(*ai) * alias->in.nents); + size_mul(sizeof(*ai), alias->in.nents)); if (err) { vfree(ai); return -EFAULT; @@ -1169,7 +1166,7 @@ static int kbase_api_mem_alias(struct kbase_context *kctx, union kbase_ioctl_mem static int kbase_api_mem_import(struct kbase_context *kctx, union kbase_ioctl_mem_import *import) { int ret; - u64 flags = import->in.flags; + base_mem_alloc_flags flags = import->in.flags; if (flags & BASEP_MEM_FLAGS_KERNEL_ONLY) return -ENOMEM; @@ -1281,7 +1278,8 @@ static int kbase_api_sticky_resource_map(struct kbase_context *kctx, if (!map->count || map->count > BASE_EXT_RES_COUNT_MAX) return -EOVERFLOW; - ret = copy_from_user(gpu_addr, u64_to_user_ptr(map->address), sizeof(u64) * map->count); + ret = copy_from_user(gpu_addr, u64_to_user_ptr(map->address), + size_mul(sizeof(u64), map->count)); if (ret != 0) return -EFAULT; @@ -1320,7 +1318,8 @@ static int kbase_api_sticky_resource_unmap(struct kbase_context *kctx, if (!unmap->count || unmap->count > BASE_EXT_RES_COUNT_MAX) return -EOVERFLOW; - ret = copy_from_user(gpu_addr, u64_to_user_ptr(unmap->address), sizeof(u64) * unmap->count); + ret = copy_from_user(gpu_addr, u64_to_user_ptr(unmap->address), + size_mul(sizeof(u64), unmap->count)); if (ret != 0) return -EFAULT; @@ -1402,7 +1401,6 @@ static int kbasep_cs_queue_group_create_1_6(struct kbase_context *kctx, union kbase_ioctl_cs_queue_group_create_1_6 *create) { int ret; - size_t i; union kbase_ioctl_cs_queue_group_create new_create = { .in = { .tiler_mask = create->in.tiler_mask, @@ -1415,13 +1413,7 @@ static int kbasep_cs_queue_group_create_1_6(struct kbase_context *kctx, .compute_max = create->in.compute_max, } }; - for (i = 0; i < ARRAY_SIZE(create->in.padding); i++) { - if (create->in.padding[i] != 0) - return -EINVAL; - } - ret = kbase_csf_queue_group_create(kctx, &new_create); - create->out.group_handle = new_create.out.group_handle; create->out.group_uid = new_create.out.group_uid; @@ -1432,7 +1424,6 @@ static int kbasep_cs_queue_group_create_1_18(struct kbase_context *kctx, union kbase_ioctl_cs_queue_group_create_1_18 *create) { int ret; - size_t i; union kbase_ioctl_cs_queue_group_create new_create = { .in = { .tiler_mask = create->in.tiler_mask, @@ -1447,13 +1438,7 @@ static int kbasep_cs_queue_group_create_1_18(struct kbase_context *kctx, .dvs_buf = create->in.dvs_buf, } }; - for (i = 0; i < ARRAY_SIZE(create->in.padding); i++) { - if (create->in.padding[i] != 0) - return -EINVAL; - } - ret = kbase_csf_queue_group_create(kctx, &new_create); - create->out.group_handle = new_create.out.group_handle; create->out.group_uid = new_create.out.group_uid; @@ -1580,14 +1565,15 @@ static int kbase_ioctl_cs_get_glb_iface(struct kbase_context *kctx, ¶m->out.prfcnt_size, ¶m->out.instr_features); if (copy_to_user(user_groups, group_data, - MIN(max_group_num, param->out.group_num) * sizeof(*group_data))) + size_mul(MIN(max_group_num, param->out.group_num), + sizeof(*group_data)))) err = -EFAULT; } if (!err) if (copy_to_user(user_streams, stream_data, - MIN(max_total_stream_num, param->out.total_stream_num) * - sizeof(*stream_data))) + size_mul(MIN(max_total_stream_num, param->out.total_stream_num), + sizeof(*stream_data)))) err = -EFAULT; kfree(group_data); @@ -1611,10 +1597,6 @@ static int kbase_ioctl_read_user_page(struct kbase_context *kctx, if (unlikely(user_page->in.offset != LATEST_FLUSH)) return -EINVAL; - /* Validating padding that must be zero */ - if (unlikely(user_page->in.padding != 0)) - return -EINVAL; - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); if (!kbdev->pm.backend.gpu_powered) user_page->out.val_lo = POWER_DOWN_LATEST_FLUSH_VALUE; @@ -1641,65 +1623,6 @@ kbasep_ioctl_context_priority_check(struct kbase_context *kctx, return 0; } -#define KBASE_HANDLE_IOCTL(cmd, function, arg) \ - do { \ - int ret; \ - BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_NONE); \ - dev_dbg(arg->kbdev->dev, "Enter ioctl %s\n", #function); \ - ret = function(arg); \ - dev_dbg(arg->kbdev->dev, "Return %d from ioctl %s\n", ret, #function); \ - return ret; \ - } while (0) - -#define KBASE_HANDLE_IOCTL_IN(cmd, function, type, arg) \ - do { \ - type param; \ - int ret, err; \ - dev_dbg(arg->kbdev->dev, "Enter ioctl %s\n", #function); \ - BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_WRITE); \ - BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ - err = copy_from_user(¶m, uarg, sizeof(param)); \ - if (err) \ - return -EFAULT; \ - ret = function(arg, ¶m); \ - dev_dbg(arg->kbdev->dev, "Return %d from ioctl %s\n", ret, #function); \ - return ret; \ - } while (0) - -#define KBASE_HANDLE_IOCTL_OUT(cmd, function, type, arg) \ - do { \ - type param; \ - int ret, err; \ - dev_dbg(arg->kbdev->dev, "Enter ioctl %s\n", #function); \ - BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_READ); \ - BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ - memset(¶m, 0, sizeof(param)); \ - ret = function(arg, ¶m); \ - err = copy_to_user(uarg, ¶m, sizeof(param)); \ - if (err) \ - return -EFAULT; \ - dev_dbg(arg->kbdev->dev, "Return %d from ioctl %s\n", ret, #function); \ - return ret; \ - } while (0) - -#define KBASE_HANDLE_IOCTL_INOUT(cmd, function, type, arg) \ - do { \ - type param; \ - int ret, err; \ - dev_dbg(arg->kbdev->dev, "Enter ioctl %s\n", #function); \ - BUILD_BUG_ON(_IOC_DIR(cmd) != (_IOC_WRITE | _IOC_READ)); \ - BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ - err = copy_from_user(¶m, uarg, sizeof(param)); \ - if (err) \ - return -EFAULT; \ - ret = function(arg, ¶m); \ - err = copy_to_user(uarg, ¶m, sizeof(param)); \ - if (err) \ - return -EFAULT; \ - dev_dbg(arg->kbdev->dev, "Return %d from ioctl %s\n", ret, #function); \ - return ret; \ - } while (0) - static int kbasep_ioctl_set_limited_core_count( struct kbase_context *kctx, struct kbase_ioctl_set_limited_core_count *set_limited_core_count) @@ -2418,30 +2341,41 @@ static int core_mask_parse(struct kbase_device *const kbdev, const char *const b static int core_mask_set(struct kbase_device *kbdev, struct kbase_core_mask *const new_mask) { u64 new_core_mask = new_mask->new_core_mask; - u64 shader_present = kbdev->gpu_props.shader_present; + u64 shader_present; + unsigned long flags; + int ret = 0; - lockdep_assert_held(&kbdev->pm.lock); - lockdep_assert_held(&kbdev->hwaccess_lock); + kbase_csf_scheduler_lock(kbdev); + kbase_pm_lock(kbdev); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + shader_present = kbdev->gpu_props.shader_present; if ((new_core_mask & shader_present) != new_core_mask) { dev_err(kbdev->dev, - "Invalid core mask 0x%llX: Includes non-existent cores (present = 0x%llX)", + "Invalid requested core mask 0x%llX: Includes non-existent cores (present = 0x%llX)", new_core_mask, shader_present); - return -EINVAL; - + ret = -EINVAL; + goto exit; } else if (!(new_core_mask & shader_present & kbdev->pm.backend.ca_cores_enabled)) { dev_err(kbdev->dev, - "Invalid core mask 0x%llX: No intersection with currently available cores (present = 0x%llX, CA enabled = 0x%llX)", + "Invalid requested core mask 0x%llX: No intersection with currently available cores (present = 0x%llX, CA enabled = 0x%llX)", new_core_mask, kbdev->gpu_props.shader_present, kbdev->pm.backend.ca_cores_enabled); - return -EINVAL; + ret = -EINVAL; + goto exit; } if (kbdev->pm.debug_core_mask != new_core_mask) kbase_pm_set_debug_core_mask(kbdev, new_core_mask); - return 0; +exit: + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + kbase_pm_unlock(kbdev); + kbase_csf_scheduler_unlock(kbdev); + + return ret; } #else struct kbase_core_mask { @@ -2478,15 +2412,23 @@ static int core_mask_set(struct kbase_device *kbdev, struct kbase_core_mask *con { u64 shader_present = kbdev->gpu_props.shader_present; u64 group_core_mask = kbdev->gpu_props.coherency_info.group.core_mask; - u64 *new_core_mask = &new_mask->new_core_mask[0]; + u64 *new_core_mask; + unsigned long flags; + int ret = 0; size_t i; + kbase_pm_lock(kbdev); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + new_core_mask = &new_mask->new_core_mask[0]; + for (i = 0; i < BASE_JM_MAX_NR_SLOTS; ++i) { if ((new_core_mask[i] & shader_present) != new_core_mask[i]) { dev_err(kbdev->dev, "Invalid core mask 0x%llX for JS %zu: Includes non-existent cores (present = 0x%llX)", new_core_mask[i], i, shader_present); - return -EINVAL; + ret = -EINVAL; + goto exit; } else if (!(new_core_mask[i] & shader_present & kbdev->pm.backend.ca_cores_enabled)) { @@ -2494,17 +2436,20 @@ static int core_mask_set(struct kbase_device *kbdev, struct kbase_core_mask *con "Invalid core mask 0x%llX for JS %zu: No intersection with currently available cores (present = 0x%llX, CA enabled = 0x%llX)", new_core_mask[i], i, kbdev->gpu_props.shader_present, kbdev->pm.backend.ca_cores_enabled); - return -EINVAL; + ret = -EINVAL; + goto exit; } else if (!(new_core_mask[i] & group_core_mask)) { dev_err(kbdev->dev, "Invalid core mask 0x%llX for JS %zu: No intersection with group 0 core mask 0x%llX", new_core_mask[i], i, group_core_mask); - return -EINVAL; + ret = -EINVAL; + goto exit; } else if (!(new_core_mask[i] & kbdev->gpu_props.curr_config.shader_present)) { dev_err(kbdev->dev, "Invalid core mask 0x%llX for JS %zu: No intersection with current core mask 0x%llX", new_core_mask[i], i, kbdev->gpu_props.curr_config.shader_present); - return -EINVAL; + ret = -EINVAL; + goto exit; } } @@ -2515,7 +2460,11 @@ static int core_mask_set(struct kbase_device *kbdev, struct kbase_core_mask *con } } - return 0; +exit: + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + kbase_pm_unlock(kbdev); + + return ret; } #endif @@ -2539,7 +2488,6 @@ static ssize_t core_mask_store(struct device *dev, struct device_attribute *attr struct kbase_core_mask core_mask = {}; int err; - unsigned long flags; CSTD_UNUSED(attr); @@ -2552,14 +2500,8 @@ static ssize_t core_mask_store(struct device *dev, struct device_attribute *attr if (err) return err; - mutex_lock(&kbdev->pm.lock); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - err = core_mask_set(kbdev, &core_mask); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kbdev->pm.lock); - if (err) return err; @@ -4385,7 +4327,8 @@ void registers_unmap(struct kbase_device *kbdev) kbase_common_reg_unmap(kbdev); } -#if defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF) +#if defined(CONFIG_OF) + static bool kbase_is_pm_enabled(const struct device_node *gpu_node) { const struct device_node *power_model_node; @@ -4424,13 +4367,13 @@ static bool kbase_is_full_coherency_enabled(const struct device_node *gpu_node) } return false; } -#endif /* defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF) */ +#endif /* defined(CONFIG_OF) */ int kbase_device_backend_init(struct kbase_device *kbdev) { int err = 0; -#if defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF) +#if defined(CONFIG_OF) /* * Attempt to initialize arbitration. * If the platform is not suitable for arbitration, return -EPERM. @@ -4439,13 +4382,13 @@ int kbase_device_backend_init(struct kbase_device *kbdev) */ if (kbase_is_pm_enabled(kbdev->dev->of_node)) { /* Arbitration AND power management invalid */ - dev_err(kbdev->dev, "Invalid combination of arbitration AND power management\n"); + dev_dbg(kbdev->dev, "Arbitration not supported with power management"); return -EPERM; } if (kbase_is_full_coherency_enabled(kbdev->dev->of_node)) { /* Arbitration AND full coherency invalid */ - dev_err(kbdev->dev, "Invalid combination of arbitration AND full coherency\n"); + dev_dbg(kbdev->dev, "Arbitration not supported with full coherency"); return -EPERM; } @@ -4466,21 +4409,19 @@ int kbase_device_backend_init(struct kbase_device *kbdev) if (product_model != GPU_ID_PRODUCT_TGOX && product_model != GPU_ID_PRODUCT_TNOX && product_model != GPU_ID_PRODUCT_TBAX) { kbase_arbiter_pm_early_term(kbdev); - dev_err(kbdev->dev, "GPU platform not suitable for arbitration\n"); + dev_dbg(kbdev->dev, "GPU platform not suitable for arbitration"); return -EPERM; } #endif /* !MALI_USE_CSF */ - dev_info(kbdev->dev, "Arbitration interface enabled\n"); + dev_info(kbdev->dev, "Arbitration interface enabled"); } -#endif /* defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF) */ +#endif /* defined(CONFIG_OF) */ return err; } void kbase_device_backend_term(struct kbase_device *kbdev) { -#ifdef CONFIG_MALI_ARBITER_SUPPORT kbase_arbiter_pm_early_term(kbdev); -#endif } int power_control_init(struct kbase_device *kbdev) @@ -4898,7 +4839,7 @@ static struct dentry *init_debugfs(struct kbase_device *kbdev) if (IS_ERR_OR_NULL(dentry)) return dentry; - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE)) { + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE)) { dentry = debugfs_create_file("protected_debug_mode", 0444, kbdev->mali_debugfs_directory, kbdev, &fops_protected_debug_mode); @@ -5785,11 +5726,11 @@ static int kbase_platform_device_probe(struct platform_device *pdev) #if (KERNEL_VERSION(5, 3, 0) <= LINUX_VERSION_CODE) mutex_unlock(&kbase_probe_mutex); #endif -#ifdef CONFIG_MALI_ARBITER_SUPPORT - mutex_lock(&kbdev->pm.lock); - kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_INITIALIZED_EVT); - mutex_unlock(&kbdev->pm.lock); -#endif + if (kbase_has_arbiter(kbdev)) { + mutex_lock(&kbdev->pm.lock); + kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_INITIALIZED_EVT); + mutex_unlock(&kbdev->pm.lock); + } } return err; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_cs_experimental.h b/drivers/gpu/arm/bifrost/mali_kbase_cs_experimental.h index 0c794e2e90bc..e6222979b72c 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_cs_experimental.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_cs_experimental.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -30,9 +30,6 @@ */ static inline void mali_kbase_print_cs_experimental(void) { -#if MALI_INCREMENTAL_RENDERING_JM - pr_info("mali_kbase: INCREMENTAL_RENDERING_JM (experimental) enabled"); -#endif /* MALI_INCREMENTAL_RENDERING_JM */ } #endif /* _KBASE_CS_EXPERIMENTAL_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_allocs.c b/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_allocs.c index c92fb9e0957e..4b7f6a186ac0 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_allocs.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_allocs.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2022-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2022-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -58,7 +58,7 @@ static void debug_zone_mem_allocs_show(struct kbase_reg_zone *zone, struct seq_f for (p = rb_first(rbtree); p; p = rb_next(p)) { reg = rb_entry(p, struct kbase_va_region, rblink); if (!(reg->flags & KBASE_REG_FREE)) { - seq_printf(sfile, "%16llx, %16zx, %16zx, %8lx, %s\n", + seq_printf(sfile, "%16llx, %16zx, %16zx, %8llx, %s\n", reg->start_pfn << PAGE_SHIFT, reg->nr_pages << PAGE_SHIFT, kbase_reg_current_backed_size(reg) << PAGE_SHIFT, reg->flags, type_names[reg->gpu_alloc->type]); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_defs.h b/drivers/gpu/arm/bifrost/mali_kbase_defs.h index 2335e0b8e449..b97df15f7a17 100755 --- a/drivers/gpu/arm/bifrost/mali_kbase_defs.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_defs.h @@ -28,8 +28,8 @@ #define _KBASE_DEFS_H_ #include -#include -#include +#include +#include #include #include #include @@ -52,10 +52,6 @@ #include -#if IS_ENABLED(CONFIG_DEBUG_FS) -#include -#endif /* CONFIG_DEBUG_FS */ - #ifdef CONFIG_MALI_BIFROST_DEVFREQ #include #endif /* CONFIG_MALI_BIFROST_DEVFREQ */ @@ -64,9 +60,7 @@ #include #endif -#ifdef CONFIG_MALI_ARBITER_SUPPORT #include -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ #include #include @@ -78,6 +72,7 @@ #include #include #include +#include #include /** Number of milliseconds before we time out on a GPU soft/hard reset */ @@ -297,24 +292,33 @@ struct kbase_fault { #define MAX_PAGES_FOR_FREE_PGDS ((size_t)9) /* Maximum number of pointers to free PGDs */ -#define MAX_FREE_PGDS ((PAGE_SIZE / sizeof(struct page *)) * MAX_PAGES_FOR_FREE_PGDS) +#define MAX_FREE_PGDS ((PAGE_SIZE / sizeof(phys_addr_t)) * MAX_PAGES_FOR_FREE_PGDS) /** * struct kbase_mmu_table - object representing a set of GPU page tables - * @mmu_lock: Lock to serialize the accesses made to multi level GPU - * page tables - * @pgd: Physical address of the page allocated for the top - * level page table of the context, this is used for - * MMU HW programming as the address translation will - * start from the top level page table. - * @group_id: A memory group ID to be passed to a platform-specific - * memory group manager. - * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). - * @kctx: If this set of MMU tables belongs to a context then - * this is a back-reference to the context, otherwise - * it is NULL. - * @scratch_mem: Scratch memory used for MMU operations, which are - * serialized by the @mmu_lock. + * @mmu_lock: Lock to serialize the accesses made to multi level GPU + * page tables + * @pgd: Physical address of the page allocated for the top + * level page table of the context, this is used for + * MMU HW programming as the address translation will + * start from the top level page table. + * @group_id: A memory group ID to be passed to a platform-specific + * memory group manager. + * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). + * @kctx: If this set of MMU tables belongs to a context then + * this is a back-reference to the context, otherwise + * it is NULL. + * @scratch_mem: Scratch memory used for MMU operations, which are + * serialized by the @mmu_lock. + * @pgd_pages_list: List head to link all 16K/64K pages allocated for the PGDs of mmut. + * These pages will be used to allocate 4KB PGD pages for + * the GPU page table. + * Linked with &kbase_page_metadata.data.pt_mapped.pgd_link. + * @last_allocated_pgd_page: Pointer to PGD page from where the last sub page + * was allocated for mmut. + * @last_freed_pgd_page: Pointer to PGD page to which the last freed 4K sub page + * was returned for mmut. + * @num_free_pgd_sub_pages: The total number of free 4K PGD pages in the mmut. */ struct kbase_mmu_table { struct mutex mmu_lock; @@ -332,7 +336,7 @@ struct kbase_mmu_table { * @levels: Array of PGD pages, large enough to copy one PGD * for each level of the MMU table. */ - u64 levels[MIDGARD_MMU_BOTTOMLEVEL][PAGE_SIZE / sizeof(u64)]; + u64 levels[MIDGARD_MMU_BOTTOMLEVEL][GPU_PAGE_SIZE / sizeof(u64)]; } teardown_pages; /** * @free_pgds: Scratch memory used for insertion, update and teardown @@ -341,11 +345,18 @@ struct kbase_mmu_table { */ struct { /** @pgds: Array of pointers to PGDs to free. */ - struct page *pgds[MAX_FREE_PGDS]; + phys_addr_t pgds[MAX_FREE_PGDS]; /** @head_index: Index of first free element in the PGDs array. */ size_t head_index; } free_pgds; } scratch_mem; + +#if GPU_PAGES_PER_CPU_PAGE > 1 + struct list_head pgd_pages_list; + struct page *last_allocated_pgd_page; + struct page *last_freed_pgd_page; + u32 num_free_pgd_sub_pages; +#endif }; #if MALI_USE_CSF @@ -371,14 +382,9 @@ static inline int kbase_as_has_page_fault(struct kbase_as *as, struct kbase_faul * * @used_pages: Tracks usage of OS shared memory. Updated when OS memory is * allocated/freed. - * @ir_threshold: Fraction of the maximum size of an allocation that grows - * on GPU page fault that can be used before the driver - * switches to incremental rendering, in 1/256ths. - * 0 means disabled. */ struct kbasep_mem_device { atomic_t used_pages; - atomic_t ir_threshold; }; struct kbase_clk_rate_listener; @@ -483,9 +489,7 @@ struct kbase_pm_device_data { #if MALI_USE_CSF bool runtime_active; #endif -#ifdef CONFIG_MALI_ARBITER_SUPPORT atomic_t gpu_lost; -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ wait_queue_head_t zero_active_count_wait; wait_queue_head_t resume_wait; @@ -501,10 +505,8 @@ struct kbase_pm_device_data { void (*callback_power_runtime_term)(struct kbase_device *kbdev); u32 dvfs_period; struct kbase_pm_backend_data backend; -#ifdef CONFIG_MALI_ARBITER_SUPPORT struct kbase_arbiter_vm_state *arb_vm_state; atomic_t gpu_users_waiting; -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ struct kbase_clk_rate_trace_manager clk_rtm; }; @@ -947,7 +949,7 @@ struct kbase_mem_migrate { * @ipa.last_sample_time: Records the time when counters, used for dynamic * energy estimation, were last sampled. * @previous_frequency: Previous frequency of GPU clock used for - * BASE_HW_ISSUE_GPU2017_1336 workaround, This clock is + * KBASE_HW_ISSUE_GPU2017_1336 workaround, This clock is * restored when L2 is powered on. * @job_fault_debug: Flag to control the dumping of debug data for job faults, * set when the 'job_fault' debugfs file is opened. @@ -1069,7 +1071,8 @@ struct kbase_mem_migrate { * KCPU queue. These structures may outlive kbase module * itself. Therefore, in such a case, a warning should be * be produced. - * @va_region_slab: kmem_cache (slab) for allocated kbase_va_region structures. + * @va_region_slab: kmem_cache (slab) for allocated @kbase_va_region structures. + * @page_metadata_slab: kmem_cache (slab) for allocated @kbase_page_metadata structures. * @fence_signal_timeout_enabled: Global flag for whether fence signal timeout tracking * is enabled. * @pcm_prioritized_process_nb: Notifier block for the Priority Control Manager @@ -1134,8 +1137,8 @@ struct kbase_device { struct kbase_gpu_props gpu_props; - unsigned long hw_issues_mask[(BASE_HW_ISSUE_END + BITS_PER_LONG - 1) / BITS_PER_LONG]; - unsigned long hw_features_mask[(BASE_HW_FEATURE_END + BITS_PER_LONG - 1) / BITS_PER_LONG]; + unsigned long hw_issues_mask[(KBASE_HW_ISSUE_END + BITS_PER_LONG - 1) / BITS_PER_LONG]; + unsigned long hw_features_mask[(KBASE_HW_FEATURE_END + BITS_PER_LONG - 1) / BITS_PER_LONG]; struct { atomic_t count; @@ -1151,6 +1154,12 @@ struct kbase_device { */ u8 pbha_propagate_bits; + /** + * @mma_wa_id: The PBHA ID to use for the PBHA OVERRIDE based workaround for MMA violation. + * + */ + u32 mma_wa_id; + #if MALI_USE_CSF struct kbase_hwcnt_backend_csf_if hwcnt_backend_csf_if_fw; #else @@ -1242,7 +1251,6 @@ struct kbase_device { atomic_t job_fault_debug; #endif /* !MALI_USE_CSF */ -#if IS_ENABLED(CONFIG_DEBUG_FS) struct dentry *mali_debugfs_directory; struct dentry *debugfs_ctx_directory; struct dentry *debugfs_instr_directory; @@ -1264,7 +1272,6 @@ struct kbase_device { u32 reg_offset; } regs_dump_debugfs_data; #endif /* !MALI_CUSTOMER_RELEASE */ -#endif /* CONFIG_DEBUG_FS */ atomic_t ctx_num; @@ -1355,9 +1362,7 @@ struct kbase_device { } dummy_job_wa; bool dummy_job_wa_loaded; -#ifdef CONFIG_MALI_ARBITER_SUPPORT struct kbase_arbiter_device arb; -#endif /* Priority Control Manager device */ struct priority_control_manager_device *pcm_dev; @@ -1382,6 +1387,9 @@ struct kbase_device { atomic_t live_fence_metadata; #endif struct kmem_cache *va_region_slab; +#if GPU_PAGES_PER_CPU_PAGE > 1 + struct kmem_cache *page_metadata_slab; +#endif #if IS_ENABLED(CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD) /** @@ -2131,6 +2139,18 @@ static inline u64 kbase_get_lock_region_min_size_log2(struct kbase_gpu_props con return 15; /* 32 kB */ } +/** + * kbase_has_arbiter - Check whether GPU has an arbiter. + * + * @kbdev: KBase device. + * + * Return: True if there is an arbiter, False otherwise. + */ +static inline bool kbase_has_arbiter(struct kbase_device *kbdev) +{ + return (bool)kbdev->arb.arb_if; +} + /* Conversion helpers for setting up high resolution timers */ #define HR_TIMER_DELAY_MSEC(x) (ns_to_ktime(((u64)(x)) * 1000000U)) #define HR_TIMER_DELAY_NSEC(x) (ns_to_ktime(x)) diff --git a/drivers/gpu/arm/bifrost/mali_kbase_dummy_job_wa.c b/drivers/gpu/arm/bifrost/mali_kbase_dummy_job_wa.c index 9c39f0e20f76..7b578c81af60 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_dummy_job_wa.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_dummy_job_wa.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -218,7 +218,7 @@ static bool wa_blob_load_needed(struct kbase_device *kbdev) if (of_machine_is_compatible("arm,juno")) return false; - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_3485)) + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TTRX_3485)) return true; return false; @@ -311,7 +311,7 @@ int kbase_dummy_job_wa_load(struct kbase_device *kbdev) while (blob_offset) { const struct wa_blob *blob; size_t nr_pages; - u64 flags; + base_mem_alloc_flags flags; u64 gpu_va; struct kbase_va_region *va_region; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gpu_metrics.c b/drivers/gpu/arm/bifrost/mali_kbase_gpu_metrics.c index 8a2e13f03683..60ad1c272f84 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_gpu_metrics.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_gpu_metrics.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -39,9 +39,6 @@ static inline void validate_tracepoint_data(struct kbase_gpu_metrics_ctx *gpu_me u64 start_time, u64 end_time, u64 total_active) { #if 0 - WARN(total_active > NSEC_PER_SEC, "total_active %llu > 1 second for aid %u active_cnt %u", - total_active, gpu_metrics_ctx->aid, gpu_metrics_ctx->active_cnt); - WARN(start_time >= end_time, "start_time %llu >= end_time %llu for aid %u active_cnt %u", start_time, end_time, gpu_metrics_ctx->aid, gpu_metrics_ctx->active_cnt); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.c b/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.c index 25ee8c1042a2..9719580837cc 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2011-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -437,7 +437,7 @@ int kbase_gpuprops_update_l2_features(struct kbase_device *kbdev) { int err = 0; - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_L2_CONFIG)) { + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_L2_CONFIG)) { struct kbasep_gpuprops_regdump *regdump = &PRIV_DATA_REGDUMP(kbdev); /* Check for L2 cache size & hash overrides */ @@ -699,7 +699,7 @@ static void kbase_populate_user_data(struct kbase_device *kbdev, struct gpu_prop data->thread_props.max_thread_group_split = THREAD_MTGS_DEFAULT; } - if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_THREAD_GROUP_SPLIT)) + if (!kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_THREAD_GROUP_SPLIT)) data->thread_props.max_thread_group_split = 0; /* Raw Register Values */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gwt.c b/drivers/gpu/arm/bifrost/mali_kbase_gwt.c index 5e59bf60aa38..99558b82ba7b 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_gwt.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_gwt.c @@ -178,6 +178,10 @@ int kbase_gpu_gwt_dump(struct kbase_context *kctx, union kbase_ioctl_cinstr_gwt_ __user void *user_addr = (__user void *)(uintptr_t)gwt_dump->in.addr_buffer; __user void *user_sizes = (__user void *)(uintptr_t)gwt_dump->in.size_buffer; + /* We don't have any valid user space buffer to copy the write modified addresses. */ + if (!gwt_dump->in.len || !gwt_dump->in.addr_buffer || !gwt_dump->in.size_buffer) + return -EINVAL; + kbase_gpu_vm_lock(kctx); if (!kctx->gwt_enabled) { @@ -186,14 +190,6 @@ int kbase_gpu_gwt_dump(struct kbase_context *kctx, union kbase_ioctl_cinstr_gwt_ return -EPERM; } - if (!gwt_dump->in.len || !gwt_dump->in.addr_buffer || !gwt_dump->in.size_buffer) { - kbase_gpu_vm_unlock(kctx); - /* We don't have any valid user space buffer to copy the - * write modified addresses. - */ - return -EINVAL; - } - if (list_empty(&kctx->gwt_snapshot_list) && !list_empty(&kctx->gwt_current_list)) { list_replace_init(&kctx->gwt_current_list, &kctx->gwt_snapshot_list); @@ -227,14 +223,14 @@ int kbase_gpu_gwt_dump(struct kbase_context *kctx, union kbase_ioctl_cinstr_gwt_ if (count) { err = copy_to_user((user_addr + (ubuf_count * sizeof(u64))), - (void *)addr_buffer, count * sizeof(u64)); + (void *)addr_buffer, size_mul(count, sizeof(u64))); if (err) { dev_err(kctx->kbdev->dev, "Copy to user failure\n"); kbase_gpu_vm_unlock(kctx); return err; } err = copy_to_user((user_sizes + (ubuf_count * sizeof(u64))), - (void *)num_page_buffer, count * sizeof(u64)); + (void *)num_page_buffer, size_mul(count, sizeof(u64))); if (err) { dev_err(kctx->kbdev->dev, "Copy to user failure\n"); kbase_gpu_vm_unlock(kctx); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hw.c b/drivers/gpu/arm/bifrost/mali_kbase_hw.c index 1fde75b996c4..e04aad2422c7 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_hw.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_hw.c @@ -23,8 +23,8 @@ * Run-time work-arounds helpers */ -#include -#include +#include +#include #include #include "mali_kbase.h" #include "mali_kbase_hw.h" @@ -92,7 +92,7 @@ void kbase_hw_set_features_mask(struct kbase_device *kbdev) break; } - for (; *features != BASE_HW_FEATURE_END; features++) + for (; *features != KBASE_HW_FEATURE_END; features++) set_bit(*features, &kbdev->hw_features_mask[0]); #if defined(CONFIG_MALI_VECTOR_DUMP) @@ -103,8 +103,8 @@ void kbase_hw_set_features_mask(struct kbase_device *kbdev) * in the implementation of flush reduction optimization due to * unclear or ambiguous ARCH spec. */ - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_CLEAN_ONLY_SAFE)) - clear_bit(BASE_HW_FEATURE_FLUSH_REDUCTION, &kbdev->hw_features_mask[0]); + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_CLEAN_ONLY_SAFE)) + clear_bit(KBASE_HW_FEATURE_FLUSH_REDUCTION, &kbdev->hw_features_mask[0]); #endif } @@ -113,7 +113,7 @@ void kbase_hw_set_features_mask(struct kbase_device *kbdev) * @kbdev: Device pointer * * Return: pointer to an array of hardware issues, terminated by - * BASE_HW_ISSUE_END. + * KBASE_HW_ISSUE_END. * * In debugging versions of the driver, unknown versions of a known GPU will * be treated as the most recent known version not later than the actual @@ -424,7 +424,7 @@ int kbase_hw_set_issues_mask(struct kbase_device *kbdev) gpu_id->product_major, gpu_id->arch_major, gpu_id->arch_minor, gpu_id->arch_rev, gpu_id->version_major, gpu_id->version_minor, gpu_id->version_status); - for (; *issues != BASE_HW_ISSUE_END; issues++) + for (; *issues != KBASE_HW_ISSUE_END; issues++) set_bit(*issues, &kbdev->hw_issues_mask[0]); return 0; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hw.h b/drivers/gpu/arm/bifrost/mali_kbase_hw.h index 44e1ee4a4a50..f14e5fb6d9ab 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_hw.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_hw.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2012-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2012-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -23,8 +23,8 @@ * DOC: Run-time work-arounds helpers */ -#ifndef _KBASE_HW_H_ -#define _KBASE_HW_H_ +#ifndef _MALI_KBASE_HW_H_ +#define _MALI_KBASE_HW_H_ #include "mali_kbase_defs.h" @@ -47,7 +47,7 @@ * @kbdev: Device pointer */ #define kbase_hw_has_l2_slice_hash_feature(kbdev) \ - test_bit(BASE_HW_FEATURE_L2_SLICE_HASH, &(kbdev)->hw_features_mask[0]) + test_bit(KBASE_HW_FEATURE_L2_SLICE_HASH, &(kbdev)->hw_features_mask[0]) /** * kbase_hw_set_issues_mask - Set the hardware issues mask based on the GPU ID @@ -73,4 +73,4 @@ int kbase_hw_set_issues_mask(struct kbase_device *kbdev); */ void kbase_hw_set_features_mask(struct kbase_device *kbdev); -#endif /* _KBASE_HW_H_ */ +#endif /* _MALI_KBASE_HW_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwconfig_features.h b/drivers/gpu/arm/bifrost/mali_kbase_hwconfig_features.h new file mode 100644 index 000000000000..265cb9585cc6 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwconfig_features.h @@ -0,0 +1,158 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +#ifndef _KBASE_HWCONFIG_FEATURES_H_ +#define _KBASE_HWCONFIG_FEATURES_H_ + +#include + +enum base_hw_feature { + KBASE_HW_FEATURE_FLUSH_REDUCTION, + KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + KBASE_HW_FEATURE_TLS_HASHING, + KBASE_HW_FEATURE_THREAD_GROUP_SPLIT, + KBASE_HW_FEATURE_CLEAN_ONLY_SAFE, + KBASE_HW_FEATURE_IDVS_GROUP_SIZE, + KBASE_HW_FEATURE_L2_CONFIG, + KBASE_HW_FEATURE_L2_SLICE_HASH, + KBASE_HW_FEATURE_GPU_SLEEP, + KBASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER, + KBASE_HW_FEATURE_CORE_FEATURES, + KBASE_HW_FEATURE_PBHA_HWU, + KBASE_HW_FEATURE_LARGE_PAGE_ALLOC, + KBASE_HW_FEATURE_THREAD_TLS_ALLOC, + KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_generic[] = { + KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tMIx[] = { + KBASE_HW_FEATURE_THREAD_GROUP_SPLIT, KBASE_HW_FEATURE_FLUSH_REDUCTION, KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tHEx[] = { + KBASE_HW_FEATURE_THREAD_GROUP_SPLIT, KBASE_HW_FEATURE_FLUSH_REDUCTION, + KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tSIx[] = { + KBASE_HW_FEATURE_THREAD_GROUP_SPLIT, KBASE_HW_FEATURE_FLUSH_REDUCTION, + KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tDVx[] = { + KBASE_HW_FEATURE_THREAD_GROUP_SPLIT, KBASE_HW_FEATURE_FLUSH_REDUCTION, + KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tNOx[] = { + KBASE_HW_FEATURE_THREAD_GROUP_SPLIT, KBASE_HW_FEATURE_FLUSH_REDUCTION, + KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, KBASE_HW_FEATURE_TLS_HASHING, + KBASE_HW_FEATURE_IDVS_GROUP_SIZE, KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tGOx[] = { + KBASE_HW_FEATURE_THREAD_GROUP_SPLIT, KBASE_HW_FEATURE_FLUSH_REDUCTION, + KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, KBASE_HW_FEATURE_TLS_HASHING, + KBASE_HW_FEATURE_IDVS_GROUP_SIZE, KBASE_HW_FEATURE_CORE_FEATURES, + KBASE_HW_FEATURE_THREAD_TLS_ALLOC, KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tTRx[] = { + KBASE_HW_FEATURE_FLUSH_REDUCTION, KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + KBASE_HW_FEATURE_IDVS_GROUP_SIZE, KBASE_HW_FEATURE_CLEAN_ONLY_SAFE, + KBASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER, KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tNAx[] = { + KBASE_HW_FEATURE_FLUSH_REDUCTION, KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + KBASE_HW_FEATURE_IDVS_GROUP_SIZE, KBASE_HW_FEATURE_CLEAN_ONLY_SAFE, + KBASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER, KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tBEx[] = { + KBASE_HW_FEATURE_FLUSH_REDUCTION, + KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + KBASE_HW_FEATURE_IDVS_GROUP_SIZE, + KBASE_HW_FEATURE_L2_CONFIG, + KBASE_HW_FEATURE_CLEAN_ONLY_SAFE, + KBASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER, + KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tBAx[] = { + KBASE_HW_FEATURE_FLUSH_REDUCTION, + KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + KBASE_HW_FEATURE_IDVS_GROUP_SIZE, + KBASE_HW_FEATURE_L2_CONFIG, + KBASE_HW_FEATURE_CLEAN_ONLY_SAFE, + KBASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER, + KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tODx[] = { + KBASE_HW_FEATURE_FLUSH_REDUCTION, KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + KBASE_HW_FEATURE_L2_CONFIG, KBASE_HW_FEATURE_CLEAN_ONLY_SAFE, KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tGRx[] = { + KBASE_HW_FEATURE_FLUSH_REDUCTION, KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + KBASE_HW_FEATURE_L2_CONFIG, KBASE_HW_FEATURE_CLEAN_ONLY_SAFE, + KBASE_HW_FEATURE_CORE_FEATURES, KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tVAx[] = { + KBASE_HW_FEATURE_FLUSH_REDUCTION, KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + KBASE_HW_FEATURE_L2_CONFIG, KBASE_HW_FEATURE_CLEAN_ONLY_SAFE, + KBASE_HW_FEATURE_CORE_FEATURES, KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tTUx[] = { + KBASE_HW_FEATURE_FLUSH_REDUCTION, KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + KBASE_HW_FEATURE_L2_CONFIG, KBASE_HW_FEATURE_CLEAN_ONLY_SAFE, + KBASE_HW_FEATURE_L2_SLICE_HASH, KBASE_HW_FEATURE_GPU_SLEEP, + KBASE_HW_FEATURE_CORE_FEATURES, KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tTIx[] = { + KBASE_HW_FEATURE_FLUSH_REDUCTION, + KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + KBASE_HW_FEATURE_L2_CONFIG, + KBASE_HW_FEATURE_CLEAN_ONLY_SAFE, + KBASE_HW_FEATURE_L2_SLICE_HASH, + KBASE_HW_FEATURE_GPU_SLEEP, + KBASE_HW_FEATURE_CORE_FEATURES, + KBASE_HW_FEATURE_PBHA_HWU, + KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tKRx[] = { + KBASE_HW_FEATURE_FLUSH_REDUCTION, KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + KBASE_HW_FEATURE_L2_CONFIG, KBASE_HW_FEATURE_CLEAN_ONLY_SAFE, + KBASE_HW_FEATURE_L2_SLICE_HASH, KBASE_HW_FEATURE_GPU_SLEEP, + KBASE_HW_FEATURE_CORE_FEATURES, KBASE_HW_FEATURE_PBHA_HWU, + KBASE_HW_FEATURE_LARGE_PAGE_ALLOC, KBASE_HW_FEATURE_END +}; + + +#endif /* _KBASE_HWCONFIG_FEATURES_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwconfig_issues.h b/drivers/gpu/arm/bifrost/mali_kbase_hwconfig_issues.h new file mode 100644 index 000000000000..b1a3a41b232b --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwconfig_issues.h @@ -0,0 +1,609 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +#ifndef _KBASE_HWCONFIG_ISSUES_H_ +#define _KBASE_HWCONFIG_ISSUES_H_ + +#include + +enum base_hw_issue { + KBASE_HW_ISSUE_5736, + KBASE_HW_ISSUE_9435, + KBASE_HW_ISSUE_10682, + KBASE_HW_ISSUE_11054, + KBASE_HW_ISSUE_T76X_3953, + KBASE_HW_ISSUE_TMIX_7891, + KBASE_HW_ISSUE_TMIX_7940, + KBASE_HW_ISSUE_TMIX_8042, + KBASE_HW_ISSUE_TMIX_8133, + KBASE_HW_ISSUE_TMIX_8138, + KBASE_HW_ISSUE_TMIX_8206, + KBASE_HW_ISSUE_TMIX_8343, + KBASE_HW_ISSUE_TMIX_8463, + KBASE_HW_ISSUE_TMIX_8456, + KBASE_HW_ISSUE_TSIX_1116, + KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TMIX_8438, + KBASE_HW_ISSUE_TNOX_1194, + KBASE_HW_ISSUE_TGOX_R1_1234, + KBASE_HW_ISSUE_TTRX_1337, + KBASE_HW_ISSUE_TSIX_1792, + KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_3076, + KBASE_HW_ISSUE_TTRX_921, + KBASE_HW_ISSUE_TTRX_3414, + KBASE_HW_ISSUE_GPU2017_1336, + KBASE_HW_ISSUE_TTRX_3083, + KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, + KBASE_HW_ISSUE_TTRX_3485, + KBASE_HW_ISSUE_GPU2019_3212, + KBASE_HW_ISSUE_TURSEHW_1997, + KBASE_HW_ISSUE_GPU2019_3878, + KBASE_HW_ISSUE_TURSEHW_2716, + KBASE_HW_ISSUE_GPU2019_3901, + KBASE_HW_ISSUE_GPU2021PRO_290, + KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_TITANHW_2679, + KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2922, + KBASE_HW_ISSUE_TITANHW_2952, + KBASE_HW_ISSUE_KRAKEHW_2151, + KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_KRAKEHW_2269, + KBASE_HW_ISSUE_TURSEHW_2934, + KBASE_HW_ISSUE_KRAKEHW_2321, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_generic[] = { KBASE_HW_ISSUE_END }; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tMIx_r0p0_05dev0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_10682, KBASE_HW_ISSUE_11054, + KBASE_HW_ISSUE_T76X_3953, KBASE_HW_ISSUE_TMIX_7891, KBASE_HW_ISSUE_TMIX_8042, + KBASE_HW_ISSUE_TMIX_8133, KBASE_HW_ISSUE_TMIX_8138, KBASE_HW_ISSUE_TMIX_8206, + KBASE_HW_ISSUE_TMIX_8343, KBASE_HW_ISSUE_TMIX_8463, KBASE_HW_ISSUE_TMIX_8456, + KBASE_HW_ISSUE_TMIX_8438, KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_921, + KBASE_HW_ISSUE_GPU2017_1336, KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tMIx_r0p0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_10682, KBASE_HW_ISSUE_11054, + KBASE_HW_ISSUE_TMIX_7891, KBASE_HW_ISSUE_TMIX_7940, KBASE_HW_ISSUE_TMIX_8042, + KBASE_HW_ISSUE_TMIX_8133, KBASE_HW_ISSUE_TMIX_8138, KBASE_HW_ISSUE_TMIX_8206, + KBASE_HW_ISSUE_TMIX_8343, KBASE_HW_ISSUE_TMIX_8463, KBASE_HW_ISSUE_TMIX_8456, + KBASE_HW_ISSUE_TMIX_8438, KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_921, + KBASE_HW_ISSUE_GPU2017_1336, KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tMIx_r0p1[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_10682, KBASE_HW_ISSUE_11054, + KBASE_HW_ISSUE_TMIX_7891, KBASE_HW_ISSUE_TMIX_7940, KBASE_HW_ISSUE_TMIX_8042, + KBASE_HW_ISSUE_TMIX_8133, KBASE_HW_ISSUE_TMIX_8138, KBASE_HW_ISSUE_TMIX_8206, + KBASE_HW_ISSUE_TMIX_8343, KBASE_HW_ISSUE_TMIX_8463, KBASE_HW_ISSUE_TMIX_8456, + KBASE_HW_ISSUE_TMIX_8438, KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_921, + KBASE_HW_ISSUE_GPU2017_1336, KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tMIx[] = { + KBASE_HW_ISSUE_5736, KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TMIX_7891, + KBASE_HW_ISSUE_TMIX_7940, KBASE_HW_ISSUE_TMIX_8042, KBASE_HW_ISSUE_TMIX_8133, + KBASE_HW_ISSUE_TMIX_8138, KBASE_HW_ISSUE_TMIX_8206, KBASE_HW_ISSUE_TMIX_8343, + KBASE_HW_ISSUE_TMIX_8456, KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tHEx_r0p0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_10682, KBASE_HW_ISSUE_11054, + KBASE_HW_ISSUE_TMIX_7891, KBASE_HW_ISSUE_TMIX_8042, KBASE_HW_ISSUE_TMIX_8133, + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_GPU2017_1336, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tHEx_r0p1[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_10682, KBASE_HW_ISSUE_11054, + KBASE_HW_ISSUE_TMIX_7891, KBASE_HW_ISSUE_TMIX_8042, KBASE_HW_ISSUE_TMIX_8133, + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_GPU2017_1336, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tHEx_r0p2[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_10682, KBASE_HW_ISSUE_11054, + KBASE_HW_ISSUE_TMIX_7891, KBASE_HW_ISSUE_TMIX_8042, KBASE_HW_ISSUE_TMIX_8133, + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_GPU2017_1336, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tHEx_r0p3[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_10682, KBASE_HW_ISSUE_TMIX_7891, + KBASE_HW_ISSUE_TMIX_8042, KBASE_HW_ISSUE_TMIX_8133, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_GPU2017_1336, KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tHEx[] = { + KBASE_HW_ISSUE_5736, KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TMIX_7891, + KBASE_HW_ISSUE_TMIX_8042, KBASE_HW_ISSUE_TMIX_8133, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tSIx_r0p0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_11054, KBASE_HW_ISSUE_TMIX_8133, + KBASE_HW_ISSUE_TSIX_1116, KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TSIX_1792, + KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_GPU2017_1336, KBASE_HW_ISSUE_TTRX_3464, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tSIx_r0p1[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_11054, KBASE_HW_ISSUE_TMIX_8133, + KBASE_HW_ISSUE_TSIX_1116, KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TSIX_1792, + KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_GPU2017_1336, KBASE_HW_ISSUE_TTRX_3464, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tSIx_r1p0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_11054, KBASE_HW_ISSUE_TMIX_8133, + KBASE_HW_ISSUE_TSIX_1116, KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_921, + KBASE_HW_ISSUE_GPU2017_1336, KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tSIx_r1p1[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TMIX_8133, KBASE_HW_ISSUE_TSIX_1116, + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_GPU2017_1336, + KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tSIx[] = { + KBASE_HW_ISSUE_5736, KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TMIX_8133, + KBASE_HW_ISSUE_TSIX_1116, KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_3464, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tDVx_r0p0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TMIX_8133, KBASE_HW_ISSUE_TSIX_1116, + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_GPU2017_1336, + KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tDVx[] = { + KBASE_HW_ISSUE_5736, KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TMIX_8133, + KBASE_HW_ISSUE_TSIX_1116, KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_3464, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tNOx_r0p0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TMIX_8133, + KBASE_HW_ISSUE_TSIX_1116, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TNOX_1194, KBASE_HW_ISSUE_TTRX_921, + KBASE_HW_ISSUE_GPU2017_1336, KBASE_HW_ISSUE_TTRX_3464, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_KRAKEHW_2321, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tNOx[] = { + KBASE_HW_ISSUE_5736, KBASE_HW_ISSUE_9435, + KBASE_HW_ISSUE_TMIX_8133, KBASE_HW_ISSUE_TSIX_1116, + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_3464, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_KRAKEHW_2321, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tGOx_r0p0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TMIX_8133, + KBASE_HW_ISSUE_TSIX_1116, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TNOX_1194, KBASE_HW_ISSUE_TTRX_921, + KBASE_HW_ISSUE_GPU2017_1336, KBASE_HW_ISSUE_TTRX_3464, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_KRAKEHW_2321, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tGOx_r1p0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TMIX_8133, + KBASE_HW_ISSUE_TSIX_1116, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TGOX_R1_1234, KBASE_HW_ISSUE_TTRX_921, + KBASE_HW_ISSUE_GPU2017_1336, KBASE_HW_ISSUE_TTRX_3464, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_KRAKEHW_2321, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tGOx[] = { + KBASE_HW_ISSUE_5736, KBASE_HW_ISSUE_9435, + KBASE_HW_ISSUE_TMIX_8133, KBASE_HW_ISSUE_TSIX_1116, + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_3464, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_KRAKEHW_2321, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tTRx_r0p0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_3076, KBASE_HW_ISSUE_TTRX_921, + KBASE_HW_ISSUE_TTRX_3414, KBASE_HW_ISSUE_GPU2017_1336, + KBASE_HW_ISSUE_TTRX_3083, KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TTRX_3485, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tTRx_r0p1[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_3076, KBASE_HW_ISSUE_TTRX_921, + KBASE_HW_ISSUE_TTRX_3414, KBASE_HW_ISSUE_GPU2017_1336, + KBASE_HW_ISSUE_TTRX_3083, KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TTRX_3485, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tTRx_r0p2[] = { + KBASE_HW_ISSUE_9435, + KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, + KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_3076, + KBASE_HW_ISSUE_TTRX_921, + KBASE_HW_ISSUE_TTRX_3414, + KBASE_HW_ISSUE_GPU2017_1336, + KBASE_HW_ISSUE_TTRX_3083, + KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, + KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tTRx[] = { + KBASE_HW_ISSUE_5736, KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TTRX_3414, KBASE_HW_ISSUE_TTRX_3083, + KBASE_HW_ISSUE_TTRX_3470, KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tNAx_r0p0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_3076, KBASE_HW_ISSUE_TTRX_921, + KBASE_HW_ISSUE_TTRX_3414, KBASE_HW_ISSUE_GPU2017_1336, + KBASE_HW_ISSUE_TTRX_3083, KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TTRX_3485, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tNAx_r0p1[] = { + KBASE_HW_ISSUE_9435, + KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, + KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_3076, + KBASE_HW_ISSUE_TTRX_921, + KBASE_HW_ISSUE_TTRX_3414, + KBASE_HW_ISSUE_GPU2017_1336, + KBASE_HW_ISSUE_TTRX_3083, + KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, + KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tNAx[] = { + KBASE_HW_ISSUE_5736, KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TTRX_3414, KBASE_HW_ISSUE_TTRX_3083, + KBASE_HW_ISSUE_TTRX_3470, KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tBEx_r0p0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_TTRX_3414, + KBASE_HW_ISSUE_TTRX_3083, KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TTRX_3485, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_KRAKEHW_2321, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tBEx_r0p1[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_TTRX_3414, + KBASE_HW_ISSUE_TTRX_3083, KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tBEx_r1p0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_TTRX_3414, + KBASE_HW_ISSUE_TTRX_3083, KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tBEx_r1p1[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_TTRX_3414, + KBASE_HW_ISSUE_TTRX_3083, KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tBEx[] = { + KBASE_HW_ISSUE_5736, KBASE_HW_ISSUE_9435, + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, + KBASE_HW_ISSUE_TTRX_3414, KBASE_HW_ISSUE_TTRX_3083, + KBASE_HW_ISSUE_TTRX_3470, KBASE_HW_ISSUE_TTRX_3464, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_KRAKEHW_2321, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_lBEx_r1p0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_TTRX_3414, + KBASE_HW_ISSUE_TTRX_3083, KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TTRX_3485, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_KRAKEHW_2321, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_lBEx_r1p1[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_TTRX_3414, + KBASE_HW_ISSUE_TTRX_3083, KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tBAx_r0p0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_TTRX_3414, + KBASE_HW_ISSUE_TTRX_3083, KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tBAx_r0p1[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_TTRX_3414, + KBASE_HW_ISSUE_TTRX_3083, KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tBAx_r0p2[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_TTRX_3414, + KBASE_HW_ISSUE_TTRX_3083, KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tBAx[] = { + KBASE_HW_ISSUE_5736, KBASE_HW_ISSUE_9435, + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, + KBASE_HW_ISSUE_TTRX_3414, KBASE_HW_ISSUE_TTRX_3083, + KBASE_HW_ISSUE_TTRX_3470, KBASE_HW_ISSUE_TTRX_3464, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_KRAKEHW_2321, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tODx_r0p0[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, + KBASE_HW_ISSUE_GPU2019_3212, KBASE_HW_ISSUE_GPU2019_3878, + KBASE_HW_ISSUE_GPU2019_3901, KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tODx[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, + KBASE_HW_ISSUE_GPU2019_3212, KBASE_HW_ISSUE_GPU2019_3878, + KBASE_HW_ISSUE_GPU2019_3901, KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tGRx_r0p0[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_GPU2019_3878, + KBASE_HW_ISSUE_GPU2019_3901, KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tGRx[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_GPU2019_3878, + KBASE_HW_ISSUE_GPU2019_3901, KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tVAx_r0p0[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_GPU2019_3878, + KBASE_HW_ISSUE_GPU2019_3901, KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tVAx_r0p1[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_GPU2019_3878, + KBASE_HW_ISSUE_GPU2019_3901, KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tVAx[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_GPU2019_3878, + KBASE_HW_ISSUE_GPU2019_3901, KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r0p0[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TURSEHW_1997, + KBASE_HW_ISSUE_GPU2019_3878, KBASE_HW_ISSUE_TURSEHW_2716, KBASE_HW_ISSUE_GPU2019_3901, + KBASE_HW_ISSUE_GPU2021PRO_290, KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_TITANHW_2679, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_TURSEHW_2934, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r0p1[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TURSEHW_1997, + KBASE_HW_ISSUE_GPU2019_3878, KBASE_HW_ISSUE_TURSEHW_2716, KBASE_HW_ISSUE_GPU2019_3901, + KBASE_HW_ISSUE_GPU2021PRO_290, KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_TITANHW_2679, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2922, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_TURSEHW_2934, KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tTUx[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_GPU2019_3878, + KBASE_HW_ISSUE_TURSEHW_2716, KBASE_HW_ISSUE_GPU2019_3901, KBASE_HW_ISSUE_GPU2021PRO_290, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_TITANHW_2679, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2922, KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_TURSEHW_2934, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r1p0[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_GPU2019_3878, + KBASE_HW_ISSUE_TURSEHW_2716, KBASE_HW_ISSUE_GPU2019_3901, KBASE_HW_ISSUE_GPU2021PRO_290, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_TITANHW_2679, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2922, KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_TURSEHW_2934, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r1p1[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_GPU2019_3878, + KBASE_HW_ISSUE_TURSEHW_2716, KBASE_HW_ISSUE_GPU2019_3901, KBASE_HW_ISSUE_GPU2021PRO_290, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_TITANHW_2679, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2922, KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_TURSEHW_2934, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r1p2[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_GPU2019_3878, + KBASE_HW_ISSUE_TURSEHW_2716, KBASE_HW_ISSUE_GPU2019_3901, KBASE_HW_ISSUE_GPU2021PRO_290, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_TITANHW_2679, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2922, KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_TURSEHW_2934, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r1p3[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_GPU2019_3878, + KBASE_HW_ISSUE_TURSEHW_2716, KBASE_HW_ISSUE_GPU2019_3901, KBASE_HW_ISSUE_GPU2021PRO_290, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_TITANHW_2679, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2922, KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_TURSEHW_2934, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tTIx[] = { + KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, + KBASE_HW_ISSUE_TURSEHW_2716, + KBASE_HW_ISSUE_GPU2021PRO_290, + KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_TITANHW_2679, + KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2922, + KBASE_HW_ISSUE_TITANHW_2952, + KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_TURSEHW_2934, + KBASE_HW_ISSUE_KRAKEHW_2321, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tTIx_r0p0[] = { + KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, + KBASE_HW_ISSUE_TURSEHW_2716, + KBASE_HW_ISSUE_GPU2021PRO_290, + KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_TITANHW_2679, + KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2922, + KBASE_HW_ISSUE_TITANHW_2952, + KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_TURSEHW_2934, + KBASE_HW_ISSUE_KRAKEHW_2321, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tTIx_r0p1[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TURSEHW_2716, + KBASE_HW_ISSUE_GPU2021PRO_290, KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_TITANHW_2679, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_TURSEHW_2934, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tKRx_r0p0[] = { + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TURSEHW_2716, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_KRAKEHW_2151, KBASE_HW_ISSUE_KRAKEHW_2269, KBASE_HW_ISSUE_TITANHW_2922, + KBASE_HW_ISSUE_TURSEHW_2934, KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tKRx_r0p1[] = { + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TURSEHW_2716, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_KRAKEHW_2269, KBASE_HW_ISSUE_TURSEHW_2934, KBASE_HW_ISSUE_KRAKEHW_2321, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tKRx[] = { + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TURSEHW_2716, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_KRAKEHW_2151, KBASE_HW_ISSUE_KRAKEHW_2269, KBASE_HW_ISSUE_TURSEHW_2934, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + + +#endif /* _KBASE_HWCONFIG_ISSUES_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_ioctl_helpers.h b/drivers/gpu/arm/bifrost/mali_kbase_ioctl_helpers.h new file mode 100644 index 000000000000..e87925bab9b0 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_ioctl_helpers.h @@ -0,0 +1,542 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * + * (C) COPYRIGHT 2024 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +#ifndef _KBASE_IOCTL_HELPERS_H_ +#define _KBASE_IOCTL_HELPERS_H_ + +#include + +/* Macro for IOCTLs that don't have IOCTL struct */ +#define KBASE_HANDLE_IOCTL(cmd, function, arg) \ + do { \ + int ret; \ + BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_NONE); \ + dev_dbg(arg->kbdev->dev, "Enter ioctl %s\n", #function); \ + ret = function(arg); \ + dev_dbg(arg->kbdev->dev, "Return %d from ioctl %s\n", ret, #function); \ + return ret; \ + } while (0) + +/* Macro for IOCTLs that have input IOCTL struct */ +#define KBASE_HANDLE_IOCTL_IN(cmd, function, type, arg) \ + do { \ + type param; \ + int ret, err; \ + dev_dbg(arg->kbdev->dev, "Enter ioctl %s\n", #function); \ + BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_WRITE); \ + BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ + err = copy_from_user(¶m, uarg, sizeof(param)); \ + if (unlikely(err)) \ + return -EFAULT; \ + err = check_padding_##cmd(¶m); \ + if (unlikely(err)) \ + return -EINVAL; \ + ret = function(arg, ¶m); \ + dev_dbg(arg->kbdev->dev, "Return %d from ioctl %s\n", ret, #function); \ + return ret; \ + } while (0) + +/* Macro for IOCTLs that have output IOCTL struct */ +#define KBASE_HANDLE_IOCTL_OUT(cmd, function, type, arg) \ + do { \ + type param; \ + int ret, err; \ + dev_dbg(arg->kbdev->dev, "Enter ioctl %s\n", #function); \ + BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_READ); \ + BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ + memset(¶m, 0, sizeof(param)); \ + ret = function(arg, ¶m); \ + err = copy_to_user(uarg, ¶m, sizeof(param)); \ + if (unlikely(err)) \ + return -EFAULT; \ + dev_dbg(arg->kbdev->dev, "Return %d from ioctl %s\n", ret, #function); \ + return ret; \ + } while (0) + +/* Macro for IOCTLs that have input and output IOCTL struct */ +#define KBASE_HANDLE_IOCTL_INOUT(cmd, function, type, arg) \ + do { \ + type param; \ + int ret, err; \ + dev_dbg(arg->kbdev->dev, "Enter ioctl %s\n", #function); \ + BUILD_BUG_ON(_IOC_DIR(cmd) != (_IOC_WRITE | _IOC_READ)); \ + BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ + err = copy_from_user(¶m, uarg, sizeof(param)); \ + if (unlikely(err)) \ + return -EFAULT; \ + err = check_padding_##cmd(¶m); \ + if (unlikely(err)) \ + return -EINVAL; \ + ret = function(arg, ¶m); \ + err = copy_to_user(uarg, ¶m, sizeof(param)); \ + if (unlikely(err)) \ + return -EFAULT; \ + dev_dbg(arg->kbdev->dev, "Return %d from ioctl %s\n", ret, #function); \ + return ret; \ + } while (0) + +/* Inline functions to check padding bytes in the input IOCTL struct. + * Return 0 if all padding bytes are zero, non-zero otherwise. + */ +static inline int check_padding_KBASE_IOCTL_VERSION_CHECK(struct kbase_ioctl_version_check *p) +{ + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_VERSION_CHECK_RESERVED(struct kbase_ioctl_version_check *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_SET_FLAGS(struct kbase_ioctl_set_flags *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_GET_GPUPROPS(struct kbase_ioctl_get_gpuprops *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_MEM_ALLOC(union kbase_ioctl_mem_alloc *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_MEM_QUERY(union kbase_ioctl_mem_query *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_MEM_FREE(struct kbase_ioctl_mem_free *p) +{ + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_HWCNT_READER_SETUP(struct kbase_ioctl_hwcnt_reader_setup *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_HWCNT_SET(struct kbase_ioctl_hwcnt_values *p) +{ + return p->padding; +} + +static inline int check_padding_KBASE_IOCTL_GET_DDK_VERSION(struct kbase_ioctl_get_ddk_version *p) +{ + return p->padding; +} + +static inline int check_padding_KBASE_IOCTL_MEM_JIT_INIT(struct kbase_ioctl_mem_jit_init *p) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(p->padding); i++) { + if (p->padding[i]) + return -1; + } + + return 0; +} + +static inline int check_padding_KBASE_IOCTL_MEM_SYNC(struct kbase_ioctl_mem_sync *p) +{ + size_t i; + + /* + * Checking p->padding is deferred till the support window for backward-compatibility ends. + * GPUCORE-42000 will add the checking. + * + * To avoid the situation with old version of base which might not set padding bytes as 0, + * padding bytes are set as zero here on behalf on user space. + */ + for (i = 0; i < ARRAY_SIZE(p->padding); i++) + p->padding[i] = 0; + + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_MEM_FIND_CPU_OFFSET(union kbase_ioctl_mem_find_cpu_offset *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_TLSTREAM_ACQUIRE(struct kbase_ioctl_tlstream_acquire *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_MEM_COMMIT(struct kbase_ioctl_mem_commit *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_MEM_ALIAS(union kbase_ioctl_mem_alias *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_MEM_IMPORT(union kbase_ioctl_mem_import *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_MEM_FLAGS_CHANGE(struct kbase_ioctl_mem_flags_change *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_STREAM_CREATE(struct kbase_ioctl_stream_create *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_FENCE_VALIDATE(struct kbase_ioctl_fence_validate *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_MEM_PROFILE_ADD(struct kbase_ioctl_mem_profile_add *p) +{ + return p->padding; +} + +static inline int +check_padding_KBASE_IOCTL_STICKY_RESOURCE_MAP(struct kbase_ioctl_sticky_resource_map *p) +{ + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_STICKY_RESOURCE_UNMAP(struct kbase_ioctl_sticky_resource_unmap *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_MEM_FIND_GPU_START_AND_OFFSET( + union kbase_ioctl_mem_find_gpu_start_and_offset *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_CINSTR_GWT_DUMP(union kbase_ioctl_cinstr_gwt_dump *p) +{ + /* + * Checking p->padding is deferred till the support window for backward-compatibility ends. + * GPUCORE-42000 will add the checking. + * + * To avoid the situation with old version of base which might not set padding bytes as 0, + * padding bytes are set as zero here on behalf on user space. + */ + p->in.padding = 0; + return 0; +} + +static inline int check_padding_KBASE_IOCTL_MEM_EXEC_INIT(struct kbase_ioctl_mem_exec_init *p) +{ + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_GET_CPU_GPU_TIMEINFO(union kbase_ioctl_get_cpu_gpu_timeinfo *p) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(p->in.paddings); i++) { + if (p->in.paddings[i]) + return -1; + } + + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_CONTEXT_PRIORITY_CHECK(struct kbase_ioctl_context_priority_check *p) +{ + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_SET_LIMITED_CORE_COUNT(struct kbase_ioctl_set_limited_core_count *p) +{ + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_KINSTR_PRFCNT_ENUM_INFO(struct kbase_ioctl_kinstr_prfcnt_enum_info *p) +{ + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_KINSTR_PRFCNT_SETUP(union kbase_ioctl_kinstr_prfcnt_setup *p) +{ + return 0; +} + +#if MALI_UNIT_TEST +#endif /* MALI_UNIT_TEST */ + +#if MALI_USE_CSF + +static inline int +check_padding_KBASE_IOCTL_CS_QUEUE_REGISTER(struct kbase_ioctl_cs_queue_register *p) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(p->padding); i++) { + if (p->padding[i]) + return -1; + } + + return 0; +} + +static inline int check_padding_KBASE_IOCTL_CS_QUEUE_KICK(struct kbase_ioctl_cs_queue_kick *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_CS_QUEUE_BIND(union kbase_ioctl_cs_queue_bind *p) +{ + size_t i; + + /* + * Checking p->padding is deferred till the support window for backward-compatibility ends. + * GPUCORE-42000 will add the checking. + * + * To avoid the situation with old version of base which might not set padding bytes as 0, + * padding bytes are set as zero here on behalf on user space. + */ + for (i = 0; i < ARRAY_SIZE(p->in.padding); i++) + p->in.padding[i] = 0; + + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_CS_QUEUE_REGISTER_EX(struct kbase_ioctl_cs_queue_register_ex *p) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(p->padding); i++) { + if (p->padding[i]) + return -1; + } + + for (i = 0; i < ARRAY_SIZE(p->ex_padding); i++) { + if (p->ex_padding[i]) + return -1; + } + + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_CS_QUEUE_TERMINATE(struct kbase_ioctl_cs_queue_terminate *p) +{ + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_CS_QUEUE_GROUP_CREATE_1_6(union kbase_ioctl_cs_queue_group_create_1_6 *p) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(p->in.padding); i++) { + if (p->in.padding[i]) + return -1; + } + + return 0; +} + +static inline int check_padding_KBASE_IOCTL_CS_QUEUE_GROUP_CREATE_1_18( + union kbase_ioctl_cs_queue_group_create_1_18 *p) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(p->in.padding); i++) { + if (p->in.padding[i]) + return -1; + } + + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_CS_QUEUE_GROUP_CREATE(union kbase_ioctl_cs_queue_group_create *p) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(p->in.padding); i++) { + if (p->in.padding[i]) + return -1; + } + + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_CS_QUEUE_GROUP_TERMINATE(struct kbase_ioctl_cs_queue_group_term *p) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(p->padding); i++) { + if (p->padding[i]) + return -1; + } + + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_KCPU_QUEUE_DELETE(struct kbase_ioctl_kcpu_queue_delete *p) +{ + size_t i; + + /* + * Checking p->padding is deferred till the support window for backward-compatibility ends. + * GPUCORE-42000 will add the checking. + * + * To avoid the situation with old version of base which might not set padding bytes as 0, + * padding bytes are set as zero here on behalf on user space. + */ + for (i = 0; i < ARRAY_SIZE(p->padding); i++) + p->padding[i] = 0; + + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_KCPU_QUEUE_ENQUEUE(struct kbase_ioctl_kcpu_queue_enqueue *p) +{ + size_t i; + + /* + * Checking p->padding is deferred till the support window for backward-compatibility ends. + * GPUCORE-42000 will add the checking. + * + * To avoid the situation with old version of base which might not set padding bytes as 0, + * padding bytes are set as zero here on behalf on user space. + */ + for (i = 0; i < ARRAY_SIZE(p->padding); i++) + p->padding[i] = 0; + + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_CS_TILER_HEAP_INIT(union kbase_ioctl_cs_tiler_heap_init *p) +{ + return p->in.padding; +} + +static inline int +check_padding_KBASE_IOCTL_CS_TILER_HEAP_INIT_1_13(union kbase_ioctl_cs_tiler_heap_init_1_13 *p) +{ + return p->in.padding; +} + +static inline int +check_padding_KBASE_IOCTL_CS_TILER_HEAP_TERM(struct kbase_ioctl_cs_tiler_heap_term *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_CS_GET_GLB_IFACE(union kbase_ioctl_cs_get_glb_iface *p) +{ + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_CS_CPU_QUEUE_DUMP(struct kbase_ioctl_cs_cpu_queue_info *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_MEM_ALLOC_EX(union kbase_ioctl_mem_alloc_ex *p) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(p->in.extra); i++) { + if (p->in.extra[i]) + return -1; + } + + return 0; +} + +static inline int check_padding_KBASE_IOCTL_READ_USER_PAGE(union kbase_ioctl_read_user_page *p) +{ + return p->in.padding; +} + +static inline int +check_padding_KBASE_IOCTL_QUEUE_GROUP_CLEAR_FAULTS(struct kbase_ioctl_queue_group_clear_faults *p) +{ + size_t i; + + /* + * Checking p->padding is deferred till the support window for backward-compatibility ends. + * GPUCORE-42000 will add the checking. + * + * To avoid the situation with old version of base which might not set padding bytes as 0, + * padding bytes are set as zero here on behalf on user space. + */ + for (i = 0; i < ARRAY_SIZE(p->padding); i++) + p->padding[i] = 0; + + return 0; +} + +#else /* MALI_USE_CSF */ + +static inline int check_padding_KBASE_IOCTL_JOB_SUBMIT(struct kbase_ioctl_job_submit *p) +{ + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_SOFT_EVENT_UPDATE(struct kbase_ioctl_soft_event_update *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_KINSTR_JM_FD(union kbase_kinstr_jm_fd *p) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(p->in.padding); i++) { + if (p->in.padding[i]) + return -1; + } + + return 0; +} + +#endif /* !MALI_USE_CSF */ + +#endif /* _KBASE_IOCTL_HELPERS_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_jd.c b/drivers/gpu/arm/bifrost/mali_kbase_jd.c index 418a1913b241..4da7fa377bd7 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_jd.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_jd.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -39,9 +39,6 @@ #include #include #include - -#include - #include /* Return whether katom will run on the GPU or not. Currently only soft jobs and @@ -209,7 +206,7 @@ static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, } if (copy_from_user(input_extres, get_compat_pointer(katom->kctx, user_atom->extres_list), - sizeof(*input_extres) * katom->nr_extres) != 0) { + size_mul(sizeof(*input_extres), katom->nr_extres)) != 0) { err = -EINVAL; goto failed_input_copy; } @@ -697,7 +694,6 @@ static void jd_trace_atom_submit(struct kbase_context *const kctx, static bool jd_submit_atom(struct kbase_context *const kctx, const struct base_jd_atom *const user_atom, - const struct base_jd_fragment *const user_jc_incr, struct kbase_jd_atom *const katom) { struct kbase_device *kbdev = kctx->kbdev; @@ -755,8 +751,6 @@ static bool jd_submit_atom(struct kbase_context *const kctx, } #endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - katom->renderpass_id = user_atom->renderpass_id; - /* Implicitly sets katom->protected_state.enter as well. */ katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; @@ -875,20 +869,7 @@ static bool jd_submit_atom(struct kbase_context *const kctx, /* Create a new atom. */ jd_trace_atom_submit(kctx, katom, &katom->sched_priority); -#if !MALI_INCREMENTAL_RENDERING_JM - /* Reject atoms for incremental rendering if not supported */ - if (katom->core_req & (BASE_JD_REQ_START_RENDERPASS | BASE_JD_REQ_END_RENDERPASS)) { - dev_err(kctx->kbdev->dev, "Rejecting atom with unsupported core_req 0x%x\n", - katom->core_req); - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - return kbase_jd_done_nolock(katom, true); - } -#endif /* !MALI_INCREMENTAL_RENDERING_JM */ - - if (katom->core_req & BASE_JD_REQ_END_RENDERPASS) { - WARN_ON(katom->jc != 0); - katom->jc_fragment = *user_jc_incr; - } else if (!katom->jc && (katom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) { + if (!katom->jc && (katom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) { /* Reject atoms with job chain = NULL, as these cause issues * with soft-stop */ @@ -1018,8 +999,7 @@ int kbase_jd_submit(struct kbase_context *kctx, void __user *user_addr, u32 nr_a struct kbase_device *kbdev; u32 latest_flush; - bool jd_atom_is_v2 = (stride == sizeof(struct base_jd_atom_v2) || - stride == offsetof(struct base_jd_atom_v2, renderpass_id)); + bool jd_atom_is_v2 = (stride == sizeof(struct base_jd_atom_v2)); CSTD_UNUSED(uk6_atom); @@ -1035,10 +1015,7 @@ int kbase_jd_submit(struct kbase_context *kctx, void __user *user_addr, u32 nr_a return -EINVAL; } - if (stride != offsetof(struct base_jd_atom_v2, renderpass_id) && - stride != sizeof(struct base_jd_atom_v2) && - stride != offsetof(struct base_jd_atom, renderpass_id) && - stride != sizeof(struct base_jd_atom)) { + if (stride != sizeof(struct base_jd_atom_v2) && stride != sizeof(struct base_jd_atom)) { dev_err(kbdev->dev, "Stride %u passed to job_submit isn't supported by the kernel\n", stride); return -EINVAL; @@ -1057,7 +1034,6 @@ int kbase_jd_submit(struct kbase_context *kctx, void __user *user_addr, u32 nr_a struct base_jd_atom user_atom = { .seq_nr = 0, }; - struct base_jd_fragment user_jc_incr; struct kbase_jd_atom *katom; if (unlikely(jd_atom_is_v2)) { @@ -1082,44 +1058,6 @@ int kbase_jd_submit(struct kbase_context *kctx, void __user *user_addr, u32 nr_a } } - if (stride == offsetof(struct base_jd_atom_v2, renderpass_id)) { - dev_dbg(kbdev->dev, "No renderpass ID: use 0\n"); - user_atom.renderpass_id = 0; - } else { - /* Ensure all padding bytes are 0 for potential future - * extension - */ - size_t j; - - dev_dbg(kbdev->dev, "Renderpass ID is %d\n", user_atom.renderpass_id); - for (j = 0; j < sizeof(user_atom.padding); j++) { - if (user_atom.padding[j]) { - dev_err(kbdev->dev, "Bad padding byte %zu: %d\n", j, - user_atom.padding[j]); - err = -EINVAL; - break; - } - } - if (err) - break; - } - - /* In this case 'jc' is the CPU address of a struct - * instead of a GPU address of a job chain. - */ - if (user_atom.core_req & BASE_JD_REQ_END_RENDERPASS) { - if (copy_from_user(&user_jc_incr, u64_to_user_ptr(user_atom.jc), - sizeof(user_jc_incr))) { - dev_err(kbdev->dev, - "Invalid jc address 0x%llx passed to job_submit\n", - user_atom.jc); - err = -EFAULT; - break; - } - dev_dbg(kbdev->dev, "Copied IR jobchain addresses\n"); - user_atom.jc = 0; - } - user_addr = (void __user *)((uintptr_t)user_addr + stride); mutex_lock(&jctx->lock); @@ -1172,8 +1110,7 @@ int kbase_jd_submit(struct kbase_context *kctx, void __user *user_addr, u32 nr_a mutex_lock(&jctx->lock); } KBASE_TLSTREAM_TL_JD_SUBMIT_ATOM_START(kbdev, katom); - need_to_try_schedule_context |= - jd_submit_atom(kctx, &user_atom, &user_jc_incr, katom); + need_to_try_schedule_context |= jd_submit_atom(kctx, &user_atom, katom); KBASE_TLSTREAM_TL_JD_SUBMIT_ATOM_END(kbdev, katom); /* Register a completed job as a disjoint event when the GPU is in a disjoint state * (ie. being reset). @@ -1579,9 +1516,6 @@ int kbase_jd_init(struct kbase_context *kctx) #endif } - for (i = 0; i < BASE_JD_RP_COUNT; i++) - kctx->jctx.renderpasses[i].state = KBASE_JD_RP_COMPLETE; - mutex_init(&kctx->jctx.lock); init_waitqueue_head(&kctx->jctx.zero_jobs_wait); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_js.c b/drivers/gpu/arm/bifrost/mali_kbase_js.c index 3e6b6b5eb066..d42fde37db2a 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_js.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_js.c @@ -333,19 +333,6 @@ static void jsctx_queue_foreach_prio(struct kbase_context *kctx, unsigned int js rb_erase(node, &queue->runnable_tree); callback(kctx->kbdev, entry); - - /* Runnable end-of-renderpass atoms can also be in the linked - * list of atoms blocked on cross-slot dependencies. Remove them - * to avoid calling the callback twice. - */ - if (entry->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST) { - WARN_ON(!(entry->core_req & BASE_JD_REQ_END_RENDERPASS)); - dev_dbg(kctx->kbdev->dev, "Del runnable atom %pK from X_DEP list\n", - (void *)entry); - - list_del(&entry->queue); - entry->atom_flags &= ~KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; - } } while (!list_empty(&queue->x_dep_head)) { @@ -1230,7 +1217,7 @@ static bool kbase_js_ctx_pullable(struct kbase_context *kctx, unsigned int js, b dev_dbg(kbdev->dev, "JS: Atom %pK is blocked in js_ctx_pullable\n", (void *)katom); return false; /* next atom blocked */ } - if (kbase_js_atom_blocked_on_x_dep(katom)) { + if (katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) { if (katom->x_pre_dep->gpu_rb_state == KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB || katom->x_pre_dep->will_fail_event_code) { dev_dbg(kbdev->dev, @@ -1371,9 +1358,6 @@ static bool kbase_js_dep_validate(struct kbase_context *kctx, struct kbase_jd_at (dep_atom->status != KBASE_JD_ATOM_STATE_UNUSED)) { katom->atom_flags |= KBASE_KATOM_FLAG_X_DEP_BLOCKED; - dev_dbg(kbdev->dev, "Set X_DEP flag on atom %pK\n", - (void *)katom); - katom->x_pre_dep = dep_atom; dep_atom->x_post_dep = katom; if (kbase_jd_katom_dep_type(&katom->dep[i]) == @@ -1447,110 +1431,12 @@ void kbase_js_update_ctx_priority(struct kbase_context *kctx) } KBASE_EXPORT_TEST_API(kbase_js_update_ctx_priority); -/** - * js_add_start_rp() - Add an atom that starts a renderpass to the job scheduler - * @start_katom: Pointer to the atom to be added. - * Return: 0 if successful or a negative value on failure. - */ -static int js_add_start_rp(struct kbase_jd_atom *const start_katom) -{ - struct kbase_context *const kctx = start_katom->kctx; - struct kbase_jd_renderpass *rp; - struct kbase_device *const kbdev = kctx->kbdev; - unsigned long flags; - - lockdep_assert_held(&kctx->jctx.lock); - - if (WARN_ON(!(start_katom->core_req & BASE_JD_REQ_START_RENDERPASS))) - return -EINVAL; - - if (start_katom->core_req & BASE_JD_REQ_END_RENDERPASS) - return -EINVAL; - - compiletime_assert((1ull << (sizeof(start_katom->renderpass_id) * 8)) <= - ARRAY_SIZE(kctx->jctx.renderpasses), - "Should check invalid access to renderpasses"); - - rp = &kctx->jctx.renderpasses[start_katom->renderpass_id]; - - if (rp->state != KBASE_JD_RP_COMPLETE) - return -EINVAL; - - dev_dbg(kctx->kbdev->dev, "JS add start atom %pK of RP %d\n", (void *)start_katom, - start_katom->renderpass_id); - - /* The following members are read when updating the job slot - * ringbuffer/fifo therefore they require additional locking. - */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - rp->state = KBASE_JD_RP_START; - rp->start_katom = start_katom; - rp->end_katom = NULL; - INIT_LIST_HEAD(&rp->oom_reg_list); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - return 0; -} - -/** - * js_add_end_rp() - Add an atom that ends a renderpass to the job scheduler - * @end_katom: Pointer to the atom to be added. - * Return: 0 if successful or a negative value on failure. - */ -static int js_add_end_rp(struct kbase_jd_atom *const end_katom) -{ - struct kbase_context *const kctx = end_katom->kctx; - struct kbase_jd_renderpass *rp; - struct kbase_device *const kbdev = kctx->kbdev; - - lockdep_assert_held(&kctx->jctx.lock); - - if (WARN_ON(!(end_katom->core_req & BASE_JD_REQ_END_RENDERPASS))) - return -EINVAL; - - if (end_katom->core_req & BASE_JD_REQ_START_RENDERPASS) - return -EINVAL; - - compiletime_assert((1ull << (sizeof(end_katom->renderpass_id) * 8)) <= - ARRAY_SIZE(kctx->jctx.renderpasses), - "Should check invalid access to renderpasses"); - - rp = &kctx->jctx.renderpasses[end_katom->renderpass_id]; - - dev_dbg(kbdev->dev, "JS add end atom %pK in state %d of RP %d\n", (void *)end_katom, - (int)rp->state, end_katom->renderpass_id); - - if (rp->state == KBASE_JD_RP_COMPLETE) - return -EINVAL; - - if (rp->end_katom == NULL) { - /* We can't be in a retry state until the fragment job chain - * has completed. - */ - unsigned long flags; - - WARN_ON(rp->state == KBASE_JD_RP_RETRY); - WARN_ON(rp->state == KBASE_JD_RP_RETRY_PEND_OOM); - WARN_ON(rp->state == KBASE_JD_RP_RETRY_OOM); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - rp->end_katom = end_katom; - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - } else - WARN_ON(rp->end_katom != end_katom); - - return 0; -} - bool kbasep_js_add_job(struct kbase_context *kctx, struct kbase_jd_atom *atom) { unsigned long flags; struct kbasep_js_kctx_info *js_kctx_info; struct kbase_device *kbdev; struct kbasep_js_device_data *js_devdata; - int err = 0; bool enqueue_required = false; bool timer_sync = false; @@ -1566,17 +1452,6 @@ bool kbasep_js_add_job(struct kbase_context *kctx, struct kbase_jd_atom *atom) mutex_lock(&js_devdata->queue_mutex); mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - if (atom->core_req & BASE_JD_REQ_START_RENDERPASS) - err = js_add_start_rp(atom); - else if (atom->core_req & BASE_JD_REQ_END_RENDERPASS) - err = js_add_end_rp(atom); - - if (err < 0) { - atom->event_code = BASE_JD_EVENT_JOB_INVALID; - atom->status = KBASE_JD_ATOM_STATE_COMPLETED; - goto out_unlock; - } - /* * Begin Runpool transaction */ @@ -1860,10 +1735,7 @@ kbasep_js_runpool_release_ctx_internal(struct kbase_device *kbdev, struct kbase_ kbasep_js_ctx_attr_ctx_release_atom(kbdev, kctx, katom_retained_state); if (new_ref_count == 2 && kbase_ctx_flag(kctx, KCTX_PRIVILEGED) && -#ifdef CONFIG_MALI_ARBITER_SUPPORT - !kbase_pm_is_gpu_lost(kbdev) && -#endif - !kbase_pm_is_suspending(kbdev)) { + !kbase_pm_is_gpu_lost(kbdev) && !kbase_pm_is_suspending(kbdev)) { /* Context is kept scheduled into an address space even when * there are no jobs, in this case we have to handle the * situation where all jobs have been evicted from the GPU and @@ -1880,10 +1752,7 @@ kbasep_js_runpool_release_ctx_internal(struct kbase_device *kbdev, struct kbase_ * which was previously acquired by kbasep_js_schedule_ctx(). */ if (new_ref_count == 1 && (!kbasep_js_is_submit_allowed(js_devdata, kctx) || -#ifdef CONFIG_MALI_ARBITER_SUPPORT - kbase_pm_is_gpu_lost(kbdev) || -#endif - kbase_pm_is_suspending(kbdev))) { + kbase_pm_is_gpu_lost(kbdev) || kbase_pm_is_suspending(kbdev))) { int num_slots = kbdev->gpu_props.num_job_slots; unsigned int slot; @@ -2189,11 +2058,7 @@ static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, struct kbase_cont * of it being called strictly after the suspend flag is set, and will * wait for this lock to drop) */ -#ifdef CONFIG_MALI_ARBITER_SUPPORT if (kbase_pm_is_suspending(kbdev) || kbase_pm_is_gpu_lost(kbdev)) { -#else - if (kbase_pm_is_suspending(kbdev)) { -#endif /* Cause it to leave at some later point */ bool retained; CSTD_UNUSED(retained); @@ -2267,7 +2132,6 @@ void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, struct kbase_ js_devdata = &kbdev->js_data; js_kctx_info = &kctx->jctx.sched_info; -#ifdef CONFIG_MALI_ARBITER_SUPPORT /* This should only happen in response to a system call * from a user-space thread. * In a non-arbitrated environment this can never happen @@ -2279,18 +2143,10 @@ void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, struct kbase_ * the wait event for KCTX_SCHEDULED, since no context * can be scheduled until we have the GPU again. */ - if (kbdev->arb.arb_if == NULL) + if (!kbase_has_arbiter(kbdev)) { if (WARN_ON(kbase_pm_is_suspending(kbdev))) return; -#else - /* This should only happen in response to a system call - * from a user-space thread. - * In a non-arbitrated environment this can never happen - * whilst suspending. - */ - if (WARN_ON(kbase_pm_is_suspending(kbdev))) - return; -#endif + } mutex_lock(&js_devdata->queue_mutex); mutex_lock(&js_kctx_info->ctx.jsctx_mutex); @@ -2416,63 +2272,63 @@ void kbasep_js_resume(struct kbase_device *kbdev) struct kbase_context *kctx, *n; unsigned long flags; -#ifndef CONFIG_MALI_ARBITER_SUPPORT - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + if (kbase_has_arbiter(kbdev)) { + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - list_for_each_entry_safe(kctx, n, - &kbdev->js_data.ctx_list_unpullable[js][prio], - jctx.sched_info.ctx.ctx_list_entry[js]) { - struct kbasep_js_kctx_info *js_kctx_info; + list_for_each_entry_safe( + kctx, n, &kbdev->js_data.ctx_list_unpullable[js][prio], + jctx.sched_info.ctx.ctx_list_entry[js]) { + struct kbasep_js_kctx_info *js_kctx_info; + bool timer_sync = false; + + /* Drop lock so we can take kctx mutexes */ + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + js_kctx_info = &kctx->jctx.sched_info; + + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + mutex_lock(&js_devdata->runpool_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED) && + kbase_js_ctx_pullable(kctx, js, false)) + timer_sync = kbase_js_ctx_list_add_pullable_nolock( + kbdev, kctx, js); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + if (timer_sync) + kbase_backend_ctx_count_changed(kbdev); + + mutex_unlock(&js_devdata->runpool_mutex); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + /* Take lock before accessing list again */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } else { bool timer_sync = false; - /* Drop lock so we can take kctx mutexes */ - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - js_kctx_info = &kctx->jctx.sched_info; - - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - mutex_lock(&js_devdata->runpool_mutex); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED) && - kbase_js_ctx_pullable(kctx, js, false)) - timer_sync = kbase_js_ctx_list_add_pullable_nolock( - kbdev, kctx, js); + list_for_each_entry_safe( + kctx, n, &kbdev->js_data.ctx_list_unpullable[js][prio], + jctx.sched_info.ctx.ctx_list_entry[js]) { + if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED) && + kbase_js_ctx_pullable(kctx, js, false)) + timer_sync |= kbase_js_ctx_list_add_pullable_nolock( + kbdev, kctx, js); + } spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - if (timer_sync) + if (timer_sync) { + mutex_lock(&js_devdata->runpool_mutex); kbase_backend_ctx_count_changed(kbdev); - - mutex_unlock(&js_devdata->runpool_mutex); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - - /* Take lock before accessing list again */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + mutex_unlock(&js_devdata->runpool_mutex); + } } - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#else - bool timer_sync = false; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - list_for_each_entry_safe(kctx, n, - &kbdev->js_data.ctx_list_unpullable[js][prio], - jctx.sched_info.ctx.ctx_list_entry[js]) { - if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED) && - kbase_js_ctx_pullable(kctx, js, false)) - timer_sync |= kbase_js_ctx_list_add_pullable_nolock( - kbdev, kctx, js); - } - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - if (timer_sync) { - mutex_lock(&js_devdata->runpool_mutex); - kbase_backend_ctx_count_changed(kbdev); - mutex_unlock(&js_devdata->runpool_mutex); - } -#endif } } mutex_unlock(&js_devdata->queue_mutex); @@ -2515,7 +2371,7 @@ static unsigned int kbase_js_get_slot(struct kbase_device *kbdev, struct kbase_j bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, struct kbase_jd_atom *katom) { - bool enqueue_required, add_required = true; + bool enqueue_required; katom->slot_nr = kbase_js_get_slot(kctx->kbdev, katom); @@ -2525,10 +2381,7 @@ bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, struct kbase_jd_at /* If slot will transition from unpullable to pullable then add to * pullable list */ - if (jsctx_rb_none_to_pull(kctx, katom->slot_nr)) - enqueue_required = true; - else - enqueue_required = false; + enqueue_required = jsctx_rb_none_to_pull(kctx, katom->slot_nr); if ((katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) || (katom->pre_dep && @@ -2541,15 +2394,9 @@ bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, struct kbase_jd_at list_add_tail(&katom->queue, &queue->x_dep_head); katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; - if (kbase_js_atom_blocked_on_x_dep(katom)) { - enqueue_required = false; - add_required = false; - } + enqueue_required = false; } else { dev_dbg(kctx->kbdev->dev, "Atom %pK not added to X_DEP list\n", (void *)katom); - } - - if (add_required) { /* Check if there are lower priority jobs to soft stop */ kbase_job_slot_ctx_priority_check_locked(kctx, katom); @@ -2575,30 +2422,22 @@ bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, struct kbase_jd_at */ static void kbase_js_move_to_tree(struct kbase_jd_atom *katom) { - struct kbase_context *const kctx = katom->kctx; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + lockdep_assert_held(&katom->kctx->kbdev->hwaccess_lock); while (katom) { WARN_ON(!(katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST)); - if (!kbase_js_atom_blocked_on_x_dep(katom)) { - dev_dbg(kctx->kbdev->dev, + if (!(katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED)) { + dev_dbg(katom->kctx->kbdev->dev, "Del atom %pK from X_DEP list in js_move_to_tree\n", (void *)katom); list_del(&katom->queue); katom->atom_flags &= ~KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; - /* For incremental rendering, an end-of-renderpass atom - * may have had its dependency on start-of-renderpass - * ignored and may therefore already be in the tree. - */ - if (!(katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE)) { - jsctx_tree_add(kctx, katom); - katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_TREE; - } + jsctx_tree_add(katom->kctx, katom); + katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_TREE; } else { - dev_dbg(kctx->kbdev->dev, "Atom %pK blocked on x-dep in js_move_to_tree\n", - (void *)katom); + dev_dbg(katom->kctx->kbdev->dev, + "Atom %pK blocked on x-dep in js_move_to_tree\n", (void *)katom); break; } @@ -2671,11 +2510,7 @@ struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, unsigned int js) dev_dbg(kbdev->dev, "JS: No submit allowed for kctx %pK\n", (void *)kctx); return NULL; } -#ifdef CONFIG_MALI_ARBITER_SUPPORT if (kbase_pm_is_suspending(kbdev) || kbase_pm_is_gpu_lost(kbdev)) -#else - if (kbase_pm_is_suspending(kbdev)) -#endif return NULL; katom = jsctx_rb_peek(kctx, js); @@ -2705,7 +2540,7 @@ struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, unsigned int js) return NULL; } - if (kbase_js_atom_blocked_on_x_dep(katom)) { + if (katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) { if (katom->x_pre_dep->gpu_rb_state == KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB || katom->x_pre_dep->will_fail_event_code) { dev_dbg(kbdev->dev, @@ -2745,190 +2580,6 @@ struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, unsigned int js) return katom; } -/** - * js_return_of_start_rp() - Handle soft-stop of an atom that starts a - * renderpass - * @start_katom: Pointer to the start-of-renderpass atom that was soft-stopped - * - * This function is called to switch to incremental rendering if the tiler job - * chain at the start of a renderpass has used too much memory. It prevents the - * tiler job being pulled for execution in the job scheduler again until the - * next phase of incremental rendering is complete. - * - * If the end-of-renderpass atom is already in the job scheduler (because a - * previous attempt at tiling used too much memory during the same renderpass) - * then it is unblocked; otherwise, it is run by handing it to the scheduler. - */ -static void js_return_of_start_rp(struct kbase_jd_atom *const start_katom) -{ - struct kbase_context *const kctx = start_katom->kctx; - struct kbase_device *const kbdev = kctx->kbdev; - struct kbase_jd_renderpass *rp; - struct kbase_jd_atom *end_katom; - unsigned long flags; - - lockdep_assert_held(&kctx->jctx.lock); - - if (WARN_ON(!(start_katom->core_req & BASE_JD_REQ_START_RENDERPASS))) - return; - - compiletime_assert((1ull << (sizeof(start_katom->renderpass_id) * 8)) <= - ARRAY_SIZE(kctx->jctx.renderpasses), - "Should check invalid access to renderpasses"); - - rp = &kctx->jctx.renderpasses[start_katom->renderpass_id]; - - if (WARN_ON(rp->start_katom != start_katom)) - return; - - dev_dbg(kctx->kbdev->dev, "JS return start atom %pK in state %d of RP %d\n", - (void *)start_katom, (int)rp->state, start_katom->renderpass_id); - - if (WARN_ON(rp->state == KBASE_JD_RP_COMPLETE)) - return; - - /* The tiler job might have been soft-stopped for some reason other - * than running out of memory. - */ - if (rp->state == KBASE_JD_RP_START || rp->state == KBASE_JD_RP_RETRY) { - dev_dbg(kctx->kbdev->dev, "JS return isn't OOM in state %d of RP %d\n", - (int)rp->state, start_katom->renderpass_id); - return; - } - - dev_dbg(kctx->kbdev->dev, "JS return confirm OOM in state %d of RP %d\n", (int)rp->state, - start_katom->renderpass_id); - - if (WARN_ON(rp->state != KBASE_JD_RP_PEND_OOM && rp->state != KBASE_JD_RP_RETRY_PEND_OOM)) - return; - - /* Prevent the tiler job being pulled for execution in the - * job scheduler again. - */ - dev_dbg(kbdev->dev, "Blocking start atom %pK\n", (void *)start_katom); - atomic_inc(&start_katom->blocked); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - rp->state = (rp->state == KBASE_JD_RP_PEND_OOM) ? KBASE_JD_RP_OOM : KBASE_JD_RP_RETRY_OOM; - - /* Was the fragment job chain submitted to kbase yet? */ - end_katom = rp->end_katom; - if (end_katom) { - dev_dbg(kctx->kbdev->dev, "JS return add end atom %pK\n", (void *)end_katom); - - if (rp->state == KBASE_JD_RP_RETRY_OOM) { - /* Allow the end of the renderpass to be pulled for - * execution again to continue incremental rendering. - */ - dev_dbg(kbdev->dev, "Unblocking end atom %pK\n", (void *)end_katom); - atomic_dec(&end_katom->blocked); - WARN_ON(!(end_katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE)); - WARN_ON(end_katom->status != KBASE_JD_ATOM_STATE_IN_JS); - - kbase_js_ctx_list_add_pullable_nolock(kbdev, kctx, end_katom->slot_nr); - - /* Expect the fragment job chain to be scheduled without - * further action because this function is called when - * returning an atom to the job scheduler ringbuffer. - */ - end_katom = NULL; - } else { - WARN_ON(end_katom->status != KBASE_JD_ATOM_STATE_QUEUED && - end_katom->status != KBASE_JD_ATOM_STATE_IN_JS); - } - } - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - if (end_katom) - kbase_jd_dep_clear_locked(end_katom); -} - -/** - * js_return_of_end_rp() - Handle completion of an atom that ends a renderpass - * @end_katom: Pointer to the end-of-renderpass atom that was completed - * - * This function is called to continue incremental rendering if the tiler job - * chain at the start of a renderpass used too much memory. It resets the - * mechanism for detecting excessive memory usage then allows the soft-stopped - * tiler job chain to be pulled for execution again. - * - * The start-of-renderpass atom must already been submitted to kbase. - */ -static void js_return_of_end_rp(struct kbase_jd_atom *const end_katom) -{ - struct kbase_context *const kctx = end_katom->kctx; - struct kbase_device *const kbdev = kctx->kbdev; - struct kbase_jd_renderpass *rp; - struct kbase_jd_atom *start_katom; - unsigned long flags; - - lockdep_assert_held(&kctx->jctx.lock); - - if (WARN_ON(!(end_katom->core_req & BASE_JD_REQ_END_RENDERPASS))) - return; - - compiletime_assert((1ull << (sizeof(end_katom->renderpass_id) * 8)) <= - ARRAY_SIZE(kctx->jctx.renderpasses), - "Should check invalid access to renderpasses"); - - rp = &kctx->jctx.renderpasses[end_katom->renderpass_id]; - - if (WARN_ON(rp->end_katom != end_katom)) - return; - - dev_dbg(kctx->kbdev->dev, "JS return end atom %pK in state %d of RP %d\n", - (void *)end_katom, (int)rp->state, end_katom->renderpass_id); - - if (WARN_ON(rp->state != KBASE_JD_RP_OOM && rp->state != KBASE_JD_RP_RETRY_OOM)) - return; - - /* Reduce the number of mapped pages in the memory regions that - * triggered out-of-memory last time so that we can detect excessive - * memory usage again. - */ - kbase_gpu_vm_lock(kctx); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - while (!list_empty(&rp->oom_reg_list)) { - struct kbase_va_region *reg = - list_first_entry(&rp->oom_reg_list, struct kbase_va_region, link); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - dev_dbg(kbdev->dev, "Reset backing to %zu pages for region %pK\n", - reg->threshold_pages, (void *)reg); - - if (!WARN_ON(reg->flags & KBASE_REG_VA_FREED)) - kbase_mem_shrink(kctx, reg, reg->threshold_pages); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - dev_dbg(kbdev->dev, "Deleting region %pK from list\n", (void *)reg); - list_del_init(®->link); - kbase_va_region_alloc_put(kctx, reg); - } - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - kbase_gpu_vm_unlock(kctx); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - rp->state = KBASE_JD_RP_RETRY; - dev_dbg(kbdev->dev, "Changed state to %d for retry\n", rp->state); - - /* Allow the start of the renderpass to be pulled for execution again - * to begin/continue incremental rendering. - */ - start_katom = rp->start_katom; - if (!WARN_ON(!start_katom)) { - dev_dbg(kbdev->dev, "Unblocking start atom %pK\n", (void *)start_katom); - atomic_dec(&start_katom->blocked); - (void)kbase_js_ctx_list_add_pullable_head_nolock(kbdev, kctx, start_katom->slot_nr); - } - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -} - static void js_return_worker(struct work_struct *data) { struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work); @@ -2949,9 +2600,7 @@ static void js_return_worker(struct work_struct *data) katom->event_code); KBASE_KTRACE_ADD_JM(kbdev, JS_RETURN_WORKER, kctx, katom, katom->jc, 0); - - if (katom->event_code != BASE_JD_EVENT_END_RP_DONE) - KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_EX(kbdev, katom); + KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_EX(kbdev, katom); kbase_backend_complete_wq(kbdev, katom); @@ -2960,8 +2609,7 @@ static void js_return_worker(struct work_struct *data) mutex_lock(&js_devdata->queue_mutex); mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - if (katom->event_code != BASE_JD_EVENT_END_RP_DONE) - atomic_dec(&katom->blocked); + atomic_dec(&katom->blocked); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); @@ -3026,16 +2674,6 @@ static void js_return_worker(struct work_struct *data) mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); mutex_unlock(&js_devdata->queue_mutex); - if (katom->core_req & BASE_JD_REQ_START_RENDERPASS) { - mutex_lock(&kctx->jctx.lock); - js_return_of_start_rp(katom); - mutex_unlock(&kctx->jctx.lock); - } else if (katom->event_code == BASE_JD_EVENT_END_RP_DONE) { - mutex_lock(&kctx->jctx.lock); - js_return_of_end_rp(katom); - mutex_unlock(&kctx->jctx.lock); - } - dev_dbg(kbdev->dev, "JS: retained state %s finished", kbasep_js_has_atom_finished(&retained_state) ? "has" : "hasn't"); @@ -3071,144 +2709,6 @@ void kbase_js_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom) queue_work(kctx->jctx.job_done_wq, &katom->work); } -/** - * js_complete_start_rp() - Handle completion of atom that starts a renderpass - * @kctx: Context pointer - * @start_katom: Pointer to the atom that completed - * - * Put any references to virtual memory regions that might have been added by - * kbase_job_slot_softstop_start_rp() because the tiler job chain completed - * despite any pending soft-stop request. - * - * If the atom that just completed was soft-stopped during a previous attempt to - * run it then there should be a blocked end-of-renderpass atom waiting for it, - * which we must unblock to process the output of the tiler job chain. - * - * Return: true if caller should call kbase_backend_ctx_count_changed() - */ -static bool js_complete_start_rp(struct kbase_context *kctx, - struct kbase_jd_atom *const start_katom) -{ - struct kbase_device *const kbdev = kctx->kbdev; - struct kbase_jd_renderpass *rp; - bool timer_sync = false; - - lockdep_assert_held(&kctx->jctx.lock); - - if (WARN_ON(!(start_katom->core_req & BASE_JD_REQ_START_RENDERPASS))) - return false; - - compiletime_assert((1ull << (sizeof(start_katom->renderpass_id) * 8)) <= - ARRAY_SIZE(kctx->jctx.renderpasses), - "Should check invalid access to renderpasses"); - - rp = &kctx->jctx.renderpasses[start_katom->renderpass_id]; - - if (WARN_ON(rp->start_katom != start_katom)) - return false; - - dev_dbg(kctx->kbdev->dev, "Start atom %pK is done in state %d of RP %d\n", - (void *)start_katom, (int)rp->state, start_katom->renderpass_id); - - if (WARN_ON(rp->state == KBASE_JD_RP_COMPLETE)) - return false; - - if (rp->state == KBASE_JD_RP_PEND_OOM || rp->state == KBASE_JD_RP_RETRY_PEND_OOM) { - unsigned long flags; - - dev_dbg(kctx->kbdev->dev, "Start atom %pK completed before soft-stop\n", - (void *)start_katom); - - kbase_gpu_vm_lock(kctx); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - while (!list_empty(&rp->oom_reg_list)) { - struct kbase_va_region *reg = - list_first_entry(&rp->oom_reg_list, struct kbase_va_region, link); - - WARN_ON(reg->flags & KBASE_REG_VA_FREED); - dev_dbg(kctx->kbdev->dev, "Deleting region %pK from list\n", (void *)reg); - list_del_init(®->link); - kbase_va_region_alloc_put(kctx, reg); - } - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - kbase_gpu_vm_unlock(kctx); - } else { - dev_dbg(kctx->kbdev->dev, "Start atom %pK did not exceed memory threshold\n", - (void *)start_katom); - - WARN_ON(rp->state != KBASE_JD_RP_START && rp->state != KBASE_JD_RP_RETRY); - } - - if (rp->state == KBASE_JD_RP_RETRY || rp->state == KBASE_JD_RP_RETRY_PEND_OOM) { - struct kbase_jd_atom *const end_katom = rp->end_katom; - - if (!WARN_ON(!end_katom)) { - unsigned long flags; - - /* Allow the end of the renderpass to be pulled for - * execution again to continue incremental rendering. - */ - dev_dbg(kbdev->dev, "Unblocking end atom %pK!\n", (void *)end_katom); - atomic_dec(&end_katom->blocked); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - timer_sync = kbase_js_ctx_list_add_pullable_nolock(kbdev, kctx, - end_katom->slot_nr); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - } - } - - return timer_sync; -} - -/** - * js_complete_end_rp() - Handle final completion of atom that ends a renderpass - * @kctx: Context pointer - * @end_katom: Pointer to the atom that completed for the last time - * - * This function must only be called if the renderpass actually completed - * without the tiler job chain at the start using too much memory; otherwise - * completion of the end-of-renderpass atom is handled similarly to a soft-stop. - */ -static void js_complete_end_rp(struct kbase_context *kctx, struct kbase_jd_atom *const end_katom) -{ - struct kbase_device *const kbdev = kctx->kbdev; - unsigned long flags; - struct kbase_jd_renderpass *rp; - - lockdep_assert_held(&kctx->jctx.lock); - - if (WARN_ON(!(end_katom->core_req & BASE_JD_REQ_END_RENDERPASS))) - return; - - compiletime_assert((1ull << (sizeof(end_katom->renderpass_id) * 8)) <= - ARRAY_SIZE(kctx->jctx.renderpasses), - "Should check invalid access to renderpasses"); - - rp = &kctx->jctx.renderpasses[end_katom->renderpass_id]; - - if (WARN_ON(rp->end_katom != end_katom)) - return; - - dev_dbg(kbdev->dev, "End atom %pK is done in state %d of RP %d\n", (void *)end_katom, - (int)rp->state, end_katom->renderpass_id); - - if (WARN_ON(rp->state == KBASE_JD_RP_COMPLETE) || WARN_ON(rp->state == KBASE_JD_RP_OOM) || - WARN_ON(rp->state == KBASE_JD_RP_RETRY_OOM)) - return; - - /* Rendering completed without running out of memory. - */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - WARN_ON(!list_empty(&rp->oom_reg_list)); - rp->state = KBASE_JD_RP_COMPLETE; - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - dev_dbg(kbdev->dev, "Renderpass %d is complete\n", end_katom->renderpass_id); -} - bool kbase_js_complete_atom_wq(struct kbase_context *kctx, struct kbase_jd_atom *katom) { struct kbasep_js_kctx_info *js_kctx_info; @@ -3225,13 +2725,6 @@ bool kbase_js_complete_atom_wq(struct kbase_context *kctx, struct kbase_jd_atom dev_dbg(kbdev->dev, "%s for atom %pK (s:%u)\n", __func__, (void *)katom, atom_slot); - /* Update the incremental rendering state machine. - */ - if (katom->core_req & BASE_JD_REQ_START_RENDERPASS) - timer_sync |= js_complete_start_rp(kctx, katom); - else if (katom->core_req & BASE_JD_REQ_END_RENDERPASS) - js_complete_end_rp(kctx, katom); - js_kctx_info = &kctx->jctx.sched_info; js_devdata = &kbdev->js_data; @@ -3320,61 +2813,6 @@ bool kbase_js_complete_atom_wq(struct kbase_context *kctx, struct kbase_jd_atom return context_idle; } -/** - * js_end_rp_is_complete() - Check whether an atom that ends a renderpass has - * completed for the last time. - * - * @end_katom: Pointer to the atom that completed on the hardware. - * - * An atom that ends a renderpass may be run on the hardware several times - * before notifying userspace or allowing dependent atoms to be executed. - * - * This function is used to decide whether or not to allow end-of-renderpass - * atom completion. It only returns false if the atom at the start of the - * renderpass was soft-stopped because it used too much memory during the most - * recent attempt at tiling. - * - * Return: True if the atom completed for the last time. - */ -static bool js_end_rp_is_complete(struct kbase_jd_atom *const end_katom) -{ - struct kbase_context *const kctx = end_katom->kctx; - struct kbase_device *const kbdev = kctx->kbdev; - struct kbase_jd_renderpass *rp; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - if (WARN_ON(!(end_katom->core_req & BASE_JD_REQ_END_RENDERPASS))) - return true; - - compiletime_assert((1ull << (sizeof(end_katom->renderpass_id) * 8)) <= - ARRAY_SIZE(kctx->jctx.renderpasses), - "Should check invalid access to renderpasses"); - - rp = &kctx->jctx.renderpasses[end_katom->renderpass_id]; - - if (WARN_ON(rp->end_katom != end_katom)) - return true; - - dev_dbg(kbdev->dev, "JS complete end atom %pK in state %d of RP %d\n", (void *)end_katom, - (int)rp->state, end_katom->renderpass_id); - - if (WARN_ON(rp->state == KBASE_JD_RP_COMPLETE)) - return true; - - /* Failure of end-of-renderpass atoms must not return to the - * start of the renderpass. - */ - if (end_katom->event_code != BASE_JD_EVENT_DONE) - return true; - - if (rp->state != KBASE_JD_RP_OOM && rp->state != KBASE_JD_RP_RETRY_OOM) - return true; - - dev_dbg(kbdev->dev, "Suppressing end atom completion\n"); - return false; -} - struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, ktime_t *end_timestamp) { struct kbase_device *kbdev; @@ -3387,12 +2825,6 @@ struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, ktime_ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - if ((katom->core_req & BASE_JD_REQ_END_RENDERPASS) && !js_end_rp_is_complete(katom)) { - katom->event_code = BASE_JD_EVENT_END_RP_DONE; - kbase_js_unpull(kctx, katom); - return NULL; - } - if (katom->will_fail_event_code) katom->event_code = katom->will_fail_event_code; @@ -3442,70 +2874,6 @@ struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, ktime_ return NULL; } -/** - * kbase_js_atom_blocked_on_x_dep - Decide whether to ignore a cross-slot - * dependency - * @katom: Pointer to an atom in the slot ringbuffer - * - * A cross-slot dependency is ignored if necessary to unblock incremental - * rendering. If the atom at the start of a renderpass used too much memory - * and was soft-stopped then the atom at the end of a renderpass is submitted - * to hardware regardless of its dependency on the start-of-renderpass atom. - * This can happen multiple times for the same pair of atoms. - * - * Return: true to block the atom or false to allow it to be submitted to - * hardware - */ -bool kbase_js_atom_blocked_on_x_dep(struct kbase_jd_atom *const katom) -{ - struct kbase_context *const kctx = katom->kctx; - struct kbase_device *kbdev = kctx->kbdev; - struct kbase_jd_renderpass *rp; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (!(katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED)) { - dev_dbg(kbdev->dev, "Atom %pK is not blocked on a cross-slot dependency", - (void *)katom); - return false; - } - - if (!(katom->core_req & BASE_JD_REQ_END_RENDERPASS)) { - dev_dbg(kbdev->dev, "Atom %pK is blocked on a cross-slot dependency", - (void *)katom); - return true; - } - - compiletime_assert((1ull << (sizeof(katom->renderpass_id) * 8)) <= - ARRAY_SIZE(kctx->jctx.renderpasses), - "Should check invalid access to renderpasses"); - - rp = &kctx->jctx.renderpasses[katom->renderpass_id]; - /* We can read a subset of renderpass state without holding - * higher-level locks (but not end_katom, for example). - */ - - WARN_ON(rp->state == KBASE_JD_RP_COMPLETE); - - dev_dbg(kbdev->dev, "End atom has cross-slot dep in state %d\n", (int)rp->state); - - if (rp->state != KBASE_JD_RP_OOM && rp->state != KBASE_JD_RP_RETRY_OOM) - return true; - - /* Tiler ran out of memory so allow the fragment job chain to run - * if it only depends on the tiler job chain. - */ - if (katom->x_pre_dep != rp->start_katom) { - dev_dbg(kbdev->dev, "Dependency is on %pK not start atom %pK\n", - (void *)katom->x_pre_dep, (void *)rp->start_katom); - return true; - } - - dev_dbg(kbdev->dev, "Ignoring cross-slot dep on atom %pK\n", (void *)katom->x_pre_dep); - - return false; -} - void kbase_js_sched(struct kbase_device *kbdev, unsigned int js_mask) { struct kbasep_js_device_data *js_devdata; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm.c b/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm.c index aae4df83e98d..87085912bd6c 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -516,7 +516,8 @@ static ssize_t reader_changes_copy_to_user(struct reader_changes *const changes, do { changes_tail = changes->tail; changes_count = reader_changes_count_locked(changes); - read_size = min(changes_count * entry_size, buffer_size & ~(entry_size - 1)); + read_size = + min(size_mul(changes_count, entry_size), buffer_size & ~(entry_size - 1)); if (!read_size) break; @@ -743,7 +744,6 @@ int kbase_kinstr_jm_get_fd(struct kbase_kinstr_jm *const ctx, union kbase_kinstr size_t const change_size = sizeof(struct kbase_kinstr_jm_atom_state_change); int status; int fd; - size_t i; if (!ctx || !jm_fd_arg) return -EINVAL; @@ -753,10 +753,6 @@ int kbase_kinstr_jm_get_fd(struct kbase_kinstr_jm *const ctx, union kbase_kinstr if (!is_power_of_2(in->count)) return -EINVAL; - for (i = 0; i < sizeof(in->padding); ++i) - if (in->padding[i]) - return -EINVAL; - status = reader_init(&reader, ctx, in->count); if (status < 0) return status; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem.c b/drivers/gpu/arm/bifrost/mali_kbase_mem.c index 7470a94a44ad..1436d8290ebc 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -42,7 +43,6 @@ #include #include #include -#include #include #include @@ -52,6 +52,11 @@ static DEFINE_STATIC_KEY_FALSE(large_pages_static_key); #define VA_REGION_SLAB_NAME_PREFIX "va-region-slab-" #define VA_REGION_SLAB_NAME_SIZE (DEVNAME_SIZE + sizeof(VA_REGION_SLAB_NAME_PREFIX) + 1) +#if GPU_PAGES_PER_CPU_PAGE > 1 +#define PAGE_METADATA_SLAB_NAME_PREFIX "page-metadata-slab-" +#define PAGE_METADATA_SLAB_NAME_SIZE (DEVNAME_SIZE + sizeof(PAGE_METADATA_SLAB_NAME_PREFIX) + 1) +#endif + #if MALI_JIT_PRESSURE_LIMIT_BASE /* @@ -152,7 +157,7 @@ static void kbasep_mem_page_size_init(struct kbase_device *kbdev) switch (large_page_conf) { case LARGE_PAGE_AUTO: { - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_LARGE_PAGE_ALLOC)) + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_LARGE_PAGE_ALLOC)) static_branch_inc(&large_pages_static_key); dev_info(kbdev->dev, "Large page allocation set to %s after hardware feature check", static_branch_unlikely(&large_pages_static_key) ? "true" : "false"); @@ -160,7 +165,7 @@ static void kbasep_mem_page_size_init(struct kbase_device *kbdev) } case LARGE_PAGE_ON: { static_branch_inc(&large_pages_static_key); - if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_LARGE_PAGE_ALLOC)) + if (!kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_LARGE_PAGE_ALLOC)) dev_warn(kbdev->dev, "Enabling large page allocations on unsupporting GPU!"); else @@ -196,16 +201,16 @@ KBASE_EXPORT_TEST_API(kbase_is_large_pages_enabled); int kbase_mem_init(struct kbase_device *kbdev) { int err = 0; - struct kbasep_mem_device *memdev; char va_region_slab_name[VA_REGION_SLAB_NAME_SIZE]; +#if GPU_PAGES_PER_CPU_PAGE > 1 + char page_metadata_slab_name[PAGE_METADATA_SLAB_NAME_SIZE]; +#endif #if IS_ENABLED(CONFIG_OF) struct device_node *mgm_node = NULL; #endif KBASE_DEBUG_ASSERT(kbdev); - memdev = &kbdev->memdev; - kbasep_mem_page_size_init(kbdev); scnprintf(va_region_slab_name, VA_REGION_SLAB_NAME_SIZE, VA_REGION_SLAB_NAME_PREFIX "%s", @@ -219,6 +224,17 @@ int kbase_mem_init(struct kbase_device *kbdev) return -ENOMEM; } +#if GPU_PAGES_PER_CPU_PAGE > 1 + scnprintf(page_metadata_slab_name, PAGE_METADATA_SLAB_NAME_SIZE, + PAGE_METADATA_SLAB_NAME_PREFIX "%s", kbdev->devname); + kbdev->page_metadata_slab = kmem_cache_create( + page_metadata_slab_name, sizeof(struct kbase_page_metadata), 0, 0, NULL); + if (kbdev->page_metadata_slab == NULL) { + dev_err(kbdev->dev, "Failed to create page_metadata_slab"); + return -ENOMEM; + } +#endif + kbase_mem_migrate_init(kbdev); kbase_mem_pool_group_config_set_max_size(&kbdev->mem_pool_defaults, KBASE_MEM_POOL_MAX_SIZE_KCTX); @@ -228,12 +244,6 @@ int kbase_mem_init(struct kbase_device *kbdev) kbdev->dma_buf_root = RB_ROOT; mutex_init(&kbdev->dma_buf_lock); -#ifdef IR_THRESHOLD - atomic_set(&memdev->ir_threshold, IR_THRESHOLD); -#else - atomic_set(&memdev->ir_threshold, DEFAULT_IR_THRESHOLD); -#endif - kbdev->mgm_dev = &kbase_native_mgm_dev; #if IS_ENABLED(CONFIG_OF) @@ -299,6 +309,10 @@ void kbase_mem_term(struct kbase_device *kbdev) kbase_mem_migrate_term(kbdev); +#if GPU_PAGES_PER_CPU_PAGE > 1 + kmem_cache_destroy(kbdev->page_metadata_slab); + kbdev->page_metadata_slab = NULL; +#endif kmem_cache_destroy(kbdev->va_region_slab); kbdev->va_region_slab = NULL; @@ -1221,6 +1235,7 @@ int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pa * to satisfy the memory allocation request. */ size_t nr_pages_to_account = 0; + size_t nr_pages_from_partials = 0; if (WARN_ON(alloc->type != KBASE_MEM_TYPE_NATIVE) || WARN_ON(alloc->imported.native.kctx == NULL) || @@ -1279,6 +1294,7 @@ int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pa *tp++ = as_tagged_tag(page_to_phys(sa->page + pidx), FROM_PARTIAL); nr_left--; + nr_pages_from_partials++; if (bitmap_full(sa->sub_pages, NUM_PAGES_IN_2MB_LARGE_PAGE)) { @@ -1386,6 +1402,13 @@ alloc_failed: alloc->nents += nr_pages_to_free; kbase_free_phy_pages_helper(alloc, nr_pages_to_free); + + /* Notice that the sub-pages from "partials" are not subtracted + * from the counter by the free pages helper, because they just go + * back to the "partials" they belong to, therefore they must be + * subtracted from the counter here. + */ + nr_left += nr_pages_from_partials; } /* Undo the preliminary memory accounting that was done early on @@ -1994,11 +2017,13 @@ out_term: } KBASE_EXPORT_TEST_API(kbase_alloc_phy_pages); -void kbase_set_phy_alloc_page_status(struct kbase_mem_phy_alloc *alloc, +void kbase_set_phy_alloc_page_status(struct kbase_context *kctx, struct kbase_mem_phy_alloc *alloc, enum kbase_page_status status) { u32 i = 0; + lockdep_assert_held(&kctx->reg_lock); + for (; i < alloc->nents; i++) { struct tagged_addr phys = alloc->pages[i]; struct kbase_page_metadata *page_md = kbase_page_private(as_page(phys)); @@ -2018,7 +2043,7 @@ void kbase_set_phy_alloc_page_status(struct kbase_mem_phy_alloc *alloc, } } -bool kbase_check_alloc_flags(unsigned long flags) +bool kbase_check_alloc_flags(struct kbase_context *kctx, unsigned long flags) { /* Only known input flags should be set. */ if (flags & ~BASE_MEM_FLAGS_INPUT_MASK) @@ -2094,6 +2119,36 @@ bool kbase_check_alloc_flags(unsigned long flags) return false; #endif + /* Cannot be set only allocation, only with base_mem_set */ + if ((flags & BASE_MEM_DONT_NEED) && + (mali_kbase_supports_reject_alloc_mem_dont_need(kctx->api_version))) + return false; + + /* Cannot directly allocate protected memory, it is imported instead */ + if ((flags & BASE_MEM_PROTECTED) && + (mali_kbase_supports_reject_alloc_mem_protected_in_unprotected_allocs( + kctx->api_version))) + return false; + +/* No unused bits are valid for allocations */ +#if MALI_USE_CSF + if ((flags & BASE_MEM_UNUSED_BIT_20) && + (mali_kbase_supports_reject_alloc_mem_unused_bit_20(kctx->api_version))) + return false; + + if ((flags & BASE_MEM_UNUSED_BIT_27) && + (mali_kbase_supports_reject_alloc_mem_unused_bit_27(kctx->api_version))) + return false; +#else /* MALI_USE_CSF */ + if ((flags & BASE_MEM_UNUSED_BIT_8) && + (mali_kbase_supports_reject_alloc_mem_unused_bit_8(kctx->api_version))) + return false; + + if ((flags & BASE_MEM_UNUSED_BIT_19) && + (mali_kbase_supports_reject_alloc_mem_unused_bit_19(kctx->api_version))) + return false; +#endif /* MALI_USE_CSF */ + return true; } @@ -2258,7 +2313,7 @@ KBASE_EXPORT_TEST_API(kbase_gpu_vm_lock); void kbase_gpu_vm_lock_with_pmode_sync(struct kbase_context *kctx) { #if MALI_USE_CSF - down_read(&kctx->kbdev->csf.pmode_sync_sem); + down_read(&kctx->kbdev->csf.mmu_sync_sem); #endif kbase_gpu_vm_lock(kctx); } @@ -2274,7 +2329,7 @@ void kbase_gpu_vm_unlock_with_pmode_sync(struct kbase_context *kctx) { kbase_gpu_vm_unlock(kctx); #if MALI_USE_CSF - up_read(&kctx->kbdev->csf.pmode_sync_sem); + up_read(&kctx->kbdev->csf.mmu_sync_sem); #endif } @@ -3230,15 +3285,17 @@ struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, if (kbase_is_page_migration_enabled()) { kbase_gpu_vm_lock(kctx); mutex_lock(&kctx->jit_evict_lock); - kbase_set_phy_alloc_page_status(reg->gpu_alloc, ALLOCATED_MAPPED); + kbase_set_phy_alloc_page_status(kctx, reg->gpu_alloc, + ALLOCATED_MAPPED); mutex_unlock(&kctx->jit_evict_lock); kbase_gpu_vm_unlock(kctx); } } } else { /* No suitable JIT allocation was found so create a new one */ - u64 flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR | - BASE_MEM_GROW_ON_GPF | BASE_MEM_COHERENT_LOCAL | BASEP_MEM_NO_USER_FREE; + base_mem_alloc_flags flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_RD | + BASE_MEM_PROT_GPU_WR | BASE_MEM_GROW_ON_GPF | + BASE_MEM_COHERENT_LOCAL | BASEP_MEM_NO_USER_FREE; u64 gpu_addr; #if !MALI_USE_CSF @@ -3335,6 +3392,7 @@ end: return reg; } +KBASE_EXPORT_TEST_API(kbase_jit_allocate); void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg) { @@ -3376,13 +3434,30 @@ void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg) trace_jit_stats(kctx, reg->jit_bin_id, UINT_MAX); + kbase_gpu_vm_lock_with_pmode_sync(kctx); + if (unlikely(atomic_read(®->cpu_alloc->kernel_mappings))) { + WARN_ON(atomic64_read(®->no_user_free_count) > 1); + kbase_va_region_no_user_free_dec(reg); + mutex_lock(&kctx->jit_evict_lock); + list_del(®->jit_node); + mutex_unlock(&kctx->jit_evict_lock); + kbase_mem_free_region(kctx, reg); + kbase_gpu_vm_unlock_with_pmode_sync(kctx); + return; + } kbase_mem_evictable_mark_reclaim(reg->gpu_alloc); - - kbase_gpu_vm_lock(kctx); reg->flags |= KBASE_REG_DONT_NEED; reg->flags &= ~KBASE_REG_ACTIVE_JIT_ALLOC; kbase_mem_shrink_cpu_mapping(kctx, reg, 0, reg->gpu_alloc->nents); - kbase_gpu_vm_unlock(kctx); + + /* Inactive JIT regions should be freed by the shrinker and not impacted + * by page migration. Once freed, they will enter into the page migration + * state machine via the mempools. + */ + if (kbase_is_page_migration_enabled()) + kbase_set_phy_alloc_page_status(kctx, reg->gpu_alloc, NOT_MOVABLE); + + kbase_gpu_vm_unlock_with_pmode_sync(kctx); /* * Add the allocation to the eviction list and the jit pool, after this @@ -3397,14 +3472,9 @@ void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg) list_move(®->jit_node, &kctx->jit_pool_head); - /* Inactive JIT regions should be freed by the shrinker and not impacted - * by page migration. Once freed, they will enter into the page migration - * state machine via the mempools. - */ - if (kbase_is_page_migration_enabled()) - kbase_set_phy_alloc_page_status(reg->gpu_alloc, NOT_MOVABLE); mutex_unlock(&kctx->jit_evict_lock); } +KBASE_EXPORT_TEST_API(kbase_jit_free); void kbase_jit_backing_lost(struct kbase_va_region *reg) { diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem.h b/drivers/gpu/arm/bifrost/mali_kbase_mem.h index 19b50f5bf08b..880b8525ae37 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem.h @@ -104,8 +104,8 @@ static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, int /* Index of chosen MEMATTR for this region (0..7) */ #define KBASE_REG_MEMATTR_MASK (7ul << 16) -#define KBASE_REG_MEMATTR_INDEX(x) (((x)&7) << 16) -#define KBASE_REG_MEMATTR_VALUE(x) (((x)&KBASE_REG_MEMATTR_MASK) >> 16) +#define KBASE_REG_MEMATTR_INDEX(x) (((x) & 7) << 16) +#define KBASE_REG_MEMATTR_VALUE(x) (((x) & KBASE_REG_MEMATTR_MASK) >> 16) /* AS_MEMATTR values from MMU_MEMATTR_STAGE1: */ /* Use GPU implementation-defined caching policy. */ @@ -482,6 +482,26 @@ struct kbase_page_metadata { struct kbase_mmu_table *mmut; /* GPU virtual page frame number info is in GPU_PAGE_SIZE units */ u64 pgd_vpfn_level; +#if GPU_PAGES_PER_CPU_PAGE > 1 + /** + * @pgd_link: Link to the &kbase_mmu_table.pgd_pages_list + */ + struct list_head pgd_link; + /** + * @pgd_page: Back pointer to the PGD page that the metadata is + * associated with + */ + struct page *pgd_page; + /** + * @allocated_sub_pages: Bitmap representing the allocation status + * of sub pages in the @pgd_page + */ + DECLARE_BITMAP(allocated_sub_pages, GPU_PAGES_PER_CPU_PAGE); + /** + * @num_allocated_sub_pages: The number of allocated sub pages in @pgd_page + */ + s8 num_allocated_sub_pages; +#endif } pt_mapped; struct { struct kbase_device *kbdev; @@ -510,6 +530,7 @@ enum kbase_jit_report_flags { KBASE_JIT_REPORT_ON_ALLOC_OR_FREE = (1u << 0) }; /** * kbase_set_phy_alloc_page_status - Set the page migration status of the underlying * physical allocation. + * @kctx: Pointer to Kbase context. * @alloc: the physical allocation containing the pages whose metadata is going * to be modified * @status: the status the pages should end up in @@ -518,7 +539,7 @@ enum kbase_jit_report_flags { KBASE_JIT_REPORT_ON_ALLOC_OR_FREE = (1u << 0) }; * proper states are set. Instead, it is only used when we change the allocation * to NOT_MOVABLE or from NOT_MOVABLE to ALLOCATED_MAPPED */ -void kbase_set_phy_alloc_page_status(struct kbase_mem_phy_alloc *alloc, +void kbase_set_phy_alloc_page_status(struct kbase_context *kctx, struct kbase_mem_phy_alloc *alloc, enum kbase_page_status status); static inline void kbase_mem_phy_alloc_gpu_mapped(struct kbase_mem_phy_alloc *alloc) @@ -620,9 +641,6 @@ static inline struct kbase_mem_phy_alloc *kbase_mem_phy_alloc_put(struct kbase_m * @nr_pages: The size of the region in pages. * @initial_commit: Initial commit, for aligning the start address and * correctly growing KBASE_REG_TILER_ALIGN_TOP regions. - * @threshold_pages: If non-zero and the amount of memory committed to a region - * that can grow on page fault exceeds this number of pages - * then the driver switches to incremental rendering. * @flags: Flags * @extension: Number of pages allocated on page fault. * @cpu_alloc: The physical memory we mmap to the CPU when mapping this region. @@ -659,8 +677,7 @@ struct kbase_va_region { void *user_data; size_t nr_pages; size_t initial_commit; - size_t threshold_pages; - unsigned long flags; + base_mem_alloc_flags flags; size_t extension; struct kbase_mem_phy_alloc *cpu_alloc; struct kbase_mem_phy_alloc *gpu_alloc; @@ -914,10 +931,12 @@ static inline struct kbase_mem_phy_alloc *kbase_alloc_create(struct kbase_contex atomic_set(&alloc->gpu_mappings, 0); atomic_set(&alloc->kernel_mappings, 0); alloc->nents = 0; - alloc->pages = (void *)(alloc + 1); - /* fill pages with invalid address value */ - for (i = 0; i < nr_pages; i++) - alloc->pages[i] = as_tagged(KBASE_INVALID_PHYSICAL_ADDRESS); + if (type != KBASE_MEM_TYPE_ALIAS) { + alloc->pages = (void *)(alloc + 1); + /* fill pages with invalid address value */ + for (i = 0; i < nr_pages; i++) + alloc->pages[i] = as_tagged(KBASE_INVALID_PHYSICAL_ADDRESS); + } INIT_LIST_HEAD(&alloc->mappings); alloc->type = type; alloc->group_id = group_id; @@ -1307,7 +1326,7 @@ struct page *kbase_mem_alloc_page(struct kbase_mem_pool *pool, const bool alloc_ */ void kbase_mem_pool_free_page(struct kbase_mem_pool *pool, struct page *p); -bool kbase_check_alloc_flags(unsigned long flags); +bool kbase_check_alloc_flags(struct kbase_context *kctx, unsigned long flags); bool kbase_check_import_flags(unsigned long flags); static inline bool kbase_import_size_is_valid(struct kbase_device *kbdev, u64 va_pages) @@ -1799,8 +1818,8 @@ static inline dma_addr_t kbase_dma_addr_from_tagged(struct tagged_addr tagged_pa phys_addr_t pa = as_phys_addr_t(tagged_pa); struct page *page = pfn_to_page(PFN_DOWN(pa)); dma_addr_t dma_addr = (is_huge(tagged_pa) || is_partial(tagged_pa)) ? - kbase_dma_addr_as_priv(page) : - kbase_dma_addr(page); + kbase_dma_addr_as_priv(page) : + kbase_dma_addr(page); return dma_addr; } diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c b/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c index 7c319abf381b..9f26c088630c 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c @@ -83,8 +83,6 @@ #define KBASE_MEM_ION_SYNC_WORKAROUND #endif -#define IR_THRESHOLD_STEPS (256u) - /* * fully_backed_gpf_memory - enable full physical backing of all grow-on-GPU-page-fault * allocations in the kernel. @@ -295,7 +293,7 @@ void kbase_phy_alloc_mapping_put(struct kbase_context *kctx, struct kbase_vmap_s } struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages, u64 commit_pages, - u64 extension, u64 *flags, u64 *gpu_va, + u64 extension, base_mem_alloc_flags *flags, u64 *gpu_va, enum kbase_caller_mmu_sync_info mmu_sync_info) { struct kbase_va_region *reg; @@ -320,9 +318,8 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages else dev_dbg(dev, "Keeping requested GPU VA of 0x%llx\n", (unsigned long long)*gpu_va); - if (!kbase_check_alloc_flags(*flags)) { - dev_warn(dev, "%s called with bad flags (%llx)", __func__, - (unsigned long long)*flags); + if (!kbase_check_alloc_flags(kctx, *flags)) { + dev_warn(dev, "%s called with bad flags (%llx)", __func__, *flags); goto bad_flags; } @@ -335,6 +332,12 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages } #endif + /* Ensure GPU cached if CPU cached */ + if ((*flags & BASE_MEM_CACHED_CPU) != 0) { + dev_warn_once(dev, "Clearing BASE_MEM_UNCACHED_GPU flag to avoid MMA violation\n"); + *flags &= ~BASE_MEM_UNCACHED_GPU; + } + if ((*flags & BASE_MEM_UNCACHED_GPU) != 0 && (*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0) { /* Remove COHERENT_SYSTEM_REQUIRED flag if uncached GPU mapping is requested */ @@ -406,17 +409,8 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages *flags &= ~BASE_MEM_CACHED_CPU; if (*flags & BASE_MEM_GROW_ON_GPF) { - unsigned int const ir_threshold = - (unsigned int)atomic_read(&kctx->kbdev->memdev.ir_threshold); - - reg->threshold_pages = - ((va_pages * ir_threshold) + (IR_THRESHOLD_STEPS / 2)) / IR_THRESHOLD_STEPS; - } else - reg->threshold_pages = 0; - - if (*flags & BASE_MEM_GROW_ON_GPF) { - /* kbase_check_alloc_sizes() already checks extension is valid for - * assigning to reg->extension + /* kbase_check_alloc_sizes() already checks extension is valid for assigning to + * reg->extension. */ reg->extension = extension; #if !MALI_USE_CSF @@ -597,11 +591,11 @@ int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, u64 query, u64 *co *out |= BASE_MEM_COHERENT_SYSTEM; if (KBASE_REG_SHARE_IN & reg->flags) *out |= BASE_MEM_COHERENT_LOCAL; - if (mali_kbase_supports_mem_dont_need(kctx->api_version)) { + if (mali_kbase_supports_query_mem_dont_need(kctx->api_version)) { if (KBASE_REG_DONT_NEED & reg->flags) *out |= BASE_MEM_DONT_NEED; } - if (mali_kbase_supports_mem_grow_on_gpf(kctx->api_version)) { + if (mali_kbase_supports_query_mem_grow_on_gpf(kctx->api_version)) { /* Prior to this version, this was known about by * user-side but we did not return them. Returning * it caused certain clients that were not expecting @@ -611,7 +605,7 @@ int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, u64 query, u64 *co if (KBASE_REG_PF_GROW & reg->flags) *out |= BASE_MEM_GROW_ON_GPF; } - if (mali_kbase_supports_mem_protected(kctx->api_version)) { + if (mali_kbase_supports_query_mem_protected(kctx->api_version)) { /* Prior to this version, this was known about by * user-side but we did not return them. Returning * it caused certain clients that were not expecting @@ -640,17 +634,17 @@ int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, u64 query, u64 *co #endif /* MALI_USE_CSF */ if (KBASE_REG_GPU_VA_SAME_4GB_PAGE & reg->flags) *out |= BASE_MEM_GPU_VA_SAME_4GB_PAGE; - if (mali_kbase_supports_mem_import_sync_on_map_unmap(kctx->api_version)) { + if (mali_kbase_supports_query_mem_import_sync_on_map_unmap(kctx->api_version)) { if (reg->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM) { if (reg->gpu_alloc->imported.umm.need_sync) *out |= BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP; } } - if (mali_kbase_supports_mem_kernel_sync(kctx->api_version)) { + if (mali_kbase_supports_query_mem_kernel_sync(kctx->api_version)) { if (unlikely(reg->cpu_alloc != reg->gpu_alloc)) *out |= BASE_MEM_KERNEL_SYNC; } - if (mali_kbase_supports_mem_same_va(kctx->api_version)) { + if (mali_kbase_supports_query_mem_same_va(kctx->api_version)) { if (kbase_bits_to_zone(reg->flags) == SAME_VA_ZONE) { /* Imported memory is an edge case, where declaring it SAME_VA * would be ambiguous. @@ -746,7 +740,7 @@ static unsigned long kbase_mem_evictable_reclaim_scan_objects(struct shrinker *s kctx = KBASE_GET_KBASE_DATA_FROM_SHRINKER(s, struct kbase_context, reclaim); #if MALI_USE_CSF - if (!down_read_trylock(&kctx->kbdev->csf.pmode_sync_sem)) { + if (!down_read_trylock(&kctx->kbdev->csf.mmu_sync_sem)) { dev_warn(kctx->kbdev->dev, "Can't shrink GPU memory when P.Mode entrance is in progress"); return 0; @@ -791,7 +785,7 @@ static unsigned long kbase_mem_evictable_reclaim_scan_objects(struct shrinker *s mutex_unlock(&kctx->jit_evict_lock); #if MALI_USE_CSF - up_read(&kctx->kbdev->csf.pmode_sync_sem); + up_read(&kctx->kbdev->csf.mmu_sync_sem); #endif return freed; } @@ -886,7 +880,7 @@ void kbase_mem_evictable_make(struct kbase_mem_phy_alloc *gpu_alloc) /* Indicate to page migration that the memory can be reclaimed by the shrinker. */ if (kbase_is_page_migration_enabled()) - kbase_set_phy_alloc_page_status(gpu_alloc, NOT_MOVABLE); + kbase_set_phy_alloc_page_status(kctx, gpu_alloc, NOT_MOVABLE); mutex_unlock(&kctx->jit_evict_lock); kbase_mem_evictable_mark_reclaim(gpu_alloc); @@ -944,7 +938,7 @@ bool kbase_mem_evictable_unmake(struct kbase_mem_phy_alloc *gpu_alloc) * from. */ if (kbase_is_page_migration_enabled()) - kbase_set_phy_alloc_page_status(gpu_alloc, ALLOCATED_MAPPED); + kbase_set_phy_alloc_page_status(kctx, gpu_alloc, ALLOCATED_MAPPED); } } @@ -962,7 +956,8 @@ bool kbase_mem_evictable_unmake(struct kbase_mem_phy_alloc *gpu_alloc) * * Return: 0 on success, error code otherwise. */ -static int kbase_mem_flags_change_imported_umm(struct kbase_context *kctx, unsigned int flags, +static int kbase_mem_flags_change_imported_umm(struct kbase_context *kctx, + base_mem_alloc_flags flags, struct kbase_va_region *reg) { unsigned int real_flags = 0; @@ -1045,7 +1040,7 @@ static int kbase_mem_flags_change_imported_umm(struct kbase_context *kctx, unsig * * Return: 0 on success, error code otherwise. */ -static int kbase_mem_flags_change_native(struct kbase_context *kctx, unsigned int flags, +static int kbase_mem_flags_change_native(struct kbase_context *kctx, base_mem_alloc_flags flags, struct kbase_va_region *reg) { bool kbase_reg_dont_need_flag = (KBASE_REG_DONT_NEED & reg->flags); @@ -1077,8 +1072,8 @@ static int kbase_mem_flags_change_native(struct kbase_context *kctx, unsigned in return ret; } -int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned int flags, - unsigned int mask) +int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, base_mem_alloc_flags flags, + base_mem_alloc_flags mask) { struct kbase_va_region *reg; int ret = -EINVAL; @@ -1468,7 +1463,7 @@ static int get_umm_memory_group_id(struct kbase_context *kctx, struct dma_buf *d * object that wraps the dma-buf. */ static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx, int fd, u64 *va_pages, - u64 *flags, u32 padding) + base_mem_alloc_flags *flags, u32 padding) { struct kbase_va_region *reg; struct dma_buf *dma_buf; @@ -1614,7 +1609,8 @@ u32 kbase_get_cache_line_alignment(struct kbase_device *kbdev) static struct kbase_va_region *kbase_mem_from_user_buffer(struct kbase_context *kctx, unsigned long address, unsigned long size, - u64 *va_pages, u64 *flags) + u64 *va_pages, + base_mem_alloc_flags *flags) { struct kbase_va_region *reg; enum kbase_memory_zone zone = CUSTOM_VA_ZONE; @@ -1746,7 +1742,7 @@ bad_size: return NULL; } -u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, u64 nents, +u64 kbase_mem_alias(struct kbase_context *kctx, base_mem_alloc_flags *flags, u64 stride, u64 nents, struct base_mem_aliasing_info *ai, u64 *num_pages) { struct kbase_va_region *reg; @@ -1968,7 +1964,8 @@ bad_flags: } int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, - void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages, u64 *flags) + void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages, + base_mem_alloc_flags *flags) { struct kbase_va_region *reg; @@ -3107,7 +3104,7 @@ static int kbase_vmap_phy_pages(struct kbase_context *kctx, struct kbase_va_regi return -ENOMEM; } - if (reg->flags & KBASE_REG_DONT_NEED) + if (kbase_is_region_shrinkable(reg)) return -EINVAL; prot = PAGE_KERNEL; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.h b/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.h index 28666037d8c6..a4b3db7fdf89 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -51,7 +51,7 @@ struct kbase_hwc_dma_mapping { * Return: 0 on success or error code */ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages, u64 commit_pages, - u64 extension, u64 *flags, u64 *gpu_va, + u64 extension, base_mem_alloc_flags *flags, u64 *gpu_va, enum kbase_caller_mmu_sync_info mmu_sync_info); /** @@ -84,7 +84,8 @@ int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, u64 query, u64 *co * Return: 0 on success or error code */ int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, - void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages, u64 *flags); + void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages, + base_mem_alloc_flags *flags); /** * kbase_mem_alias - Create a new allocation for GPU, aliasing one or more @@ -99,7 +100,7 @@ int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, * * Return: 0 on failure or otherwise the GPU VA for the alias */ -u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, u64 nents, +u64 kbase_mem_alias(struct kbase_context *kctx, base_mem_alloc_flags *flags, u64 stride, u64 nents, struct base_mem_aliasing_info *ai, u64 *num_pages); /** @@ -112,8 +113,8 @@ u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, u64 nent * * Return: 0 on success or error code */ -int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned int flags, - unsigned int mask); +int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, base_mem_alloc_flags flags, + base_mem_alloc_flags mask); /** * kbase_mem_commit - Change the physical backing size of a region diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_migrate.c b/drivers/gpu/arm/bifrost/mali_kbase_mem_migrate.c index f0ce0cf4e56c..eecab323f59f 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem_migrate.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_migrate.c @@ -68,6 +68,12 @@ bool kbase_alloc_page_metadata(struct kbase_device *kbdev, struct page *p, dma_a if (!IS_ENABLED(CONFIG_PAGE_MIGRATION_SUPPORT)) return false; + /* Composite large-page is excluded from migration, trigger a warn if a development + * wrongly leads to it. + */ + if (is_huge_head(as_tagged(page_to_phys(p))) || is_partial(as_tagged(page_to_phys(p)))) + dev_WARN(kbdev->dev, "%s: migration-metadata attempted on large-page.", __func__); + page_md = kzalloc(sizeof(struct kbase_page_metadata), GFP_KERNEL); if (!page_md) return false; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_native_mgm.c b/drivers/gpu/arm/bifrost/mali_kbase_native_mgm.c index d688509cee03..f9a3788a2ecf 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_native_mgm.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_native_mgm.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -122,12 +122,16 @@ static vm_fault_t kbase_native_mgm_vmf_insert_pfn_prot(struct memory_group_manag } static u64 kbase_native_mgm_update_gpu_pte(struct memory_group_manager_device *mgm_dev, - unsigned int group_id, int mmu_level, u64 pte) + unsigned int group_id, unsigned int pbha_id, + unsigned int pte_flags, int mmu_level, u64 pte) { if (WARN_ON(group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS)) return pte; - pte |= ((u64)group_id << PTE_PBHA_SHIFT) & PTE_PBHA_MASK; + if ((pte_flags & BIT(MMA_VIOLATION)) && pbha_id) { + pr_warn_once("MMA violation! Applying PBHA override workaround to PTE\n"); + pte |= ((u64)pbha_id << PTE_PBHA_SHIFT) & PTE_PBHA_MASK; + } /* Address could be translated into a different bus address here */ pte |= ((u64)1 << PTE_RES_BIT_MULTI_AS_SHIFT); @@ -150,14 +154,24 @@ static u64 kbase_native_mgm_pte_to_original_pte(struct memory_group_manager_devi return pte; } +static bool kbase_native_mgm_get_import_memory_cached_access_permitted( + struct memory_group_manager_device *mgm_dev, + struct memory_group_manager_import_data *import_data) +{ + CSTD_UNUSED(mgm_dev); + CSTD_UNUSED(import_data); + + return true; +} + struct memory_group_manager_device kbase_native_mgm_dev = { - .ops = { - .mgm_alloc_page = kbase_native_mgm_alloc, - .mgm_free_page = kbase_native_mgm_free, - .mgm_get_import_memory_id = NULL, - .mgm_vmf_insert_pfn_prot = kbase_native_mgm_vmf_insert_pfn_prot, - .mgm_update_gpu_pte = kbase_native_mgm_update_gpu_pte, - .mgm_pte_to_original_pte = kbase_native_mgm_pte_to_original_pte, - }, + .ops = { .mgm_alloc_page = kbase_native_mgm_alloc, + .mgm_free_page = kbase_native_mgm_free, + .mgm_get_import_memory_id = NULL, + .mgm_vmf_insert_pfn_prot = kbase_native_mgm_vmf_insert_pfn_prot, + .mgm_update_gpu_pte = kbase_native_mgm_update_gpu_pte, + .mgm_pte_to_original_pte = kbase_native_mgm_pte_to_original_pte, + .mgm_get_import_memory_cached_access_permitted = + kbase_native_mgm_get_import_memory_cached_access_permitted }, .data = NULL }; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_pbha.c b/drivers/gpu/arm/bifrost/mali_kbase_pbha.c index c5b6fada2451..ea79811ea293 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_pbha.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_pbha.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2021-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2021-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -220,6 +220,24 @@ void kbase_pbha_write_settings(struct kbase_device *kbdev) for (i = 0; i < GPU_SYSC_ALLOC_COUNT; ++i) kbase_reg_write32(kbdev, GPU_SYSC_ALLOC_OFFSET(i), kbdev->sysc_alloc[i]); } + + if (kbdev->mma_wa_id) { + /* PBHA OVERRIDE register index (0-3) */ + uint reg_index = kbdev->mma_wa_id >> 2; + /* PBHA index within a PBHA OVERRIDE register (0-3) */ + uint pbha_index = kbdev->mma_wa_id & 0x3; + /* 4 bits of read attributes + 4 bits of write attributes for each PBHA */ + uint pbha_shift = pbha_index * 8; + /* Noncacheable read = noncacheable write = b0001*/ + uint pbha_override_rw_noncacheable = 0x01 | 0x10; + + u32 pbha_override_val = + kbase_reg_read32(kbdev, GPU_SYSC_PBHA_OVERRIDE_OFFSET(reg_index)); + pbha_override_val &= ~((u32)0xFF << pbha_shift); + pbha_override_val |= ((u32)pbha_override_rw_noncacheable << pbha_shift); + kbase_reg_write32(kbdev, GPU_SYSC_PBHA_OVERRIDE_OFFSET(reg_index), + pbha_override_val); + } #else CSTD_UNUSED(kbdev); #endif /* MALI_USE_CSF */ @@ -280,7 +298,7 @@ static int kbase_pbha_read_propagate_bits_property(struct kbase_device *kbdev, u8 bits = 0; int err; - if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PBHA_HWU)) + if (!kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_PBHA_HWU)) return 0; err = of_property_read_u8(pbha_node, "propagate-bits", &bits); @@ -310,6 +328,43 @@ static int kbase_pbha_read_propagate_bits_property(struct kbase_device *kbdev, kbdev->pbha_propagate_bits = bits; return 0; } + +static int kbase_pbha_read_mma_wa_id_property(struct kbase_device *kbdev, + const struct device_node *pbha_node) +{ + u32 mma_wa_id = 0; + int err; + + /* Skip if kbdev->mma_wa_id has already been set via the module parameter */ + if ((kbdev->gpu_props.gpu_id.arch_id < GPU_ID_ARCH_MAKE(14, 8, 0)) || kbdev->mma_wa_id != 0) + return 0; + + err = of_property_read_u32(pbha_node, "mma-wa-id", &mma_wa_id); + + /* Property does not exist. This is not a mandatory property, ignore this error */ + if (err == -EINVAL) + return 0; + + if (err == -ENODATA) { + dev_err(kbdev->dev, "DTB property mma-wa-id has no value\n"); + return err; + } + + if (err == -EOVERFLOW) { + dev_err(kbdev->dev, "DTB value for mma-wa-id is out of range\n"); + return err; + } + + if (mma_wa_id == 0 || mma_wa_id > 15) { + dev_err(kbdev->dev, + "Invalid DTB value for mma-wa-id: %u. Valid range is between 1 and 15.\n", + mma_wa_id); + return -EINVAL; + } + + kbdev->mma_wa_id = mma_wa_id; + return 0; +} #endif /* MALI_USE_CSF */ int kbase_pbha_read_dtb(struct kbase_device *kbdev) @@ -331,6 +386,12 @@ int kbase_pbha_read_dtb(struct kbase_device *kbdev) return err; err = kbase_pbha_read_propagate_bits_property(kbdev, pbha_node); + + if (err < 0) + return err; + + err = kbase_pbha_read_mma_wa_id_property(kbdev, pbha_node); + return err; #else return 0; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_pbha_debugfs.c b/drivers/gpu/arm/bifrost/mali_kbase_pbha_debugfs.c index 8ab0d1823165..81f2df5ea977 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_pbha_debugfs.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_pbha_debugfs.c @@ -234,7 +234,7 @@ void kbase_pbha_debugfs_init(struct kbase_device *kbdev) debugfs_create_file("int_id_overrides", mode, debugfs_pbha_dir, kbdev, &pbha_int_id_overrides_fops); #if MALI_USE_CSF - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PBHA_HWU)) + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_PBHA_HWU)) debugfs_create_file("propagate_bits", mode, debugfs_pbha_dir, kbdev, &pbha_propagate_bits_fops); #endif /* MALI_USE_CSF */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_pm.c b/drivers/gpu/arm/bifrost/mali_kbase_pm.c index 17c34f334aad..6719a120c1f3 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_pm.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_pm.c @@ -31,9 +31,7 @@ #include #include -#ifdef CONFIG_MALI_ARBITER_SUPPORT #include -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ #include @@ -62,10 +60,11 @@ int kbase_pm_context_active_handle_suspend_locked(struct kbase_device *kbdev, current->pid); lockdep_assert_held(&kbdev->pm.lock); -#ifdef CONFIG_MALI_ARBITER_SUPPORT + /* If there is an Arbiter, wait for Arbiter to grant GPU back to KBase + * so suspend request can be handled. + */ if (kbase_arbiter_pm_ctx_active_handle_suspend(kbdev, suspend_handler)) return 1; -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ if (kbase_pm_is_suspending(kbdev)) { switch (suspend_handler) { @@ -91,9 +90,7 @@ int kbase_pm_context_active_handle_suspend_locked(struct kbase_device *kbdev, * any cores requested by the policy */ kbase_hwaccess_pm_gpu_active(kbdev); -#ifdef CONFIG_MALI_ARBITER_SUPPORT kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_REF_EVENT); -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ kbase_clk_rate_trace_manager_gpu_active(kbdev); } @@ -222,8 +219,7 @@ int kbase_pm_driver_suspend(struct kbase_device *kbdev) kbdev->pm.suspending = true; mutex_unlock(&kbdev->pm.lock); -#ifdef CONFIG_MALI_ARBITER_SUPPORT - if (kbdev->arb.arb_if) { + if (kbase_has_arbiter(kbdev)) { unsigned long flags; #if MALI_USE_CSF @@ -241,7 +237,6 @@ int kbase_pm_driver_suspend(struct kbase_device *kbdev) spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); #endif } -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ /* From now on, the active count will drop towards zero. Sometimes, * it'll go up briefly before going down again. However, once @@ -287,19 +282,17 @@ int kbase_pm_driver_suspend(struct kbase_device *kbdev) */ if (kbase_hwaccess_pm_suspend(kbdev)) { /* No early return yet */ - if (IS_ENABLED(CONFIG_MALI_ARBITER_SUPPORT)) + if (kbase_has_arbiter(kbdev)) WARN_ON_ONCE(1); else goto exit; } -#ifdef CONFIG_MALI_ARBITER_SUPPORT - if (kbdev->arb.arb_if) { + if (kbase_has_arbiter(kbdev)) { mutex_lock(&kbdev->pm.arb_vm_state->vm_state_lock); kbase_arbiter_pm_vm_stopped(kbdev); mutex_unlock(&kbdev->pm.arb_vm_state->vm_state_lock); } -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ #if MALI_USE_CSF kbase_backend_invalidate_gpu_timestamp_offset(kbdev); @@ -339,14 +332,13 @@ void kbase_pm_driver_resume(struct kbase_device *kbdev, bool arb_gpu_start) kbase_hwaccess_pm_resume(kbdev); /* Initial active call, to power on the GPU/cores if needed */ -#ifdef CONFIG_MALI_ARBITER_SUPPORT - if (kbase_pm_context_active_handle_suspend( - kbdev, (arb_gpu_start ? KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED : - KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE))) - return; -#else - kbase_pm_context_active(kbdev); -#endif + if (kbase_has_arbiter(kbdev)) { + if (kbase_pm_context_active_handle_suspend( + kbdev, (arb_gpu_start ? KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED : + KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE))) + return; + } else + kbase_pm_context_active(kbdev); resume_job_scheduling(kbdev); @@ -371,26 +363,18 @@ int kbase_pm_suspend(struct kbase_device *kbdev) { int result = 0; -#ifdef CONFIG_MALI_ARBITER_SUPPORT - if (kbdev->arb.arb_if) + if (kbase_has_arbiter(kbdev)) kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_OS_SUSPEND_EVENT); else result = kbase_pm_driver_suspend(kbdev); -#else - result = kbase_pm_driver_suspend(kbdev); -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ return result; } void kbase_pm_resume(struct kbase_device *kbdev) { -#ifdef CONFIG_MALI_ARBITER_SUPPORT - if (kbdev->arb.arb_if) + if (kbase_has_arbiter(kbdev)) kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_OS_RESUME_EVENT); else kbase_pm_driver_resume(kbdev, false); -#else - kbase_pm_driver_resume(kbdev, false); -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ } diff --git a/drivers/gpu/arm/bifrost/mali_kbase_pm.h b/drivers/gpu/arm/bifrost/mali_kbase_pm.h index 0c3575bfd54a..25e4732a8d08 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_pm.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_pm.h @@ -120,12 +120,10 @@ enum kbase_pm_suspend_handler { * (e.g. guarantee it's going to be idled very soon after) */ KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE, -#ifdef CONFIG_MALI_ARBITER_SUPPORT /** Special case when Arbiter has notified we can use GPU. * Active count should always start at 0 in this case. */ KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED, -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ }; /** @@ -234,7 +232,7 @@ void kbase_pm_vsync_callback(int buffer_updated, void *data); * kbase components to complete the suspend. * * Despite kbase_pm_suspend(), it will ignore to update Arbiter - * status if MALI_ARBITER_SUPPORT is enabled. + * status if there is one. * * @note the mechanisms used here rely on all user-space threads being frozen * by the OS before we suspend. Otherwise, an IOCTL could occur that powers up @@ -258,11 +256,10 @@ int kbase_pm_driver_suspend(struct kbase_device *kbdev); * Also called when using VM arbiter, when GPU access has been granted. * * Despite kbase_pm_resume(), it will ignore to update Arbiter - * status if MALI_ARBITER_SUPPORT is enabled. + * status if there is one. */ void kbase_pm_driver_resume(struct kbase_device *kbdev, bool arb_gpu_start); -#ifdef CONFIG_MALI_ARBITER_SUPPORT /** * kbase_pm_handle_gpu_lost() - Handle GPU Lost for the VM * @kbdev: Device pointer @@ -273,6 +270,5 @@ void kbase_pm_driver_resume(struct kbase_device *kbdev, bool arb_gpu_start); * Kill any running tasks and put the driver into a GPU powered-off state. */ void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev); -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ #endif /* _KBASE_PM_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_reg_track.c b/drivers/gpu/arm/bifrost/mali_kbase_reg_track.c index 3128292a9a30..e490a2a3d179 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_reg_track.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_reg_track.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -648,7 +648,7 @@ int kbase_add_va_region_rbtree(struct kbase_device *kbdev, struct kbase_va_regio } else if (!kbase_is_region_free(tmp)) { dev_warn( dev, - "!(tmp->flags & KBASE_REG_FREE): tmp->start_pfn=0x%llx tmp->flags=0x%lx tmp->nr_pages=0x%zx gpu_pfn=0x%llx nr_pages=0x%zx\n", + "!(tmp->flags & KBASE_REG_FREE): tmp->start_pfn=0x%llx tmp->flags=0x%llx tmp->nr_pages=0x%zx gpu_pfn=0x%llx nr_pages=0x%zx\n", tmp->start_pfn, tmp->flags, tmp->nr_pages, gpu_pfn, nr_pages); err = -ENOMEM; goto exit; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c b/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c index c5c16b497369..bae1630c94a9 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c @@ -143,9 +143,8 @@ static int kbase_dump_cpu_gpu_time(struct kbase_jd_atom *katom) * delay suspend until we process the atom (which may be at the end of a * long chain of dependencies */ -#ifdef CONFIG_MALI_ARBITER_SUPPORT - atomic_inc(&kctx->kbdev->pm.gpu_users_waiting); -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + if (kbase_has_arbiter(kctx->kbdev)) + atomic_inc(&kctx->kbdev->pm.gpu_users_waiting); pm_active_err = kbase_pm_context_active_handle_suspend( kctx->kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE); if (pm_active_err) { @@ -163,11 +162,8 @@ static int kbase_dump_cpu_gpu_time(struct kbase_jd_atom *katom) kbasep_add_waiting_soft_job(katom); return pm_active_err; - } -#ifdef CONFIG_MALI_ARBITER_SUPPORT - else + } else if (kbase_has_arbiter(kctx->kbdev)) atomic_dec(&kctx->kbdev->pm.gpu_users_waiting); -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ kbase_backend_get_gpu_time(kctx->kbdev, &cycle_counter, &system_time, &ts); @@ -553,7 +549,7 @@ static int kbase_debug_copy_prepare(struct kbase_jd_atom *katom) goto out_cleanup; } - ret = copy_from_user(user_buffers, user_structs, sizeof(*user_buffers) * nr); + ret = copy_from_user(user_buffers, user_structs, size_mul(sizeof(*user_buffers), nr)); if (ret) { ret = -EFAULT; goto out_cleanup; @@ -1235,7 +1231,7 @@ static int kbase_jit_free_prepare(struct kbase_jd_atom *katom) goto free_info; } - if (copy_from_user(ids, data, sizeof(*ids) * count) != 0) { + if (copy_from_user(ids, data, size_mul(sizeof(*ids), count)) != 0) { ret = -EINVAL; goto free_info; } @@ -1688,9 +1684,8 @@ void kbase_resume_suspended_soft_jobs(struct kbase_device *kbdev) if (kbase_process_soft_job(katom_iter) == 0) { kbase_finish_soft_job(katom_iter); resched |= kbase_jd_done_nolock(katom_iter, true); -#ifdef CONFIG_MALI_ARBITER_SUPPORT - atomic_dec(&kbdev->pm.gpu_users_waiting); -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + if (kbase_has_arbiter(kctx->kbdev)) + atomic_dec(&kbdev->pm.gpu_users_waiting); } mutex_unlock(&kctx->jctx.lock); } diff --git a/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_csf.c b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_csf.c index bd5f3914b8c7..196d481d6827 100644 --- a/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_csf.c +++ b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_csf.c @@ -100,7 +100,7 @@ void kbase_mmu_report_mcu_as_fault_and_reset(struct kbase_device *kbdev, struct u32 as_no; /* terminal fault, print info about the fault */ - if (kbdev->gpu_props.gpu_id.product_model <= GPU_ID_MODEL_MAKE(13, 0)) { + if (kbdev->gpu_props.gpu_id.product_model < GPU_ID_MODEL_MAKE(14, 0)) { dev_err(kbdev->dev, "Unexpected Page fault in firmware address space at VA 0x%016llX\n" "raw fault status: 0x%X\n" @@ -115,22 +115,6 @@ void kbase_mmu_report_mcu_as_fault_and_reset(struct kbase_device *kbdev, struct fault_source_id_internal_requester_get(kbdev, source_id), fault_source_id_core_type_description_get(kbdev, source_id), fault_source_id_internal_requester_get_str(kbdev, source_id, access_type)); - } else { - dev_err(kbdev->dev, - "Unexpected Page fault in firmware address space at VA 0x%016llX\n" - "raw fault status: 0x%X\n" - "exception type 0x%X: %s\n" - "access type 0x%X: %s\n" - "source id 0x%X (type:idx:IR 0x%X:0x%X:0x%X): %s %u, %s\n", - fault->addr, fault->status, exception_type, - kbase_gpu_exception_name(exception_type), access_type, - kbase_gpu_access_type_name(fault->status), source_id, - FAULT_SOURCE_ID_CORE_TYPE_GET(source_id), - FAULT_SOURCE_ID_CORE_INDEX_GET(source_id), - fault_source_id_internal_requester_get(kbdev, source_id), - fault_source_id_core_type_description_get(kbdev, source_id), - FAULT_SOURCE_ID_CORE_INDEX_GET(source_id), - fault_source_id_internal_requester_get_str(kbdev, source_id, access_type)); } kbase_debug_csf_fault_notify(kbdev, NULL, DF_GPU_PAGE_FAULT); @@ -163,7 +147,7 @@ void kbase_gpu_report_bus_fault_and_kill(struct kbase_context *kctx, struct kbas const uintptr_t fault_addr = fault->addr; /* terminal fault, print info about the fault */ - if (kbdev->gpu_props.gpu_id.product_model <= GPU_ID_MODEL_MAKE(13, 0)) { + if (kbdev->gpu_props.gpu_id.product_model < GPU_ID_MODEL_MAKE(14, 0)) { dev_err(kbdev->dev, "GPU bus fault in AS%u at PA %pK\n" "PA_VALID: %s\n" @@ -181,25 +165,6 @@ void kbase_gpu_report_bus_fault_and_kill(struct kbase_context *kctx, struct kbas fault_source_id_core_type_description_get(kbdev, source_id), fault_source_id_internal_requester_get_str(kbdev, source_id, access_type), kctx->pid); - } else { - dev_err(kbdev->dev, - "GPU bus fault in AS%u at PA %pK\n" - "PA_VALID: %s\n" - "raw fault status: 0x%X\n" - "exception type 0x%X: %s\n" - "access type 0x%X: %s\n" - "source id 0x%X (type:idx:IR 0x%X:0x%X:0x%X): %s %u, %s\n" - "pid: %d\n", - as_no, (void *)fault_addr, addr_valid, status, exception_type, - kbase_gpu_exception_name(exception_type), access_type, - kbase_gpu_access_type_name(access_type), source_id, - FAULT_SOURCE_ID_CORE_TYPE_GET(source_id), - FAULT_SOURCE_ID_CORE_INDEX_GET(source_id), - fault_source_id_internal_requester_get(kbdev, source_id), - fault_source_id_core_type_description_get(kbdev, source_id), - FAULT_SOURCE_ID_CORE_INDEX_GET(source_id), - fault_source_id_internal_requester_get_str(kbdev, source_id, access_type), - kctx->pid); } /* AS transaction begin */ @@ -246,7 +211,7 @@ void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, struct kbase_as unsigned int as_no = as->number; /* terminal fault, print info about the fault */ - if (kbdev->gpu_props.gpu_id.product_model <= GPU_ID_MODEL_MAKE(13, 0)) { + if (kbdev->gpu_props.gpu_id.product_model < GPU_ID_MODEL_MAKE(14, 0)) { dev_err(kbdev->dev, "Unhandled Page fault in AS%u at VA 0x%016llX\n" "Reason: %s\n" @@ -265,26 +230,6 @@ void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, struct kbase_as fault_source_id_internal_requester_get_str(kbdev, source_id, access_type), kctx->pid); - } else { - dev_err(kbdev->dev, - "Unhandled Page fault in AS%u at VA 0x%016llX\n" - "Reason: %s\n" - "raw fault status: 0x%X\n" - "exception type 0x%X: %s\n" - "access type 0x%X: %s\n" - "source id 0x%X (type:idx:IR 0x%X:0x%X:0x%X): %s %u, %s\n" - "pid: %d\n", - as_no, fault->addr, reason_str, status, exception_type, - kbase_gpu_exception_name(exception_type), access_type, - kbase_gpu_access_type_name(status), source_id, - FAULT_SOURCE_ID_CORE_TYPE_GET(source_id), - FAULT_SOURCE_ID_CORE_INDEX_GET(source_id), - fault_source_id_internal_requester_get(kbdev, source_id), - fault_source_id_core_type_description_get(kbdev, source_id), - FAULT_SOURCE_ID_CORE_INDEX_GET(source_id), - fault_source_id_internal_requester_get_str(kbdev, source_id, - access_type), - kctx->pid); } } @@ -294,6 +239,14 @@ void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, struct kbase_as * will abort all jobs and stop any hw counter dumping */ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + /* Update the page fault counter value in firmware visible memory, just before disabling + * the MMU which would in turn unblock the MCU firmware. + */ + if (kbdev->csf.page_fault_cnt_ptr) { + spin_lock(&kbdev->mmu_mask_change); + *kbdev->csf.page_fault_cnt_ptr = ++kbdev->csf.page_fault_cnt; + spin_unlock(&kbdev->mmu_mask_change); + } kbase_mmu_disable(kctx); kbase_ctx_flag_set(kctx, KCTX_AS_DISABLED_ON_FAULT); kbase_debug_csf_fault_notify(kbdev, kctx, DF_GPU_PAGE_FAULT); @@ -487,15 +440,6 @@ void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat) spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); } -int kbase_mmu_switch_to_ir(struct kbase_context *const kctx, struct kbase_va_region *const reg) -{ - CSTD_UNUSED(kctx); - CSTD_UNUSED(reg); - - /* Can't soft-stop the provoking job */ - return -EPERM; -} - /** * kbase_mmu_gpu_fault_worker() - Process a GPU fault for the device. * diff --git a/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_jm.c b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_jm.c index e8e136117fd8..a7f3f40ef325 100644 --- a/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_jm.c +++ b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_jm.c @@ -119,23 +119,37 @@ void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, struct kbase_as const u32 access_type = AS_FAULTSTATUS_ACCESS_TYPE_GET(status); const u32 source_id = AS_FAULTSTATUS_SOURCE_ID_GET(status); /* terminal fault, print info about the fault */ - dev_err(kbdev->dev, - "Unhandled Page fault in AS%u at VA 0x%016llX\n" - "Reason: %s\n" - "raw fault status: 0x%X\n" - "exception type 0x%X: %s\n" - "access type 0x%X: %s\n" - "source id 0x%X (core_id:utlb:IR 0x%X:0x%X:0x%X): %s, %s\n" - "pid: %d\n", - as_no, fault->addr, reason_str, status, exception_type, - kbase_gpu_exception_name(exception_type), access_type, - kbase_gpu_access_type_name(status), source_id, - FAULT_SOURCE_ID_CORE_ID_GET(source_id), - FAULT_SOURCE_ID_UTLB_ID_GET(source_id), - fault_source_id_internal_requester_get(kbdev, source_id), - fault_source_id_core_type_description_get(kbdev, source_id), - fault_source_id_internal_requester_get_str(kbdev, source_id, access_type), - kctx->pid); + if (kbdev->gpu_props.gpu_id.product_model < GPU_ID_MODEL_MAKE(9, 0)) { + dev_err(kbdev->dev, + "Unhandled Page fault in AS%u at VA 0x%016llX\n" + "Reason: %s\n" + "raw fault status: 0x%X\n" + "exception type 0x%X: %s\n" + "access type 0x%X: %s\n" + "pid: %d\n", + as_no, fault->addr, reason_str, status, exception_type, + kbase_gpu_exception_name(exception_type), access_type, + kbase_gpu_access_type_name(status), kctx->pid); + } else { + dev_err(kbdev->dev, + "Unhandled Page fault in AS%u at VA 0x%016llX\n" + "Reason: %s\n" + "raw fault status: 0x%X\n" + "exception type 0x%X: %s\n" + "access type 0x%X: %s\n" + "source id 0x%X (core_id:utlb:IR 0x%X:0x%X:0x%X): %s, %s\n" + "pid: %d\n", + as_no, fault->addr, reason_str, status, exception_type, + kbase_gpu_exception_name(exception_type), access_type, + kbase_gpu_access_type_name(status), source_id, + FAULT_SOURCE_ID_CORE_ID_GET(source_id), + FAULT_SOURCE_ID_UTLB_ID_GET(source_id), + fault_source_id_internal_requester_get(kbdev, source_id), + fault_source_id_core_type_description_get(kbdev, source_id), + fault_source_id_internal_requester_get_str(kbdev, source_id, + access_type), + kctx->pid); + } } /* hardware counters dump fault handling */ @@ -271,7 +285,7 @@ static void validate_protected_page_fault(struct kbase_device *kbdev) */ u32 protected_debug_mode = 0; - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE)) { + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE)) { protected_debug_mode = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_STATUS)) & GPU_STATUS_GPU_DBG_ENABLED; } @@ -387,13 +401,6 @@ void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat) dev_dbg(kbdev->dev, "Leaving %s irq_stat %u\n", __func__, irq_stat); } -int kbase_mmu_switch_to_ir(struct kbase_context *const kctx, struct kbase_va_region *const reg) -{ - dev_dbg(kctx->kbdev->dev, "Switching to incremental rendering for region %pK\n", - (void *)reg); - return kbase_job_slot_softstop_start_rp(kctx, reg); -} - int kbase_mmu_as_init(struct kbase_device *kbdev, unsigned int i) { kbdev->as[i].number = i; diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.c b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.c index 8d52e90f9579..4963d990054f 100644 --- a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.c +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.c @@ -40,7 +40,6 @@ #include #include #include -#include #include #include #if !MALI_USE_CSF @@ -58,6 +57,257 @@ /* Macro to convert updated PDGs to flags indicating levels skip in flush */ #define pgd_level_to_skip_flush(dirty_pgds) (~(dirty_pgds)&0xF) +/** + * kmap_pgd() - Map a PGD page and return the address of it + * + * @p: Pointer to the PGD page to be mapped. + * @pgd: The physical address of the PGD. May not be PAGE_SIZE aligned but shall be + * GPU_PAGE_SIZE aligned. + * + * Return: The mapped address of the @pgd, adjusted by the offset of @pgd from the start of page. + */ +static inline void *kmap_pgd(struct page *p, phys_addr_t pgd) +{ +#if GPU_PAGES_PER_CPU_PAGE > 1 + return kbase_kmap(p) + (pgd & ~PAGE_MASK); +#else + CSTD_UNUSED(pgd); + return kbase_kmap(p); +#endif +} + +/** + * kmap_atomic_pgd() - Variant of kmap_pgd for atomic mapping + * + * @p: Pointer to the PGD page to be mapped. + * @pgd: The physical address of the PGD. May not be PAGE_SIZE aligned but shall be + * GPU_PAGE_SIZE aligned. + * + * Return: The mapped address of the @pgd. + */ +static inline void *kmap_atomic_pgd(struct page *p, phys_addr_t pgd) +{ +#if GPU_PAGES_PER_CPU_PAGE > 1 + return kbase_kmap_atomic(p) + (pgd & ~PAGE_MASK); +#else + CSTD_UNUSED(pgd); + return kbase_kmap_atomic(p); +#endif +} + +/** + * kunmap_pgd() - Unmap a PGD page + * + * @p: Pointer to the PGD page to be unmapped. + * @pgd_address: The address of the PGD. May not be PAGE_SIZE aligned but shall be + * GPU_PAGE_SIZE aligned. + */ +static inline void kunmap_pgd(struct page *p, void *pgd_address) +{ + /* It is okay to not align pgd_address to PAGE_SIZE boundary */ + kbase_kunmap(p, pgd_address); +} + +/** + * kunmap_atomic_pgd() - Variant of kunmap_pgd for atomic unmapping + * + * @pgd_address: The address of the PGD. May not be PAGE_SIZE aligned but shall be + * GPU_PAGE_SIZE aligned. + */ +static inline void kunmap_atomic_pgd(void *pgd_address) +{ + /* It is okay to not align pgd_address to PAGE_SIZE boundary */ + kbase_kunmap_atomic(pgd_address); +} + +/** + * pgd_dma_addr() - Return dma addr of a PGD + * + * @p: Pointer to the PGD page. + * @pgd: The physical address of the PGD. + * + * Return: DMA address of the PGD + */ +static inline dma_addr_t pgd_dma_addr(struct page *p, phys_addr_t pgd) +{ +#if GPU_PAGES_PER_CPU_PAGE > 1 + return kbase_page_private(p)->dma_addr + (pgd & ~PAGE_MASK); +#else + CSTD_UNUSED(pgd); + return kbase_dma_addr(p); +#endif +} + +/** + * get_pgd_sub_page_index() - Return the index of a sub PGD page in the PGD page. + * + * @pgd: The physical address of the PGD. + * + * Return: The index value ranging from 0 to (GPU_PAGES_PER_CPU_PAGE - 1) + */ +static inline u32 get_pgd_sub_page_index(phys_addr_t pgd) +{ + return (pgd & ~PAGE_MASK) / GPU_PAGE_SIZE; +} + +#if GPU_PAGES_PER_CPU_PAGE > 1 +/** + * alloc_pgd_page_metadata() - Allocate page metadata for a PGD. + * + * @kbdev: Pointer to the instance of a kbase device. + * @mmut: Structure holding details of the MMU table for a kcontext. + * @p: PGD page. + * + * The PGD page, @p is linked to &kbase_mmu_table.pgd_pages_list for allocating + * sub PGD pages from the list. + * + * Return: True on success. + */ +static bool alloc_pgd_page_metadata(struct kbase_device *kbdev, struct kbase_mmu_table *mmut, + struct page *p) +{ + struct kbase_page_metadata *page_md; + + if (!kbase_is_page_migration_enabled()) { + page_md = kmem_cache_zalloc(kbdev->page_metadata_slab, GFP_KERNEL); + if (!page_md) + return false; + + page_md->dma_addr = kbase_dma_addr_as_priv(p); + set_page_private(p, (unsigned long)page_md); + } else { + page_md = kbase_page_private(p); + } + + page_md->data.pt_mapped.num_allocated_sub_pages = 1; + set_bit(0, page_md->data.pt_mapped.allocated_sub_pages); + page_md->data.pt_mapped.pgd_page = p; + list_add(&page_md->data.pt_mapped.pgd_link, &mmut->pgd_pages_list); + + return true; +} + +/** + * free_pgd_page_metadata() - Free page metadata for a PGD. + * + * @kbdev: Pointer to the instance of a kbase device. + * @p: PGD page where the metadata belongs to. + * + * The PGD page, @p is removed from &kbase_mmu_table.pgd_pages_list. + */ +static void free_pgd_page_metadata(struct kbase_device *kbdev, struct page *p) +{ + struct kbase_page_metadata *page_md = kbase_page_private(p); + + WARN_ON_ONCE(page_md->data.pt_mapped.num_allocated_sub_pages); + page_md->data.pt_mapped.pgd_page = NULL; + list_del_init(&page_md->data.pt_mapped.pgd_link); + + if (kbase_is_page_migration_enabled()) + return; + + set_page_private(p, (unsigned long)page_md->dma_addr); + kmem_cache_free(kbdev->page_metadata_slab, page_md); +} + +/** + * allocate_pgd_sub_page() - Allocate a PGD sub page + * + * @page_md: Page metadata of a PGD page where a sub page is allocated from. + * + * Return: Physical address of allocated PGD sub page on success. + * KBASE_INVALID_PHYSICAL_ADDRESS on failure. + */ +static inline phys_addr_t allocate_pgd_sub_page(struct kbase_page_metadata *page_md) +{ + unsigned long sub_page_index; + + if (page_md->data.pt_mapped.num_allocated_sub_pages == GPU_PAGES_PER_CPU_PAGE) + return KBASE_INVALID_PHYSICAL_ADDRESS; + sub_page_index = find_first_zero_bit(page_md->data.pt_mapped.allocated_sub_pages, + GPU_PAGES_PER_CPU_PAGE); + +#ifdef CONFIG_MALI_BIFROST_DEBUG + if (WARN_ON_ONCE(sub_page_index >= GPU_PAGES_PER_CPU_PAGE)) + return KBASE_INVALID_PHYSICAL_ADDRESS; + if (WARN_ON_ONCE(page_md->data.pt_mapped.num_allocated_sub_pages > GPU_PAGES_PER_CPU_PAGE)) + return KBASE_INVALID_PHYSICAL_ADDRESS; +#endif + set_bit(sub_page_index, page_md->data.pt_mapped.allocated_sub_pages); + page_md->data.pt_mapped.num_allocated_sub_pages++; + + return (page_to_phys(page_md->data.pt_mapped.pgd_page) + (sub_page_index * GPU_PAGE_SIZE)); +} + +/** + * free_pgd_sub_page() - Free a PGD sub page + * + * @pgd: Sub PGD to be freed. + * + * Return: The number of remaining allocated sub pages in the PGD. + */ +static int free_pgd_sub_page(phys_addr_t pgd) +{ + struct page *p = pfn_to_page(PFN_DOWN(pgd)); + struct kbase_page_metadata *page_md = kbase_page_private(p); + const u32 sub_page_index = get_pgd_sub_page_index(pgd); + +#ifdef CONFIG_MALI_BIFROST_DEBUG + if (WARN_ON_ONCE(!test_bit(sub_page_index, page_md->data.pt_mapped.allocated_sub_pages))) + return page_md->data.pt_mapped.num_allocated_sub_pages; +#endif + clear_bit(sub_page_index, page_md->data.pt_mapped.allocated_sub_pages); + if (!WARN_ON_ONCE(page_md->data.pt_mapped.num_allocated_sub_pages <= 0)) + page_md->data.pt_mapped.num_allocated_sub_pages--; + + return page_md->data.pt_mapped.num_allocated_sub_pages; +} + +/** + * allocate_from_pgd_pages_list() - Allocate a PGD from the PGD pages list + * + * @mmut: Structure holding details of the MMU table for a kcontext. + * + * Return: Physical address of the allocated PGD. + */ +static inline phys_addr_t allocate_from_pgd_pages_list(struct kbase_mmu_table *mmut) +{ + struct list_head *entry; + phys_addr_t pgd; + + lockdep_assert_held(&mmut->mmu_lock); + + if (unlikely(!mmut->num_free_pgd_sub_pages)) + return KBASE_INVALID_PHYSICAL_ADDRESS; + + if (mmut->last_allocated_pgd_page) { + pgd = allocate_pgd_sub_page(kbase_page_private(mmut->last_allocated_pgd_page)); + if (pgd != KBASE_INVALID_PHYSICAL_ADDRESS) + goto success; + } + + if (mmut->last_freed_pgd_page) { + pgd = allocate_pgd_sub_page(kbase_page_private(mmut->last_freed_pgd_page)); + if (pgd != KBASE_INVALID_PHYSICAL_ADDRESS) + goto success; + } + + list_for_each(entry, &mmut->pgd_pages_list) { + struct kbase_page_metadata *page_md = + list_entry(entry, struct kbase_page_metadata, data.pt_mapped.pgd_link); + + pgd = allocate_pgd_sub_page(page_md); + if (pgd != KBASE_INVALID_PHYSICAL_ADDRESS) + goto success; + } + + return KBASE_INVALID_PHYSICAL_ADDRESS; +success: + mmut->num_free_pgd_sub_pages--; + return pgd; +} +#endif + static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mmu_table *mmut, const u64 start_vpfn, struct tagged_addr *phys, size_t nr, unsigned long flags, int const group_id, u64 *dirty_pgds, @@ -417,7 +667,7 @@ static bool kbase_mmu_handle_isolated_pgd_page(struct kbase_device *kbdev, * PGD page, which is done inside kbase_mmu_free_pgd() for the * PGD page that did not get isolated. */ - dma_sync_single_for_device(kbdev->dev, kbase_dma_addr(p), PAGE_SIZE, + dma_sync_single_for_device(kbdev->dev, pgd_dma_addr(p, page_to_phys(p)), PAGE_SIZE, DMA_BIDIRECTIONAL); kbase_mmu_account_freed_pgd(kbdev, mmut); } @@ -444,6 +694,20 @@ static void kbase_mmu_free_pgd(struct kbase_device *kbdev, struct kbase_mmu_tabl lockdep_assert_held(&mmut->mmu_lock); p = pfn_to_page(PFN_DOWN(pgd)); +#if GPU_PAGES_PER_CPU_PAGE > 1 + if (free_pgd_sub_page(pgd)) { + mmut->num_free_pgd_sub_pages++; + mmut->last_freed_pgd_page = p; + return; + } + + mmut->num_free_pgd_sub_pages -= (GPU_PAGES_PER_CPU_PAGE - 1); + if (p == mmut->last_freed_pgd_page) + mmut->last_freed_pgd_page = NULL; + if (p == mmut->last_allocated_pgd_page) + mmut->last_allocated_pgd_page = NULL; + free_pgd_page_metadata(kbdev, p); +#endif page_is_isolated = kbase_mmu_handle_isolated_pgd_page(kbdev, mmut, p); if (likely(!page_is_isolated)) { @@ -473,19 +737,19 @@ static void kbase_mmu_free_pgds_list(struct kbase_device *kbdev, struct kbase_mm lockdep_assert_held(&mmut->mmu_lock); for (i = 0; i < mmut->scratch_mem.free_pgds.head_index; i++) - kbase_mmu_free_pgd(kbdev, mmut, page_to_phys(mmut->scratch_mem.free_pgds.pgds[i])); + kbase_mmu_free_pgd(kbdev, mmut, mmut->scratch_mem.free_pgds.pgds[i]); mmut->scratch_mem.free_pgds.head_index = 0; } -static void kbase_mmu_add_to_free_pgds_list(struct kbase_mmu_table *mmut, struct page *p) +static void kbase_mmu_add_to_free_pgds_list(struct kbase_mmu_table *mmut, phys_addr_t pgd) { lockdep_assert_held(&mmut->mmu_lock); if (WARN_ON_ONCE(mmut->scratch_mem.free_pgds.head_index > (MAX_FREE_PGDS - 1))) return; - mmut->scratch_mem.free_pgds.pgds[mmut->scratch_mem.free_pgds.head_index++] = p; + mmut->scratch_mem.free_pgds.pgds[mmut->scratch_mem.free_pgds.head_index++] = pgd; } static inline void kbase_mmu_reset_free_pgds_list(struct kbase_mmu_table *mmut) @@ -666,6 +930,14 @@ static void kbase_gpu_mmu_handle_write_fault(struct kbase_context *kctx, return; } + if (unlikely(region->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS)) { + kbase_gpu_vm_unlock(kctx); + kbase_mmu_report_fault_and_kill( + kctx, faulting_as, "Unexpected write permission fault on an alias region", + &faulting_as->pf_data); + return; + } + pfn_offset = fault_pfn - region->start_pfn; fault_phys_addr = &kbase_get_gpu_phy_pages(region)[pfn_offset]; @@ -781,6 +1053,7 @@ static size_t estimate_pool_space_required(struct kbase_mem_pool *pool, const si * either small or 2 MiB pages, depending on the number of pages requested. * @grow_2mb_pool: Pointer to variable to store which pool needs to grow - true for 2 MiB, false for * pool of small pages. + * @fallback_to_small: Whether fallback to small pages or not * @prealloc_sas: Pointer to kbase_sub_alloc structures * * This function will try to allocate as many pages as possible from the context pool, then if @@ -798,7 +1071,7 @@ static size_t estimate_pool_space_required(struct kbase_mem_pool *pool, const si */ static bool page_fault_try_alloc(struct kbase_context *kctx, struct kbase_va_region *region, size_t new_pages, size_t *pages_to_grow, bool *grow_2mb_pool, - struct kbase_sub_alloc **prealloc_sas) + bool fallback_to_small, struct kbase_sub_alloc **prealloc_sas) { size_t total_gpu_pages_alloced = 0; size_t total_cpu_pages_alloced = 0; @@ -816,7 +1089,8 @@ static bool page_fault_try_alloc(struct kbase_context *kctx, struct kbase_va_reg return false; } - if (kbase_is_large_pages_enabled() && new_pages >= NUM_PAGES_IN_2MB_LARGE_PAGE) { + if (kbase_is_large_pages_enabled() && new_pages >= NUM_PAGES_IN_2MB_LARGE_PAGE && + !fallback_to_small) { root_pool = &kctx->mem_pools.large[region->gpu_alloc->group_id]; *grow_2mb_pool = true; } else { @@ -964,6 +1238,7 @@ void kbase_mmu_page_fault_worker(struct work_struct *data) bool grown = false; size_t pages_to_grow; bool grow_2mb_pool = false; + bool fallback_to_small = false; struct kbase_sub_alloc *prealloc_sas[2] = { NULL, NULL }; int i; size_t current_backed_size; @@ -1004,13 +1279,11 @@ void kbase_mmu_page_fault_worker(struct work_struct *data) #endif #endif -#ifdef CONFIG_MALI_ARBITER_SUPPORT /* check if we still have GPU */ if (unlikely(kbase_is_gpu_removed(kbdev))) { dev_dbg(kbdev->dev, "%s: GPU has been removed", __func__); goto fault_done; } -#endif if (unlikely(fault->protected_mode)) { kbase_mmu_report_fault_and_kill(kctx, faulting_as, "Protected mode fault", fault); @@ -1133,7 +1406,7 @@ void kbase_mmu_page_fault_worker(struct work_struct *data) } page_fault_retry: - if (kbase_is_large_pages_enabled()) { + if (kbase_is_large_pages_enabled() && !fallback_to_small) { /* Preallocate (or re-allocate) memory for the sub-allocation structs if necessary */ for (i = 0; i != ARRAY_SIZE(prealloc_sas); ++i) { if (!prealloc_sas[i]) { @@ -1170,6 +1443,14 @@ page_fault_retry: goto fault_done; } + if (unlikely(region->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS)) { + kbase_gpu_vm_unlock(kctx); + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Unexpected page fault on an alias region", + &faulting_as->pf_data); + goto fault_done; + } + if (region->gpu_alloc->group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS) { kbase_gpu_vm_unlock(kctx); kbase_mmu_report_fault_and_kill(kctx, faulting_as, "Bad physical memory group ID", @@ -1297,7 +1578,7 @@ page_fault_retry: spin_lock(&kctx->mem_partials_lock); grown = page_fault_try_alloc(kctx, region, new_pages, &pages_to_grow, &grow_2mb_pool, - prealloc_sas); + fallback_to_small, prealloc_sas); spin_unlock(&kctx->mem_partials_lock); if (grown) { @@ -1341,22 +1622,6 @@ page_fault_retry: else trace_mali_mmu_page_fault_grow(region, fault, new_pages); -#if MALI_INCREMENTAL_RENDERING_JM - /* Switch to incremental rendering if we have nearly run out of - * memory in a JIT memory allocation. - */ - if (region->threshold_pages && - kbase_reg_current_backed_size(region) > region->threshold_pages) { - dev_dbg(kctx->kbdev->dev, "%zu pages exceeded IR threshold %zu", - new_pages + current_backed_size, region->threshold_pages); - - if (kbase_mmu_switch_to_ir(kctx, region) >= 0) { - dev_dbg(kctx->kbdev->dev, "Get region %pK for IR", (void *)region); - kbase_va_region_alloc_get(kctx, region); - } - } -#endif - /* AS transaction begin */ /* clear MMU interrupt - this needs to be done after updating @@ -1440,6 +1705,15 @@ page_fault_retry: lp_mem_pool->order; ret = kbase_mem_pool_grow(lp_mem_pool, pages_to_grow, kctx->task); + /* Retry handling the fault with small pages if required + * number of 2MB pages couldn't be allocated. + */ + if (ret < 0) { + fallback_to_small = true; + dev_dbg(kbdev->dev, + "No room for 2MB pages, fallback to small pages"); + goto page_fault_retry; + } } else { struct kbase_mem_pool *const mem_pool = &kctx->mem_pools.small[group_id]; @@ -1484,12 +1758,32 @@ fault_done: dev_dbg(kbdev->dev, "Leaving page_fault_worker %pK", (void *)data); } +/** + * kbase_mmu_alloc_pgd() - Allocate a PGD + * + * @kbdev: Pointer to the instance of a kbase device. + * @mmut: Structure holding details of the MMU table for a kcontext. + * + * A 4KB sized PGD page is allocated for the PGD from the memory pool if PAGE_SIZE is 4KB. + * Otherwise PGD is sub-allocated from a page that is allocated from the memory pool or + * from one of the pages earlier allocated for the PGD of @mmut. + * + * Return: Physical address of the allocated PGD. + */ static phys_addr_t kbase_mmu_alloc_pgd(struct kbase_device *kbdev, struct kbase_mmu_table *mmut) { u64 *page; struct page *p; phys_addr_t pgd; + lockdep_assert_held(&mmut->mmu_lock); + +#if GPU_PAGES_PER_CPU_PAGE > 1 + pgd = allocate_from_pgd_pages_list(mmut); + if (pgd != KBASE_INVALID_PHYSICAL_ADDRESS) + return pgd; +#endif + p = kbase_mem_pool_alloc(&kbdev->mem_pools.small[mmut->group_id]); if (!p) return KBASE_INVALID_PHYSICAL_ADDRESS; @@ -1499,6 +1793,15 @@ static phys_addr_t kbase_mmu_alloc_pgd(struct kbase_device *kbdev, struct kbase_ if (page == NULL) goto alloc_free; +#if GPU_PAGES_PER_CPU_PAGE > 1 + if (!alloc_pgd_page_metadata(kbdev, mmut, p)) { + kbase_kunmap(p, page); + goto alloc_free; + } + mmut->num_free_pgd_sub_pages += (GPU_PAGES_PER_CPU_PAGE - 1); + mmut->last_allocated_pgd_page = p; +#endif + pgd = page_to_phys(p); /* If the MMU tables belong to a context then account the memory usage @@ -1517,12 +1820,12 @@ static phys_addr_t kbase_mmu_alloc_pgd(struct kbase_device *kbdev, struct kbase_ kbase_trace_gpu_mem_usage_inc(kbdev, mmut->kctx, 1); - kbdev->mmu_mode->entries_invalidate(page, KBASE_MMU_PAGE_ENTRIES); + kbdev->mmu_mode->entries_invalidate(page, KBASE_MMU_PAGE_ENTRIES * GPU_PAGES_PER_CPU_PAGE); /* As this page is newly created, therefore there is no content to * clean or invalidate in the GPU caches. */ - kbase_mmu_sync_pgd_cpu(kbdev, kbase_dma_addr(p), PAGE_SIZE); + kbase_mmu_sync_pgd_cpu(kbdev, pgd_dma_addr(p, pgd), PAGE_SIZE); kbase_kunmap(p, page); return pgd; @@ -1564,7 +1867,7 @@ static int mmu_get_next_pgd(struct kbase_device *kbdev, struct kbase_mmu_table * vpfn &= 0x1FF; p = pfn_to_page(PFN_DOWN(*pgd)); - page = kbase_kmap(p); + page = kmap_pgd(p, *pgd); if (page == NULL) { dev_err(kbdev->dev, "%s: kmap failure", __func__); return -EINVAL; @@ -1573,7 +1876,7 @@ static int mmu_get_next_pgd(struct kbase_device *kbdev, struct kbase_mmu_table * if (!kbdev->mmu_mode->pte_is_valid(page[vpfn], level)) { dev_dbg(kbdev->dev, "%s: invalid PTE at level %d vpfn 0x%llx", __func__, level, vpfn); - kbase_kunmap(p, page); + kunmap_pgd(p, page); return -EFAULT; } else { target_pgd = kbdev->mmu_mode->pte_to_phy_addr( @@ -1581,7 +1884,7 @@ static int mmu_get_next_pgd(struct kbase_device *kbdev, struct kbase_mmu_table * kbdev->mgm_dev, MGM_DEFAULT_PTE_GROUP, level, page[vpfn])); } - kbase_kunmap(p, page); + kunmap_pgd(p, page); *pgd = target_pgd; return 0; @@ -1713,10 +2016,10 @@ static void mmu_insert_pages_failure_recovery(struct kbase_device *kbdev, for (level = MIDGARD_MMU_TOPLEVEL; level <= MIDGARD_MMU_BOTTOMLEVEL; level++) { idx = (vpfn >> ((3 - level) * 9)) & 0x1FF; pgds[level] = pgd; - page = kbase_kmap(p); + page = kmap_pgd(p, pgd); if (mmu_mode->ate_is_valid(page[idx], level)) break; /* keep the mapping */ - kbase_kunmap(p, page); + kunmap_pgd(p, page); pgd = mmu_mode->pte_to_phy_addr(kbdev->mgm_dev->ops.mgm_pte_to_original_pte( kbdev->mgm_dev, MGM_DEFAULT_PTE_GROUP, level, page[idx])); p = phys_to_page(pgd); @@ -1751,7 +2054,7 @@ static void mmu_insert_pages_failure_recovery(struct kbase_device *kbdev, if (!num_of_valid_entries) { mmu_mode->set_num_valid_entries(page, 0); - kbase_kunmap(p, page); + kunmap_pgd(p, page); kbase_mmu_update_and_free_parent_pgds(kbdev, mmut, pgds, vpfn, level - 1, KBASE_MMU_OP_NONE, dirty_pgds, 0); @@ -1762,7 +2065,7 @@ static void mmu_insert_pages_failure_recovery(struct kbase_device *kbdev, * returned to the memory pool. */ - kbase_mmu_add_to_free_pgds_list(mmut, p); + kbase_mmu_add_to_free_pgds_list(mmut, pgd); vpfn += count; continue; @@ -1774,9 +2077,9 @@ static void mmu_insert_pages_failure_recovery(struct kbase_device *kbdev, * going to be done by the caller */ kbase_mmu_sync_pgd(kbdev, mmut->kctx, pgd + (idx * sizeof(u64)), - kbase_dma_addr(p) + sizeof(u64) * idx, sizeof(u64) * pcount, + pgd_dma_addr(p, pgd) + sizeof(u64) * idx, sizeof(u64) * pcount, KBASE_MMU_OP_NONE); - kbase_kunmap(p, page); + kunmap_pgd(p, page); next: vpfn += count; } @@ -1895,7 +2198,7 @@ static int update_parent_pgds(struct kbase_device *kbdev, struct kbase_mmu_table goto failure_recovery; } - parent_page_va = kbase_kmap(parent_page); + parent_page_va = kmap_pgd(parent_page, parent_pgd); if (unlikely(parent_page_va == NULL)) { dev_err(kbdev->dev, "%s: kmap failure", __func__); @@ -1907,15 +2210,17 @@ static int update_parent_pgds(struct kbase_device *kbdev, struct kbase_mmu_table kbdev->mmu_mode->entry_set_pte(&pte, target_pgd); parent_page_va[parent_vpfn] = kbdev->mgm_dev->ops.mgm_update_gpu_pte( - kbdev->mgm_dev, MGM_DEFAULT_PTE_GROUP, parent_index, pte); + kbdev->mgm_dev, MGM_DEFAULT_PTE_GROUP, PBHA_ID_DEFAULT, PTE_FLAGS_NONE, + parent_index, pte); kbdev->mmu_mode->set_num_valid_entries(parent_page_va, current_valid_entries + 1); - kbase_kunmap(parent_page, parent_page_va); + kunmap_pgd(parent_page, parent_page_va); if (parent_index != insert_level) { /* Newly allocated PGDs */ - kbase_mmu_sync_pgd_cpu( - kbdev, kbase_dma_addr(parent_page) + (parent_vpfn * sizeof(u64)), - sizeof(u64)); + kbase_mmu_sync_pgd_cpu(kbdev, + pgd_dma_addr(parent_page, parent_pgd) + + (parent_vpfn * sizeof(u64)), + sizeof(u64)); } else { /* A new valid entry is added to an existing PGD. Perform the * invalidate operation for GPU cache as it could be having a @@ -1923,7 +2228,7 @@ static int update_parent_pgds(struct kbase_device *kbdev, struct kbase_mmu_table */ kbase_mmu_sync_pgd( kbdev, mmut->kctx, parent_pgd + (parent_vpfn * sizeof(u64)), - kbase_dma_addr(parent_page) + (parent_vpfn * sizeof(u64)), + pgd_dma_addr(parent_page, parent_pgd) + (parent_vpfn * sizeof(u64)), sizeof(u64), KBASE_MMU_OP_FLUSH_PT); } @@ -1934,6 +2239,9 @@ static int update_parent_pgds(struct kbase_device *kbdev, struct kbase_mmu_table spin_lock(&page_md->migrate_lock); +#if GPU_PAGES_PER_CPU_PAGE > 1 + page_md->status = PAGE_STATUS_SET(page_md->status, NOT_MOVABLE); +#else WARN_ON_ONCE(PAGE_STATUS_GET(page_md->status) != ALLOCATE_IN_PROGRESS || IS_PAGE_ISOLATED(page_md->status)); @@ -1945,6 +2253,7 @@ static int update_parent_pgds(struct kbase_device *kbdev, struct kbase_mmu_table } else { page_md->status = PAGE_STATUS_SET(page_md->status, NOT_MOVABLE); } +#endif spin_unlock(&page_md->migrate_lock); } @@ -1957,11 +2266,11 @@ failure_recovery: for (; pgd_index < cur_level; pgd_index++) { phys_addr_t pgd = pgds_to_insert[pgd_index]; struct page *pgd_page = pfn_to_page(PFN_DOWN(pgd)); - u64 *pgd_page_va = kbase_kmap(pgd_page); + u64 *pgd_page_va = kmap_pgd(pgd_page, pgd); u64 vpfn = (insert_vpfn >> ((3 - pgd_index) * 9)) & 0x1FF; kbdev->mmu_mode->entries_invalidate(&pgd_page_va[vpfn], 1); - kbase_kunmap(pgd_page, pgd_page_va); + kunmap_pgd(pgd_page, pgd_page_va); } return err; @@ -1977,6 +2286,8 @@ failure_recovery: * @level_high: The higher bound for the levels for which the PGD allocs are required * @new_pgds: Ptr to an array (size MIDGARD_MMU_BOTTOMLEVEL+1) to write the * newly allocated PGD addresses to. + * @pool_grown: True if new PGDs required the memory pool to grow to allocate more pages, + * or false otherwise * * Numerically, level_low < level_high, not to be confused with top level and * bottom level concepts for MMU PGDs. They are only used as low and high bounds @@ -1987,19 +2298,22 @@ failure_recovery: * * -ENOMEM - allocation failed for a PGD. */ static int mmu_insert_alloc_pgds(struct kbase_device *kbdev, struct kbase_mmu_table *mmut, - phys_addr_t *new_pgds, int level_low, int level_high) + phys_addr_t *new_pgds, int level_low, int level_high, + bool *pool_grown) { int err = 0; int i; lockdep_assert_held(&mmut->mmu_lock); + *pool_grown = false; for (i = level_low; i <= level_high; i++) { + if (new_pgds[i] != KBASE_INVALID_PHYSICAL_ADDRESS) + continue; do { new_pgds[i] = kbase_mmu_alloc_pgd(kbdev, mmut); if (new_pgds[i] != KBASE_INVALID_PHYSICAL_ADDRESS) break; - mutex_unlock(&mmut->mmu_lock); err = kbase_mem_pool_grow(&kbdev->mem_pools.small[mmut->group_id], (size_t)level_high, NULL); @@ -2007,17 +2321,9 @@ static int mmu_insert_alloc_pgds(struct kbase_device *kbdev, struct kbase_mmu_ta if (err) { dev_err(kbdev->dev, "%s: kbase_mem_pool_grow() returned error %d", __func__, err); - - /* Free all PGDs allocated in previous successful iterations - * from (i-1) to level_low - */ - for (i = (i - 1); i >= level_low; i--) { - if (new_pgds[i] != KBASE_INVALID_PHYSICAL_ADDRESS) - kbase_mmu_free_pgd(kbdev, mmut, new_pgds[i]); - } - return err; } + *pool_grown = true; } while (1); } @@ -2047,6 +2353,8 @@ static int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 start_vp if (WARN_ON(kctx == NULL)) return -EINVAL; + lockdep_assert_held(&kctx->reg_lock); + /* 64-bit address range is the max */ KBASE_DEBUG_ASSERT(start_vpfn <= (U64_MAX / PAGE_SIZE)); @@ -2082,6 +2390,7 @@ static int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 start_vp struct page *p; register unsigned int num_of_valid_entries; bool newly_created_pgd = false; + bool pool_grown; if (count > remain) count = remain; @@ -2089,6 +2398,10 @@ static int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 start_vp cur_level = MIDGARD_MMU_BOTTOMLEVEL; insert_level = cur_level; + for (l = MIDGARD_MMU_TOPLEVEL + 1; l <= cur_level; l++) + new_pgds[l] = KBASE_INVALID_PHYSICAL_ADDRESS; + +repeat_page_table_walk: /* * Repeatedly calling mmu_get_lowest_valid_pgd() is clearly * suboptimal. We don't have to re-parse the whole tree @@ -2103,7 +2416,7 @@ static int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 start_vp if (err) { dev_err(kbdev->dev, "%s: mmu_get_lowest_valid_pgd() returned error %d", __func__, err); - goto fail_unlock; + goto fail_unlock_free_pgds; } /* No valid pgd at cur_level */ @@ -2112,9 +2425,12 @@ static int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 start_vp * down to the lowest valid pgd at insert_level */ err = mmu_insert_alloc_pgds(kbdev, mmut, new_pgds, (insert_level + 1), - cur_level); + cur_level, &pool_grown); if (err) - goto fail_unlock; + goto fail_unlock_free_pgds; + + if (pool_grown) + goto repeat_page_table_walk; newly_created_pgd = true; @@ -2129,7 +2445,7 @@ static int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 start_vp p = pfn_to_page(PFN_DOWN(pgd)); - pgd_page = kbase_kmap(p); + pgd_page = kmap_pgd(p, pgd); if (!pgd_page) { dev_err(kbdev->dev, "%s: kmap failure", __func__); err = -ENOMEM; @@ -2168,8 +2484,8 @@ static int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 start_vp flush_op = newly_created_pgd ? KBASE_MMU_OP_NONE : KBASE_MMU_OP_FLUSH_PT; kbase_mmu_sync_pgd(kbdev, kctx, pgd + (vindex * sizeof(u64)), - kbase_dma_addr(p) + (vindex * sizeof(u64)), count * sizeof(u64), - flush_op); + pgd_dma_addr(p, pgd) + (vindex * sizeof(u64)), + count * sizeof(u64), flush_op); if (newly_created_pgd) { err = update_parent_pgds(kbdev, mmut, cur_level, insert_level, insert_vpfn, @@ -2180,14 +2496,14 @@ static int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 start_vp kbdev->mmu_mode->entries_invalidate(&pgd_page[vindex], count); - kbase_kunmap(p, pgd_page); + kunmap_pgd(p, pgd_page); goto fail_unlock_free_pgds; } } insert_vpfn += count; remain -= count; - kbase_kunmap(p, pgd_page); + kunmap_pgd(p, pgd_page); } mutex_unlock(&mmut->mmu_lock); @@ -2200,9 +2516,9 @@ static int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 start_vp fail_unlock_free_pgds: /* Free the pgds allocated by us from insert_level+1 to bottom level */ for (l = cur_level; l > insert_level; l--) - kbase_mmu_free_pgd(kbdev, mmut, new_pgds[l]); + if (new_pgds[l] != KBASE_INVALID_PHYSICAL_ADDRESS) + kbase_mmu_free_pgd(kbdev, mmut, new_pgds[l]); -fail_unlock: if (insert_vpfn != (start_vpfn * GPU_PAGES_PER_CPU_PAGE)) { /* Invalidate the pages we have partially completed */ mmu_insert_pages_failure_recovery(kbdev, mmut, start_vpfn * GPU_PAGES_PER_CPU_PAGE, @@ -2326,10 +2642,15 @@ u64 kbase_mmu_create_ate(struct kbase_device *const kbdev, struct tagged_addr co unsigned long const flags, int const level, int const group_id) { u64 entry; + unsigned int pte_flags = 0; kbdev->mmu_mode->entry_set_ate(&entry, phy, flags, level); - return kbdev->mgm_dev->ops.mgm_update_gpu_pte(kbdev->mgm_dev, (unsigned int)group_id, level, - entry); + + if ((flags & KBASE_REG_GPU_CACHED) && !(flags & KBASE_REG_CPU_CACHED)) + pte_flags |= BIT(MMA_VIOLATION); + + return kbdev->mgm_dev->ops.mgm_update_gpu_pte(kbdev->mgm_dev, (unsigned int)group_id, + kbdev->mma_wa_id, pte_flags, level, entry); } static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mmu_table *mmut, @@ -2348,6 +2669,9 @@ static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mm int l, cur_level, insert_level; struct tagged_addr *start_phys = phys; + if (mmut->kctx) + lockdep_assert_held(&mmut->kctx->reg_lock); + /* Note that 0 is a valid start_vpfn */ /* 64-bit address range is the max */ KBASE_DEBUG_ASSERT(start_vpfn <= (U64_MAX / PAGE_SIZE)); @@ -2370,6 +2694,7 @@ static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mm register unsigned int num_of_valid_entries; bool newly_created_pgd = false; enum kbase_mmu_op_type flush_op; + bool pool_grown; if (count > remain) count = remain; @@ -2389,6 +2714,10 @@ static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mm insert_level = cur_level; + for (l = MIDGARD_MMU_TOPLEVEL + 1; l <= cur_level; l++) + new_pgds[l] = KBASE_INVALID_PHYSICAL_ADDRESS; + +repeat_page_table_walk: /* * Repeatedly calling mmu_get_lowest_valid_pgd() is clearly * suboptimal. We don't have to re-parse the whole tree @@ -2403,7 +2732,7 @@ static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mm if (err) { dev_err(kbdev->dev, "%s: mmu_get_lowest_valid_pgd() returned error %d", __func__, err); - goto fail_unlock; + goto fail_unlock_free_pgds; } /* No valid pgd at cur_level */ @@ -2412,9 +2741,12 @@ static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mm * down to the lowest valid pgd at insert_level */ err = mmu_insert_alloc_pgds(kbdev, mmut, new_pgds, (insert_level + 1), - cur_level); + cur_level, &pool_grown); if (err) - goto fail_unlock; + goto fail_unlock_free_pgds; + + if (pool_grown) + goto repeat_page_table_walk; newly_created_pgd = true; @@ -2428,7 +2760,7 @@ static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mm } p = pfn_to_page(PFN_DOWN(pgd)); - pgd_page = kbase_kmap(p); + pgd_page = kmap_pgd(p, pgd); if (!pgd_page) { dev_err(kbdev->dev, "%s: kmap failure", __func__); @@ -2498,8 +2830,8 @@ static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mm flush_op = newly_created_pgd ? KBASE_MMU_OP_NONE : KBASE_MMU_OP_FLUSH_PT; kbase_mmu_sync_pgd(kbdev, mmut->kctx, pgd + (vindex * sizeof(u64)), - kbase_dma_addr(p) + (vindex * sizeof(u64)), count * sizeof(u64), - flush_op); + pgd_dma_addr(p, pgd) + (vindex * sizeof(u64)), + count * sizeof(u64), flush_op); if (newly_created_pgd) { err = update_parent_pgds(kbdev, mmut, cur_level, insert_level, insert_vpfn, @@ -2510,7 +2842,7 @@ static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mm kbdev->mmu_mode->entries_invalidate(&pgd_page[vindex], count); - kbase_kunmap(p, pgd_page); + kunmap_pgd(p, pgd_page); goto fail_unlock_free_pgds; } } @@ -2518,7 +2850,7 @@ static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mm phys += (count / GPU_PAGES_PER_CPU_PAGE); insert_vpfn += count; remain -= count; - kbase_kunmap(p, pgd_page); + kunmap_pgd(p, pgd_page); } mutex_unlock(&mmut->mmu_lock); @@ -2528,9 +2860,9 @@ static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mm fail_unlock_free_pgds: /* Free the pgds allocated by us from insert_level+1 to bottom level */ for (l = cur_level; l > insert_level; l--) - kbase_mmu_free_pgd(kbdev, mmut, new_pgds[l]); + if (new_pgds[l] != KBASE_INVALID_PHYSICAL_ADDRESS) + kbase_mmu_free_pgd(kbdev, mmut, new_pgds[l]); -fail_unlock: if (insert_vpfn != (start_vpfn * GPU_PAGES_PER_CPU_PAGE)) { /* Invalidate the pages we have partially completed */ mmu_insert_pages_failure_recovery(kbdev, mmut, start_vpfn * GPU_PAGES_PER_CPU_PAGE, @@ -2773,7 +3105,7 @@ static void kbase_mmu_update_and_free_parent_pgds(struct kbase_device *kbdev, { phys_addr_t current_pgd = pgds[level]; struct page *p = phys_to_page(current_pgd); - u64 *current_page = kbase_kmap(p); + u64 *current_page = kmap_pgd(p, current_pgd); unsigned int current_valid_entries = kbdev->mmu_mode->get_num_valid_entries(current_page); unsigned int index = (vpfn >> ((3 - level) * 9)) & 0x1FFU; @@ -2787,7 +3119,7 @@ static void kbase_mmu_update_and_free_parent_pgds(struct kbase_device *kbdev, if (current_valid_entries == 1 && level != MIDGARD_MMU_LEVEL(0)) { kbdev->mmu_mode->set_num_valid_entries(current_page, 0); - kbase_kunmap(p, current_page); + kunmap_pgd(p, current_page); kbase_mmu_update_and_free_parent_pgds(kbdev, mmut, pgds, vpfn, level - 1, flush_op, dirty_pgds, as_nr); @@ -2795,7 +3127,8 @@ static void kbase_mmu_update_and_free_parent_pgds(struct kbase_device *kbdev, /* Check if fine grained GPU cache maintenance is being used */ if (flush_op == KBASE_MMU_OP_FLUSH_PT) { /* Ensure the invalidated PTE is visible in memory right away */ - kbase_mmu_sync_pgd_cpu(kbdev, kbase_dma_addr(p) + (index * sizeof(u64)), + kbase_mmu_sync_pgd_cpu(kbdev, + pgd_dma_addr(p, current_pgd) + (index * sizeof(u64)), sizeof(u64)); /* Invalidate the GPU cache for the whole PGD page and not just for * the cacheline containing the invalidated PTE, as the PGD page is @@ -2807,17 +3140,17 @@ static void kbase_mmu_update_and_free_parent_pgds(struct kbase_device *kbdev, KBASE_MMU_OP_FLUSH_PT); } - kbase_mmu_add_to_free_pgds_list(mmut, p); + kbase_mmu_add_to_free_pgds_list(mmut, current_pgd); } else { current_valid_entries--; kbdev->mmu_mode->set_num_valid_entries(current_page, current_valid_entries); - kbase_kunmap(p, current_page); + kunmap_pgd(p, current_page); kbase_mmu_sync_pgd(kbdev, mmut->kctx, current_pgd + (index * sizeof(u64)), - kbase_dma_addr(p) + (index * sizeof(u64)), sizeof(u64), - flush_op); + pgd_dma_addr(p, current_pgd) + (index * sizeof(u64)), + sizeof(u64), flush_op); /* When fine grained GPU cache maintenance is used then invalidate the MMU caches * now as the top most level PGD entry, affected by the teardown operation, has @@ -2921,7 +3254,7 @@ static int kbase_mmu_teardown_pgd_pages(struct kbase_device *kbdev, struct kbase phys_addr_t next_pgd; index = (vpfn >> ((3 - level) * 9)) & 0x1FF; - page = kbase_kmap(p); + page = kmap_pgd(p, pgd); if (mmu_mode->ate_is_valid(page[index], level)) break; /* keep the mapping */ else if (!mmu_mode->pte_is_valid(page[index], level)) { @@ -2936,7 +3269,7 @@ static int kbase_mmu_teardown_pgd_pages(struct kbase_device *kbdev, struct kbase next_pgd = mmu_mode->pte_to_phy_addr( kbdev->mgm_dev->ops.mgm_pte_to_original_pte( kbdev->mgm_dev, MGM_DEFAULT_PTE_GROUP, level, page[index])); - kbase_kunmap(p, page); + kunmap_pgd(p, page); pgds[level] = pgd; pgd = next_pgd; p = phys_to_page(pgd); @@ -2947,7 +3280,7 @@ static int kbase_mmu_teardown_pgd_pages(struct kbase_device *kbdev, struct kbase case MIDGARD_MMU_LEVEL(1): dev_warn(kbdev->dev, "%s: No support for ATEs at level %d", __func__, level); - kbase_kunmap(p, page); + kunmap_pgd(p, page); goto out; case MIDGARD_MMU_LEVEL(2): /* can only teardown if count >= 512 */ @@ -2987,7 +3320,7 @@ static int kbase_mmu_teardown_pgd_pages(struct kbase_device *kbdev, struct kbase if (!num_of_valid_entries) { mmu_mode->set_num_valid_entries(page, 0); - kbase_kunmap(p, page); + kunmap_pgd(p, page); /* To avoid the invalid ATEs from the PGD page (that is going to be freed) * from getting reloaded into the GPU L2 cache whilst the teardown is @@ -3002,7 +3335,7 @@ static int kbase_mmu_teardown_pgd_pages(struct kbase_device *kbdev, struct kbase if (flush_op == KBASE_MMU_OP_FLUSH_PT) { /* Ensure the invalidated ATEs are visible in memory right away */ kbase_mmu_sync_pgd_cpu(kbdev, - kbase_dma_addr(p) + (index * sizeof(u64)), + pgd_dma_addr(p, pgd) + (index * sizeof(u64)), pcount * sizeof(u64)); /* Invalidate the GPU cache for the whole PGD page and not just for * the cachelines containing the invalidated ATEs, as the PGD page @@ -3014,7 +3347,7 @@ static int kbase_mmu_teardown_pgd_pages(struct kbase_device *kbdev, struct kbase KBASE_MMU_OP_FLUSH_PT); } - kbase_mmu_add_to_free_pgds_list(mmut, p); + kbase_mmu_add_to_free_pgds_list(mmut, pgd); vpfn += count; nr -= count; @@ -3024,8 +3357,8 @@ static int kbase_mmu_teardown_pgd_pages(struct kbase_device *kbdev, struct kbase mmu_mode->set_num_valid_entries(page, num_of_valid_entries); kbase_mmu_sync_pgd(kbdev, mmut->kctx, pgd + (index * sizeof(u64)), - kbase_dma_addr(p) + (index * sizeof(u64)), pcount * sizeof(u64), - flush_op); + pgd_dma_addr(p, pgd) + (index * sizeof(u64)), + pcount * sizeof(u64), flush_op); /* When fine grained GPU cache maintenance is used then invalidation of MMU cache * is done inline for every bottom level PGD touched in the teardown. @@ -3033,7 +3366,7 @@ static int kbase_mmu_teardown_pgd_pages(struct kbase_device *kbdev, struct kbase if (flush_op == KBASE_MMU_OP_FLUSH_PT) mmu_invalidate_on_teardown(kbdev, mmut->kctx, vpfn, pcount, level, as_nr); next: - kbase_kunmap(p, page); + kunmap_pgd(p, page); vpfn += count; nr -= count; } @@ -3238,7 +3571,7 @@ static int kbase_mmu_update_pages_no_flush(struct kbase_device *kbdev, struct kb goto fail_unlock; p = pfn_to_page(PFN_DOWN(pgd)); - pgd_page = kbase_kmap(p); + pgd_page = kmap_pgd(p, pgd); if (!pgd_page) { dev_warn(kbdev->dev, "kmap failure on update_pages"); err = -ENOMEM; @@ -3258,7 +3591,7 @@ static int kbase_mmu_update_pages_no_flush(struct kbase_device *kbdev, struct kb pgd_page[level_index] = kbase_mmu_create_ate( kbdev, *target_phys, flags, MIDGARD_MMU_LEVEL(2), group_id); kbase_mmu_sync_pgd(kbdev, mmut->kctx, pgd + (level_index * sizeof(u64)), - kbase_dma_addr(p) + (level_index * sizeof(u64)), + pgd_dma_addr(p, pgd) + (level_index * sizeof(u64)), sizeof(u64), KBASE_MMU_OP_NONE); } else { for (i = 0; i < count; i += GPU_PAGES_PER_CPU_PAGE) { @@ -3283,7 +3616,7 @@ static int kbase_mmu_update_pages_no_flush(struct kbase_device *kbdev, struct kb * will be done by the caller. */ kbase_mmu_sync_pgd(kbdev, mmut->kctx, pgd + (index * sizeof(u64)), - kbase_dma_addr(p) + (index * sizeof(u64)), + pgd_dma_addr(p, pgd) + (index * sizeof(u64)), count * sizeof(u64), KBASE_MMU_OP_NONE); } @@ -3296,7 +3629,7 @@ static int kbase_mmu_update_pages_no_flush(struct kbase_device *kbdev, struct kb vpfn += count; nr -= count; - kbase_kunmap(p, pgd_page); + kunmap_pgd(p, pgd_page); } mutex_unlock(&mmut->mmu_lock); @@ -3532,7 +3865,7 @@ int kbase_mmu_migrate_page(struct tagged_addr old_phys, struct tagged_addr new_p goto get_pgd_at_level_error; } - pgd_page = kbase_kmap(phys_to_page(pgd)); + pgd_page = kmap_pgd(phys_to_page(pgd), pgd); if (!pgd_page) { dev_warn(kbdev->dev, "%s: kmap failure for PGD page.", __func__); ret = -EINVAL; @@ -3641,8 +3974,10 @@ int kbase_mmu_migrate_page(struct tagged_addr old_phys, struct tagged_addr new_p kbdev->mgm_dev, MGM_DEFAULT_PTE_GROUP, level, pgd_page[index]))); #endif kbdev->mmu_mode->entry_set_pte(&managed_pte, as_phys_addr_t(new_phys)); - *target = kbdev->mgm_dev->ops.mgm_update_gpu_pte( - kbdev->mgm_dev, MGM_DEFAULT_PTE_GROUP, level, managed_pte); + *target = kbdev->mgm_dev->ops.mgm_update_gpu_pte(kbdev->mgm_dev, + MGM_DEFAULT_PTE_GROUP, + PBHA_ID_DEFAULT, PTE_FLAGS_NONE, + level, managed_pte); } kbdev->mmu_mode->set_num_valid_entries(pgd_page, num_of_valid_entries); @@ -3653,29 +3988,35 @@ int kbase_mmu_migrate_page(struct tagged_addr old_phys, struct tagged_addr new_p * maintenance is necessary. */ kbase_mmu_sync_pgd(kbdev, mmut->kctx, pgd + (index * sizeof(u64)), - kbase_dma_addr(phys_to_page(pgd)) + (index * sizeof(u64)), + pgd_dma_addr(phys_to_page(pgd), pgd) + (index * sizeof(u64)), pgd_entries_to_sync * sizeof(u64), KBASE_MMU_OP_FLUSH_PT); /* Unlock MMU region. * - * Notice that GPUs which don't issue flush commands via GPU control - * still need an additional GPU cache flush here, this time only - * for the page table, because the function call above to sync PGDs - * won't have any effect on them. + * For GPUs without FLUSH_PA_RANGE support, the GPU caches were completely + * cleaned and invalidated after locking the virtual address range affected + * by the migration. As long as the lock is in place, GPU access to the + * locked range would remain blocked. So there is no need to clean and + * invalidate the GPU caches again after the copying the page contents + * of old page and updating the page table entry to point to new page. + * + * For GPUs with FLUSH_PA_RANGE support, the contents of old page would + * have been evicted from the GPU caches after locking the virtual address + * range. The page table entry contents also would have been invalidated + * from the GPU's L2 cache by kbase_mmu_sync_pgd() after the page table + * update. + * + * If kbase_mmu_hw_do_unlock_no_addr() fails, GPU reset will be triggered which + * would remove the MMU lock and so there is no need to rollback page migration + * and the failure can be ignored. */ spin_lock_irqsave(&kbdev->hwaccess_lock, hwaccess_flags); if (kbdev->pm.backend.gpu_ready && mmut->kctx->as_nr >= 0) { int as_nr = mmut->kctx->as_nr; struct kbase_as *as = &kbdev->as[as_nr]; + int local_ret = kbase_mmu_hw_do_unlock_no_addr(kbdev, as, &op_param); - if (mmu_flush_cache_on_gpu_ctrl(kbdev)) { - ret = kbase_mmu_hw_do_unlock(kbdev, as, &op_param); - } else { - ret = kbase_gpu_cache_flush_and_busy_wait(kbdev, - GPU_COMMAND_CACHE_CLN_INV_L2); - if (!ret) - ret = kbase_mmu_hw_do_unlock_no_addr(kbdev, as, &op_param); - } + CSTD_UNUSED(local_ret); } /* Release the transition prevention in L2 by ending the transaction */ @@ -3684,22 +4025,27 @@ int kbase_mmu_migrate_page(struct tagged_addr old_phys, struct tagged_addr new_p /* Releasing locks before checking the migration transaction error state */ mutex_unlock(&kbdev->mmu_hw_mutex); - /* Checking the final migration transaction error state */ - if (ret < 0) { - dev_err(kbdev->dev, "%s: failed to unlock MMU region.", __func__); - goto undo_mappings; - } - /* Undertaking metadata transfer, while we are holding the mmu_lock */ spin_lock(&page_md->migrate_lock); if (level == MIDGARD_MMU_BOTTOMLEVEL) { - size_t page_array_index = (page_md->data.mapped.vpfn / GPU_PAGES_PER_CPU_PAGE) - - page_md->data.mapped.reg->start_pfn; + enum kbase_page_status page_status = PAGE_STATUS_GET(page_md->status); - WARN_ON(PAGE_STATUS_GET(page_md->status) != ALLOCATED_MAPPED); + if (page_status == ALLOCATED_MAPPED) { + /* Replace page in array of pages of the physical allocation. */ + size_t page_array_index = + div_u64(page_md->data.mapped.vpfn, GPU_PAGES_PER_CPU_PAGE) - + page_md->data.mapped.reg->start_pfn; - /* Replace page in array of pages of the physical allocation. */ - page_md->data.mapped.reg->gpu_alloc->pages[page_array_index] = new_phys; + page_md->data.mapped.reg->gpu_alloc->pages[page_array_index] = new_phys; + } else if (page_status == NOT_MOVABLE) { + dev_dbg(kbdev->dev, + "%s: migration completed and page has become NOT_MOVABLE.", + __func__); + } else { + dev_WARN(kbdev->dev, + "%s: migration completed but page has moved to status %d.", + __func__, page_status); + } } /* Update the new page dma_addr with the transferred metadata from the old_page */ page_md->dma_addr = new_dma_addr; @@ -3710,7 +4056,7 @@ int kbase_mmu_migrate_page(struct tagged_addr old_phys, struct tagged_addr new_p set_page_private(as_page(old_phys), 0); l2_state_defer_out: - kbase_kunmap(phys_to_page(pgd), pgd_page); + kunmap_pgd(phys_to_page(pgd), pgd_page); pgd_page_map_error: get_pgd_at_level_error: page_state_change_out: @@ -3725,7 +4071,7 @@ old_page_map_error: undo_mappings: /* Unlock the MMU table and undo mappings. */ mutex_unlock(&mmut->mmu_lock); - kbase_kunmap(phys_to_page(pgd), pgd_page); + kunmap_pgd(phys_to_page(pgd), pgd_page); kbase_kunmap(as_page(new_phys), new_page); kbase_kunmap(as_page(old_phys), old_page); @@ -3744,7 +4090,7 @@ static void mmu_teardown_level(struct kbase_device *kbdev, struct kbase_mmu_tabl lockdep_assert_held(&mmut->mmu_lock); - pgd_page = kbase_kmap_atomic(p); + pgd_page = kmap_atomic_pgd(p, pgd); /* kmap_atomic should NEVER fail. */ if (WARN_ON_ONCE(pgd_page == NULL)) return; @@ -3753,7 +4099,7 @@ static void mmu_teardown_level(struct kbase_device *kbdev, struct kbase_mmu_tabl * kmap_atomic usage */ pgd_page_buffer = mmut->scratch_mem.teardown_pages.levels[level]; - memcpy(pgd_page_buffer, pgd_page, PAGE_SIZE); + memcpy(pgd_page_buffer, pgd_page, GPU_PAGE_SIZE); } /* When page migration is enabled, kbase_region_tracker_term() would ensure @@ -3764,7 +4110,7 @@ static void mmu_teardown_level(struct kbase_device *kbdev, struct kbase_mmu_tabl WARN_ON_ONCE(kbdev->mmu_mode->get_num_valid_entries(pgd_page)); /* Invalidate page after copying */ mmu_mode->entries_invalidate(pgd_page, KBASE_MMU_PAGE_ENTRIES); - kbase_kunmap_atomic(pgd_page); + kunmap_atomic_pgd(pgd_page); pgd_page = pgd_page_buffer; if (level < MIDGARD_MMU_BOTTOMLEVEL) { @@ -3783,13 +4129,20 @@ static void mmu_teardown_level(struct kbase_device *kbdev, struct kbase_mmu_tabl kbase_mmu_free_pgd(kbdev, mmut, pgd); } -static void kbase_mmu_mark_non_movable(struct page *page) +static void kbase_mmu_mark_non_movable(struct kbase_device *const kbdev, struct page *page) { struct kbase_page_metadata *page_md; if (!kbase_is_page_migration_enabled()) return; + /* Composite large-page is excluded from migration, trigger a warn if a development + * wrongly leads to it. + */ + if (is_huge_head(as_tagged(page_to_phys(page))) || + is_partial(as_tagged(page_to_phys(page)))) + dev_WARN(kbdev->dev, "%s: migration on large-page attempted.", __func__); + page_md = kbase_page_private(page); spin_lock(&page_md->migrate_lock); @@ -3817,6 +4170,10 @@ int kbase_mmu_init(struct kbase_device *const kbdev, struct kbase_mmu_table *con mmut->kctx = kctx; mmut->pgd = KBASE_INVALID_PHYSICAL_ADDRESS; +#if GPU_PAGES_PER_CPU_PAGE > 1 + INIT_LIST_HEAD(&mmut->pgd_pages_list); +#endif + /* We allocate pages into the kbdev memory pool, then * kbase_mmu_alloc_pgd will allocate out of that pool. This is done to * avoid allocations from the kernel happening with the lock held. @@ -3831,10 +4188,12 @@ int kbase_mmu_init(struct kbase_device *const kbdev, struct kbase_mmu_table *con return -ENOMEM; } + mutex_lock(&mmut->mmu_lock); mmut->pgd = kbase_mmu_alloc_pgd(kbdev, mmut); + mutex_unlock(&mmut->mmu_lock); } - kbase_mmu_mark_non_movable(pfn_to_page(PFN_DOWN(mmut->pgd))); + kbase_mmu_mark_non_movable(kbdev, pfn_to_page(PFN_DOWN(mmut->pgd))); return 0; } @@ -3892,6 +4251,7 @@ static size_t kbasep_mmu_dump_level(struct kbase_context *kctx, phys_addr_t pgd, size_t dump_size; struct kbase_device *kbdev; struct kbase_mmu_mode const *mmu_mode; + struct page *p; if (WARN_ON(kctx == NULL)) return 0; @@ -3900,7 +4260,8 @@ static size_t kbasep_mmu_dump_level(struct kbase_context *kctx, phys_addr_t pgd, kbdev = kctx->kbdev; mmu_mode = kbdev->mmu_mode; - pgd_page = kbase_kmap(pfn_to_page(PFN_DOWN(pgd))); + p = pfn_to_page(PFN_DOWN(pgd)); + pgd_page = kmap_pgd(p, pgd); if (!pgd_page) { dev_warn(kbdev->dev, "%s: kmap failure", __func__); return 0; @@ -3934,7 +4295,7 @@ static size_t kbasep_mmu_dump_level(struct kbase_context *kctx, phys_addr_t pgd, dump_size = kbasep_mmu_dump_level(kctx, target_pgd, level + 1, buffer, size_left); if (!dump_size) { - kbase_kunmap(pfn_to_page(PFN_DOWN(pgd)), pgd_page); + kunmap_pgd(p, pgd_page); return 0; } size += dump_size; @@ -3942,7 +4303,7 @@ static size_t kbasep_mmu_dump_level(struct kbase_context *kctx, phys_addr_t pgd, } } - kbase_kunmap(pfn_to_page(PFN_DOWN(pgd)), pgd_page); + kunmap_pgd(p, pgd_page); return size; } @@ -4048,7 +4409,6 @@ void kbase_mmu_bus_fault_worker(struct work_struct *data) return; } -#ifdef CONFIG_MALI_ARBITER_SUPPORT /* check if we still have GPU */ if (unlikely(kbase_is_gpu_removed(kbdev))) { dev_dbg(kbdev->dev, "%s: GPU has been removed", __func__); @@ -4056,7 +4416,6 @@ void kbase_mmu_bus_fault_worker(struct work_struct *data) atomic_dec(&kbdev->faults_pending); return; } -#endif if (unlikely(fault->protected_mode)) { kbase_mmu_report_fault_and_kill(kctx, faulting_as, "Permission failure", fault); diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_faults_decoder.c b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_faults_decoder.c index e1b72ed24321..548d88cf216e 100644 --- a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_faults_decoder.c +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_faults_decoder.c @@ -65,7 +65,7 @@ const char *fault_source_id_internal_requester_get_str(struct kbase_device *kbde if (older_source_id_fmt) utlb_id = FAULT_SOURCE_ID_UTLB_ID_GET(source_id); - if (strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "shader") == 0) { + if (!strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "shader")) { if (utlb_id == 0) { if (access_type == AS_FAULTSTATUS_ACCESS_TYPE_READ) return decode_fault_source_shader_r_t( @@ -75,7 +75,7 @@ const char *fault_source_id_internal_requester_get_str(struct kbase_device *kbde ir, kbdev->gpu_props.gpu_id.arch_id); } else return "Load/store cache"; - } else if (strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "tiler")) { + } else if (!strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "tiler")) { #if MALI_USE_CSF if (utlb_id == 0) { if (access_type == AS_FAULTSTATUS_ACCESS_TYPE_READ) @@ -92,14 +92,14 @@ const char *fault_source_id_internal_requester_get_str(struct kbase_device *kbde #endif } #if MALI_USE_CSF - else if (strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "csf")) { + else if (!strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "csf")) { if (access_type == AS_FAULTSTATUS_ACCESS_TYPE_READ) return decode_fault_source_csf_r_t(ir, kbdev->gpu_props.gpu_id.arch_id); else return decode_fault_source_csf_w_t(ir, kbdev->gpu_props.gpu_id.arch_id); } #else - else if (strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "jm")) + else if (!strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "jm")) return decode_fault_source_jm_t(ir, kbdev->gpu_props.gpu_id.arch_id); #endif else if (!strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "I2c") || diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw.h b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw.h index c2b377de54a9..560baceafe8a 100644 --- a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw.h +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -42,7 +42,7 @@ struct kbase_context; * enum kbase_mmu_fault_type - MMU fault type descriptor. * @KBASE_MMU_FAULT_TYPE_UNKNOWN: unknown fault * @KBASE_MMU_FAULT_TYPE_PAGE: page fault - * @KBASE_MMU_FAULT_TYPE_BUS: nus fault + * @KBASE_MMU_FAULT_TYPE_BUS: bus fault * @KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED: page_unexpected fault * @KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED: bus_unexpected fault */ diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw_direct.c b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw_direct.c index ba67ae0e01e9..46c04f2b1fc1 100644 --- a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw_direct.c +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw_direct.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -529,8 +529,8 @@ int kbase_mmu_hw_do_flush(struct kbase_device *kbdev, struct kbase_as *as, return ret; #if MALI_USE_CSF && !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) - /* WA for the BASE_HW_ISSUE_GPU2019_3901. */ - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_GPU2019_3901) && + /* WA for the KBASE_HW_ISSUE_GPU2019_3901. */ + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_GPU2019_3901) && mmu_cmd == AS_COMMAND_COMMAND_FLUSH_MEM) { ret = apply_hw_issue_GPU2019_3901_wa(kbdev, &mmu_cmd, as->number); if (ret) { @@ -635,6 +635,15 @@ void kbase_mmu_hw_clear_fault(struct kbase_device *kbdev, struct kbase_as *as, #endif kbase_reg_write32(kbdev, MMU_CONTROL_ENUM(IRQ_CLEAR), pf_bf_mask); +#if MALI_USE_CSF + /* For valid page faults, this function is called just before unblocking the MMU (which + * would in turn unblock the MCU firmware) and so this is an opportune location to + * update the page fault counter value in firmware visible memory. + */ + if (likely(type == KBASE_MMU_FAULT_TYPE_PAGE) && kbdev->csf.page_fault_cnt_ptr) + *kbdev->csf.page_fault_cnt_ptr = ++kbdev->csf.page_fault_cnt; +#endif + unlock: spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); } diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_internal.h b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_internal.h index 4c2c1a64ca41..8b68791e4c77 100644 --- a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_internal.h +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_internal.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -38,19 +38,6 @@ void kbase_gpu_report_bus_fault_and_kill(struct kbase_context *kctx, struct kbas void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, struct kbase_as *as, const char *reason_str, struct kbase_fault *fault); -/** - * kbase_mmu_switch_to_ir() - Switch to incremental rendering if possible - * @kctx: kbase_context for the faulting address space. - * @reg: of a growable GPU memory region in the same context. - * Takes ownership of the reference if successful. - * - * Used to switch to incremental rendering if we have nearly run out of - * virtual address space in a growable memory region. - * - * Return: 0 if successful, otherwise a negative error code. - */ -int kbase_mmu_switch_to_ir(struct kbase_context *kctx, struct kbase_va_region *reg); - /** * kbase_mmu_page_fault_worker() - Process a page fault. * diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_aarch64.c b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_aarch64.c index e3ad78daed59..7aace473011f 100644 --- a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_aarch64.c +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_aarch64.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -179,7 +179,7 @@ static void set_num_valid_entries(u64 *pgd, unsigned int num_of_valid_entries) static void entry_set_pte(u64 *entry, phys_addr_t phy) { - page_table_entry_set(entry, (phy & PAGE_MASK) | ENTRY_ACCESS_BIT | ENTRY_IS_PTE); + page_table_entry_set(entry, (phy & GPU_PAGE_MASK) | ENTRY_ACCESS_BIT | ENTRY_IS_PTE); } static void entries_invalidate(u64 *entry, u32 count) diff --git a/drivers/gpu/arm/bifrost/tests/Kconfig b/drivers/gpu/arm/bifrost/tests/Kconfig index aa011bac8990..88a4194c5cd7 100644 --- a/drivers/gpu/arm/bifrost/tests/Kconfig +++ b/drivers/gpu/arm/bifrost/tests/Kconfig @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2017, 2020-2023 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2017, 2020-2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c index 6c343cf9f73b..0598d4397e2a 100644 --- a/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c +++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -425,7 +425,7 @@ static const char *kutf_clk_trace_do_get_platform(struct kutf_context *context, const void *arbiter_if_node = NULL; const void *power_node = NULL; const char *platform = "GPU"; -#if defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF) +#if defined(CONFIG_OF) struct kutf_clk_rate_trace_fixture_data *data = context->fixture; arbiter_if_node = of_get_property(data->kbdev->dev->of_node, "arbiter-if", NULL); diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_mgm_integration_test/mali_kutf_mgm_integration_test_main.c b/drivers/gpu/arm/bifrost/tests/mali_kutf_mgm_integration_test/mali_kutf_mgm_integration_test_main.c index 8937d69f182f..f341a411324e 100644 --- a/drivers/gpu/arm/bifrost/tests/mali_kutf_mgm_integration_test/mali_kutf_mgm_integration_test_main.c +++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_mgm_integration_test/mali_kutf_mgm_integration_test_main.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2022-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2022-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -95,7 +95,9 @@ static void mali_kutf_mgm_pte_translation_test(struct kutf_context *context) data->group_id, mmu_level, original_pte); translated_pte = mgm_dev->ops.mgm_update_gpu_pte(mgm_dev, data->group_id, - mmu_level, original_pte); + PBHA_ID_DEFAULT, + PTE_FLAGS_NONE, mmu_level, + original_pte); if (translated_pte == original_pte) { snprintf( msg_buf, sizeof(msg_buf), diff --git a/drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_csf.c b/drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_csf.c index a91278dd3bef..f254aa84dc20 100644 --- a/drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_csf.c +++ b/drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_csf.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -35,7 +35,7 @@ void kbase_create_timeline_objects(struct kbase_device *kbdev) u32 const num_sb_entries = kbdev->gpu_props.gpu_id.arch_major >= 11 ? 16 : 8; u32 const supports_gpu_sleep = #ifdef KBASE_PM_RUNTIME - kbdev->pm.backend.gpu_sleep_supported; + test_bit(KBASE_GPU_SUPPORTS_GPU_SLEEP, &kbdev->pm.backend.gpu_sleep_allowed); #else false; #endif /* KBASE_PM_RUNTIME */ diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline_io.c b/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline_io.c index d98e22880419..719e26124409 100644 --- a/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline_io.c +++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline_io.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -29,6 +29,7 @@ #include #include #include +#include /* Explicitly include epoll header for old kernels. Not required from 4.16. */ #if KERNEL_VERSION(4, 16, 0) > LINUX_VERSION_CODE @@ -169,7 +170,7 @@ static inline int copy_stream_header(char __user *buffer, size_t size, ssize_t * const char *hdr, size_t hdr_size, size_t *hdr_btc) { const size_t offset = hdr_size - *hdr_btc; - const size_t copy_size = MIN((size_t)((ssize_t)size - *copy_len), *hdr_btc); + const size_t copy_size = MIN(size_sub((ssize_t)size, *copy_len), *hdr_btc); if (!*hdr_btc) return 0; diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.c b/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.c index 34cabbd6e535..d4465c44addb 100644 --- a/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.c +++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.c @@ -62,6 +62,7 @@ enum tl_msg_id_obj { KBASE_TL_EVENT_ATOM_SOFTJOB_START, KBASE_TL_EVENT_ATOM_SOFTJOB_END, KBASE_TL_ARBITER_GRANTED, + KBASE_TL_ARBITER_LOST, KBASE_TL_ARBITER_STARTED, KBASE_TL_ARBITER_STOP_REQUESTED, KBASE_TL_ARBITER_STOPPED, @@ -272,6 +273,10 @@ enum tl_msg_id_obj { "Arbiter has granted gpu access", \ "@p", \ "gpu") \ + TRACEPOINT_DESC(KBASE_TL_ARBITER_LOST, \ + "Received a gpu lost event from the arbiter", \ + "@p", \ + "gpu") \ TRACEPOINT_DESC(KBASE_TL_ARBITER_STARTED, \ "Driver is running again and able to process jobs", \ "@p", \ @@ -1546,6 +1551,29 @@ void __kbase_tlstream_tl_arbiter_granted( kbase_tlstream_msgbuf_release(stream, acq_flags); } +void __kbase_tlstream_tl_arbiter_lost( + struct kbase_tlstream *stream, + const void *gpu +) +{ + const u32 msg_id = KBASE_TL_ARBITER_LOST; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(gpu) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &gpu, sizeof(gpu)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + void __kbase_tlstream_tl_arbiter_started( struct kbase_tlstream *stream, const void *gpu diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.h b/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.h index 987d4d3dbf27..6dd4b44ea6b2 100644 --- a/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.h +++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.h @@ -271,6 +271,11 @@ void __kbase_tlstream_tl_arbiter_granted( const void *gpu ); +void __kbase_tlstream_tl_arbiter_lost( + struct kbase_tlstream *stream, + const void *gpu +); + void __kbase_tlstream_tl_arbiter_started( struct kbase_tlstream *stream, const void *gpu @@ -1550,6 +1555,25 @@ struct kbase_tlstream; ); \ } while (0) +/** + * KBASE_TLSTREAM_TL_ARBITER_LOST - Received a gpu lost event from the arbiter + * + * @kbdev: Kbase device + * @gpu: Name of the GPU object + */ +#define KBASE_TLSTREAM_TL_ARBITER_LOST( \ + kbdev, \ + gpu \ + ) \ + do { \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_arbiter_lost( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + gpu \ + ); \ + } while (0) + /** * KBASE_TLSTREAM_TL_ARBITER_STARTED - Driver is running again and able to process jobs * diff --git a/drivers/hwtracing/coresight/mali/build.bp b/drivers/hwtracing/coresight/mali/build.bp index 33dcd22fa364..d69148c8cb70 100644 --- a/drivers/hwtracing/coresight/mali/build.bp +++ b/drivers/hwtracing/coresight/mali/build.bp @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2022-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2022-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/drivers/hwtracing/coresight/mali/sources/itm/coresight_mali_source_itm_core.c b/drivers/hwtracing/coresight/mali/sources/itm/coresight_mali_source_itm_core.c index 59d5cd314c2f..727e5c7a552a 100644 --- a/drivers/hwtracing/coresight/mali/sources/itm/coresight_mali_source_itm_core.c +++ b/drivers/hwtracing/coresight/mali/sources/itm/coresight_mali_source_itm_core.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2022-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2022-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -85,14 +85,14 @@ static struct kbase_debug_coresight_csf_op dwt_itm_enable_ops[] = { }; static struct kbase_debug_coresight_csf_op dwt_itm_disable_ops[] = { - // Disable ITM/DWT functionality via DEMCR register - WRITE_IMM_OP(CS_SCS_BASE_ADDR + SCS_DEMCR, 0x00000000), // Unlock ITM configuration WRITE_IMM_OP(CS_ITM_BASE_ADDR + CORESIGHT_LAR, CS_MALI_UNLOCK_COMPONENT), // Check ITM is disabled POLL_OP(CS_ITM_BASE_ADDR + ITM_TCR, ITM_TCR_BUSY_BIT, 0x0), // Lock WRITE_IMM_OP(CS_ITM_BASE_ADDR + CORESIGHT_LAR, 0x00000000), + // Disable ITM/DWT functionality via DEMCR register + WRITE_IMM_OP(CS_SCS_BASE_ADDR + SCS_DEMCR, 0x00000000), // Set enabled bit off at the end of sequence BIT_AND_OP(&itm_state.enabled, 0x0), }; diff --git a/include/linux/mali_hw_access.h b/include/linux/mali_hw_access.h index 4ed9da994e57..106393fc3372 100644 --- a/include/linux/mali_hw_access.h +++ b/include/linux/mali_hw_access.h @@ -30,6 +30,13 @@ #define mali_writel(val, addr) writel(val, addr) +#define mali_ioremap(addr, size) ioremap(addr, size) + +#define mali_iounmap(addr) iounmap(addr) + +#define mali_arch_timer_get_cntfrq() arch_timer_get_cntfrq() + + #define mali_readq(addr) ((u64)mali_readl(addr) | ((u64)mali_readl(addr + 4) << 32)) static inline u64 mali_readq_coherent(const void __iomem *addr) @@ -45,17 +52,11 @@ static inline u64 mali_readq_coherent(const void __iomem *addr) return lo | (((u64)hi1) << 32); } -#define mali_writeq(val, addr) \ - do { \ - mali_writel(val & 0xFFFFFFFF, addr); \ - mali_writel(val >> 32, addr + 4); \ +#define mali_writeq(val, addr) \ + do { \ + u64 __val = (u64)val; \ + mali_writel((u32)(__val & 0xFFFFFFFF), addr); \ + mali_writel((u32)(__val >> 32), addr + 4); \ } while (0) -#define mali_ioremap(addr, size) ioremap(addr, size) - -#define mali_iounmap(addr) iounmap(addr) - -#define mali_arch_timer_get_cntfrq() arch_timer_get_cntfrq() - - #endif /* _MALI_HW_ACCESS_H_ */ diff --git a/include/linux/memory_group_manager.h b/include/linux/memory_group_manager.h index e92d3dea0178..ec55d74f56ad 100644 --- a/include/linux/memory_group_manager.h +++ b/include/linux/memory_group_manager.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -35,10 +35,16 @@ typedef int vm_fault_t; #define PTE_PBHA_SHIFT (59) #define PTE_PBHA_MASK ((uint64_t)0xf << PTE_PBHA_SHIFT) #define PTE_RES_BIT_MULTI_AS_SHIFT (63) +#define PTE_FLAGS_NONE (0) +#define PBHA_ID_DEFAULT (0) struct memory_group_manager_device; struct memory_group_manager_import_data; +enum mgm_pte_flags { + MMA_VIOLATION = 0, +}; + /** * struct memory_group_manager_ops - Callbacks for memory group manager * operations @@ -50,6 +56,8 @@ struct memory_group_manager_import_data; * @mgm_pte_to_original_pte: Callback to get the original PTE entry as given * to mgm_update_gpu_pte * @mgm_vmf_insert_pfn_prot: Callback to map a physical memory page for the CPU + * @mgm_get_import_memory_cached_access_permitted: Callback to query if a given imported + * memory is allowed to be accessed as cached or not by the GPU */ struct memory_group_manager_ops { /* @@ -119,6 +127,11 @@ struct memory_group_manager_ops { * @group_id: A physical memory group ID. The meaning of this is * defined by the systems integrator. Its valid range is * 0 .. MEMORY_GROUP_MANAGER_NR_GROUPS-1. + * + * @pbha_id: PBHA Overrride ID to encode into the PTE + * @pte_flags: PTE related flags, defined in enum mgm_pte_flags + * + * * @mmu_level: The level of the page table entry in @ate. * @pte: The page table entry to modify, in LPAE or AArch64 format * (depending on the driver's configuration). This should be @@ -128,13 +141,14 @@ struct memory_group_manager_ops { * This function allows the memory group manager to modify a GPU page * table entry before it is stored by the kbase module (controller * driver). It may set certain bits in the page table entry attributes - * or modify the physical address, based on the physical memory group ID - * and/or additional data in struct memory_group_manager_device. + * or modify the physical address, based on the physical memory group ID, + * PBHA ID, PTE flags and/or additional data in struct memory_group_manager_device. * * Return: A modified GPU page table entry to be stored in a page table. */ u64 (*mgm_update_gpu_pte)(struct memory_group_manager_device *mgm_dev, - unsigned int group_id, int mmu_level, u64 pte); + unsigned int group_id, unsigned int pbha_id, + unsigned int pte_flags, int mmu_level, u64 pte); /* * mgm_pte_to_original_pte - Undo any modification done during mgm_update_gpu_pte() @@ -182,6 +196,20 @@ struct memory_group_manager_ops { unsigned int group_id, struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, pgprot_t pgprot); + + /* + * mgm_get_import_memory_cached_access_permitted - Check if a given imported memory + * is allowed to be accessed as cached or not by the GPU + * + * @mgm_dev: The memory group manager through which the request + * is being made. + * @import_data: Pointer to the data which describes imported memory. + * + * Return: true if cached access is permitted, false otherwise + */ + bool (*mgm_get_import_memory_cached_access_permitted)( + struct memory_group_manager_device *mgm_dev, + struct memory_group_manager_import_data *import_data); }; /** diff --git a/include/linux/version_compat_defs.h b/include/linux/version_compat_defs.h index 46f227e108f6..d94a39ebbadc 100644 --- a/include/linux/version_compat_defs.h +++ b/include/linux/version_compat_defs.h @@ -26,6 +26,9 @@ #include #include #include +#include +#include +#include #if (KERNEL_VERSION(4, 4, 267) < LINUX_VERSION_CODE) #include @@ -176,6 +179,7 @@ static inline void kbase_kunmap_atomic(void *address) */ #define check_mul_overflow(a, b, d) __builtin_mul_overflow(a, b, d) #define check_add_overflow(a, b, d) __builtin_add_overflow(a, b, d) +#define check_sub_overflow(a, b, d) __builtin_sub_overflow(a, b, d) #endif /* @@ -356,8 +360,10 @@ static inline long kbase_pin_user_pages_remote(struct task_struct *tsk, struct m #if KERNEL_VERSION(6, 0, 0) > LINUX_VERSION_CODE #define KBASE_REGISTER_SHRINKER(reclaim, name, priv_data) register_shrinker(reclaim) +/* clang-format off */ #elif ((KERNEL_VERSION(6, 7, 0) > LINUX_VERSION_CODE) && \ !(defined(__ANDROID_COMMON_KERNEL__) && (KERNEL_VERSION(6, 6, 0) == LINUX_VERSION_CODE))) +/* clang-format on */ #define KBASE_REGISTER_SHRINKER(reclaim, name, priv_data) register_shrinker(reclaim, name) #else @@ -368,9 +374,10 @@ static inline long kbase_pin_user_pages_remote(struct task_struct *tsk, struct m } while (0) #endif /* KERNEL_VERSION(6, 0, 0) > LINUX_VERSION_CODE */ - +/* clang-format off */ #if ((KERNEL_VERSION(6, 7, 0) > LINUX_VERSION_CODE) && \ !(defined(__ANDROID_COMMON_KERNEL__) && (KERNEL_VERSION(6, 6, 0) == LINUX_VERSION_CODE))) +/* clang-format on */ #define KBASE_UNREGISTER_SHRINKER(reclaim) unregister_shrinker(&reclaim) #define KBASE_GET_KBASE_DATA_FROM_SHRINKER(s, type, var) container_of(s, type, var) #define DEFINE_KBASE_SHRINKER struct shrinker @@ -387,6 +394,31 @@ static inline long kbase_pin_user_pages_remote(struct task_struct *tsk, struct m #endif +static inline int kbase_param_set_uint_minmax(const char *val, const struct kernel_param *kp, + unsigned int min, unsigned int max) +{ +#if (KERNEL_VERSION(5, 15, 0) > LINUX_VERSION_CODE) + uint uint_val; + int ret; + + if (!val) + return -EINVAL; + + ret = kstrtouint(val, 0, &uint_val); + + if (ret == 0) { + if (uint_val < min || uint_val > max) + return -EINVAL; + + *((uint *)kp->arg) = uint_val; + } + + return ret; +#else + return param_set_uint_minmax(val, kp, min, max); +#endif +} + #if (KERNEL_VERSION(4, 20, 0) <= LINUX_VERSION_CODE) #include #endif @@ -394,4 +426,99 @@ static inline long kbase_pin_user_pages_remote(struct task_struct *tsk, struct m #define __maybe_unused __attribute__((unused)) #endif +#if KERNEL_VERSION(5, 4, 103) <= LINUX_VERSION_CODE +#define mali_sysfs_emit(buf, fmt, ...) sysfs_emit(buf, fmt, __VA_ARGS__) +#else +#define mali_sysfs_emit(buf, fmt, ...) scnprintf(buf, PAGE_SIZE, fmt, __VA_ARGS__) +#endif + +#if KERNEL_VERSION(5, 10, 0) > LINUX_VERSION_CODE +#include +#include + +static inline struct devfreq *devfreq_get_devfreq_by_node(struct device_node *node) +{ + struct platform_device *pdev = of_find_device_by_node(node); + + if (!pdev || !node) + return NULL; + + return devfreq_get_devfreq_by_phandle(&pdev->dev, 0); +} +#endif + +#if (KERNEL_VERSION(5, 16, 0) <= LINUX_VERSION_CODE && \ + KERNEL_VERSION(5, 18, 0) > LINUX_VERSION_CODE) || \ + (KERNEL_VERSION(5, 11, 0) <= LINUX_VERSION_CODE && \ + KERNEL_VERSION(5, 15, 85) >= LINUX_VERSION_CODE) || \ + (KERNEL_VERSION(5, 10, 200) >= LINUX_VERSION_CODE) +/* + * Kernel revisions + * - up to 5.10.200 + * - between 5.11.0 and 5.15.85 inclusive + * - between 5.16.0 and 5.17.15 inclusive + * do not provide an implementation of + * size_add, size_sub and size_mul. + * The implementations below provides + * backward compatibility implementations of these functions. + */ + +static inline size_t __must_check size_mul(size_t factor1, size_t factor2) +{ + size_t ret_val; + + if (check_mul_overflow(factor1, factor2, &ret_val)) + return SIZE_MAX; + return ret_val; +} + +static inline size_t __must_check size_add(size_t addend1, size_t addend2) +{ + size_t ret_val; + + if (check_add_overflow(addend1, addend2, &ret_val)) + return SIZE_MAX; + return ret_val; +} + +static inline size_t __must_check size_sub(size_t minuend, size_t subtrahend) +{ + size_t ret_val; + + if (minuend == SIZE_MAX || subtrahend == SIZE_MAX || + check_sub_overflow(minuend, subtrahend, &ret_val)) + return SIZE_MAX; + return ret_val; +} +#endif + +#if KERNEL_VERSION(5, 5, 0) > LINUX_VERSION_CODE +static inline unsigned long bitmap_get_value8(const unsigned long *map, unsigned long start) +{ + const size_t index = BIT_WORD(start); + const unsigned long offset = start % BITS_PER_LONG; + + return (map[index] >> offset) & 0xFF; +} + +static inline unsigned long find_next_clump8(unsigned long *clump, const unsigned long *addr, + unsigned long size, unsigned long offset) +{ + offset = find_next_bit(addr, size, offset); + if (offset == size) + return size; + + offset = round_down(offset, 8); + *clump = bitmap_get_value8(addr, offset); + + return offset; +} + +#define find_first_clump8(clump, bits, size) find_next_clump8((clump), (bits), (size), 0) + +#define for_each_set_clump8(start, clump, bits, size) \ + for ((start) = find_first_clump8(&(clump), (bits), (size)); (start) < (size); \ + (start) = find_next_clump8(&(clump), (bits), (size), (start) + 8)) +#endif + #endif /* _VERSION_COMPAT_DEFS_H_ */ diff --git a/include/uapi/gpu/arm/bifrost/csf/mali_base_csf_kernel.h b/include/uapi/gpu/arm/bifrost/csf/mali_base_csf_kernel.h index b4b38f242bd2..2b2fd1dd7bc8 100644 --- a/include/uapi/gpu/arm/bifrost/csf/mali_base_csf_kernel.h +++ b/include/uapi/gpu/arm/bifrost/csf/mali_base_csf_kernel.h @@ -61,13 +61,15 @@ * must be less than BASE_MEM_FLAGS_NR_BITS !!! */ -/* A mask of all the flags which are only valid for allocations within kbase, - * and may not be passed from user space. +/* A mask of all the flags which are only valid within kbase, + * and may not be passed to/from user space. */ #define BASEP_MEM_FLAGS_KERNEL_ONLY (BASEP_MEM_PERMANENT_KERNEL_MAPPING | BASEP_MEM_NO_USER_FREE) -/* A mask of all flags that should not be queried */ -#define BASE_MEM_DONT_QUERY (BASE_MEM_COHERENT_SYSTEM_REQUIRED | BASE_MEM_IMPORT_SHARED) +/* A mask of flags that, when provied, cause other flags to be + * enabled but are not enabled themselves + */ +#define BASE_MEM_FLAGS_ACTION_MODIFIERS (BASE_MEM_COHERENT_SYSTEM_REQUIRED | BASE_MEM_IMPORT_SHARED) /* A mask of all currently reserved flags */ #define BASE_MEM_FLAGS_RESERVED ((base_mem_alloc_flags)0) diff --git a/include/uapi/gpu/arm/bifrost/csf/mali_kbase_csf_ioctl.h b/include/uapi/gpu/arm/bifrost/csf/mali_kbase_csf_ioctl.h index 28e7db49f676..2b5b8b25fc2c 100644 --- a/include/uapi/gpu/arm/bifrost/csf/mali_kbase_csf_ioctl.h +++ b/include/uapi/gpu/arm/bifrost/csf/mali_kbase_csf_ioctl.h @@ -116,10 +116,14 @@ * from the parent process. * 1.30: * - Implement support for setting GPU Timestamp Offset register. + * 1.31: + * - Reject non-protected allocations containing the BASE_MEM_PROTECTED memory flag. + * - Reject allocations containing the BASE_MEM_DONT_NEED memory flag (it is only settable). + * - Reject allocations containing the BASE_MEM_UNUSED_BIT_xx memory flags. */ #define BASE_UK_VERSION_MAJOR 1 -#define BASE_UK_VERSION_MINOR 30 +#define BASE_UK_VERSION_MINOR 31 /** * struct kbase_ioctl_version_check - Check version compatibility between diff --git a/include/uapi/gpu/arm/bifrost/jm/mali_base_jm_kernel.h b/include/uapi/gpu/arm/bifrost/jm/mali_base_jm_kernel.h index b5351c50aa30..fad61299b1c1 100644 --- a/include/uapi/gpu/arm/bifrost/jm/mali_base_jm_kernel.h +++ b/include/uapi/gpu/arm/bifrost/jm/mali_base_jm_kernel.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -53,15 +53,17 @@ * must be less than BASE_MEM_FLAGS_NR_BITS !!! */ -/* A mask of all the flags which are only valid for allocations within kbase, - * and may not be passed from user space. +/* A mask of all the flags which are only valid within kbase, + * and may not be passed to/from user space. */ #define BASEP_MEM_FLAGS_KERNEL_ONLY \ (BASEP_MEM_PERMANENT_KERNEL_MAPPING | BASEP_MEM_NO_USER_FREE | BASE_MEM_FLAG_MAP_FIXED | \ BASEP_MEM_PERFORM_JIT_TRIM) -/* A mask of all flags that should not be queried */ -#define BASE_MEM_DONT_QUERY (BASE_MEM_COHERENT_SYSTEM_REQUIRED | BASE_MEM_IMPORT_SHARED) +/* A mask of flags that, when provied, cause other flags to be + * enabled but are not enabled themselves + */ +#define BASE_MEM_FLAGS_ACTION_MODIFIERS (BASE_MEM_COHERENT_SYSTEM_REQUIRED | BASE_MEM_IMPORT_SHARED) /* A mask of all currently reserved flags */ #define BASE_MEM_FLAGS_RESERVED ((base_mem_alloc_flags)0) @@ -120,10 +122,6 @@ */ #define BASE_JD_ATOM_COUNT 256 -/* Maximum number of concurrent render passes. - */ -#define BASE_JD_RP_COUNT (256) - /* Set/reset values for a software event */ #define BASE_JD_SOFT_EVENT_SET ((unsigned char)1) #define BASE_JD_SOFT_EVENT_RESET ((unsigned char)0) @@ -363,40 +361,6 @@ typedef __u32 base_jd_core_req; */ #define BASE_JD_REQ_JOB_SLOT ((base_jd_core_req)1 << 17) -/* SW-only requirement: The atom is the start of a renderpass. - * - * If this bit is set then the job chain will be soft-stopped if it causes the - * GPU to write beyond the end of the physical pages backing the tiler heap, and - * committing more memory to the heap would exceed an internal threshold. It may - * be resumed after running one of the job chains attached to an atom with - * BASE_JD_REQ_END_RENDERPASS set and the same renderpass ID. It may be - * resumed multiple times until it completes without memory usage exceeding the - * threshold. - * - * Usually used with BASE_JD_REQ_T. - */ -#define BASE_JD_REQ_START_RENDERPASS ((base_jd_core_req)1 << 18) - -/* SW-only requirement: The atom is the end of a renderpass. - * - * If this bit is set then the atom incorporates the CPU address of a - * base_jd_fragment object instead of the GPU address of a job chain. - * - * Which job chain is run depends upon whether the atom with the same renderpass - * ID and the BASE_JD_REQ_START_RENDERPASS bit set completed normally or - * was soft-stopped when it exceeded an upper threshold for tiler heap memory - * usage. - * - * It also depends upon whether one of the job chains attached to the atom has - * already been run as part of the same renderpass (in which case it would have - * written unresolved multisampled and otherwise-discarded output to temporary - * buffers that need to be read back). The job chain for doing a forced read and - * forced write (from/to temporary buffers) is run as many times as necessary. - * - * Usually used with BASE_JD_REQ_FS. - */ -#define BASE_JD_REQ_END_RENDERPASS ((base_jd_core_req)1 << 19) - /* SW-only requirement: The atom needs to run on a limited core mask affinity. * * If this bit is set then the kbase_context.limited_core_mask will be applied @@ -412,7 +376,6 @@ typedef __u32 base_jd_core_req; BASE_JD_REQ_EVENT_COALESCE | BASE_JD_REQ_COHERENT_GROUP | \ BASE_JD_REQ_SPECIFIC_COHERENT_GROUP | BASE_JD_REQ_FS_AFBC | BASE_JD_REQ_PERMON | \ BASE_JD_REQ_SKIP_CACHE_START | BASE_JD_REQ_SKIP_CACHE_END | BASE_JD_REQ_JOB_SLOT | \ - BASE_JD_REQ_START_RENDERPASS | BASE_JD_REQ_END_RENDERPASS | \ BASE_JD_REQ_LIMITED_CORE_MASK)) /* Mask of all bits in base_jd_core_req that control the type of the atom. @@ -471,62 +434,6 @@ struct base_dependency { base_jd_dep_type dependency_type; }; -/** - * struct base_jd_fragment - Set of GPU fragment job chains used for rendering. - * - * @norm_read_norm_write: Job chain for full rendering. - * GPU address of a fragment job chain to render in the - * circumstance where the tiler job chain did not exceed - * its memory usage threshold and no fragment job chain - * was previously run for the same renderpass. - * It is used no more than once per renderpass. - * @norm_read_forced_write: Job chain for starting incremental - * rendering. - * GPU address of a fragment job chain to render in - * the circumstance where the tiler job chain exceeded - * its memory usage threshold for the first time and - * no fragment job chain was previously run for the - * same renderpass. - * Writes unresolved multisampled and normally- - * discarded output to temporary buffers that must be - * read back by a subsequent forced_read job chain - * before the renderpass is complete. - * It is used no more than once per renderpass. - * @forced_read_forced_write: Job chain for continuing incremental - * rendering. - * GPU address of a fragment job chain to render in - * the circumstance where the tiler job chain - * exceeded its memory usage threshold again - * and a fragment job chain was previously run for - * the same renderpass. - * Reads unresolved multisampled and - * normally-discarded output from temporary buffers - * written by a previous forced_write job chain and - * writes the same to temporary buffers again. - * It is used as many times as required until - * rendering completes. - * @forced_read_norm_write: Job chain for ending incremental rendering. - * GPU address of a fragment job chain to render in the - * circumstance where the tiler job chain did not - * exceed its memory usage threshold this time and a - * fragment job chain was previously run for the same - * renderpass. - * Reads unresolved multisampled and normally-discarded - * output from temporary buffers written by a previous - * forced_write job chain in order to complete a - * renderpass. - * It is used no more than once per renderpass. - * - * This structure is referenced by the main atom structure if - * BASE_JD_REQ_END_RENDERPASS is set in the base_jd_core_req. - */ -struct base_jd_fragment { - __u64 norm_read_norm_write; - __u64 norm_read_forced_write; - __u64 forced_read_forced_write; - __u64 forced_read_norm_write; -}; - /** * typedef base_jd_prio - Base Atom priority. * @@ -591,9 +498,7 @@ typedef __u8 base_jd_prio; * struct base_jd_atom_v2 - Node of a dependency graph used to submit a * GPU job chain or soft-job to the kernel driver. * - * @jc: GPU address of a job chain or (if BASE_JD_REQ_END_RENDERPASS - * is set in the base_jd_core_req) the CPU address of a - * base_jd_fragment object. + * @jc: GPU address of a job chain. * @udata: User data. * @extres_list: List of external resources. * @nr_extres: Number of external resources or JIT allocations. @@ -612,9 +517,6 @@ typedef __u8 base_jd_prio; * specified. * @jobslot: Job slot to use when BASE_JD_REQ_JOB_SLOT is specified. * @core_req: Core requirements. - * @renderpass_id: Renderpass identifier used to associate an atom that has - * BASE_JD_REQ_START_RENDERPASS set in its core requirements - * with an atom that has BASE_JD_REQ_END_RENDERPASS set. * @padding: Unused. Must be zero. * * This structure has changed since UK 10.2 for which base_jd_core_req was a @@ -642,8 +544,7 @@ struct base_jd_atom_v2 { __u8 device_nr; __u8 jobslot; base_jd_core_req core_req; - __u8 renderpass_id; - __u8 padding[7]; + __u8 padding[8]; }; /** @@ -651,9 +552,7 @@ struct base_jd_atom_v2 { * at the beginning. * * @seq_nr: Sequence number of logical grouping of atoms. - * @jc: GPU address of a job chain or (if BASE_JD_REQ_END_RENDERPASS - * is set in the base_jd_core_req) the CPU address of a - * base_jd_fragment object. + * @jc: GPU address of a job chain. * @udata: User data. * @extres_list: List of external resources. * @nr_extres: Number of external resources or JIT allocations. @@ -835,11 +734,6 @@ enum { * @BASE_JD_EVENT_REMOVED_FROM_NEXT: raised when an atom that was configured in * the GPU has to be retried (but it has not * started) due to e.g., GPU reset - * @BASE_JD_EVENT_END_RP_DONE: this is used for incremental rendering to signal - * the completion of a renderpass. This value - * shouldn't be returned to userspace but I haven't - * seen where it is reset back to JD_EVENT_DONE. - * * HW and low-level SW events are represented by event codes. * The status of jobs which succeeded are also represented by * an event code (see @BASE_JD_EVENT_DONE). @@ -938,8 +832,6 @@ enum base_jd_event_code { BASE_JD_EVENT_RANGE_KERNEL_ONLY_START = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | 0x000, BASE_JD_EVENT_REMOVED_FROM_NEXT = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | BASE_JD_SW_EVENT_JOB | 0x000, - BASE_JD_EVENT_END_RP_DONE = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | - BASE_JD_SW_EVENT_JOB | 0x001, BASE_JD_EVENT_RANGE_KERNEL_ONLY_END = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | BASE_JD_SW_EVENT_RESERVED | 0x3FF diff --git a/include/uapi/gpu/arm/bifrost/jm/mali_kbase_jm_ioctl.h b/include/uapi/gpu/arm/bifrost/jm/mali_kbase_jm_ioctl.h index 1c115adb8172..34da87860ee2 100644 --- a/include/uapi/gpu/arm/bifrost/jm/mali_kbase_jm_ioctl.h +++ b/include/uapi/gpu/arm/bifrost/jm/mali_kbase_jm_ioctl.h @@ -164,10 +164,16 @@ * - Re-allow child process to do supported file operations (like mmap, ioctl * read, poll) on the file descriptor of mali device that was inherited * from the parent process. - */ + * 11.46: + * - Remove renderpass_id from base_jd_atom_v2 to deprecate support for JM Incremental Rendering + * 11.47: + * - Reject non-protected allocations containing the BASE_MEM_PROTECTED memory flag. + * - Reject allocations containing the BASE_MEM_DONT_NEED memory flag (it is only settable). + * - Reject allocations containing the BASE_MEM_UNUSED_BIT_xx memory flags. + */ #define BASE_UK_VERSION_MAJOR 11 -#define BASE_UK_VERSION_MINOR 45 +#define BASE_UK_VERSION_MINOR 47 /** * struct kbase_ioctl_version_check - Check version compatibility between diff --git a/include/uapi/gpu/arm/bifrost/mali_base_common_kernel.h b/include/uapi/gpu/arm/bifrost/mali_base_common_kernel.h index c009d5ddd494..bbbee900415e 100644 --- a/include/uapi/gpu/arm/bifrost/mali_base_common_kernel.h +++ b/include/uapi/gpu/arm/bifrost/mali_base_common_kernel.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2022-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2022-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -146,7 +146,6 @@ struct base_mem_handle { */ #define BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP ((base_mem_alloc_flags)1 << 26) -/* OUT */ /* Kernel side cache sync ops required */ #define BASE_MEM_KERNEL_SYNC ((base_mem_alloc_flags)1 << 28) @@ -156,12 +155,10 @@ struct base_mem_handle { */ #define BASE_MEM_FLAGS_NR_BITS 30 -/* A mask for all output bits, excluding IN/OUT bits. - */ +/* A mask for all bits that are output from kbase, but never input. */ #define BASE_MEM_FLAGS_OUTPUT_MASK BASE_MEM_NEED_MMAP -/* A mask for all input bits, including IN/OUT bits. - */ +/* A mask for all bits that can be input to kbase. */ #define BASE_MEM_FLAGS_INPUT_MASK \ (((1 << BASE_MEM_FLAGS_NR_BITS) - 1) & ~BASE_MEM_FLAGS_OUTPUT_MASK) diff --git a/include/uapi/gpu/arm/bifrost/mali_base_kernel.h b/include/uapi/gpu/arm/bifrost/mali_base_kernel.h index 198d6b8a3942..9e7294970efb 100644 --- a/include/uapi/gpu/arm/bifrost/mali_base_kernel.h +++ b/include/uapi/gpu/arm/bifrost/mali_base_kernel.h @@ -74,7 +74,7 @@ * More flags can be added to this list, as long as they don't clash * (see BASE_MEM_FLAGS_NR_BITS for the number of the first free bit). */ -typedef __u32 base_mem_alloc_flags; +typedef __u64 base_mem_alloc_flags; #define BASE_MEM_FLAGS_MODIFIABLE_NATIVE (BASE_MEM_DONT_NEED) @@ -89,9 +89,10 @@ typedef __u32 base_mem_alloc_flags; /* A mask of all the flags that can be returned via the base_mem_get_flags() * interface. */ -#define BASE_MEM_FLAGS_QUERYABLE \ - (BASE_MEM_FLAGS_INPUT_MASK & ~(BASE_MEM_DONT_QUERY | BASE_MEM_FLAGS_RESERVED | \ - BASE_MEM_FLAGS_UNUSED | BASEP_MEM_FLAGS_KERNEL_ONLY)) +#define BASE_MEM_FLAGS_QUERYABLE \ + (BASE_MEM_FLAGS_INPUT_MASK & \ + ~(BASE_MEM_FLAGS_RESERVED | BASE_MEM_FLAGS_UNUSED | BASE_MEM_FLAGS_ACTION_MODIFIERS | \ + BASEP_MEM_FLAGS_KERNEL_ONLY)) /** * enum base_mem_import_type - Memory types supported by @a base_mem_import diff --git a/include/uapi/gpu/arm/bifrost/mali_kbase_ioctl.h b/include/uapi/gpu/arm/bifrost/mali_kbase_ioctl.h index d60745f564b0..163637c62297 100644 --- a/include/uapi/gpu/arm/bifrost/mali_kbase_ioctl.h +++ b/include/uapi/gpu/arm/bifrost/mali_kbase_ioctl.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2017-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2017-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -169,7 +169,7 @@ struct kbase_ioctl_hwcnt_reader_setup { * struct kbase_ioctl_hwcnt_values - Values to set dummy the dummy counters to. * @data: Counter samples for the dummy model. * @size: Size of the counter sample data. - * @padding: Padding. + * @padding: Currently unused, must be zero */ struct kbase_ioctl_hwcnt_values { __u64 data; @@ -193,7 +193,7 @@ struct kbase_ioctl_disjoint_query { * struct kbase_ioctl_get_ddk_version - Query the kernel version * @version_buffer: Buffer to receive the kernel version string * @size: Size of the buffer - * @padding: Padding + * @padding: Currently unused, must be zero * * The ioctl will return the number of bytes written into version_buffer * (which includes a NULL byte) or a negative error code diff --git a/include/uapi/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs_buf_size.h b/include/uapi/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs_buf_size.h index 11c51d9c2993..648c166b1e3d 100644 --- a/include/uapi/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs_buf_size.h +++ b/include/uapi/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs_buf_size.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -30,6 +30,6 @@ * KBASE_MEM_PROFILE_MAX_BUF_SIZE - The size of the buffer to accumulate the histogram report text * in @see @ref CCTXP_HIST_BUF_SIZE_MAX_LENGTH_REPORT */ -#define KBASE_MEM_PROFILE_MAX_BUF_SIZE ((size_t)(64 + ((80 + (56 * 64)) * 57) + 56)) +#define KBASE_MEM_PROFILE_MAX_BUF_SIZE ((size_t)(64 + ((80 + (56 * 64)) * 69) + 56)) #endif /*_UAPI_KBASE_MEM_PROFILE_DEBUGFS_BUF_SIZE_H_*/