shinys000114/linux
1
0
Fork 0
mirror of https://github.com/hardkernel/linux.git synced 2026-06-06 19:08:57 +09:00
Code Issues Packages Projects Releases Wiki Activity

Merge commit 'd1b62d2e45b49ef7792d43a660d048d8618dee46'

Browse Source 
* commit 'd1b62d2e45b49ef7792d43a660d048d8618dee46':
  MALI: rockchip: upgrade bifrost DDK to g25p0-00eac0, from g24p0-00eac0
  MALI: rockchip: upgrade bifrost DDK to g24p0-00eac0, from g22p0-01eac0

Change-Id: I94905342bc16df1779accc1e3692a9e0796b2d1f

Conflicts:
	drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c
This commit is contained in:
Tao Huang
2024-09-12 20:25:55 +08:00
parent 8e02576946 d1b62d2e45
commit 06c3dc397c
198 changed files with 10325 additions and 5801 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
Documentation/ABI/testing/sysfs-device-mali
View File

@@ -341,8 +341,7 @@ Description:
device-driver that supports a CSF GPU.
Used to enable firmware logs, logging levels valid values
are indicated using 'min and 'max' attribute values
values that are read-only.
are indicated using 'min' and 'max' attributes, which are read-only.
Log level can be set using the 'cur' read, write attribute,
we can use a valid log level value from min and max range values

2
Documentation/ABI/testing/sysfs-device-mali-coresight-source
View File

@@ -19,7 +19,7 @@ Description:
What: /sys/bus/coresight/devices/mali-source-etm/is_enabled
Description:
Attribute used to check if Coresight Source ITM is enabled.
Attribute used to check if Coresight Source ETM is enabled.
What: /sys/bus/coresight/devices/mali-source-etm/trcconfigr
Description:

163
Documentation/devicetree/bindings/arm/arm,coresight-mali-source.yaml Normal file
View File

@@ -0,0 +1,163 @@
# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
#
# (C) COPYRIGHT 2022-2024 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU license.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, you can access it online at
# http://www.gnu.org/licenses/gpl-2.0.html.
#
#
%YAML 1.2
---
$id: http://devicetree.org/schemas/arm/arm,coresight-mali-source.yaml#
$schema: http://devicetree.org/meta-schemas/core.yaml#
title: ARM CoreSight Mali Source integration
maintainers:
- ARM Ltd.
description: |
See Documentation/trace/coresight/coresight.rst for detailed information
about Coresight.
This documentation will cover Mali specific devicetree integration.
References to Sink ports are given as examples. Access to Sink is specific
to an implementation and would require dedicated kernel modules.
Arm Mali GPU are supporting 3 different sources: ITM, ETM, ELA
ELA source configuration via SysFS entries:
The register values used by CoreSight for ELA can be configured using SysFS
interfaces. This implicitly includes configuring the ELA for independent or
shared JCN request and response channels.
properties:
compatible:
enum:
- arm,coresight-mali-source-itm
- arm,coresight-mali-source-etm
- arm,coresight-mali-source-ela
gpu:
minItems: 1
maxItems: 1
description:
Phandle to a Mali GPU definition
port:
description:
Output connection to CoreSight Sink Trace bus.
Legacy binding between Coresight Sources and CoreSight Sink.
For Linux kernel < v4.20.
$ref: /schemas/graph.yaml#/properties/port
out-ports:
description:
Binding between Coresight Sources and CoreSight Sink.
For Linux kernel >= v4.20.
$ref: /schemas/graph.yaml#/properties/ports
properties:
port:
description: Output connection to CoreSight Sink Trace bus.
$ref: /schemas/graph.yaml#/properties/port
required:
- compatible
- gpu
- port
- out-ports
additionalProperties: false
examples:
# A Sink node without legacy CoreSight connections
- |
mali-source-itm {
compatible = "arm,coresight-mali-source-itm";
gpu = <&gpu>;
out-ports {
port {
mali_source_itm_out_port0: endpoint {
remote-endpoint = <&mali_sink_in_port0>;
};
};
};
};
mali-source-ela {
compatible = "arm,coresight-mali-source-ela";
gpu = <&gpu>;
out-ports {
port {
mali_source_ela_out_port0: endpoint {
remote-endpoint = <&mali_sink_in_port1>;
};
};
};
};
mali-source-etm {
compatible = "arm,coresight-mali-source-etm";
gpu = <&gpu>;
out-ports {
port {
mali_source_etm_out_port0: endpoint {
remote-endpoint = <&mali_sink_in_port2>;
};
};
};
};
# A Sink node with legacy CoreSight connections
- |
mali-source-itm {
compatible = "arm,coresight-mali-source-itm";
gpu = <&gpu>;
port {
mali_source_itm_out_port0: endpoint {
remote-endpoint = <&mali_sink_in_port0>;
};
};
};
mali-source-etm {
compatible = "arm,coresight-mali-source-etm";
gpu = <&gpu>;
port {
mali_source_etm_out_port0: endpoint {
remote-endpoint = <&mali_sink_in_port1>;
};
};
};
mali-source-ela {
compatible = "arm,coresight-mali-source-ela";
gpu = <&gpu>;
port {
mali_source_ela_out_port0: endpoint {
remote-endpoint = <&mali_sink_in_port2>;
};
};
};

14
Documentation/devicetree/bindings/arm/mali-bifrost.txt
View File

@@ -1,6 +1,6 @@
# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
#
# (C) COPYRIGHT 2013-2023 ARM Limited. All rights reserved.
# (C) COPYRIGHT 2013-2024 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
@@ -111,7 +111,10 @@ for details.
- idvs-group-size : Override the IDVS group size value. Tasks are sent to
cores in groups of N + 1, so i.e. 0xF means 16 tasks.
Valid values are between 0 to 0x3F (including).
- l2-size : Override L2 cache size on GPU that supports it
- l2-size : Override L2 cache size on GPU that supports it. Value should be larger than the minimum
size 1KiB and smaller than the maximum size. Maximum size is Hardware integration dependent.
The value passed should be of log2(Cache Size in Bytes).
For example for a 1KiB of cache size, 0xa should be passed.
- l2-hash : Override L2 hash function on GPU that supports it
- l2-hash-values : Override L2 hash function using provided hash values, on GPUs that supports it.
It is mutually exclusive with 'l2-hash'. Only one or the other must be
@@ -129,6 +132,10 @@ for details.
set and the setting coresponding to the SYSC_ALLOC register.
- propagate-bits: Used to write to L2_CONFIG.PBHA_HWU. This bitset establishes which
PBHA bits are propagated on the AXI bus.
- mma-wa-id: Sets the PBHA ID to be used for the PBHA override based MMA violation workaround.
The read and write allocation override bits for the PBHA are set to NONCACHEABLE
and the driver encodes the PBHA ID in the PTEs where this workaround is to be applied.
Valid values are from 1 to 15.
Example for a Mali GPU with 1 clock and 1 regulator:
@@ -237,7 +244,8 @@ gpu@0xfc010000 {
...
pbha {
int-id-override = <2 0x32>, <9 0x05>, <16 0x32>;
propagate-bits = /bits/ 4 <0x03>;
propagate-bits = /bits/ 8 <0x03>;
mma-wa-id = <2>;
};
...
};

2
drivers/base/arm/Kconfig
View File

@@ -1,6 +1,6 @@
# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
#
# (C) COPYRIGHT 2021-2023 ARM Limited. All rights reserved.
# (C) COPYRIGHT 2021-2024 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software

2
drivers/base/arm/Makefile
View File

@@ -125,6 +125,8 @@ CFLAGS_MODULE += -Wno-sign-compare
CFLAGS_MODULE += -Wno-shift-negative-value
# This flag is needed to avoid build errors on older kernels
CFLAGS_MODULE += $(call cc-option, -Wno-cast-function-type)
# The following ensures the stack frame does not get larger than a page
CFLAGS_MODULE += -Wframe-larger-than=4096
KBUILD_CPPFLAGS += -DKBUILD_EXTRA_WARN1

28
drivers/base/arm/memory_group_manager/memory_group_manager.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -51,10 +51,6 @@ static inline vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigne
}
#endif
#define PTE_PBHA_SHIFT (59)
#define PTE_PBHA_MASK ((uint64_t)0xf << PTE_PBHA_SHIFT)
#define PTE_RES_BIT_MULTI_AS_SHIFT (63)
#define IMPORTED_MEMORY_ID (MEMORY_GROUP_MANAGER_NR_GROUPS - 1)
/**
@@ -263,7 +259,7 @@ static struct page *example_mgm_alloc_page(struct memory_group_manager_device *m
} else {
struct mgm_groups *data = mgm_dev->data;
dev_err(data->dev, "alloc_pages failed\n");
dev_dbg(data->dev, "alloc_pages failed\n");
}
return p;
@@ -303,7 +299,8 @@ static int example_mgm_get_import_memory_id(struct memory_group_manager_device *
}
static u64 example_mgm_update_gpu_pte(struct memory_group_manager_device *const mgm_dev,
unsigned int const group_id, int const mmu_level, u64 pte)
unsigned int const group_id, unsigned int const pbha_id,
unsigned int pte_flags, int const mmu_level, u64 pte)
{
struct mgm_groups *const data = mgm_dev->data;
@@ -313,7 +310,10 @@ static u64 example_mgm_update_gpu_pte(struct memory_group_manager_device *const
if (WARN_ON(group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS))
return pte;
pte |= ((u64)group_id << PTE_PBHA_SHIFT) & PTE_PBHA_MASK;
if (pte_flags & BIT(MMA_VIOLATION)) {
pr_warn_once("MMA violation! Applying PBHA override workaround to PTE\n");
pte |= ((u64)pbha_id << PTE_PBHA_SHIFT) & PTE_PBHA_MASK;
}
/* Address could be translated into a different bus address here */
pte |= ((u64)1 << PTE_RES_BIT_MULTI_AS_SHIFT);
@@ -366,6 +366,16 @@ static vm_fault_t example_mgm_vmf_insert_pfn_prot(struct memory_group_manager_de
return fault;
}
static bool example_mgm_get_import_memory_cached_access_permitted(
struct memory_group_manager_device *mgm_dev,
struct memory_group_manager_import_data *import_data)
{
CSTD_UNUSED(mgm_dev);
CSTD_UNUSED(import_data);
return true;
}
static int mgm_initialize_data(struct mgm_groups *mgm_data)
{
int i;
@@ -412,6 +422,8 @@ static int memory_group_manager_probe(struct platform_device *pdev)
mgm_dev->ops.mgm_vmf_insert_pfn_prot = example_mgm_vmf_insert_pfn_prot;
mgm_dev->ops.mgm_update_gpu_pte = example_mgm_update_gpu_pte;
mgm_dev->ops.mgm_pte_to_original_pte = example_mgm_pte_to_original_pte;
mgm_dev->ops.mgm_get_import_memory_cached_access_permitted =
example_mgm_get_import_memory_cached_access_permitted;
mgm_data = kzalloc(sizeof(*mgm_data), GFP_KERNEL);
if (!mgm_data) {

10
drivers/gpu/arm/bifrost/Kbuild
View File

@@ -1,6 +1,6 @@
# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
#
# (C) COPYRIGHT 2012-2023 ARM Limited. All rights reserved.
# (C) COPYRIGHT 2012-2024 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
@@ -69,7 +69,7 @@ endif
#
# Driver version string which is returned to userspace via an ioctl
MALI_RELEASE_NAME ?= '"g22p0-01eac0"'
MALI_RELEASE_NAME ?= '"g25p0-00eac0"'
# Set up defaults if not defined by build system
ifeq ($(CONFIG_MALI_BIFROST_DEBUG), y)
MALI_UNIT_TEST = 1
@@ -104,7 +104,6 @@ endif
#
# Experimental features must default to disabled, e.g.:
# MALI_EXPERIMENTAL_FEATURE ?= 0
MALI_INCREMENTAL_RENDERING_JM ?= 0
#
# ccflags
@@ -117,7 +116,6 @@ ccflags-y = \
-DMALI_COVERAGE=$(MALI_COVERAGE) \
-DMALI_RELEASE_NAME=$(MALI_RELEASE_NAME) \
-DMALI_JIT_PRESSURE_LIMIT_BASE=$(MALI_JIT_PRESSURE_LIMIT_BASE) \
-DMALI_INCREMENTAL_RENDERING_JM=$(MALI_INCREMENTAL_RENDERING_JM) \
-DMALI_PLATFORM_DIR=$(MALI_PLATFORM_DIR)
@@ -212,6 +210,7 @@ endif
INCLUDE_SUBDIR = \
$(src)/arbiter/Kbuild \
$(src)/context/Kbuild \
$(src)/debug/Kbuild \
$(src)/device/Kbuild \
@@ -228,9 +227,6 @@ ifeq ($(CONFIG_MALI_CSF_SUPPORT),y)
INCLUDE_SUBDIR += $(src)/csf/Kbuild
endif
ifeq ($(CONFIG_MALI_ARBITER_SUPPORT),y)
INCLUDE_SUBDIR += $(src)/arbiter/Kbuild
endif
ifeq ($(CONFIG_MALI_BIFROST_DEVFREQ),y)
ifeq ($(CONFIG_DEVFREQ_THERMAL),y)

28
drivers/gpu/arm/bifrost/Kconfig
View File

@@ -1,6 +1,6 @@
# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
#
# (C) COPYRIGHT 2012-2023 ARM Limited. All rights reserved.
# (C) COPYRIGHT 2012-2024 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
@@ -63,6 +63,8 @@ config MALI_BIFROST_NO_MALI
All calls to the simulated hardware will complete immediately as if the hardware
completed the task.
endchoice
config MALI_NO_MALI_DEFAULT_GPU
string "Default GPU for No Mali"
depends on MALI_BIFROST_NO_MALI
@@ -70,8 +72,12 @@ config MALI_NO_MALI_DEFAULT_GPU
help
This option sets the default GPU to identify as for No Mali builds.
endchoice
config MALI_IS_FPGA
bool "Enable build of Mali kernel driver for FPGA"
depends on MALI_BIFROST
default n
help
This is the default HW backend.
menu "Platform specific options"
source "$(MALI_KCONFIG_EXT_PREFIX)drivers/gpu/arm/bifrost/platform/Kconfig"
@@ -214,16 +220,6 @@ config MALI_CORESTACK
If unsure, say N.
comment "Platform options"
depends on MALI_BIFROST && MALI_BIFROST_EXPERT
config MALI_BIFROST_ERROR_INJECT
bool "Enable No Mali error injection"
depends on MALI_BIFROST && MALI_BIFROST_EXPERT && MALI_BIFROST_NO_MALI
default n
help
Enables insertion of errors to test module failure and recovery mechanisms.
comment "Debug options"
depends on MALI_BIFROST && MALI_BIFROST_EXPERT
@@ -304,7 +300,7 @@ endchoice
config MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS
bool "Enable runtime selection of performance counters set via debugfs"
depends on MALI_BIFROST && MALI_BIFROST_EXPERT && DEBUG_FS
depends on MALI_BIFROST && MALI_BIFROST_EXPERT && DEBUG_FS && !MALI_CSF_SUPPORT
default n
help
Select this option to make the secondary set of performance counters
@@ -351,7 +347,7 @@ config MALI_PWRSOFT_765
changes have been backported say Y to avoid compilation errors.
config MALI_HW_ERRATA_1485982_NOT_AFFECTED
bool "Disable workaround for BASE_HW_ISSUE_GPU2017_1336"
bool "Disable workaround for KBASE_HW_ISSUE_GPU2017_1336"
depends on MALI_BIFROST && MALI_BIFROST_EXPERT
default n
help
@@ -363,7 +359,7 @@ config MALI_HW_ERRATA_1485982_NOT_AFFECTED
coherency mode requires the L2 to be turned off.
config MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE
bool "Use alternative workaround for BASE_HW_ISSUE_GPU2017_1336"
bool "Use alternative workaround for KBASE_HW_ISSUE_GPU2017_1336"
depends on MALI_BIFROST && MALI_BIFROST_EXPERT && !MALI_HW_ERRATA_1485982_NOT_AFFECTED
default n
help

14
drivers/gpu/arm/bifrost/Makefile
View File

@@ -1,6 +1,6 @@
# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
#
# (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved.
# (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
@@ -41,11 +41,12 @@ ifeq ($(MALI_KCONFIG_EXT_PREFIX),)
CONFIG_MALI_BIFROST_GATOR_SUPPORT ?= y
CONFIG_MALI_ARBITRATION ?= n
CONFIG_MALI_PARTITION_MANAGER ?= n
CONFIG_MALI_64BIT_HW_ACCESS ?= n
ifneq ($(CONFIG_MALI_BIFROST_NO_MALI),y)
# Prevent misuse when CONFIG_MALI_BIFROST_NO_MALI=y
# Prevent misuse when CONFIG_MALI_BIFROST_NO_MALI!=y
CONFIG_MALI_REAL_HW ?= y
else
CONFIG_MALI_CORESIGHT = n
endif
@@ -76,7 +77,6 @@ ifeq ($(MALI_KCONFIG_EXT_PREFIX),)
else
# Prevent misuse when CONFIG_MALI_BIFROST_NO_MALI=n
CONFIG_MALI_REAL_HW = y
CONFIG_MALI_BIFROST_ERROR_INJECT = n
endif
@@ -108,7 +108,6 @@ ifeq ($(MALI_KCONFIG_EXT_PREFIX),)
CONFIG_MALI_JOB_DUMP = n
CONFIG_MALI_BIFROST_NO_MALI = n
CONFIG_MALI_REAL_HW = y
CONFIG_MALI_BIFROST_ERROR_INJECT = n
CONFIG_MALI_HW_ERRATA_1485982_NOT_AFFECTED = n
CONFIG_MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE = n
CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS = n
@@ -157,7 +156,6 @@ ifeq ($(MALI_KCONFIG_EXT_PREFIX),)
CONFIG_MALI_BIFROST \
CONFIG_MALI_CSF_SUPPORT \
CONFIG_MALI_BIFROST_GATOR_SUPPORT \
CONFIG_MALI_ARBITER_SUPPORT \
CONFIG_MALI_ARBITRATION \
CONFIG_MALI_PARTITION_MANAGER \
CONFIG_MALI_REAL_HW \
@@ -171,7 +169,7 @@ ifeq ($(MALI_KCONFIG_EXT_PREFIX),)
CONFIG_MALI_PWRSOFT_765 \
CONFIG_MALI_JOB_DUMP \
CONFIG_MALI_BIFROST_NO_MALI \
CONFIG_MALI_BIFROST_ERROR_INJECT \
CONFIG_MALI_IS_FPGA \
CONFIG_MALI_HW_ERRATA_1485982_NOT_AFFECTED \
CONFIG_MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE \
CONFIG_MALI_PRFCNT_SET_PRIMARY \
@@ -272,6 +270,8 @@ CFLAGS_MODULE += -Wmissing-field-initializers
CFLAGS_MODULE += -Wno-type-limits
CFLAGS_MODULE += $(call cc-option, -Wmaybe-uninitialized)
CFLAGS_MODULE += $(call cc-option, -Wunused-macros)
# The following ensures the stack frame does not get larger than a page
CFLAGS_MODULE += -Wframe-larger-than=4096
KBUILD_CPPFLAGS += -DKBUILD_EXTRA_WARN2

3
drivers/gpu/arm/bifrost/arbiter/Kbuild
View File

@@ -1,6 +1,6 @@
# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
#
# (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
# (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
@@ -21,3 +21,4 @@
bifrost_kbase-y += \
arbiter/mali_kbase_arbif.o \
arbiter/mali_kbase_arbiter_pm.o

184
drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -108,6 +108,7 @@ static void on_gpu_stop(struct device *dev)
}
KBASE_TLSTREAM_TL_ARBITER_STOP_REQUESTED(kbdev, kbdev);
KBASE_KTRACE_ADD(kbdev, ARB_GPU_STOP_REQUESTED, NULL, 0);
kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_STOP_EVT);
}
@@ -133,6 +134,7 @@ static void on_gpu_granted(struct device *dev)
}
KBASE_TLSTREAM_TL_ARBITER_GRANTED(kbdev, kbdev);
KBASE_KTRACE_ADD(kbdev, ARB_GPU_GRANTED, NULL, 0);
kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_GRANTED_EVT);
}
@@ -156,10 +158,73 @@ static void on_gpu_lost(struct device *dev)
dev_err(dev, "%s(): kbdev is NULL", __func__);
return;
}
KBASE_TLSTREAM_TL_ARBITER_LOST(kbdev, kbdev);
KBASE_KTRACE_ADD(kbdev, ARB_GPU_LOST, NULL, 0);
kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_LOST_EVT);
}
static int kbase_arbif_of_init(struct kbase_device *kbdev)
{
struct arbiter_if_dev *arb_if;
struct device_node *arbiter_if_node;
struct platform_device *pdev;
if (!IS_ENABLED(CONFIG_OF)) {
/*
* Return -ENODEV in the event CONFIG_OF is not available and let the
* internal AW check for suitability for arbitration.
*/
return -ENODEV;
}
arbiter_if_node = of_parse_phandle(kbdev->dev->of_node, "arbiter-if", 0);
if (!arbiter_if_node)
arbiter_if_node = of_parse_phandle(kbdev->dev->of_node, "arbiter_if", 0);
if (!arbiter_if_node) {
dev_dbg(kbdev->dev, "No arbiter_if in Device Tree");
/* no arbiter interface defined in device tree */
kbdev->arb.arb_dev = NULL;
kbdev->arb.arb_if = NULL;
return -ENODEV;
}
pdev = of_find_device_by_node(arbiter_if_node);
if (!pdev) {
dev_err(kbdev->dev, "Failed to find arbiter_if device");
return -EPROBE_DEFER;
}
if (!pdev->dev.driver || !try_module_get(pdev->dev.driver->owner)) {
dev_err(kbdev->dev, "arbiter_if driver not available");
put_device(&pdev->dev);
return -EPROBE_DEFER;
}
kbdev->arb.arb_dev = &pdev->dev;
arb_if = platform_get_drvdata(pdev);
if (!arb_if) {
dev_err(kbdev->dev, "arbiter_if driver not ready");
module_put(pdev->dev.driver->owner);
put_device(&pdev->dev);
return -EPROBE_DEFER;
}
kbdev->arb.arb_if = arb_if;
return 0;
}
static void kbase_arbif_of_term(struct kbase_device *kbdev)
{
if (!IS_ENABLED(CONFIG_OF))
return;
if (kbdev->arb.arb_dev) {
module_put(kbdev->arb.arb_dev->driver->owner);
put_device(kbdev->arb.arb_dev);
}
kbdev->arb.arb_dev = NULL;
}
/**
* kbase_arbif_init() - Kbase Arbiter interface initialisation.
* @kbdev: The kbase device structure for the device (must be a valid pointer)
@@ -174,47 +239,21 @@ static void on_gpu_lost(struct device *dev)
*/
int kbase_arbif_init(struct kbase_device *kbdev)
{
#if IS_ENABLED(CONFIG_OF)
struct arbiter_if_arb_vm_ops ops;
struct arbiter_if_dev *arb_if;
struct device_node *arbiter_if_node;
struct platform_device *pdev;
int err;
int err = 0;
dev_dbg(kbdev->dev, "%s\n", __func__);
/* Tries to init with 'arbiter-if' if present in devicetree */
err = kbase_arbif_of_init(kbdev);
arbiter_if_node = of_parse_phandle(kbdev->dev->of_node, "arbiter-if", 0);
if (!arbiter_if_node)
arbiter_if_node = of_parse_phandle(kbdev->dev->of_node, "arbiter_if", 0);
if (!arbiter_if_node) {
dev_dbg(kbdev->dev, "No arbiter_if in Device Tree\n");
/* no arbiter interface defined in device tree */
kbdev->arb.arb_dev = NULL;
kbdev->arb.arb_if = NULL;
return 0;
if (err == -ENODEV) {
/* devicetree does not support arbitration */
return -EPERM;
}
pdev = of_find_device_by_node(arbiter_if_node);
if (!pdev) {
dev_err(kbdev->dev, "Failed to find arbiter_if device\n");
return -EPROBE_DEFER;
}
if (err)
return err;
if (!pdev->dev.driver || !try_module_get(pdev->dev.driver->owner)) {
dev_err(kbdev->dev, "arbiter_if driver not available\n");
put_device(&pdev->dev);
return -EPROBE_DEFER;
}
kbdev->arb.arb_dev = &pdev->dev;
arb_if = platform_get_drvdata(pdev);
if (!arb_if) {
dev_err(kbdev->dev, "arbiter_if driver not ready\n");
module_put(pdev->dev.driver->owner);
put_device(&pdev->dev);
return -EPROBE_DEFER;
}
kbdev->arb.arb_if = arb_if;
ops.arb_vm_gpu_stop = on_gpu_stop;
ops.arb_vm_gpu_granted = on_gpu_granted;
ops.arb_vm_gpu_lost = on_gpu_lost;
@@ -225,25 +264,35 @@ int kbase_arbif_init(struct kbase_device *kbdev)
kbdev->arb.arb_freq.freq_updated = false;
mutex_init(&kbdev->arb.arb_freq.arb_freq_lock);
/* register kbase arbiter_if callbacks */
if (arb_if->vm_ops.vm_arb_register_dev) {
err = arb_if->vm_ops.vm_arb_register_dev(arb_if, kbdev->dev, &ops);
if (err) {
dev_err(&pdev->dev, "Failed to register with arbiter. (err = %d)\n", err);
module_put(pdev->dev.driver->owner);
put_device(&pdev->dev);
if (err != -EPROBE_DEFER)
err = -EFAULT;
return err;
}
arb_if = kbdev->arb.arb_if;
if (arb_if == NULL) {
dev_err(kbdev->dev, "No arbiter interface present");
goto failure_term;
}
if (!arb_if->vm_ops.vm_arb_register_dev) {
dev_err(kbdev->dev, "arbiter_if registration callback not present");
goto failure_term;
}
/* register kbase arbiter_if callbacks */
err = arb_if->vm_ops.vm_arb_register_dev(arb_if, kbdev->dev, &ops);
if (err) {
dev_err(kbdev->dev, "Failed to register with arbiter. (err = %d)", err);
goto failure_term;
}
#else /* CONFIG_OF */
dev_dbg(kbdev->dev, "No arbiter without Device Tree support\n");
kbdev->arb.arb_dev = NULL;
kbdev->arb.arb_if = NULL;
#endif
return 0;
failure_term:
{
kbase_arbif_of_term(kbdev);
}
if (err != -EPROBE_DEFER)
err = -EFAULT;
return err;
}
/**
@@ -256,16 +305,13 @@ void kbase_arbif_destroy(struct kbase_device *kbdev)
{
struct arbiter_if_dev *arb_if = kbdev->arb.arb_if;
if (arb_if && arb_if->vm_ops.vm_arb_unregister_dev) {
dev_dbg(kbdev->dev, "%s\n", __func__);
if (arb_if && arb_if->vm_ops.vm_arb_unregister_dev)
arb_if->vm_ops.vm_arb_unregister_dev(kbdev->arb.arb_if);
{
kbase_arbif_of_term(kbdev);
}
kbdev->arb.arb_if = NULL;
if (kbdev->arb.arb_dev) {
module_put(kbdev->arb.arb_dev->driver->owner);
put_device(kbdev->arb.arb_dev);
}
kbdev->arb.arb_dev = NULL;
}
/**
@@ -278,10 +324,8 @@ void kbase_arbif_get_max_config(struct kbase_device *kbdev)
{
struct arbiter_if_dev *arb_if = kbdev->arb.arb_if;
if (arb_if && arb_if->vm_ops.vm_arb_get_max_config) {
dev_dbg(kbdev->dev, "%s\n", __func__);
if (arb_if && arb_if->vm_ops.vm_arb_get_max_config)
arb_if->vm_ops.vm_arb_get_max_config(arb_if);
}
}
/**
@@ -295,8 +339,8 @@ void kbase_arbif_gpu_request(struct kbase_device *kbdev)
struct arbiter_if_dev *arb_if = kbdev->arb.arb_if;
if (arb_if && arb_if->vm_ops.vm_arb_gpu_request) {
dev_dbg(kbdev->dev, "%s\n", __func__);
KBASE_TLSTREAM_TL_ARBITER_REQUESTED(kbdev, kbdev);
KBASE_KTRACE_ADD(kbdev, ARB_GPU_REQUESTED, NULL, 0);
arb_if->vm_ops.vm_arb_gpu_request(arb_if);
}
}
@@ -312,10 +356,12 @@ void kbase_arbif_gpu_stopped(struct kbase_device *kbdev, u8 gpu_required)
struct arbiter_if_dev *arb_if = kbdev->arb.arb_if;
if (arb_if && arb_if->vm_ops.vm_arb_gpu_stopped) {
dev_dbg(kbdev->dev, "%s\n", __func__);
KBASE_TLSTREAM_TL_ARBITER_STOPPED(kbdev, kbdev);
if (gpu_required)
KBASE_KTRACE_ADD(kbdev, ARB_GPU_STOPPED, NULL, 0);
if (gpu_required) {
KBASE_TLSTREAM_TL_ARBITER_REQUESTED(kbdev, kbdev);
KBASE_KTRACE_ADD(kbdev, ARB_GPU_REQUESTED, NULL, 0);
}
arb_if->vm_ops.vm_arb_gpu_stopped(arb_if, gpu_required);
}
}
@@ -330,10 +376,8 @@ void kbase_arbif_gpu_active(struct kbase_device *kbdev)
{
struct arbiter_if_dev *arb_if = kbdev->arb.arb_if;
if (arb_if && arb_if->vm_ops.vm_arb_gpu_active) {
dev_dbg(kbdev->dev, "%s\n", __func__);
if (arb_if && arb_if->vm_ops.vm_arb_gpu_active)
arb_if->vm_ops.vm_arb_gpu_active(arb_if);
}
}
/**
@@ -346,8 +390,6 @@ void kbase_arbif_gpu_idle(struct kbase_device *kbdev)
{
struct arbiter_if_dev *arb_if = kbdev->arb.arb_if;
if (arb_if && arb_if->vm_ops.vm_arb_gpu_idle) {
dev_dbg(kbdev->dev, "vm_arb_gpu_idle\n");
if (arb_if && arb_if->vm_ops.vm_arb_gpu_idle)
arb_if->vm_ops.vm_arb_gpu_idle(arb_if);
}
}

3
drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.h
View File

@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/*
*
* (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -50,6 +50,7 @@ enum kbase_arbif_evt {
KBASE_VM_OS_RESUME_EVENT,
};
/**
* kbase_arbif_init() - Initialize the arbiter interface functionality.
* @kbdev: The kbase device structure for the device (must be a valid pointer)

214
drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -48,7 +48,7 @@ MODULE_PARM_DESC(
"On a virtualized platform, if the GPU is not granted within this time(ms) kbase will defer the probe");
static void kbase_arbiter_pm_vm_wait_gpu_assignment(struct kbase_device *kbdev);
static inline bool kbase_arbiter_pm_vm_gpu_assigned_lockheld(struct kbase_device *kbdev);
static inline bool kbase_arbiter_pm_vm_gpu_assigned_locked(struct kbase_device *kbdev);
/**
* kbase_arbiter_pm_vm_state_str() - Helper function to get string
@@ -85,7 +85,6 @@ static inline const char *kbase_arbiter_pm_vm_state_str(enum kbase_vm_state stat
case KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT:
return "KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT";
default:
KBASE_DEBUG_ASSERT(false);
return "[UnknownState]";
}
}
@@ -117,14 +116,13 @@ static inline const char *kbase_arbiter_pm_vm_event_str(enum kbase_arbif_evt evt
case KBASE_VM_REF_EVENT:
return "KBASE_VM_REF_EVENT";
default:
KBASE_DEBUG_ASSERT(false);
return "[UnknownEvent]";
}
}
/**
* kbase_arbiter_pm_vm_set_state() - Sets new kbase_arbiter_vm_state
* @kbdev: The kbase device structure for the device (must be a valid pointer)
* @kbdev: The kbase device structure for the device
* @new_state: kbase VM new state
*
* This function sets the new state for the VM
@@ -201,6 +199,7 @@ static void kbase_arbiter_pm_resume_wq(struct work_struct *data)
arb_vm_state->vm_arb_starting = false;
mutex_unlock(&arb_vm_state->vm_state_lock);
KBASE_TLSTREAM_TL_ARBITER_STARTED(kbdev, kbdev);
KBASE_KTRACE_ADD(kbdev, ARB_GPU_STARTED, NULL, 0);
dev_dbg(kbdev->dev, "<%s\n", __func__);
}
@@ -229,7 +228,7 @@ static enum hrtimer_restart request_timer_callback(struct hrtimer *timer)
/**
* start_request_timer() - Start a timer after requesting GPU
* @kbdev: The kbase device structure for the device (must be a valid pointer)
* @kbdev: The kbase device structure for the device
*
* Start a timer to track when kbase is waiting for the GPU from the
* Arbiter. If the timer expires before GPU is granted, a warning in
@@ -245,7 +244,7 @@ static void start_request_timer(struct kbase_device *kbdev)
/**
* cancel_request_timer() - Stop the request timer
* @kbdev: The kbase device structure for the device (must be a valid pointer)
* @kbdev: The kbase device structure for the device
*
* Stops the request timer once GPU has been granted. Safe to call
* even if timer is no longer running.
@@ -260,7 +259,7 @@ static void cancel_request_timer(struct kbase_device *kbdev)
/**
* kbase_arbiter_pm_early_init() - Initialize arbiter for VM
* Paravirtualized use.
* @kbdev: The kbase device structure for the device (must be a valid pointer)
* @kbdev: The kbase device structure for the device
*
* Initialize the arbiter and other required resources during the runtime
* and request the GPU for the VM for the first time.
@@ -272,7 +271,7 @@ int kbase_arbiter_pm_early_init(struct kbase_device *kbdev)
int err;
struct kbase_arbiter_vm_state *arb_vm_state = NULL;
arb_vm_state = kmalloc(sizeof(struct kbase_arbiter_vm_state), GFP_KERNEL);
arb_vm_state = kzalloc(sizeof(struct kbase_arbiter_vm_state), GFP_KERNEL);
if (arb_vm_state == NULL)
return -ENOMEM;
@@ -297,11 +296,13 @@ int kbase_arbiter_pm_early_init(struct kbase_device *kbdev)
err = kbase_arbif_init(kbdev);
if (err) {
dev_err(kbdev->dev, "Failed to initialise arbif module. (err = %d)\n", err);
if (err != -EPERM)
dev_err(kbdev->dev, "Failed to initialise arbif module. (err = %d)", err);
goto arbif_init_fail;
}
if (kbdev->arb.arb_if) {
if (kbase_has_arbiter(kbdev)) {
kbase_arbif_gpu_request(kbdev);
dev_dbg(kbdev->dev, "Waiting for initial GPU assignment...\n");
@@ -311,7 +312,7 @@ int kbase_arbiter_pm_early_init(struct kbase_device *kbdev)
msecs_to_jiffies((unsigned int)gpu_req_timeout));
if (!err) {
dev_dbg(kbdev->dev,
dev_err(kbdev->dev,
"Kbase probe Deferred after waiting %d ms to receive GPU_GRANT\n",
gpu_req_timeout);
@@ -336,7 +337,7 @@ arbif_init_fail:
/**
* kbase_arbiter_pm_early_term() - Shutdown arbiter and free resources
* @kbdev: The kbase device structure for the device (must be a valid pointer)
* @kbdev: The kbase device structure for the device
*
* Clean up all the resources
*/
@@ -344,6 +345,14 @@ void kbase_arbiter_pm_early_term(struct kbase_device *kbdev)
{
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
if (arb_vm_state == NULL)
return;
if (!kbase_has_arbiter(kbdev))
return;
kbase_arbiter_pm_release_interrupts(kbdev);
cancel_request_timer(kbdev);
mutex_lock(&arb_vm_state->vm_state_lock);
if (arb_vm_state->vm_state > KBASE_VM_STATE_STOPPED_GPU_REQUESTED) {
@@ -358,12 +367,6 @@ void kbase_arbiter_pm_early_term(struct kbase_device *kbdev)
kbdev->pm.arb_vm_state = NULL;
}
/**
* kbase_arbiter_pm_release_interrupts() - Release the GPU interrupts
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* Releases interrupts and set the interrupt flag to false
*/
void kbase_arbiter_pm_release_interrupts(struct kbase_device *kbdev)
{
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
@@ -376,29 +379,25 @@ void kbase_arbiter_pm_release_interrupts(struct kbase_device *kbdev)
mutex_unlock(&arb_vm_state->vm_state_lock);
}
/**
* kbase_arbiter_pm_install_interrupts() - Install the GPU interrupts
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* Install interrupts and set the interrupt_install flag to true.
*
* Return: 0 if success, or a Linux error code
*/
int kbase_arbiter_pm_install_interrupts(struct kbase_device *kbdev)
{
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
int err;
int err = 0;
mutex_lock(&arb_vm_state->vm_state_lock);
arb_vm_state->interrupts_installed = true;
err = kbase_install_interrupts(kbdev);
if (arb_vm_state->interrupts_installed == false) {
arb_vm_state->interrupts_installed = true;
err = kbase_install_interrupts(kbdev);
} else {
dev_dbg(kbdev->dev, "%s: interrupts installed already", __func__);
}
mutex_unlock(&arb_vm_state->vm_state_lock);
return err;
}
/**
* kbase_arbiter_pm_vm_stopped() - Handle stop state for the VM
* @kbdev: The kbase device structure for the device (must be a valid pointer)
* @kbdev: The kbase device structure for the device
*
* Handles a stop state for the VM
*/
@@ -416,7 +415,13 @@ void kbase_arbiter_pm_vm_stopped(struct kbase_device *kbdev)
dev_dbg(kbdev->dev, "%s %s\n", __func__,
kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state));
if (arb_vm_state->interrupts_installed) {
/*
* Release the interrupts on external arb_if to address Xen requirements.
* Interrupts are not released with internal arb_if as the IRQs are required
* to handle messaging to/from Arbiter/Resource Group.
*/
if (arb_vm_state->interrupts_installed
) {
arb_vm_state->interrupts_installed = false;
kbase_release_interrupts(kbdev);
}
@@ -476,6 +481,12 @@ int kbase_arbiter_pm_gpu_assigned(struct kbase_device *kbdev)
if (!kbdev)
return result;
/* If there is no Arbiter, then there is no virtualization
* and current VM always has access to GPU.
*/
if (!kbase_has_arbiter(kbdev))
return 1;
/* First check the GPU_LOST state */
kbase_pm_lock(kbdev);
if (kbase_pm_is_gpu_lost(kbdev)) {
@@ -507,7 +518,7 @@ int kbase_arbiter_pm_gpu_assigned(struct kbase_device *kbdev)
/**
* kbase_arbiter_pm_vm_gpu_start() - Handles the start state of the VM
* @kbdev: The kbase device structure for the device (must be a valid pointer)
* @kbdev: The kbase device structure for the device
*
* Handles the start state of the VM
*/
@@ -532,7 +543,15 @@ static void kbase_arbiter_pm_vm_gpu_start(struct kbase_device *kbdev)
case KBASE_VM_STATE_STOPPED_GPU_REQUESTED:
kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_STARTING);
arb_vm_state->interrupts_installed = true;
kbase_install_interrupts(kbdev);
/*
* Re-install interrupts that were released for external arb_if to
* address Xen requirements. Interrupts are not released with internal
* arb_if as the IRQs are required to handle messaging to/from
* Arbiter/Resource Group.
*/
{
kbase_install_interrupts(kbdev);
}
/*
* GPU GRANTED received while in stop can be a result of a
* repartitioning.
@@ -561,7 +580,7 @@ static void kbase_arbiter_pm_vm_gpu_start(struct kbase_device *kbdev)
/**
* kbase_arbiter_pm_vm_gpu_stop() - Handles the stop state of the VM
* @kbdev: The kbase device structure for the device (must be a valid pointer)
* @kbdev: The kbase device structure for the device
*
* Handles the start state of the VM
*/
@@ -603,7 +622,7 @@ static void kbase_arbiter_pm_vm_gpu_stop(struct kbase_device *kbdev)
/**
* kbase_gpu_lost() - Kbase signals GPU is lost on a lost event signal
* @kbdev: The kbase device structure for the device (must be a valid pointer)
* @kbdev: The kbase device structure for the device
*
* On GPU lost event signals GPU_LOST to the aribiter
*/
@@ -658,7 +677,7 @@ static void kbase_gpu_lost(struct kbase_device *kbdev)
/**
* kbase_arbiter_pm_vm_os_suspend_ready_state() - checks if VM is ready
* to be moved to suspended state.
* @kbdev: The kbase device structure for the device (must be a valid pointer)
* @kbdev: The kbase device structure for the device
*
* Return: True if its ready to be suspended else False.
*/
@@ -678,10 +697,10 @@ static inline bool kbase_arbiter_pm_vm_os_suspend_ready_state(struct kbase_devic
/**
* kbase_arbiter_pm_vm_os_prepare_suspend() - Prepare OS to be in suspend state
* until it receives the grant message from arbiter
* @kbdev: The kbase device structure for the device (must be a valid pointer)
* @kbdev: The kbase device structure for the device
*
* Prepares OS to be in suspend state until it receives GRANT message
* from Arbiter asynchronously.
* from Arbiter asynchronously. This function assumes there is an active Arbiter.
*/
static void kbase_arbiter_pm_vm_os_prepare_suspend(struct kbase_device *kbdev)
{
@@ -689,10 +708,8 @@ static void kbase_arbiter_pm_vm_os_prepare_suspend(struct kbase_device *kbdev)
enum kbase_vm_state prev_state;
lockdep_assert_held(&arb_vm_state->vm_state_lock);
if (kbdev->arb.arb_if) {
if (kbdev->pm.arb_vm_state->vm_state == KBASE_VM_STATE_SUSPENDED)
return;
}
if (kbdev->pm.arb_vm_state->vm_state == KBASE_VM_STATE_SUSPENDED)
return;
/* Block suspend OS function until we are in a stable state
* with vm_state_lock
*/
@@ -745,7 +762,7 @@ static void kbase_arbiter_pm_vm_os_prepare_suspend(struct kbase_device *kbdev)
/**
* kbase_arbiter_pm_vm_os_resume() - Resume OS function once it receives
* a grant message from arbiter
* @kbdev: The kbase device structure for the device (must be a valid pointer)
* @kbdev: The kbase device structure for the device
*
* Resume OS function once it receives GRANT message
* from Arbiter asynchronously.
@@ -774,7 +791,7 @@ static void kbase_arbiter_pm_vm_os_resume(struct kbase_device *kbdev)
/**
* kbase_arbiter_pm_vm_event() - Dispatch VM event to the state machine.
* @kbdev: The kbase device structure for the device (must be a valid pointer)
* @kbdev: The kbase device structure for the device
* @evt: VM event
*
* The state machine function. Receives events and transitions states
@@ -784,7 +801,7 @@ void kbase_arbiter_pm_vm_event(struct kbase_device *kbdev, enum kbase_arbif_evt
{
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
if (!kbdev->arb.arb_if)
if (!kbase_has_arbiter(kbdev))
return;
mutex_lock(&arb_vm_state->vm_state_lock);
@@ -853,7 +870,7 @@ void kbase_arbiter_pm_vm_event(struct kbase_device *kbdev, enum kbase_arbif_evt
break;
default:
dev_alert(kbdev->dev, "Got Unknown Event!");
dev_err(kbdev->dev, "Got Unknown Event!");
break;
}
mutex_unlock(&arb_vm_state->vm_state_lock);
@@ -863,7 +880,7 @@ KBASE_EXPORT_TEST_API(kbase_arbiter_pm_vm_event);
/**
* kbase_arbiter_pm_vm_wait_gpu_assignment() - VM wait for a GPU assignment.
* @kbdev: The kbase device structure for the device (must be a valid pointer)
* @kbdev: The kbase device structure for the device
*
* VM waits for a GPU assignment.
*/
@@ -879,14 +896,14 @@ static void kbase_arbiter_pm_vm_wait_gpu_assignment(struct kbase_device *kbdev)
}
/**
* kbase_arbiter_pm_vm_gpu_assigned_lockheld() - Check if VM holds VM state lock
* @kbdev: The kbase device structure for the device (must be a valid pointer)
* kbase_arbiter_pm_vm_gpu_assigned_locked() - Check if VM holds VM state lock
* @kbdev: The kbase device structure for the device
*
* Checks if the virtual machine holds VM state lock.
*
* Return: true if GPU is assigned, else false.
*/
static inline bool kbase_arbiter_pm_vm_gpu_assigned_lockheld(struct kbase_device *kbdev)
static inline bool kbase_arbiter_pm_vm_gpu_assigned_locked(struct kbase_device *kbdev)
{
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
@@ -898,13 +915,14 @@ static inline bool kbase_arbiter_pm_vm_gpu_assigned_lockheld(struct kbase_device
/**
* kbase_arbiter_pm_ctx_active_handle_suspend() - Handle suspend operation for
* arbitration mode
* @kbdev: The kbase device structure for the device (must be a valid pointer)
* @kbdev: The kbase device structure for the device
* @suspend_handler: The handler code for how to handle a suspend
* that might occur
*
* This function handles a suspend event from the driver,
* communicating with the arbiter and waiting synchronously for the GPU
* to be granted again depending on the VM state.
* to be granted again depending on the VM state. Returns immediately
* with success if there is no Arbiter.
*
* Return: 0 on success else 1 suspend handler isn not possible.
*/
@@ -914,58 +932,58 @@ int kbase_arbiter_pm_ctx_active_handle_suspend(struct kbase_device *kbdev,
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
int res = 0;
if (kbdev->arb.arb_if) {
mutex_lock(&arb_vm_state->vm_state_lock);
while (!kbase_arbiter_pm_vm_gpu_assigned_lockheld(kbdev)) {
/* Update VM state since we have GPU work to do */
if (arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_IDLE)
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_STOPPING_ACTIVE);
else if (arb_vm_state->vm_state == KBASE_VM_STATE_STOPPED) {
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_STOPPED_GPU_REQUESTED);
kbase_arbif_gpu_request(kbdev);
start_request_timer(kbdev);
} else if (arb_vm_state->vm_state == KBASE_VM_STATE_INITIALIZING_WITH_GPU)
if (!kbase_has_arbiter(kbdev))
return res;
mutex_lock(&arb_vm_state->vm_state_lock);
while (!kbase_arbiter_pm_vm_gpu_assigned_locked(kbdev)) {
/* Update VM state since we have GPU work to do */
if (arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_IDLE)
kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_STOPPING_ACTIVE);
else if (arb_vm_state->vm_state == KBASE_VM_STATE_STOPPED) {
kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_STOPPED_GPU_REQUESTED);
kbase_arbif_gpu_request(kbdev);
start_request_timer(kbdev);
} else if (arb_vm_state->vm_state == KBASE_VM_STATE_INITIALIZING_WITH_GPU)
break;
if (suspend_handler != KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE) {
/* In case of GPU lost, even if
* active_count > 0, we no longer have GPU
* access
*/
if (kbase_pm_is_gpu_lost(kbdev))
res = 1;
switch (suspend_handler) {
case KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE:
res = 1;
break;
if (suspend_handler != KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE) {
/* In case of GPU lost, even if
* active_count > 0, we no longer have GPU
* access
*/
if (kbase_pm_is_gpu_lost(kbdev))
case KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE:
if (kbdev->pm.active_count == 0)
res = 1;
switch (suspend_handler) {
case KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE:
res = 1;
break;
case KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE:
if (kbdev->pm.active_count == 0)
res = 1;
break;
case KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED:
break;
default:
WARN(1, "Unknown suspend_handler\n");
res = 1;
break;
}
break;
case KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED:
break;
default:
WARN(1, "Unknown suspend_handler\n");
res = 1;
break;
}
/* Need to synchronously wait for GPU assignment */
atomic_inc(&kbdev->pm.gpu_users_waiting);
mutex_unlock(&arb_vm_state->vm_state_lock);
kbase_pm_unlock(kbdev);
kbase_arbiter_pm_vm_wait_gpu_assignment(kbdev);
kbase_pm_lock(kbdev);
mutex_lock(&arb_vm_state->vm_state_lock);
atomic_dec(&kbdev->pm.gpu_users_waiting);
break;
}
/* Need to synchronously wait for GPU assignment */
atomic_inc(&kbdev->pm.gpu_users_waiting);
mutex_unlock(&arb_vm_state->vm_state_lock);
kbase_pm_unlock(kbdev);
kbase_arbiter_pm_vm_wait_gpu_assignment(kbdev);
kbase_pm_lock(kbdev);
mutex_lock(&arb_vm_state->vm_state_lock);
atomic_dec(&kbdev->pm.gpu_users_waiting);
}
mutex_unlock(&arb_vm_state->vm_state_lock);
return res;
}

4
drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.h
View File

@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/*
*
* (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -102,7 +102,7 @@ void kbase_arbiter_pm_release_interrupts(struct kbase_device *kbdev);
*
* Install interrupts and set the interrupt_install flag to true.
*
* Return: 0 if success, or a Linux error code
* Return: 0 if success or already installed. Otherwise a Linux error code
*/
int kbase_arbiter_pm_install_interrupts(struct kbase_device *kbdev);

9
drivers/gpu/arm/bifrost/backend/gpu/Kbuild
View File

@@ -1,6 +1,6 @@
# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
#
# (C) COPYRIGHT 2014-2022 ARM Limited. All rights reserved.
# (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
@@ -47,12 +47,7 @@ endif
bifrost_kbase-$(CONFIG_MALI_BIFROST_DEVFREQ) += \
backend/gpu/mali_kbase_devfreq.o
ifneq ($(CONFIG_MALI_REAL_HW),y)
bifrost_kbase-y += backend/gpu/mali_kbase_model_linux.o
endif
bifrost_kbase-$(CONFIG_MALI_BIFROST_NO_MALI) += backend/gpu/mali_kbase_model_linux.o
# NO_MALI Dummy model interface
bifrost_kbase-$(CONFIG_MALI_BIFROST_NO_MALI) += backend/gpu/mali_kbase_model_dummy.o
# HW error simulation
bifrost_kbase-$(CONFIG_MALI_BIFROST_NO_MALI) += backend/gpu/mali_kbase_model_error_generator.o

16
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_clk_rate_trace_mgr.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -50,14 +50,22 @@ static struct kbase_clk_rate_trace_op_conf *
get_clk_rate_trace_callbacks(__maybe_unused struct kbase_device *kbdev)
{
/* base case */
const void *arbiter_if_node;
struct kbase_clk_rate_trace_op_conf *callbacks =
(struct kbase_clk_rate_trace_op_conf *)CLK_RATE_TRACE_OPS;
#if defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF)
const void *arbiter_if_node;
/* Nothing left to do here if there is no Arbiter/virtualization or if
* CONFIG_OF is not enabled.
*/
if (!IS_ENABLED(CONFIG_OF))
return callbacks;
if (WARN_ON(!kbdev) || WARN_ON(!kbdev->dev))
return callbacks;
if (!kbase_has_arbiter(kbdev))
return callbacks;
arbiter_if_node = of_get_property(kbdev->dev->of_node, "arbiter-if", NULL);
if (!arbiter_if_node)
arbiter_if_node = of_get_property(kbdev->dev->of_node, "arbiter_if", NULL);
@@ -69,8 +77,6 @@ get_clk_rate_trace_callbacks(__maybe_unused struct kbase_device *kbdev)
dev_dbg(kbdev->dev,
"Arbitration supported but disabled by platform. Leaving clk rate callbacks as default.\n");
#endif
return callbacks;
}

11
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -366,7 +366,7 @@ static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev)
err = of_property_read_u64(node, "opp-hz-real", real_freqs);
#endif
if (err < 0) {
dev_warn(kbdev->dev, "Failed to read opp-hz-real property with error %d\n",
dev_warn(kbdev->dev, "Failed to read opp-hz-real property with error %d",
err);
continue;
}
@@ -374,8 +374,8 @@ static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev)
err = of_property_read_u32_array(node, "opp-microvolt", opp_volts,
kbdev->nr_regulators);
if (err < 0) {
dev_warn(kbdev->dev,
"Failed to read opp-microvolt property with error %d\n", err);
dev_warn(kbdev->dev, "Failed to read opp-microvolt property with error %d",
err);
continue;
}
#endif
@@ -386,11 +386,12 @@ static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev)
dev_warn(
kbdev->dev,
"Ignoring OPP %llu - Dynamic Core Scaling not supported on this GPU\n",
"Ignoring OPP %llu - Dynamic Core Scaling not supported on this GPU",
opp_freq);
continue;
}
core_count_p = of_get_property(node, "opp-core-count", NULL);
if (core_count_p) {
u64 remaining_core_mask = kbdev->gpu_props.shader_present;

8
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_gpuprops_backend.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -48,7 +48,7 @@ int kbase_backend_gpuprops_get(struct kbase_device *kbdev, struct kbasep_gpuprop
/* Not a valid register on TMIX */
/* TGOx specific register */
if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_THREAD_TLS_ALLOC))
if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_THREAD_TLS_ALLOC))
regdump->thread_tls_alloc =
kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(THREAD_TLS_ALLOC));
#endif /* !MALI_USE_CSF */
@@ -64,7 +64,7 @@ int kbase_backend_gpuprops_get(struct kbase_device *kbdev, struct kbasep_gpuprop
/* AMBA_FEATURES enum is mapped to COHERENCY_FEATURES enum */
regdump->coherency_features = KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(COHERENCY_FEATURES));
if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_CORE_FEATURES))
if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_CORE_FEATURES))
regdump->core_features = KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(CORE_FEATURES));
#if MALI_USE_CSF
@@ -116,7 +116,7 @@ int kbase_backend_gpuprops_get_curr_config(struct kbase_device *kbdev,
int kbase_backend_gpuprops_get_l2_features(struct kbase_device *kbdev,
struct kbasep_gpuprops_regdump *regdump)
{
if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_L2_CONFIG)) {
if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_L2_CONFIG)) {
regdump->l2_features = KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(L2_FEATURES));
regdump->l2_config = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(L2_CONFIG));

23
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_backend.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -29,6 +29,8 @@
#include <device/mali_kbase_device.h>
#include <backend/gpu/mali_kbase_instr_internal.h>
#define WAIT_FOR_DUMP_TIMEOUT_MS 5000
static int wait_prfcnt_ready(struct kbase_device *kbdev)
{
u32 val;
@@ -163,6 +165,7 @@ int kbase_instr_hwcnt_disable_internal(struct kbase_context *kctx)
{
unsigned long flags, pm_flags;
struct kbase_device *kbdev = kctx->kbdev;
const unsigned long timeout = msecs_to_jiffies(WAIT_FOR_DUMP_TIMEOUT_MS);
while (1) {
spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags);
@@ -199,7 +202,8 @@ int kbase_instr_hwcnt_disable_internal(struct kbase_context *kctx)
spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags);
/* Ongoing dump/setup - wait for its completion */
wait_event(kbdev->hwcnt.backend.wait, kbdev->hwcnt.backend.triggered != 0);
wait_event_timeout(kbdev->hwcnt.backend.wait, kbdev->hwcnt.backend.triggered != 0,
timeout);
}
kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED;
@@ -319,8 +323,19 @@ int kbase_instr_hwcnt_wait_for_dump(struct kbase_context *kctx)
unsigned long flags;
int err;
unsigned long remaining;
const unsigned long timeout = msecs_to_jiffies(WAIT_FOR_DUMP_TIMEOUT_MS);
/* Wait for dump & cache clean to complete */
wait_event(kbdev->hwcnt.backend.wait, kbdev->hwcnt.backend.triggered != 0);
remaining = wait_event_timeout(kbdev->hwcnt.backend.wait,
kbdev->hwcnt.backend.triggered != 0, timeout);
if (remaining == 0) {
err = -ETIME;
/* Set the backend state so it's clear things have gone bad (could be a HW issue)
*/
kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_UNRECOVERABLE_ERROR;
goto timed_out;
}
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
@@ -336,7 +351,7 @@ int kbase_instr_hwcnt_wait_for_dump(struct kbase_context *kctx)
}
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
timed_out:
return err;
}

2
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_internal.h
View File

@@ -74,7 +74,7 @@ void kbase_synchronize_irqs(struct kbase_device *kbdev);
* Return: 0 on success. Error code (negative) on failure.
*/
int kbase_validate_interrupts(struct kbase_device *const kbdev);
#endif /* CONFIG_MALI_REAL_HW */
#endif /* IS_ENABLED(CONFIG_MALI_REAL_HW) */
#endif /* CONFIG_MALI_BIFROST_DEBUG */
/**

28
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_linux.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -23,6 +23,7 @@
#include <device/mali_kbase_device.h>
#include <backend/gpu/mali_kbase_irq_internal.h>
#include <linux/interrupt.h>
#if IS_ENABLED(CONFIG_MALI_REAL_HW)
@@ -163,13 +164,9 @@ static irqreturn_t kbase_gpu_irq_handler(int irq, void *data)
static irqreturn_t kbase_combined_irq_handler(int irq, void *data)
{
irqreturn_t irq_state = IRQ_NONE;
if (kbase_job_irq_handler(irq, data) == IRQ_HANDLED)
irq_state = IRQ_HANDLED;
if (kbase_mmu_irq_handler(irq, data) == IRQ_HANDLED)
irq_state = IRQ_HANDLED;
if (kbase_gpu_irq_handler(irq, data) == IRQ_HANDLED)
irq_state = IRQ_HANDLED;
irq_state |= kbase_job_irq_handler(irq, data);
irq_state |= kbase_mmu_irq_handler(irq, data);
irq_state |= kbase_gpu_irq_handler(irq, data);
return irq_state;
}
@@ -212,8 +209,7 @@ int kbase_set_custom_irq_handler(struct kbase_device *kbdev, irq_handler_t custo
if (!handler)
handler = kbase_get_interrupt_handler(kbdev, irq_tag);
if (request_irq(kbdev->irqs[irq].irq, handler,
kbdev->irqs[irq].flags | ((kbdev->nr_irqs == 1) ? 0 : IRQF_SHARED),
if (request_irq(kbdev->irqs[irq].irq, handler, kbdev->irqs[irq].flags | IRQF_SHARED,
dev_name(kbdev->dev), kbase_tag(kbdev, irq)) != 0) {
result = -EINVAL;
dev_err(kbdev->dev, "Can't request interrupt %u (index %u)\n", kbdev->irqs[irq].irq,
@@ -396,8 +392,8 @@ static int validate_interrupt(struct kbase_device *const kbdev, u32 tag)
/* restore original interrupt */
if (request_irq(kbdev->irqs[irq].irq, kbase_get_interrupt_handler(kbdev, tag),
kbdev->irqs[irq].flags | ((kbdev->nr_irqs == 1) ? 0 : IRQF_SHARED),
dev_name(kbdev->dev), kbase_tag(kbdev, irq))) {
kbdev->irqs[irq].flags | IRQF_SHARED, dev_name(kbdev->dev),
kbase_tag(kbdev, irq))) {
dev_err(kbdev->dev, "Can't restore original interrupt %u (index %u)\n",
kbdev->irqs[irq].irq, tag);
err = -EINVAL;
@@ -449,10 +445,10 @@ int kbase_install_interrupts(struct kbase_device *kbdev)
u32 i;
for (i = 0; i < kbdev->nr_irqs; i++) {
const int result = request_irq(
kbdev->irqs[i].irq, kbase_get_interrupt_handler(kbdev, i),
kbdev->irqs[i].flags | ((kbdev->nr_irqs == 1) ? 0 : IRQF_SHARED),
dev_name(kbdev->dev), kbase_tag(kbdev, i));
const int result = request_irq(kbdev->irqs[i].irq,
kbase_get_interrupt_handler(kbdev, i),
kbdev->irqs[i].flags | IRQF_SHARED,
dev_name(kbdev->dev), kbase_tag(kbdev, i));
if (result) {
dev_err(kbdev->dev, "Can't request interrupt %u (index %u)\n",
kbdev->irqs[i].irq, i);

165
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -98,82 +98,6 @@ static u64 kbase_job_write_affinity(struct kbase_device *kbdev, base_jd_core_req
return affinity;
}
/**
* select_job_chain() - Select which job chain to submit to the GPU
* @katom: Pointer to the atom about to be submitted to the GPU
*
* Selects one of the fragment job chains attached to the special atom at the
* end of a renderpass, or returns the address of the single job chain attached
* to any other type of atom.
*
* Which job chain is selected depends upon whether the tiling phase of the
* renderpass completed normally or was soft-stopped because it used too
* much memory. It also depends upon whether one of the fragment job chains
* has already been run as part of the same renderpass.
*
* Return: GPU virtual address of the selected job chain
*/
static u64 select_job_chain(struct kbase_jd_atom *katom)
{
struct kbase_context *const kctx = katom->kctx;
u64 jc = katom->jc;
struct kbase_jd_renderpass *rp;
lockdep_assert_held(&kctx->kbdev->hwaccess_lock);
if (!(katom->core_req & BASE_JD_REQ_END_RENDERPASS))
return jc;
compiletime_assert((1ull << (sizeof(katom->renderpass_id) * 8)) <=
ARRAY_SIZE(kctx->jctx.renderpasses),
"Should check invalid access to renderpasses");
rp = &kctx->jctx.renderpasses[katom->renderpass_id];
/* We can read a subset of renderpass state without holding
* higher-level locks (but not end_katom, for example).
* If the end-of-renderpass atom is running with as-yet indeterminate
* OOM state then assume that the start atom was not soft-stopped.
*/
switch (rp->state) {
case KBASE_JD_RP_OOM:
/* Tiling ran out of memory.
* Start of incremental rendering, used once.
*/
jc = katom->jc_fragment.norm_read_forced_write;
break;
case KBASE_JD_RP_START:
case KBASE_JD_RP_PEND_OOM:
/* Tiling completed successfully first time.
* Single-iteration rendering, used once.
*/
jc = katom->jc_fragment.norm_read_norm_write;
break;
case KBASE_JD_RP_RETRY_OOM:
/* Tiling ran out of memory again.
* Continuation of incremental rendering, used as
* many times as required.
*/
jc = katom->jc_fragment.forced_read_forced_write;
break;
case KBASE_JD_RP_RETRY:
case KBASE_JD_RP_RETRY_PEND_OOM:
/* Tiling completed successfully this time.
* End of incremental rendering, used once.
*/
jc = katom->jc_fragment.forced_read_norm_write;
break;
default:
WARN_ON(1);
break;
}
dev_dbg(kctx->kbdev->dev, "Selected job chain 0x%llx for end atom %pK in state %d\n", jc,
(void *)katom, (int)rp->state);
katom->jc = jc;
return jc;
}
static inline bool kbasep_jm_wait_js_free(struct kbase_device *kbdev, unsigned int js,
struct kbase_context *kctx)
{
@@ -196,7 +120,7 @@ int kbase_job_hw_submit(struct kbase_device *kbdev, struct kbase_jd_atom *katom,
{
struct kbase_context *kctx;
u32 cfg;
u64 const jc_head = select_job_chain(katom);
u64 jc_head = katom->jc;
u64 affinity;
struct slot_rb *ptr_slot_rb = &kbdev->hwaccess.backend.slot_rb[js];
@@ -220,21 +144,21 @@ int kbase_job_hw_submit(struct kbase_device *kbdev, struct kbase_jd_atom *katom,
*/
cfg = (u32)kctx->as_nr;
if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION) &&
if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_FLUSH_REDUCTION) &&
!(kbdev->serialize_jobs & KBASE_SERIALIZE_RESET))
cfg |= JS_CONFIG_ENABLE_FLUSH_REDUCTION;
if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_START)) {
/* Force a cache maintenance operation if the newly submitted
* katom to the slot is from a different kctx. For a JM GPU
* that has the feature BASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER,
* that has the feature KBASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER,
* applies a FLUSH_INV_SHADER_OTHER. Otherwise, do a
* FLUSH_CLEAN_INVALIDATE.
*/
u64 tagged_kctx = ptr_slot_rb->last_kctx_tagged;
if (tagged_kctx != SLOT_RB_NULL_TAG_VAL && tagged_kctx != SLOT_RB_TAG_KCTX(kctx)) {
if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER))
if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER))
cfg |= JS_CONFIG_START_FLUSH_INV_SHADER_OTHER;
else
cfg |= JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE;
@@ -246,15 +170,14 @@ int kbase_job_hw_submit(struct kbase_device *kbdev, struct kbase_jd_atom *katom,
if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_END) &&
!(kbdev->serialize_jobs & KBASE_SERIALIZE_RESET))
cfg |= JS_CONFIG_END_FLUSH_NO_ACTION;
else if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_CLEAN_ONLY_SAFE))
else if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_CLEAN_ONLY_SAFE))
cfg |= JS_CONFIG_END_FLUSH_CLEAN;
else
cfg |= JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE;
cfg |= JS_CONFIG_THREAD_PRI(8);
if ((katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED) ||
(katom->core_req & BASE_JD_REQ_END_RENDERPASS))
if (katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED)
cfg |= JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK;
if (!ptr_slot_rb->job_chain_flag) {
@@ -268,7 +191,7 @@ int kbase_job_hw_submit(struct kbase_device *kbdev, struct kbase_jd_atom *katom,
kbase_reg_write32(kbdev, JOB_SLOT_OFFSET(js, CONFIG_NEXT), cfg);
if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION))
if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_FLUSH_REDUCTION))
kbase_reg_write32(kbdev, JOB_SLOT_OFFSET(js, FLUSH_ID_NEXT), katom->flush_id);
/* Write an approximate start timestamp.
@@ -440,7 +363,7 @@ void kbase_job_done(struct kbase_device *kbdev, u32 done)
* jobs to hang. Reset GPU before allowing
* any other jobs on the slot to continue.
*/
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_3076)) {
if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TTRX_3076)) {
if (completion_code == BASE_JD_EVENT_JOB_BUS_FAULT) {
if (kbase_prepare_to_reset_gpu_locked(
kbdev, RESET_FLAGS_NONE))
@@ -740,66 +663,6 @@ void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx,
}
}
static int softstop_start_rp_nolock(struct kbase_context *kctx, struct kbase_va_region *reg)
{
struct kbase_device *const kbdev = kctx->kbdev;
struct kbase_jd_atom *katom;
struct kbase_jd_renderpass *rp;
lockdep_assert_held(&kbdev->hwaccess_lock);
katom = kbase_gpu_inspect(kbdev, 1, 0);
if (!katom) {
dev_dbg(kctx->kbdev->dev, "No atom on job slot\n");
return -ESRCH;
}
if (!(katom->core_req & BASE_JD_REQ_START_RENDERPASS)) {
dev_dbg(kctx->kbdev->dev, "Atom %pK on job slot is not start RP\n", (void *)katom);
return -EPERM;
}
compiletime_assert((1ull << (sizeof(katom->renderpass_id) * 8)) <=
ARRAY_SIZE(kctx->jctx.renderpasses),
"Should check invalid access to renderpasses");
rp = &kctx->jctx.renderpasses[katom->renderpass_id];
if (WARN_ON(rp->state != KBASE_JD_RP_START && rp->state != KBASE_JD_RP_RETRY))
return -EINVAL;
dev_dbg(kctx->kbdev->dev, "OOM in state %d with region %pK\n", (int)rp->state, (void *)reg);
if (WARN_ON(katom != rp->start_katom))
return -EINVAL;
dev_dbg(kctx->kbdev->dev, "Adding region %pK to list %pK\n", (void *)reg,
(void *)&rp->oom_reg_list);
list_move_tail(&reg->link, &rp->oom_reg_list);
dev_dbg(kctx->kbdev->dev, "Added region to list\n");
rp->state = (rp->state == KBASE_JD_RP_START ? KBASE_JD_RP_PEND_OOM :
KBASE_JD_RP_RETRY_PEND_OOM);
kbase_job_slot_softstop(kbdev, 1, katom);
return 0;
}
int kbase_job_slot_softstop_start_rp(struct kbase_context *const kctx,
struct kbase_va_region *const reg)
{
struct kbase_device *const kbdev = kctx->kbdev;
int err;
unsigned long flags;
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
err = softstop_start_rp_nolock(kctx, reg);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
return err;
}
void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx)
{
struct kbase_device *kbdev = kctx->kbdev;
@@ -839,7 +702,7 @@ u32 kbase_backend_get_current_flush_id(struct kbase_device *kbdev)
{
u32 flush_id = 0;
if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION)) {
if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_FLUSH_REDUCTION)) {
mutex_lock(&kbdev->pm.lock);
if (kbdev->pm.backend.gpu_powered)
flush_id = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(LATEST_FLUSH));
@@ -1085,7 +948,7 @@ static void kbasep_reset_timeout_worker(struct work_struct *data)
/* The flush has completed so reset the active indicator */
kbdev->irq_reset_flush = false;
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TMIX_8463)) {
if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TMIX_8463)) {
u64 val;
const u32 timeout_us =
kbase_get_timeout_ms(kbdev, KBASE_CLEAN_CACHE_TIMEOUT) * USEC_PER_MSEC;
@@ -1268,14 +1131,12 @@ bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev, unsigned int
{
unsigned int i;
#ifdef CONFIG_MALI_ARBITER_SUPPORT
if (kbase_pm_is_gpu_lost(kbdev)) {
/* GPU access has been removed, reset will be done by
* Arbiter instead
*/
return false;
}
#endif
if (flags & RESET_FLAGS_HWC_UNRECOVERABLE_ERROR)
kbase_instr_hwcnt_on_unrecoverable_error(kbdev);
@@ -1328,7 +1189,7 @@ void kbase_reset_gpu(struct kbase_device *kbdev)
if (!kbase_is_quick_reset_enabled(kbdev))
dev_err(kbdev->dev,
"Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n",
"Preparing to soft-reset GPU: Waiting (up to %d ms) for all jobs to complete soft-stop\n",
kbdev->reset_timeout_ms);
hrtimer_start(&kbdev->hwaccess.backend.reset_timer,
@@ -1350,7 +1211,7 @@ void kbase_reset_gpu_locked(struct kbase_device *kbdev)
if (!kbase_is_quick_reset_enabled(kbdev))
dev_err(kbdev->dev,
"Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n",
"Preparing to soft-reset GPU: Waiting (up to %d ms) for all jobs to complete soft-stop\n",
kbdev->reset_timeout_ms);
hrtimer_start(&kbdev->hwaccess.backend.reset_timer,
HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms), HRTIMER_MODE_REL);

27
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -425,7 +425,7 @@ static void kbase_gpu_release_atom(struct kbase_device *kbdev, struct kbase_jd_a
}
}
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TGOX_R1_1234)) {
if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TGOX_R1_1234)) {
if (katom->atom_flags & KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT) {
kbase_pm_protected_l2_override(kbdev, false);
katom->atom_flags &= ~KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT;
@@ -698,7 +698,7 @@ static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, struct kbas
kbase_pm_protected_entry_override_disable(kbdev);
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TGOX_R1_1234)) {
if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TGOX_R1_1234)) {
/*
* Power on L2 caches; this will also result in the
* correct value written to coherency enable register.
@@ -714,13 +714,13 @@ static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, struct kbas
katom[idx]->protected_state.enter = KBASE_ATOM_ENTER_PROTECTED_FINISHED;
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TGOX_R1_1234))
if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TGOX_R1_1234))
return -EAGAIN;
/* ***TRANSITION TO HIGHER STATE*** */
fallthrough;
case KBASE_ATOM_ENTER_PROTECTED_FINISHED:
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TGOX_R1_1234)) {
if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TGOX_R1_1234)) {
/*
* Check that L2 caches are powered and, if so,
* enter protected mode.
@@ -864,11 +864,7 @@ void kbase_backend_slot_update(struct kbase_device *kbdev)
lockdep_assert_held(&kbdev->hwaccess_lock);
#ifdef CONFIG_MALI_ARBITER_SUPPORT
if (kbase_reset_gpu_is_active(kbdev) || kbase_is_gpu_removed(kbdev))
#else
if (kbase_reset_gpu_is_active(kbdev))
#endif
if (kbase_reset_gpu_is_active(kbdev) || (kbase_is_gpu_removed(kbdev)))
return;
for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
@@ -896,7 +892,7 @@ void kbase_backend_slot_update(struct kbase_device *kbdev)
break;
case KBASE_ATOM_GPU_RB_WAITING_BLOCKED:
if (kbase_js_atom_blocked_on_x_dep(katom[idx]))
if (katom[idx]->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED)
break;
katom[idx]->gpu_rb_state =
@@ -1236,7 +1232,7 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, unsigned int js, u32 comp
* When a hard-stop is followed close after a soft-stop, the completion
* code may be set to STOPPED, even though the job is terminated
*/
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TMIX_8438)) {
if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TMIX_8438)) {
if (completion_code == BASE_JD_EVENT_STOPPED &&
(katom->atom_flags & KBASE_KATOM_FLAG_BEEN_HARD_STOPPED)) {
completion_code = BASE_JD_EVENT_TERMINATED;
@@ -1331,6 +1327,9 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, unsigned int js, u32 comp
dev_dbg(kbdev->dev, "Update job chain address of atom %pK to resume from 0x%llx\n",
(void *)katom, job_tail);
/* Some of the job has been executed, so we update the job chain address to where
* we should resume from
*/
katom->jc = job_tail;
KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_UPDATE_HEAD, katom->kctx, katom, job_tail, js);
}
@@ -1381,6 +1380,8 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, unsigned int js, u32 comp
dev_dbg(kbdev->dev, "Cross-slot dependency %pK has become runnable.\n",
(void *)katom);
/* Cross-slot dependency has now become runnable. Try to submit it. */
/* Check if there are lower priority jobs to soft stop */
kbase_job_slot_ctx_priority_check_locked(kctx, katom);
@@ -1437,7 +1438,7 @@ void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp)
* then leave it in the RB and next time we're kicked
* it will be processed again from the starting state.
*/
if (keep_in_jm_rb) {
if (!kbase_is_gpu_removed(kbdev) && keep_in_jm_rb) {
katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK;
/* As the atom was not removed, increment the
* index so that we read the correct atom in the

6
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_backend.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -46,7 +46,7 @@ static inline bool timer_callback_should_run(struct kbase_device *kbdev, int nr_
}
#endif /* CONFIG_MALI_BIFROST_DEBUG */
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_9435)) {
if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_9435)) {
/* Timeouts would have to be 4x longer (due to micro-
* architectural design) to support OpenCL conformance tests, so
* only run the timer when there's:
@@ -100,7 +100,7 @@ static enum hrtimer_restart timer_callback(struct hrtimer *timer)
/* The current version of the model doesn't support
* Soft-Stop
*/
if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_5736)) {
if (!kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_5736)) {
u32 ticks = atom->ticks++;
#if !defined(CONFIG_MALI_JOB_DUMP) && !defined(CONFIG_MALI_VECTOR_DUMP)

59
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_dummy.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -25,42 +25,8 @@
* insmod'ing mali_kbase.ko with no arguments after a build with "scons
* gpu=tXYZ" will yield the expected GPU ID for tXYZ. This can always be
* overridden by passing the 'no_mali_gpu' argument to insmod.
*
* - if CONFIG_MALI_BIFROST_ERROR_INJECT is defined the error injection system is
* activated.
*/
/* Implementation of failure injection system:
*
* Error conditions are generated by gpu_generate_error().
* According to CONFIG_MALI_BIFROST_ERROR_INJECT definition gpu_generate_error() either
* generates an error HW condition randomly (CONFIG_MALI_ERROR_INJECT_RANDOM) or
* checks if there is (in error_track_list) an error configuration to be set for
* the current job chain (CONFIG_MALI_ERROR_INJECT_RANDOM not defined).
* Each error condition will trigger a specific "state" for a certain set of
* registers as per Midgard Architecture Specifications doc.
*
* According to Midgard Architecture Specifications doc the following registers
* are always affected by error conditions:
*
* JOB Exception:
* JOB_IRQ_RAWSTAT
* JOB<n> STATUS AREA
*
* MMU Exception:
* MMU_IRQ_RAWSTAT
* AS<n>_FAULTSTATUS
* AS<n>_FAULTADDRESS
*
* GPU Exception:
* GPU_IRQ_RAWSTAT
* GPU_FAULTSTATUS
* GPU_FAULTADDRESS
*
* For further clarification on the model behaviour upon specific error
* conditions the user may refer to the Midgard Architecture Specification
* document
*/
#include <mali_kbase.h>
#include <device/mali_kbase_device.h>
#include <hw_access/mali_kbase_hw_access_regmap.h>
@@ -126,7 +92,7 @@ struct error_status_t hw_error_status;
*/
struct control_reg_values_t {
const char *name;
u32 gpu_id;
u64 gpu_id;
u32 as_present;
u32 thread_max_threads;
u32 thread_max_workgroup_size;
@@ -524,7 +490,7 @@ MODULE_PARM_DESC(no_mali_gpu, "GPU to identify as");
static u32 gpu_model_get_prfcnt_value(enum kbase_ipa_core_type core_type, u32 cnt_idx,
bool is_low_word)
{
u64 *counters_data;
u64 *counters_data = NULL;
u32 core_count = 0;
u32 event_index;
u64 value = 0;
@@ -580,6 +546,9 @@ static u32 gpu_model_get_prfcnt_value(enum kbase_ipa_core_type core_type, u32 cn
break;
}
if (unlikely(counters_data == NULL))
return 0;
for (core = 0; core < core_count; core++) {
value += counters_data[event_index];
event_index += KBASE_DUMMY_MODEL_COUNTER_PER_CORE;
@@ -1172,9 +1141,6 @@ static void midgard_model_update(void *h)
/*this job is done assert IRQ lines */
signal_int(dummy, i);
#ifdef CONFIG_MALI_BIFROST_ERROR_INJECT
midgard_set_error(i);
#endif /* CONFIG_MALI_BIFROST_ERROR_INJECT */
update_register_statuses(dummy, i);
/*if this job slot returned failures we cannot use it */
if (hw_error_status.job_irq_rawstat & (1u << (i + 16))) {
@@ -1564,6 +1530,7 @@ void midgard_model_write_reg(void *h, u32 addr, u32 value)
case L2_PWROFF_HI:
case PWR_KEY:
case PWR_OVERRIDE0:
case PWR_OVERRIDE1:
#if MALI_USE_CSF
case SHADER_PWRFEATURES:
case CSF_CONFIG:
@@ -1607,8 +1574,7 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value)
#else /* !MALI_USE_CSF */
if (addr == GPU_CONTROL_REG(GPU_ID)) {
#endif /* !MALI_USE_CSF */
*value = dummy->control_reg_values->gpu_id;
*value = dummy->control_reg_values->gpu_id & U32_MAX;
} else if (addr == JOB_CONTROL_REG(JOB_IRQ_RAWSTAT)) {
*value = hw_error_status.job_irq_rawstat;
pr_debug("%s", "JS_IRQ_RAWSTAT being read");
@@ -1987,7 +1953,8 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value)
*value = dummy->control_reg_values->gpu_features_lo;
} else if (addr == GPU_CONTROL_REG(GPU_FEATURES_HI)) {
*value = dummy->control_reg_values->gpu_features_hi;
} else {
}
else {
model_error_log(
KBASE_CORE,
"Dummy model register access: Reading unsupported register 0x%x. Returning 0\n",
@@ -2166,9 +2133,3 @@ int gpu_model_control(void *model, struct kbase_model_control_params *params)
return 0;
}
u64 midgard_model_arch_timer_get_cntfrq(void *h)
{
CSTD_UNUSED(h);
return arch_timer_get_cntfrq();
}

172
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_error_generator.c
View File

@@ -1,172 +0,0 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2014-2015, 2018-2023 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU license.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
*/
#include <mali_kbase.h>
#include <linux/random.h>
#include "backend/gpu/mali_kbase_model_linux.h"
static struct kbase_error_atom *error_track_list;
#ifdef CONFIG_MALI_ERROR_INJECT_RANDOM
/** Kernel 6.1.0 has dropped prandom_u32(), use get_random_u32() */
#if (KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE)
#define prandom_u32 get_random_u32
#endif
/*following error probability are set quite high in order to stress the driver*/
static unsigned int error_probability = 50; /* to be set between 0 and 100 */
/* probability to have multiple error give that there is an error */
static unsigned int multiple_error_probability = 50;
/* all the error conditions supported by the model */
#define TOTAL_FAULTS 27
/* maximum number of levels in the MMU translation table tree */
#define MAX_MMU_TABLE_LEVEL 4
/* worst case scenario is <1 MMU fault + 1 job fault + 2 GPU faults> */
#define MAX_CONCURRENT_FAULTS 3
/**
* gpu_generate_error - Generate GPU error
*/
static void gpu_generate_error(void)
{
unsigned int errors_num = 0;
/*is there at least one error? */
if ((prandom_u32() % 100) < error_probability) {
/* pick up a faulty mmu address space */
hw_error_status.faulty_mmu_as = prandom_u32() % NUM_MMU_AS;
/* pick up an mmu table level */
hw_error_status.mmu_table_level = 1 + (prandom_u32() % MAX_MMU_TABLE_LEVEL);
hw_error_status.errors_mask = (u32)(1 << (prandom_u32() % TOTAL_FAULTS));
/*is there also one or more errors? */
if ((prandom_u32() % 100) < multiple_error_probability) {
errors_num = 1 + (prandom_u32() % (MAX_CONCURRENT_FAULTS - 1));
while (errors_num-- > 0) {
u32 temp_mask;
temp_mask = (u32)(1 << (prandom_u32() % TOTAL_FAULTS));
/* below we check that no bit of the same error
* type is set again in the error mask
*/
if ((temp_mask & IS_A_JOB_ERROR) &&
(hw_error_status.errors_mask & IS_A_JOB_ERROR)) {
errors_num++;
continue;
}
if ((temp_mask & IS_A_MMU_ERROR) &&
(hw_error_status.errors_mask & IS_A_MMU_ERROR)) {
errors_num++;
continue;
}
if ((temp_mask & IS_A_GPU_ERROR) &&
(hw_error_status.errors_mask & IS_A_GPU_ERROR)) {
errors_num++;
continue;
}
/* this error mask is already set */
if ((hw_error_status.errors_mask | temp_mask) ==
hw_error_status.errors_mask) {
errors_num++;
continue;
}
hw_error_status.errors_mask |= temp_mask;
}
}
}
}
#endif
int job_atom_inject_error(struct kbase_error_params *params)
{
struct kbase_error_atom *new_elem;
KBASE_DEBUG_ASSERT(params);
new_elem = kzalloc(sizeof(*new_elem), GFP_KERNEL);
if (!new_elem) {
model_error_log(KBASE_CORE,
"\njob_atom_inject_error: kzalloc failed for new_elem\n");
return -ENOMEM;
}
new_elem->params.jc = params->jc;
new_elem->params.errors_mask = params->errors_mask;
new_elem->params.mmu_table_level = params->mmu_table_level;
new_elem->params.faulty_mmu_as = params->faulty_mmu_as;
/*circular list below */
if (error_track_list == NULL) { /*no elements */
error_track_list = new_elem;
new_elem->next = error_track_list;
} else {
struct kbase_error_atom *walker = error_track_list;
while (walker->next != error_track_list)
walker = walker->next;
new_elem->next = error_track_list;
walker->next = new_elem;
}
return 0;
}
void midgard_set_error(u32 job_slot)
{
#ifdef CONFIG_MALI_ERROR_INJECT_RANDOM
gpu_generate_error();
#else
struct kbase_error_atom *walker, *auxiliar;
if (error_track_list != NULL) {
walker = error_track_list->next;
auxiliar = error_track_list;
do {
if (walker->params.jc == hw_error_status.current_jc) {
/* found a faulty atom matching with the
* current one
*/
hw_error_status.errors_mask = walker->params.errors_mask;
hw_error_status.mmu_table_level = walker->params.mmu_table_level;
hw_error_status.faulty_mmu_as = walker->params.faulty_mmu_as;
hw_error_status.current_job_slot = job_slot;
if (walker->next == walker) {
/* only one element */
kfree(error_track_list);
error_track_list = NULL;
} else {
auxiliar->next = walker->next;
if (walker == error_track_list)
error_track_list = walker->next;
kfree(walker);
}
break;
}
auxiliar = walker;
walker = walker->next;
} while (auxiliar->next != error_track_list);
}
#endif /* CONFIG_MALI_ERROR_INJECT_RANDOM */
}

14
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_linux.h
View File

@@ -48,12 +48,8 @@
/*
* Include Model definitions
*/
#if IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI)
#include <backend/gpu/mali_kbase_model_dummy.h>
#endif /* IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) */
#if !IS_ENABLED(CONFIG_MALI_REAL_HW)
/**
* kbase_gpu_device_create() - Generic create function.
*
@@ -116,15 +112,6 @@ void midgard_model_write_reg(void *h, u32 addr, u32 value);
*/
void midgard_model_read_reg(void *h, u32 addr, u32 *const value);
/**
* midgard_model_arch_timer_get_cntfrq - Get Model specific System Timer Frequency
*
* @h: Model handle.
*
* Return: Frequency in Hz
*/
u64 midgard_model_arch_timer_get_cntfrq(void *h);
/**
* gpu_device_raise_irq() - Private IRQ raise function.
*
@@ -155,6 +142,5 @@ void gpu_device_set_data(void *model, void *data);
* Return: Pointer to the data carried by model.
*/
void *gpu_device_get_data(void *model);
#endif /* !IS_ENABLED(CONFIG_MALI_REAL_HW) */
#endif /* _KBASE_MODEL_LINUX_H_ */

123
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -36,6 +36,7 @@
#include <linux/version_compat_defs.h>
#include <linux/pm_runtime.h>
#include <mali_kbase_reset_gpu.h>
#include <csf/mali_kbase_csf_scheduler.h>
#endif /* !MALI_USE_CSF */
#include <hwcnt/mali_kbase_hwcnt_context.h>
#include <backend/gpu/mali_kbase_pm_internal.h>
@@ -97,10 +98,8 @@ void kbase_pm_register_access_enable(struct kbase_device *kbdev)
if (callbacks)
callbacks->power_on_callback(kbdev);
#ifdef CONFIG_MALI_ARBITER_SUPPORT
if (WARN_ON(kbase_pm_is_gpu_lost(kbdev)))
dev_err(kbdev->dev, "Attempting to power on while GPU lost\n");
#endif
kbdev->pm.backend.gpu_powered = true;
}
@@ -133,9 +132,7 @@ int kbase_hwaccess_pm_init(struct kbase_device *kbdev)
INIT_WORK(&kbdev->pm.backend.gpu_poweroff_wait_work, kbase_pm_gpu_poweroff_wait_wq);
kbdev->pm.backend.ca_cores_enabled = ~0ull;
#ifdef CONFIG_MALI_ARBITER_SUPPORT
kbase_pm_set_gpu_lost(kbdev, false);
#endif
init_waitqueue_head(&kbdev->pm.backend.gpu_in_desired_state_wait);
#if !MALI_USE_CSF
@@ -177,15 +174,18 @@ int kbase_hwaccess_pm_init(struct kbase_device *kbdev)
kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx);
#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME)
kbdev->pm.backend.gpu_sleep_supported =
kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_GPU_SLEEP) &&
!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TURSEHW_1997) &&
kbdev->pm.backend.callback_power_runtime_gpu_active &&
kbdev->pm.backend.callback_power_runtime_gpu_idle;
kbdev->pm.backend.gpu_sleep_allowed = 0;
if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GPU_SLEEP) &&
!kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TURSEHW_1997) &&
kbdev->pm.backend.callback_power_runtime_gpu_active &&
kbdev->pm.backend.callback_power_runtime_gpu_idle)
set_bit(KBASE_GPU_SUPPORTS_GPU_SLEEP, &kbdev->pm.backend.gpu_sleep_allowed);
kbdev->pm.backend.apply_hw_issue_TITANHW_2938_wa =
kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TITANHW_2938) &&
kbdev->pm.backend.gpu_sleep_supported;
kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TITANHW_2938) &&
test_bit(KBASE_GPU_SUPPORTS_GPU_SLEEP, &kbdev->pm.backend.gpu_sleep_allowed);
/* FW Sleep-on-Idle is feature is kept disabled */
#endif
if (IS_ENABLED(CONFIG_MALI_HW_ERRATA_1485982_NOT_AFFECTED))
@@ -193,14 +193,14 @@ int kbase_hwaccess_pm_init(struct kbase_device *kbdev)
/* WA1: L2 always_on for GPUs being affected by GPU2017-1336 */
if (!IS_ENABLED(CONFIG_MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE)) {
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_GPU2017_1336))
if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_GPU2017_1336))
kbdev->pm.backend.l2_always_on = true;
return 0;
}
/* WA3: Clock slow down for GPUs being affected by GPU2017-1336 */
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_GPU2017_1336)) {
if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_GPU2017_1336)) {
kbdev->pm.backend.gpu_clock_slow_down_wa = true;
kbdev->pm.backend.gpu_clock_slow_down_desired = true;
INIT_WORK(&kbdev->pm.backend.gpu_clock_control_work,
@@ -345,13 +345,11 @@ static void pm_handle_power_off(struct kbase_device *kbdev)
*/
wait_for_mmu_fault_handling_in_gpu_poweroff_wait_wq(kbdev);
#ifdef CONFIG_MALI_ARBITER_SUPPORT
/* poweron_required may have changed while pm lock
* was released.
*/
if (kbase_pm_is_gpu_lost(kbdev))
backend->poweron_required = false;
#endif
/* Turn off clock now that fault have been handled. We
* dropped locks so poweron_required may have changed -
@@ -393,7 +391,7 @@ static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data)
backend->poweron_required = false;
kbdev->pm.backend.l2_desired = true;
#if MALI_USE_CSF
kbdev->pm.backend.mcu_desired = true;
kbdev->pm.backend.mcu_desired = kbdev->pm.backend.mcu_poweron_required;
#endif
kbase_pm_update_state(kbdev);
kbase_pm_update_cores_state_nolock(kbdev);
@@ -860,9 +858,11 @@ void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, u64 new_core_mask)
}
KBASE_EXPORT_TEST_API(kbase_pm_set_debug_core_mask);
#else
void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, u64 new_core_mask_js0,
u64 new_core_mask_js1, u64 new_core_mask_js2)
void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, u64 *new_core_mask,
size_t new_core_mask_size)
{
size_t i;
lockdep_assert_held(&kbdev->hwaccess_lock);
lockdep_assert_held(&kbdev->pm.lock);
@@ -870,13 +870,14 @@ void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, u64 new_core_mask_
dev_warn_once(
kbdev->dev,
"Change of core mask not supported for slot 0 as dummy job WA is enabled");
new_core_mask_js0 = kbdev->pm.debug_core_mask[0];
new_core_mask[0] = kbdev->pm.debug_core_mask[0];
}
kbdev->pm.debug_core_mask[0] = new_core_mask_js0;
kbdev->pm.debug_core_mask[1] = new_core_mask_js1;
kbdev->pm.debug_core_mask[2] = new_core_mask_js2;
kbdev->pm.debug_core_mask_all = new_core_mask_js0 | new_core_mask_js1 | new_core_mask_js2;
kbdev->pm.debug_core_mask_all = 0;
for (i = 0; i < new_core_mask_size; i++) {
kbdev->pm.debug_core_mask[i] = new_core_mask[i];
kbdev->pm.debug_core_mask_all |= new_core_mask[i];
}
kbase_pm_update_dynamic_cores_onoff(kbdev);
}
@@ -942,13 +943,11 @@ void kbase_hwaccess_pm_resume(struct kbase_device *kbdev)
/* System resume callback has begun */
kbdev->pm.resuming = true;
kbdev->pm.suspending = false;
#ifdef CONFIG_MALI_ARBITER_SUPPORT
if (kbase_pm_is_gpu_lost(kbdev)) {
dev_dbg(kbdev->dev, "%s: GPU lost in progress\n", __func__);
kbase_pm_unlock(kbdev);
return;
}
#endif
kbase_pm_do_poweron(kbdev, true);
#if !MALI_USE_CSF
@@ -958,17 +957,20 @@ void kbase_hwaccess_pm_resume(struct kbase_device *kbdev)
kbase_pm_unlock(kbdev);
}
#ifdef CONFIG_MALI_ARBITER_SUPPORT
void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev)
{
unsigned long flags;
#if !MALI_USE_CSF
#if MALI_USE_CSF
unsigned long flags_sched;
#else
ktime_t end_timestamp = ktime_get_raw();
#endif
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
if (!kbdev->arb.arb_if)
if (!kbase_has_arbiter(kbdev)) {
dev_warn(kbdev->dev, "%s called with no active arbiter!\n", __func__);
return;
}
mutex_lock(&kbdev->pm.lock);
mutex_lock(&arb_vm_state->vm_state_lock);
@@ -981,24 +983,45 @@ void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev)
*/
WARN(!kbase_is_gpu_removed(kbdev), "GPU is still available after GPU lost event\n");
/* Full GPU reset will have been done by hypervisor, so
* cancel
*/
#if MALI_USE_CSF
/* Full GPU reset will have been done by hypervisor, so cancel */
if (kbase_reset_gpu_prevent_and_wait(kbdev))
dev_warn(kbdev->dev, "Failed to prevent GPU reset.");
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
kbase_csf_scheduler_spin_lock(kbdev, &flags_sched);
atomic_set(&kbdev->hwaccess.backend.reset_gpu, KBASE_RESET_GPU_NOT_PENDING);
kbase_csf_scheduler_spin_unlock(kbdev, flags_sched);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
kbase_synchronize_irqs(kbdev);
/* Scheduler reset happens outside of spinlock due to the mutex it acquires */
kbase_csf_scheduler_reset(kbdev);
/* Update kbase status */
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
kbdev->protected_mode = false;
kbase_pm_update_state(kbdev);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
/* Cancel any pending HWC dumps */
kbase_hwcnt_backend_csf_on_unrecoverable_error(&kbdev->hwcnt_gpu_iface);
#else
/* Full GPU reset will have been done by hypervisor, so cancel */
atomic_set(&kbdev->hwaccess.backend.reset_gpu, KBASE_RESET_GPU_NOT_PENDING);
hrtimer_cancel(&kbdev->hwaccess.backend.reset_timer);
kbase_synchronize_irqs(kbdev);
/* Clear all jobs running on the GPU */
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
kbdev->protected_mode = false;
#if !MALI_USE_CSF
kbase_backend_reset(kbdev, &end_timestamp);
kbase_pm_metrics_update(kbdev, NULL);
#endif
kbase_pm_update_state(kbdev);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
#if !MALI_USE_CSF
/* Cancel any pending HWC dumps */
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DUMPING ||
@@ -1008,14 +1031,12 @@ void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev)
wake_up(&kbdev->hwcnt.backend.wait);
}
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
#endif
#endif /* MALI_USE_CSF */
}
mutex_unlock(&arb_vm_state->vm_state_lock);
mutex_unlock(&kbdev->pm.lock);
}
#endif /* CONFIG_MALI_ARBITER_SUPPORT */
#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME)
int kbase_pm_force_mcu_wakeup_after_sleep(struct kbase_device *kbdev)
{
@@ -1063,26 +1084,15 @@ static int pm_handle_mcu_sleep_on_runtime_suspend(struct kbase_device *kbdev)
}
/* Check if a Doorbell mirror interrupt occurred meanwhile.
* Also check if GPU idle work item is pending. If FW had sent the GPU idle notification
* after the wake up of MCU then it can be assumed that Userspace submission didn't make
* GPU non-idle, so runtime suspend doesn't need to be aborted.
*/
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
if (kbdev->pm.backend.gpu_sleep_mode_active && kbdev->pm.backend.exit_gpu_sleep_mode &&
!work_pending(&kbdev->csf.scheduler.gpu_idle_work)) {
u32 glb_req =
kbase_csf_firmware_global_input_read(&kbdev->csf.global_iface, GLB_REQ);
u32 glb_ack = kbase_csf_firmware_global_output(&kbdev->csf.global_iface, GLB_ACK);
/* Only abort the runtime suspend if GPU idle event is not pending */
if (!((glb_req ^ glb_ack) & GLB_REQ_IDLE_EVENT_MASK)) {
dev_dbg(kbdev->dev,
"DB mirror interrupt occurred during runtime suspend after L2 power up");
kbdev->pm.backend.gpu_wakeup_override = false;
kbdev->pm.backend.runtime_suspend_abort_reason = ABORT_REASON_DB_MIRROR_IRQ;
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
return -EBUSY;
}
if (kbdev->pm.backend.gpu_sleep_mode_active && kbdev->pm.backend.exit_gpu_sleep_mode) {
dev_dbg(kbdev->dev,
"DB mirror interrupt occurred during runtime suspend after L2 power up");
kbdev->pm.backend.gpu_wakeup_override = false;
kbdev->pm.backend.runtime_suspend_abort_reason = ABORT_REASON_DB_MIRROR_IRQ;
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
return -EBUSY;
}
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
/* Need to release the kbdev->pm.lock to avoid lock ordering issue
@@ -1237,4 +1247,5 @@ out:
return ret;
}
#endif

18
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2013-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2013-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -55,11 +55,18 @@ void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask)
unsigned long flags;
#if MALI_USE_CSF
u64 old_core_mask = 0;
#endif
bool mmu_sync_needed = false;
if (!IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) &&
kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_GPU2019_3901)) {
mmu_sync_needed = true;
down_write(&kbdev->csf.mmu_sync_sem);
}
#endif
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
#if MALI_USE_CSF
if (!(core_mask & kbdev->pm.debug_core_mask)) {
dev_err(kbdev->dev,
"OPP core mask 0x%llX does not intersect with debug mask 0x%llX\n",
@@ -98,6 +105,9 @@ void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask)
old_core_mask, core_mask);
}
}
if (mmu_sync_needed)
up_write(&kbdev->csf.mmu_sync_sem);
#endif
dev_dbg(kbdev->dev, "Devfreq policy : new core mask=%llX\n", pm_backend->ca_cores_enabled);
@@ -105,6 +115,10 @@ void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask)
return;
unlock:
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
#if MALI_USE_CSF
if (mmu_sync_needed)
up_write(&kbdev->csf.mmu_sync_sem);
#endif
}
KBASE_EXPORT_TEST_API(kbase_devfreq_set_core_mask);
#endif

41
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h
View File

@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/*
*
* (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -114,6 +114,27 @@ enum kbase_pm_runtime_suspend_abort_reason {
ABORT_REASON_NON_IDLE_CGS
};
/* The following indices point to the corresponding bits stored in
* &kbase_pm_backend_data.gpu_sleep_allowed. They denote the conditions that
* would be checked against to determine the level of support for GPU sleep
* and firmware sleep-on-idle.
*/
#define KBASE_GPU_SUPPORTS_GPU_SLEEP ((uint8_t)0)
#define KBASE_GPU_SUPPORTS_FW_SLEEP_ON_IDLE ((uint8_t)1)
#define KBASE_GPU_PERF_COUNTERS_COLLECTION_ENABLED ((uint8_t)2)
#define KBASE_GPU_IGNORE_IDLE_EVENT ((uint8_t)3)
#define KBASE_GPU_NON_IDLE_OFF_SLOT_GROUPS_AVAILABLE ((uint8_t)4)
/* FW sleep-on-idle could be enabled if
* &kbase_pm_backend_data.gpu_sleep_allowed is equal to this value.
*/
#define KBASE_GPU_FW_SLEEP_ON_IDLE_ALLOWED \
((uint8_t)((1 << KBASE_GPU_SUPPORTS_GPU_SLEEP) | \
(1 << KBASE_GPU_SUPPORTS_FW_SLEEP_ON_IDLE) | \
(0 << KBASE_GPU_PERF_COUNTERS_COLLECTION_ENABLED) | \
(0 << KBASE_GPU_IGNORE_IDLE_EVENT) | \
(0 << KBASE_GPU_NON_IDLE_OFF_SLOT_GROUPS_AVAILABLE)))
/**
* struct kbasep_pm_metrics - Metrics data collected for use by the power
* management framework.
@@ -304,7 +325,7 @@ union kbase_pm_policy_data {
* called previously.
* See &struct kbase_pm_callback_conf.
* @ca_cores_enabled: Cores that are currently available
* @apply_hw_issue_TITANHW_2938_wa: Indicates if the workaround for BASE_HW_ISSUE_TITANHW_2938
* @apply_hw_issue_TITANHW_2938_wa: Indicates if the workaround for KBASE_HW_ISSUE_TITANHW_2938
* needs to be applied when unmapping memory from GPU.
* @mcu_state: The current state of the micro-control unit, only applicable
* to GPUs that have such a component
@@ -332,7 +353,11 @@ union kbase_pm_policy_data {
* cores may be different, but there should be transitions in
* progress that will eventually achieve this state (assuming
* that the policy doesn't change its mind in the mean time).
* @mcu_desired: True if the micro-control unit should be powered on
* @mcu_desired: True if the micro-control unit should be powered on by the MCU state
* machine. Updated as per the value of @mcu_poweron_required.
* @mcu_poweron_required: Boolean flag updated mainly by the CSF Scheduler code,
* before updating the PM active count, to indicate to the
* PM code that micro-control unit needs to be powered up/down.
* @policy_change_clamp_state_to_off: Signaling the backend is in PM policy
* change transition, needs the mcu/L2 to be brought back to the
* off state and remain in that state until the flag is cleared.
@@ -346,10 +371,9 @@ union kbase_pm_policy_data {
* @core_idle_work: Work item used to wait for undesired cores to become inactive.
* The work item is enqueued when Host controls the power for
* shader cores and down scaling of cores is performed.
* @gpu_sleep_supported: Flag to indicate that if GPU sleep feature can be
* supported by the kernel driver or not. If this
* flag is not set, then HW state is directly saved
* when GPU idle notification is received.
* @gpu_sleep_allowed: Bitmask to indicate the conditions that would be
* used to determine what support for GPU sleep is
* available.
* @gpu_sleep_mode_active: Flag to indicate that the GPU needs to be in sleep
* mode. It is set when the GPU idle notification is
* received and is cleared when HW state has been
@@ -485,6 +509,7 @@ struct kbase_pm_backend_data {
u64 shaders_desired_mask;
#if MALI_USE_CSF
bool mcu_desired;
bool mcu_poweron_required;
bool policy_change_clamp_state_to_off;
unsigned int csf_pm_sched_flags;
struct mutex policy_change_lock;
@@ -492,7 +517,7 @@ struct kbase_pm_backend_data {
struct work_struct core_idle_work;
#ifdef KBASE_PM_RUNTIME
bool gpu_sleep_supported;
unsigned long gpu_sleep_allowed;
bool gpu_sleep_mode_active;
bool exit_gpu_sleep_mode;
bool gpu_idled;

256
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -47,9 +47,7 @@
#include <backend/gpu/mali_kbase_pm_internal.h>
#include <backend/gpu/mali_kbase_l2_mmu_config.h>
#include <mali_kbase_dummy_job_wa.h>
#ifdef CONFIG_MALI_ARBITER_SUPPORT
#include <arbiter/mali_kbase_arbiter_pm.h>
#endif /* CONFIG_MALI_ARBITER_SUPPORT */
#if MALI_USE_CSF
#include <linux/delay.h>
@@ -70,6 +68,19 @@ MODULE_PARM_DESC(corestack_driver_control,
"to the Mali GPU is known to be problematic.");
KBASE_EXPORT_TEST_API(corestack_driver_control);
/**
* enum kbase_gpu_state - The state of data in the GPU.
*
* @GPU_STATE_INTACT: The GPU state is intact
* @GPU_STATE_LOST: The GPU state is lost
* @GPU_STATE_IN_RESET: The GPU is in reset state
*
* This enumeration is private to the file. It is used as
* the return values of platform specific PM
* callback (*power_on_callback).
*/
enum kbase_gpu_state { GPU_STATE_INTACT = 0, GPU_STATE_LOST, GPU_STATE_IN_RESET };
/**
* enum kbasep_pm_action - Actions that can be performed on a core.
*
@@ -110,7 +121,15 @@ bool kbase_pm_is_mcu_desired(struct kbase_device *kbdev)
if (kbdev->pm.backend.l2_force_off_after_mcu_halt)
return false;
if (kbdev->csf.scheduler.pm_active_count && kbdev->pm.backend.mcu_desired)
/* Check if policy changing transition needs MCU to be off. */
if (unlikely(kbdev->pm.backend.policy_change_clamp_state_to_off))
return false;
if (kbdev->pm.backend.mcu_desired)
return true;
/* For always_on policy, the MCU needs to be kept on */
if (kbase_pm_no_mcu_core_pwroff(kbdev))
return true;
#ifdef KBASE_PM_RUNTIME
@@ -119,13 +138,7 @@ bool kbase_pm_is_mcu_desired(struct kbase_device *kbdev)
return true;
#endif
/* MCU is supposed to be ON, only when scheduler.pm_active_count is
* non zero. But for always_on policy, the MCU needs to be kept on,
* unless policy changing transition needs it off.
*/
return (kbdev->pm.backend.mcu_desired && kbase_pm_no_mcu_core_pwroff(kbdev) &&
!kbdev->pm.backend.policy_change_clamp_state_to_off);
return false;
}
#endif
@@ -600,11 +613,11 @@ static void kbase_pm_l2_config_override(struct kbase_device *kbdev)
/*
* Skip if it is not supported
*/
if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_L2_CONFIG))
if (!kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_L2_CONFIG))
return;
#if MALI_USE_CSF
if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PBHA_HWU)) {
if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_PBHA_HWU)) {
val = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(L2_CONFIG));
kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(L2_CONFIG),
L2_CONFIG_PBHA_HWU_SET(val, kbdev->pbha_propagate_bits));
@@ -728,16 +741,8 @@ bool kbase_pm_is_mcu_inactive(struct kbase_device *kbdev, enum kbase_mcu_state s
}
#ifdef KBASE_PM_RUNTIME
/**
* kbase_pm_enable_mcu_db_notification - Enable the Doorbell notification on
* MCU side
*
* @kbdev: Pointer to the device.
*
* This function is called to re-enable the Doorbell notification on MCU side
* when MCU needs to beome active again.
*/
static void kbase_pm_enable_mcu_db_notification(struct kbase_device *kbdev)
void kbase_pm_enable_mcu_db_notification(struct kbase_device *kbdev)
{
u32 val = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(MCU_CONTROL));
@@ -763,7 +768,7 @@ static void wait_mcu_as_inactive(struct kbase_device *kbdev)
kbase_get_timeout_ms(kbdev, KBASE_AS_INACTIVE_TIMEOUT) * USEC_PER_MSEC;
lockdep_assert_held(&kbdev->hwaccess_lock);
if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TURSEHW_2716))
if (!kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TURSEHW_2716))
return;
/* Wait for the AS_ACTIVE_INT bit to become 0 for the AS used by MCU FW */
@@ -912,6 +917,18 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev)
if (kbase_pm_is_mcu_desired(kbdev) &&
!backend->policy_change_clamp_state_to_off &&
backend->l2_state == KBASE_L2_ON) {
kbdev->csf.mcu_halted = false;
/* Ensure that FW would not go to sleep immediately after
* resumption.
*/
kbase_csf_firmware_global_input_mask(&kbdev->csf.global_iface,
GLB_REQ,
GLB_REQ_REQ_IDLE_DISABLE,
GLB_REQ_IDLE_DISABLE_MASK);
atomic_set(&kbdev->csf.scheduler.gpu_idle_timer_enabled, false);
atomic_set(&kbdev->csf.scheduler.fw_soi_enabled, false);
kbase_csf_firmware_trigger_reload(kbdev);
backend->mcu_state = KBASE_MCU_PEND_ON_RELOAD;
}
@@ -979,8 +996,8 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev)
kbase_hwcnt_backend_csf_set_hw_availability(
&kbdev->hwcnt_gpu_iface,
kbdev->gpu_props.curr_config.l2_slices,
kbdev->gpu_props.curr_config.shader_present &
kbdev->pm.debug_core_mask);
kbdev->gpu_props.curr_config.shader_present,
kbdev->pm.debug_core_mask);
kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx);
kbase_csf_scheduler_spin_unlock(kbdev, flags);
backend->hwcnt_disabled = false;
@@ -990,7 +1007,6 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev)
case KBASE_MCU_ON:
backend->shaders_desired_mask = kbase_pm_ca_get_core_mask(kbdev);
if (!kbase_pm_is_mcu_desired(kbdev))
backend->mcu_state = KBASE_MCU_ON_HWCNT_DISABLE;
else if (kbdev->csf.firmware_hctl_core_pwr) {
@@ -1170,7 +1186,7 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev)
break;
case KBASE_MCU_POWER_DOWN:
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TITANHW_2922)) {
if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TITANHW_2922)) {
if (!kbdev->csf.firmware_hctl_core_pwr)
kbasep_pm_toggle_power_interrupt(kbdev, true);
backend->mcu_state = KBASE_MCU_OFF;
@@ -1191,7 +1207,20 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev)
#ifdef KBASE_PM_RUNTIME
case KBASE_MCU_ON_SLEEP_INITIATE:
if (!kbase_pm_is_mcu_desired(kbdev)) {
kbase_csf_firmware_trigger_mcu_sleep(kbdev);
bool db_notif_disabled = false;
if (likely(test_bit(KBASE_GPU_SUPPORTS_FW_SLEEP_ON_IDLE,
&kbdev->pm.backend.gpu_sleep_allowed)))
db_notif_disabled =
kbase_reg_read32(kbdev,
GPU_CONTROL_ENUM(MCU_CONTROL)) &
MCU_CNTRL_DOORBELL_DISABLE_MASK;
/* If DB notification is enabled on FW side then send a sleep
* request to FW.
*/
if (!db_notif_disabled)
kbase_csf_firmware_trigger_mcu_sleep(kbdev);
backend->mcu_state = KBASE_MCU_ON_PEND_SLEEP;
} else
backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE;
@@ -1225,6 +1254,16 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev)
case KBASE_MCU_IN_SLEEP:
if (kbase_pm_is_mcu_desired(kbdev) && backend->l2_state == KBASE_L2_ON) {
wait_mcu_as_inactive(kbdev);
/* Ensure that FW would not go to sleep immediately after
* resumption.
*/
kbase_csf_firmware_global_input_mask(&kbdev->csf.global_iface,
GLB_REQ,
GLB_REQ_REQ_IDLE_DISABLE,
GLB_REQ_IDLE_DISABLE_MASK);
atomic_set(&kbdev->csf.scheduler.gpu_idle_timer_enabled, false);
atomic_set(&kbdev->csf.scheduler.fw_soi_enabled, false);
KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_REQUEST_WAKEUP(
kbdev, kbase_backend_get_cycle_cnt(kbdev));
kbase_pm_enable_mcu_db_notification(kbdev);
@@ -1342,6 +1381,8 @@ static void kbase_pm_l2_clear_backend_slot_submit_kctx(struct kbase_device *kbde
static bool can_power_down_l2(struct kbase_device *kbdev)
{
lockdep_assert_held(&kbdev->hwaccess_lock);
/* Defer the power-down if MMU is in process of page migration. */
return !kbdev->mmu_page_migrate_in_progress;
}
@@ -1367,20 +1408,6 @@ static bool need_tiler_control(struct kbase_device *kbdev)
#endif
}
/**
* hctl_l2_power_down - Initiate power down of L2 cache
*
* @kbdev: The kbase device structure for the device.
*
* This function initiates the power down of L2 cache when Host controls the power
* for Tiler block. The function expects that power down of Tiler to already have
* been initiated and it triggers the L2 power down only after the power down for
* Tiler is complete.
* The function shall be called only if L2 is in ready state.
*/
static void hctl_l2_power_down(struct kbase_device *kbdev)
{
}
/**
* hctl_tiler_power_up_done - Check and/or initiate power up of Tiler
@@ -1427,7 +1454,6 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev)
u64 l2_trans = kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2);
u64 l2_ready = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2);
#ifdef CONFIG_MALI_ARBITER_SUPPORT
/*
* kbase_pm_get_ready_cores and kbase_pm_get_trans_cores
* are vulnerable to corruption if gpu is lost
@@ -1456,7 +1482,6 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev)
}
break;
}
#endif
/* mask off ready from trans in case transitions finished
* between the register reads
@@ -1557,7 +1582,7 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev)
case KBASE_L2_RESTORE_CLOCKS:
/* We always assume only GPUs being affected by
* BASE_HW_ISSUE_GPU2017_1336 fall into this state
* KBASE_HW_ISSUE_GPU2017_1336 fall into this state
*/
WARN_ON_ONCE(!kbdev->pm.backend.gpu_clock_slow_down_wa);
@@ -1659,7 +1684,7 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev)
case KBASE_L2_SLOW_DOWN_CLOCKS:
/* We always assume only GPUs being affected by
* BASE_HW_ISSUE_GPU2017_1336 fall into this state
* KBASE_HW_ISSUE_GPU2017_1336 fall into this state
*/
WARN_ON_ONCE(!kbdev->pm.backend.gpu_clock_slow_down_wa);
@@ -1708,11 +1733,6 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev)
case KBASE_L2_PEND_OFF:
if (likely(!backend->l2_always_on)) {
if (need_tiler_control(kbdev) && l2_ready) {
hctl_l2_power_down(kbdev);
break;
}
if (l2_trans || l2_ready)
break;
} else if (kbdev->cache_clean_in_progress)
@@ -1727,11 +1747,10 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev)
}
#endif
/* Disabling MCU after L2 cache power down is to address
* BASE_HW_ISSUE_TITANHW_2922 hardware issue.
* KBASE_HW_ISSUE_TITANHW_2922 hardware issue.
*/
if (backend->l2_force_off_after_mcu_halt) {
kbase_csf_firmware_disable_mcu(kbdev);
kbase_csf_firmware_disable_mcu_wait(kbdev);
kbase_csf_stop_firmware_and_wait(kbdev);
WARN_ON_ONCE(backend->mcu_state != KBASE_MCU_OFF);
backend->l2_force_off_after_mcu_halt = false;
}
@@ -1878,12 +1897,7 @@ static int kbase_pm_shaders_update_state(struct kbase_device *kbdev)
* kbase_pm_get_ready_cores and kbase_pm_get_trans_cores
* are vulnerable to corruption if gpu is lost
*/
if (kbase_is_gpu_removed(kbdev)
#ifdef CONFIG_MALI_ARBITER_SUPPORT
|| kbase_pm_is_gpu_lost(kbdev)) {
#else
) {
#endif
if (kbase_is_gpu_removed(kbdev) || kbase_pm_is_gpu_lost(kbdev)) {
backend->shaders_state = KBASE_SHADERS_OFF_CORESTACK_OFF;
dev_dbg(kbdev->dev, "GPU lost has occurred - shaders off\n");
break;
@@ -1988,9 +2002,8 @@ static int kbase_pm_shaders_update_state(struct kbase_device *kbdev)
kbdev, KBASE_PM_POLICY_EVENT_IDLE);
if (kbdev->pm.backend.protected_transition_override ||
#ifdef CONFIG_MALI_ARBITER_SUPPORT
kbase_pm_is_suspending(kbdev) || kbase_pm_is_gpu_lost(kbdev) ||
#endif /* CONFIG_MALI_ARBITER_SUPPORT */
(kbase_has_arbiter(kbdev) && (kbase_pm_is_suspending(kbdev) ||
kbase_pm_is_gpu_lost(kbdev))) ||
!stt->configured_ticks || WARN_ON(stt->cancel_queued)) {
backend->shaders_state =
KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON;
@@ -2057,10 +2070,9 @@ static int kbase_pm_shaders_update_state(struct kbase_device *kbdev)
kbdev, KBASE_PM_POLICY_EVENT_TIMER_MISS);
backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON;
#ifdef CONFIG_MALI_ARBITER_SUPPORT
} else if (kbase_pm_is_suspending(kbdev) || kbase_pm_is_gpu_lost(kbdev)) {
} else if (kbase_has_arbiter(kbdev) &&
(kbase_pm_is_suspending(kbdev) || kbase_pm_is_gpu_lost(kbdev))) {
backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON;
#endif /* CONFIG_MALI_ARBITER_SUPPORT */
}
break;
@@ -2079,7 +2091,7 @@ static int kbase_pm_shaders_update_state(struct kbase_device *kbdev)
if (!backend->partial_shaderoff)
shader_poweroff_timer_queue_cancel(kbdev);
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_921)) {
if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TTRX_921)) {
kbase_gpu_start_cache_clean_nolock(kbdev,
GPU_COMMAND_CACHE_CLN_INV_L2);
backend->shaders_state = KBASE_SHADERS_L2_FLUSHING_CORESTACK_ON;
@@ -2429,6 +2441,9 @@ void kbase_pm_reset_complete(struct kbase_device *kbdev)
backend->in_reset = false;
#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME)
backend->gpu_wakeup_override = false;
backend->db_mirror_interrupt_enabled = false;
backend->gpu_sleep_mode_active = false;
backend->exit_gpu_sleep_mode = false;
#endif
kbase_pm_update_state(kbdev);
@@ -2653,12 +2668,9 @@ static int pm_wait_for_poweroff_work_complete(struct kbase_device *kbdev, bool k
const long timeout = kbase_csf_timeout_in_jiffies(
kbase_get_timeout_ms(kbdev, CSF_PM_TIMEOUT) + extra_wait_time_ms);
#else
#ifdef CONFIG_MALI_ARBITER_SUPPORT
/* Handling of timeout error isn't supported for arbiter builds */
const long timeout = MAX_SCHEDULE_TIMEOUT;
#else
const long timeout = (long)msecs_to_jiffies(PM_TIMEOUT_MS);
#endif
const long timeout = kbase_has_arbiter(kbdev) ? MAX_SCHEDULE_TIMEOUT :
(long)msecs_to_jiffies(PM_TIMEOUT_MS);
#endif
int err = 0;
@@ -2779,7 +2791,8 @@ static void update_user_reg_page_mapping(struct kbase_device *kbdev)
* when the context (user process) needs to access to the page.
*/
unmap_mapping_range(kbdev->csf.user_reg.filp->f_inode->i_mapping,
kctx->csf.user_reg.file_offset << PAGE_SHIFT, PAGE_SIZE, 1);
(loff_t)kctx->csf.user_reg.file_offset << PAGE_SHIFT, PAGE_SIZE,
1);
list_del_init(&kctx->csf.user_reg.link);
dev_dbg(kbdev->dev, "Updated USER Reg page mapping of ctx %d_%d", kctx->tgid,
kctx->id);
@@ -2797,7 +2810,7 @@ static void update_user_reg_page_mapping(struct kbase_device *kbdev)
void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume)
{
struct kbase_pm_backend_data *backend = &kbdev->pm.backend;
bool reset_required = is_resume;
int ret = is_resume;
unsigned long flags;
KBASE_DEBUG_ASSERT(kbdev != NULL);
@@ -2806,12 +2819,10 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume)
#endif /* !MALI_USE_CSF */
lockdep_assert_held(&kbdev->pm.lock);
#ifdef CONFIG_MALI_ARBITER_SUPPORT
if (WARN_ON(kbase_pm_is_gpu_lost(kbdev))) {
dev_err(kbdev->dev, "%s: Cannot power up while GPU lost", __func__);
return;
}
#endif
if (backend->gpu_powered) {
#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME)
@@ -2836,7 +2847,7 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume)
backend->callback_power_resume(kbdev);
return;
} else if (backend->callback_power_on) {
reset_required = backend->callback_power_on(kbdev);
ret = backend->callback_power_on(kbdev);
}
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
@@ -2849,15 +2860,18 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume)
#endif
if (reset_required) {
if (ret == GPU_STATE_IN_RESET) {
/* GPU is already in reset state after power on and no
* soft-reset needed. Just reconfiguration is needed.
*/
kbase_pm_init_hw(kbdev, PM_ENABLE_IRQS | PM_NO_RESET);
} else if (ret == GPU_STATE_LOST) {
/* GPU state was lost, reset GPU to ensure it is in a
* consistent state
*/
kbase_pm_init_hw(kbdev, PM_ENABLE_IRQS);
}
#ifdef CONFIG_MALI_ARBITER_SUPPORT
else {
if (kbdev->arb.arb_if) {
} else {
if (kbase_has_arbiter(kbdev)) {
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
/* In the case that the GPU has just been granted by
@@ -2873,8 +2887,8 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume)
* that a repartitioning occurred. In this case the current config
* should be read again.
*/
kbase_gpuprops_get_curr_config_props(kbdev, &kbdev->gpu_props.curr_config);
#endif /* CONFIG_MALI_ARBITER_SUPPORT */
if (kbase_has_arbiter(kbdev))
kbase_gpuprops_get_curr_config_props(kbdev, &kbdev->gpu_props.curr_config);
mutex_lock(&kbdev->mmu_hw_mutex);
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
@@ -2898,7 +2912,7 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume)
backend->l2_desired = true;
#if MALI_USE_CSF
{
if (reset_required) {
if (ret != GPU_STATE_INTACT) {
/* GPU reset was done after the power on, so send the post
* reset event instead. This is okay as GPU power off event
* is same as pre GPU reset event.
@@ -2966,12 +2980,7 @@ bool kbase_pm_clock_off(struct kbase_device *kbdev)
}
#endif
if (kbase_is_gpu_removed(kbdev)
#ifdef CONFIG_MALI_ARBITER_SUPPORT
|| kbase_pm_is_gpu_lost(kbdev)) {
#else
) {
#endif
if (kbase_is_gpu_removed(kbdev) || kbase_pm_is_gpu_lost(kbdev)) {
/* Ensure we unblock any threads that are stuck waiting
* for the GPU
*/
@@ -2989,10 +2998,7 @@ bool kbase_pm_clock_off(struct kbase_device *kbdev)
/* GPU is about to be turned off, switch to dummy page */
update_user_reg_page_mapping(kbdev);
#endif
#ifdef CONFIG_MALI_ARBITER_SUPPORT
kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_IDLE_EVENT);
#endif /* CONFIG_MALI_ARBITER_SUPPORT */
if (kbdev->pm.backend.callback_power_off)
kbdev->pm.backend.callback_power_off(kbdev);
@@ -3046,6 +3052,7 @@ static enum hrtimer_restart kbasep_reset_timeout(struct hrtimer *timer)
return HRTIMER_NORESTART;
}
static int kbase_set_gpu_quirks(struct kbase_device *kbdev)
{
#if MALI_USE_CSF
@@ -3075,7 +3082,7 @@ static int kbase_set_gpu_quirks(struct kbase_device *kbdev)
kbdev->hw_quirks_gpu = hw_quirks_gpu;
#endif /* !MALI_USE_CSF */
if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_IDVS_GROUP_SIZE)) {
if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_IDVS_GROUP_SIZE)) {
u32 default_idvs_group_size = 0xF;
u32 group_size = 0;
@@ -3109,10 +3116,10 @@ static int kbase_set_sc_quirks(struct kbase_device *kbdev)
if (kbase_is_gpu_removed(kbdev))
return -EIO;
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_2968_TTRX_3162))
if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TTRX_2968_TTRX_3162))
hw_quirks_sc |= SC_VAR_ALGORITHM;
if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_TLS_HASHING))
if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_TLS_HASHING))
hw_quirks_sc |= SC_TLS_HASH_ENABLE;
kbdev->hw_quirks_sc = hw_quirks_sc;
@@ -3131,7 +3138,7 @@ static int kbase_set_tiler_quirks(struct kbase_device *kbdev)
return -EIO;
/* Set tiler clock gate override if required */
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3953))
if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_T76X_3953))
hw_quirks_tiler |= TC_CLOCK_GATE_OVERRIDE;
kbdev->hw_quirks_tiler = hw_quirks_tiler;
@@ -3139,6 +3146,7 @@ static int kbase_set_tiler_quirks(struct kbase_device *kbdev)
return 0;
}
static int kbase_pm_hw_issues_detect(struct kbase_device *kbdev)
{
struct device_node *np = kbdev->dev->of_node;
@@ -3191,6 +3199,7 @@ static int kbase_pm_hw_issues_detect(struct kbase_device *kbdev)
error = kbase_set_mmu_quirks(kbdev);
}
return error;
}
@@ -3210,6 +3219,7 @@ static void kbase_pm_hw_issues_apply(struct kbase_device *kbdev)
#else
kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(JM_CONFIG), kbdev->hw_quirks_gpu);
#endif
}
void kbase_pm_cache_snoop_enable(struct kbase_device *kbdev)
@@ -3257,16 +3267,10 @@ static void reenable_protected_mode_hwcnt(struct kbase_device *kbdev)
}
#endif
static int kbase_pm_do_reset(struct kbase_device *kbdev)
static int kbase_pm_do_reset_soft(struct kbase_device *kbdev)
{
struct kbasep_reset_timeout_data rtdata;
u32 reg_offset, reg_val;
int ret;
KBASE_KTRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, 0);
KBASE_TLSTREAM_JD_GPU_SOFT_RESET(kbdev, kbdev);
if (kbdev->pm.backend.callback_soft_reset) {
ret = kbdev->pm.backend.callback_soft_reset(kbdev);
if (ret < 0)
@@ -3279,12 +3283,30 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev)
GPU_COMMAND_SOFT_RESET);
}
}
return 0;
}
reg_offset = GPU_CONTROL_ENUM(GPU_IRQ_MASK);
reg_val = RESET_COMPLETED;
static int kbase_pm_do_reset(struct kbase_device *kbdev)
{
struct kbasep_reset_timeout_data rtdata;
u32 reg_offset, reg_val;
int ret;
/* Unmask the reset complete interrupt only */
kbase_reg_write32(kbdev, reg_offset, reg_val);
KBASE_KTRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, 0);
KBASE_TLSTREAM_JD_GPU_SOFT_RESET(kbdev, kbdev);
{
ret = kbase_pm_do_reset_soft(kbdev);
if (ret)
return ret;
reg_offset = GPU_CONTROL_ENUM(GPU_IRQ_MASK);
reg_val = RESET_COMPLETED;
/* Unmask the reset complete interrupt only */
kbase_reg_write32(kbdev, reg_offset, reg_val);
}
/* Initialize a structure for tracking the status of the reset */
rtdata.kbdev = kbdev;
@@ -3333,9 +3355,8 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev)
/* The GPU doesn't seem to be responding to the reset so try a hard
* reset, but only when NOT in arbitration mode.
*/
#ifdef CONFIG_MALI_ARBITER_SUPPORT
if (!kbdev->arb.arb_if) {
#endif /* CONFIG_MALI_ARBITER_SUPPORT */
if (!kbase_has_arbiter(kbdev)) {
dev_err(kbdev->dev,
"Failed to soft-reset GPU (timed out after %d ms), now attempting a hard reset\n",
RESET_TIMEOUT);
@@ -3365,9 +3386,7 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev)
dev_err(kbdev->dev, "Failed to hard-reset the GPU (timed out after %d ms)\n",
RESET_TIMEOUT);
#ifdef CONFIG_MALI_ARBITER_SUPPORT
}
#endif /* CONFIG_MALI_ARBITER_SUPPORT */
return -EINVAL;
}
@@ -3418,9 +3437,7 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags)
spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags);
/* Soft reset the GPU */
#ifdef CONFIG_MALI_ARBITER_SUPPORT
if (!(flags & PM_NO_RESET))
#endif /* CONFIG_MALI_ARBITER_SUPPORT */
err = kbdev->protected_ops->protected_mode_disable(kbdev->protected_dev);
spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags);
@@ -3441,7 +3458,6 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags)
if (err)
goto exit;
if (flags & PM_HW_ISSUES_DETECT) {
err = kbase_pm_hw_issues_detect(kbdev);
if (err)
@@ -3451,6 +3467,10 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags)
kbase_pm_hw_issues_apply(kbdev);
kbase_cache_set_coherency_mode(kbdev, kbdev->system_coherency);
kbase_amba_set_shareable_cache_support(kbdev);
#if MALI_USE_CSF
kbase_backend_update_gpu_timestamp_offset(kbdev);
kbdev->csf.compute_progress_timeout_cc = 0;
#endif
/* Sanity check protected mode was left after reset */
WARN_ON(kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_STATUS)) &

41
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h
View File

@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/*
*
* (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -821,6 +821,21 @@ bool kbase_pm_is_mcu_desired(struct kbase_device *kbdev);
*/
bool kbase_pm_is_mcu_inactive(struct kbase_device *kbdev, enum kbase_mcu_state state);
#ifdef KBASE_PM_RUNTIME
/**
* kbase_pm_enable_mcu_db_notification - Enable the Doorbell notification on
* MCU side
*
* @kbdev: Pointer to the device.
*
* This function is called to re-enable the Doorbell notification on MCU side
* when MCU needs to beome active again.
*/
void kbase_pm_enable_mcu_db_notification(struct kbase_device *kbdev);
#endif /* KBASE_PM_RUNTIME */
/**
* kbase_pm_idle_groups_sched_suspendable - Check whether the scheduler can be
* suspended to low power state when all
@@ -963,11 +978,29 @@ static inline bool kbase_pm_gpu_sleep_allowed(struct kbase_device *kbdev)
* A high positive value of autosuspend_delay can be used to keep the
* GPU in sleep state for a long time.
*/
if (unlikely(!kbdev->dev->power.autosuspend_delay ||
(kbdev->dev->power.autosuspend_delay < 0)))
if (unlikely(kbdev->dev->power.autosuspend_delay <= 0))
return false;
return kbdev->pm.backend.gpu_sleep_supported;
return test_bit(KBASE_GPU_SUPPORTS_GPU_SLEEP, &kbdev->pm.backend.gpu_sleep_allowed);
}
/**
* kbase_pm_fw_sleep_on_idle_allowed - Check if FW sleep-on-idle could be enabled
*
* @kbdev: Device pointer
*
* This function should be called whenever the conditions that impact
* FW sleep-on-idle support change so that it could be enabled/disabled
* accordingly.
*
* Return: true if FW sleep-on-idle is allowed
*/
static inline bool kbase_pm_fw_sleep_on_idle_allowed(struct kbase_device *kbdev)
{
if (unlikely(kbdev->dev->power.autosuspend_delay <= 0))
return false;
return kbdev->pm.backend.gpu_sleep_allowed == KBASE_GPU_FW_SLEEP_ON_IDLE_ALLOWED;
}
/**

20
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -77,7 +77,16 @@ void kbase_pm_policy_init(struct kbase_device *kbdev)
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
kbdev->pm.backend.pm_current_policy = default_policy;
kbdev->pm.backend.csf_pm_sched_flags = default_policy->pm_sched_flags;
#ifdef KBASE_PM_RUNTIME
if (kbase_pm_idle_groups_sched_suspendable(kbdev))
clear_bit(KBASE_GPU_IGNORE_IDLE_EVENT, &kbdev->pm.backend.gpu_sleep_allowed);
else
set_bit(KBASE_GPU_IGNORE_IDLE_EVENT, &kbdev->pm.backend.gpu_sleep_allowed);
#endif /* KBASE_PM_RUNTIME */
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
#else
CSTD_UNUSED(flags);
kbdev->pm.backend.pm_current_policy = default_policy;
@@ -127,7 +136,7 @@ void kbase_pm_update_active(struct kbase_device *kbdev)
pm->backend.poweroff_wait_in_progress = false;
pm->backend.l2_desired = true;
#if MALI_USE_CSF
pm->backend.mcu_desired = true;
pm->backend.mcu_desired = pm->backend.mcu_poweron_required;
#endif
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
@@ -400,6 +409,13 @@ void kbase_pm_set_policy(struct kbase_device *kbdev, const struct kbase_pm_polic
/* New policy in place, release the clamping on mcu/L2 off state */
kbdev->pm.backend.policy_change_clamp_state_to_off = false;
kbase_pm_update_state(kbdev);
#ifdef KBASE_PM_RUNTIME
if (kbase_pm_idle_groups_sched_suspendable(kbdev))
clear_bit(KBASE_GPU_IGNORE_IDLE_EVENT, &kbdev->pm.backend.gpu_sleep_allowed);
else
set_bit(KBASE_GPU_IGNORE_IDLE_EVENT, &kbdev->pm.backend.gpu_sleep_allowed);
#endif /* KBASE_PM_RUNTIME */
#endif
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);

121
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -30,10 +30,7 @@
#include <mali_kbase_config_defaults.h>
#include <linux/version_compat_defs.h>
#include <asm/arch_timer.h>
#if !IS_ENABLED(CONFIG_MALI_REAL_HW)
#include <backend/gpu/mali_kbase_model_linux.h>
#endif
#include <linux/mali_hw_access.h>
struct kbase_timeout_info {
char *selector_str;
@@ -41,12 +38,15 @@ struct kbase_timeout_info {
};
#if MALI_USE_CSF
#define GPU_TIMESTAMP_OFFSET_INVALID S64_MAX
static struct kbase_timeout_info timeout_info[KBASE_TIMEOUT_SELECTOR_COUNT] = {
[CSF_FIRMWARE_TIMEOUT] = { "CSF_FIRMWARE_TIMEOUT", MIN(CSF_FIRMWARE_TIMEOUT_CYCLES,
CSF_FIRMWARE_PING_TIMEOUT_CYCLES) },
[CSF_PM_TIMEOUT] = { "CSF_PM_TIMEOUT", CSF_PM_TIMEOUT_CYCLES },
[CSF_GPU_RESET_TIMEOUT] = { "CSF_GPU_RESET_TIMEOUT", CSF_GPU_RESET_TIMEOUT_CYCLES },
[CSF_CSG_SUSPEND_TIMEOUT] = { "CSF_CSG_SUSPEND_TIMEOUT", CSF_CSG_SUSPEND_TIMEOUT_CYCLES },
[CSF_CSG_TERM_TIMEOUT] = { "CSF_CSG_TERM_TIMEOUT", CSF_CSG_TERM_TIMEOUT_CYCLES },
[CSF_FIRMWARE_BOOT_TIMEOUT] = { "CSF_FIRMWARE_BOOT_TIMEOUT",
CSF_FIRMWARE_BOOT_TIMEOUT_CYCLES },
[CSF_FIRMWARE_PING_TIMEOUT] = { "CSF_FIRMWARE_PING_TIMEOUT",
@@ -82,6 +82,68 @@ static struct kbase_timeout_info timeout_info[KBASE_TIMEOUT_SELECTOR_COUNT] = {
};
#endif
#if MALI_USE_CSF
void kbase_backend_invalidate_gpu_timestamp_offset(struct kbase_device *kbdev)
{
kbdev->backend_time.gpu_timestamp_offset = GPU_TIMESTAMP_OFFSET_INVALID;
}
KBASE_EXPORT_TEST_API(kbase_backend_invalidate_gpu_timestamp_offset);
/**
* kbase_backend_compute_gpu_ts_offset() - Compute GPU TS offset.
*
* @kbdev: Kbase device.
*
* This function compute the value of GPU and CPU TS offset:
* - set to zero current TIMESTAMP_OFFSET register
* - read CPU TS and convert it to ticks
* - read GPU TS
* - calculate diff between CPU and GPU ticks
* - cache the diff as the GPU TS offset
*
* To reduce delays, preemption must be disabled during reads of both CPU and GPU TS
* this function require access to GPU register to be enabled
*/
static inline void kbase_backend_compute_gpu_ts_offset(struct kbase_device *kbdev)
{
s64 cpu_ts_ticks = 0;
s64 gpu_ts_ticks = 0;
if (kbdev->backend_time.gpu_timestamp_offset != GPU_TIMESTAMP_OFFSET_INVALID)
return;
kbase_reg_write64(kbdev, GPU_CONTROL_ENUM(TIMESTAMP_OFFSET), 0);
gpu_ts_ticks = kbase_reg_read64_coherent(kbdev, GPU_CONTROL_ENUM(TIMESTAMP));
cpu_ts_ticks = ktime_get_raw_ns();
cpu_ts_ticks = div64_u64(cpu_ts_ticks * kbdev->backend_time.divisor,
kbdev->backend_time.multiplier);
kbdev->backend_time.gpu_timestamp_offset = cpu_ts_ticks - gpu_ts_ticks;
}
void kbase_backend_update_gpu_timestamp_offset(struct kbase_device *kbdev)
{
lockdep_assert_held(&kbdev->pm.lock);
kbase_backend_compute_gpu_ts_offset(kbdev);
dev_dbg(kbdev->dev, "Setting GPU timestamp offset register to %lld (%lld ns)",
kbdev->backend_time.gpu_timestamp_offset,
div64_s64(kbdev->backend_time.gpu_timestamp_offset *
(s64)kbdev->backend_time.multiplier,
(s64)kbdev->backend_time.divisor));
kbase_reg_write64(kbdev, GPU_CONTROL_ENUM(TIMESTAMP_OFFSET),
kbdev->backend_time.gpu_timestamp_offset);
}
#if MALI_UNIT_TEST
u64 kbase_backend_read_gpu_timestamp_offset_reg(struct kbase_device *kbdev)
{
return kbase_reg_read64_coherent(kbdev, GPU_CONTROL_ENUM(TIMESTAMP_OFFSET));
}
KBASE_EXPORT_TEST_API(kbase_backend_read_gpu_timestamp_offset_reg);
#endif
#endif
void kbase_backend_get_gpu_time_norequest(struct kbase_device *kbdev, u64 *cycle_counter,
u64 *system_time, struct timespec64 *ts)
{
@@ -100,6 +162,7 @@ void kbase_backend_get_gpu_time_norequest(struct kbase_device *kbdev, u64 *cycle
ktime_get_raw_ts64(ts);
#endif
}
KBASE_EXPORT_TEST_API(kbase_backend_get_gpu_time_norequest);
#if !MALI_USE_CSF
/**
@@ -143,6 +206,7 @@ void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter,
kbase_pm_release_gpu_cycle_counter(kbdev);
#endif
}
KBASE_EXPORT_TEST_API(kbase_backend_get_gpu_time);
static u64 kbase_device_get_scaling_frequency(struct kbase_device *kbdev)
{
@@ -171,6 +235,15 @@ void kbase_device_set_timeout_ms(struct kbase_device *kbdev, enum kbase_timeout_
}
selector_str = timeout_info[selector].selector_str;
#if MALI_USE_CSF
if (IS_ENABLED(CONFIG_MALI_REAL_HW) && !IS_ENABLED(CONFIG_MALI_IS_FPGA) &&
unlikely(timeout_ms >= MAX_TIMEOUT_MS)) {
dev_warn(kbdev->dev, "%s is capped from %dms to %dms\n",
timeout_info[selector].selector_str, timeout_ms, MAX_TIMEOUT_MS);
timeout_ms = MAX_TIMEOUT_MS;
}
#endif
kbdev->backend_time.device_scaled_timeouts[selector] = timeout_ms;
dev_dbg(kbdev->dev, "\t%-35s: %ums\n", selector_str, timeout_ms);
}
@@ -282,36 +355,14 @@ u64 __maybe_unused kbase_backend_time_convert_gpu_to_cpu(struct kbase_device *kb
if (WARN_ON(!kbdev))
return 0;
return div64_u64(gpu_ts * kbdev->backend_time.multiplier, kbdev->backend_time.divisor) +
kbdev->backend_time.offset;
}
/**
* get_cpu_gpu_time() - Get current CPU and GPU timestamps.
*
* @kbdev: Kbase device.
* @cpu_ts: Output CPU timestamp.
* @gpu_ts: Output GPU timestamp.
* @gpu_cycle: Output GPU cycle counts.
*/
static void get_cpu_gpu_time(struct kbase_device *kbdev, u64 *cpu_ts, u64 *gpu_ts, u64 *gpu_cycle)
{
struct timespec64 ts;
kbase_backend_get_gpu_time(kbdev, gpu_cycle, gpu_ts, &ts);
if (cpu_ts)
*cpu_ts = (u64)(ts.tv_sec * NSEC_PER_SEC + ts.tv_nsec);
return div64_u64(gpu_ts * kbdev->backend_time.multiplier, kbdev->backend_time.divisor);
}
KBASE_EXPORT_TEST_API(kbase_backend_time_convert_gpu_to_cpu);
#endif
u64 kbase_arch_timer_get_cntfrq(struct kbase_device *kbdev)
{
u64 freq = arch_timer_get_cntfrq();
#if !IS_ENABLED(CONFIG_MALI_REAL_HW)
freq = midgard_model_arch_timer_get_cntfrq(kbdev->model);
#endif
u64 freq = mali_arch_timer_get_cntfrq();
dev_dbg(kbdev->dev, "System Timer Freq = %lluHz", freq);
@@ -322,13 +373,10 @@ int kbase_backend_time_init(struct kbase_device *kbdev)
{
int err = 0;
#if MALI_USE_CSF
u64 cpu_ts = 0;
u64 gpu_ts = 0;
u64 freq;
u64 common_factor;
kbase_pm_register_access_enable(kbdev);
get_cpu_gpu_time(kbdev, &cpu_ts, &gpu_ts, NULL);
freq = kbase_arch_timer_get_cntfrq(kbdev);
if (!freq) {
@@ -348,9 +396,8 @@ int kbase_backend_time_init(struct kbase_device *kbdev)
goto disable_registers;
}
kbdev->backend_time.offset =
(s64)(cpu_ts - div64_u64(gpu_ts * kbdev->backend_time.multiplier,
kbdev->backend_time.divisor));
kbase_backend_invalidate_gpu_timestamp_offset(
kbdev); /* force computation of GPU Timestamp offset */
#endif
if (kbase_timeout_scaling_init(kbdev)) {

29
drivers/gpu/arm/bifrost/build.bp
View File

@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/*
*
* (C) COPYRIGHT 2017-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2017-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -71,18 +71,6 @@ bob_defaults {
mali_real_hw: {
kbuild_options: ["CONFIG_MALI_REAL_HW=y"],
},
mali_error_inject_none: {
kbuild_options: ["CONFIG_MALI_ERROR_INJECT_NONE=y"],
},
mali_error_inject_track_list: {
kbuild_options: ["CONFIG_MALI_ERROR_INJECT_TRACK_LIST=y"],
},
mali_error_inject_random: {
kbuild_options: ["CONFIG_MALI_ERROR_INJECT_RANDOM=y"],
},
mali_error_inject: {
kbuild_options: ["CONFIG_MALI_BIFROST_ERROR_INJECT=y"],
},
mali_debug: {
kbuild_options: [
"CONFIG_MALI_BIFROST_DEBUG=y",
@@ -125,7 +113,7 @@ bob_defaults {
mali_hw_errata_1485982_use_clock_alternative: {
kbuild_options: ["CONFIG_MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE=y"],
},
platform_is_fpga: {
mali_is_fpga: {
kbuild_options: ["CONFIG_MALI_IS_FPGA=y"],
},
mali_coresight: {
@@ -160,7 +148,6 @@ bob_defaults {
// is an umbrella feature that would be open for inappropriate use
// (catch-all for experimental CS code without separating it into
// different features).
"MALI_INCREMENTAL_RENDERING_JM={{.incremental_rendering_jm}}",
"MALI_BASE_CSF_PERFORMANCE_TESTS={{.base_csf_performance_tests}}",
],
}
@@ -174,6 +161,9 @@ bob_kernel_module {
"*.c",
"*.h",
"Kbuild",
"arbiter/*.c",
"arbiter/*.h",
"arbiter/Kbuild",
"backend/gpu/*.c",
"backend/gpu/*.h",
"backend/gpu/Kbuild",
@@ -239,6 +229,7 @@ bob_kernel_module {
"jm/*.h",
"tl/backend/*_jm.c",
"mmu/backend/*_jm.c",
"mmu/backend/*_jm.h",
"ipa/backend/*_jm.c",
"ipa/backend/*_jm.h",
],
@@ -263,17 +254,11 @@ bob_kernel_module {
"hwcnt/backend/*_csf_*.h",
"tl/backend/*_csf.c",
"mmu/backend/*_csf.c",
"mmu/backend/*_csf.h",
"ipa/backend/*_csf.c",
"ipa/backend/*_csf.h",
],
},
mali_arbiter_support: {
srcs: [
"arbiter/*.c",
"arbiter/*.h",
"arbiter/Kbuild",
],
},
kbuild_options: [
"CONFIG_MALI_BIFROST=m",
"CONFIG_MALI_KUTF=n",

13
drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_csf.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -116,8 +116,7 @@ static void kbase_context_term_partial(struct kbase_context *kctx, unsigned int
struct kbase_context *kbase_create_context(struct kbase_device *kbdev, bool is_compat,
base_context_create_flags const flags,
unsigned long const api_version,
struct kbase_file *const kfile)
unsigned long const api_version, struct file *const filp)
{
struct kbase_context *kctx;
unsigned int i = 0;
@@ -136,7 +135,7 @@ struct kbase_context *kbase_create_context(struct kbase_device *kbdev, bool is_c
kctx->kbdev = kbdev;
kctx->api_version = api_version;
kctx->kfile = kfile;
kctx->filp = filp;
kctx->create_flags = flags;
memcpy(kctx->comm, current->comm, sizeof(current->comm));
@@ -187,11 +186,15 @@ void kbase_destroy_context(struct kbase_context *kctx)
* Customer side that a hang could occur if context termination is
* not blocked until the resume of GPU device.
*/
if (kbase_has_arbiter(kbdev))
atomic_inc(&kbdev->pm.gpu_users_waiting);
while (kbase_pm_context_active_handle_suspend(kbdev,
KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE)) {
dev_info(kbdev->dev, "Suspend in progress when destroying context");
dev_dbg(kbdev->dev, "Suspend in progress when destroying context");
wait_event(kbdev->pm.resume_wait, !kbase_pm_is_suspending(kbdev));
}
if (kbase_has_arbiter(kbdev))
atomic_dec(&kbdev->pm.gpu_users_waiting);
/* Have synchronized against the System suspend and incremented the
* pm.active_count. So any subsequent invocation of System suspend

19
drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_jm.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -168,8 +168,7 @@ static void kbase_context_term_partial(struct kbase_context *kctx, unsigned int
struct kbase_context *kbase_create_context(struct kbase_device *kbdev, bool is_compat,
base_context_create_flags const flags,
unsigned long const api_version,
struct kbase_file *const kfile)
unsigned long const api_version, struct file *const filp)
{
struct kbase_context *kctx;
unsigned int i = 0;
@@ -188,7 +187,7 @@ struct kbase_context *kbase_create_context(struct kbase_device *kbdev, bool is_c
kctx->kbdev = kbdev;
kctx->api_version = api_version;
kctx->kfile = kfile;
kctx->filp = filp;
kctx->create_flags = flags;
if (is_compat)
@@ -232,14 +231,13 @@ void kbase_destroy_context(struct kbase_context *kctx)
if (WARN_ON(!kbdev))
return;
/* Context termination could happen whilst the system suspend of
/* Context termination could happen whilst the system suspend of
* the GPU device is ongoing or has completed. It has been seen on
* Customer side that a hang could occur if context termination is
* not blocked until the resume of GPU device.
*/
#ifdef CONFIG_MALI_ARBITER_SUPPORT
atomic_inc(&kbdev->pm.gpu_users_waiting);
#endif /* CONFIG_MALI_ARBITER_SUPPORT */
if (kbase_has_arbiter(kbdev))
atomic_inc(&kbdev->pm.gpu_users_waiting);
while (kbase_pm_context_active_handle_suspend(kbdev,
KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE)) {
dev_dbg(kbdev->dev, "Suspend in progress when destroying context");
@@ -256,9 +254,8 @@ void kbase_destroy_context(struct kbase_context *kctx)
*/
wait_event(kbdev->pm.resume_wait, !kbase_pm_is_resuming(kbdev));
#ifdef CONFIG_MALI_ARBITER_SUPPORT
atomic_dec(&kbdev->pm.gpu_users_waiting);
#endif /* CONFIG_MALI_ARBITER_SUPPORT */
if (kbase_has_arbiter(kbdev))
atomic_dec(&kbdev->pm.gpu_users_waiting);
kbase_mem_pool_group_mark_dying(&kctx->mem_pools);

10
drivers/gpu/arm/bifrost/context/mali_kbase_context.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -141,7 +141,7 @@ int kbase_context_common_init(struct kbase_context *kctx)
kctx->pid = task_pid_vnr(current);
/* Check if this is a Userspace created context */
if (likely(kctx->kfile)) {
if (likely(kctx->filp)) {
struct pid *pid_struct;
rcu_read_lock();
@@ -184,6 +184,8 @@ int kbase_context_common_init(struct kbase_context *kctx)
spin_lock_init(&kctx->waiting_soft_jobs_lock);
INIT_LIST_HEAD(&kctx->waiting_soft_jobs);
init_waitqueue_head(&kctx->event_queue);
kbase_gpu_vm_lock(kctx);
bitmap_copy(kctx->cookies, &cookies_mask, BITS_PER_LONG);
kbase_gpu_vm_unlock(kctx);
@@ -195,7 +197,7 @@ int kbase_context_common_init(struct kbase_context *kctx)
mutex_unlock(&kctx->kbdev->kctx_list_lock);
if (err) {
dev_err(kctx->kbdev->dev, "(err:%d) failed to insert kctx to kbase_process", err);
if (likely(kctx->kfile)) {
if (likely(kctx->filp)) {
mmdrop(kctx->process_mm);
put_task_struct(kctx->task);
}
@@ -284,7 +286,7 @@ void kbase_context_common_term(struct kbase_context *kctx)
kbase_remove_kctx_from_process(kctx);
mutex_unlock(&kctx->kbdev->kctx_list_lock);
if (likely(kctx->kfile)) {
if (likely(kctx->filp)) {
mmdrop(kctx->process_mm);
put_task_struct(kctx->task);
}

11
drivers/gpu/arm/bifrost/context/mali_kbase_context.h
View File

@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/*
*
* (C) COPYRIGHT 2011-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2011-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -56,9 +56,9 @@ void kbase_context_debugfs_term(struct kbase_context *const kctx);
* BASEP_CONTEXT_CREATE_KERNEL_FLAGS.
* @api_version: Application program interface version, as encoded in
* a single integer by the KBASE_API_VERSION macro.
* @kfile: Pointer to the object representing the /dev/malixx device
* file instance. Shall be passed as NULL for internally created
* contexts.
* @filp: Pointer to the struct file corresponding to device file
* /dev/malixx instance, passed to the file's open method.
* Shall be passed as NULL for internally created contexts.
*
* Up to one context can be created for each client that opens the device file
* /dev/malixx. Context creation is deferred until a special ioctl() system call
@@ -68,8 +68,7 @@ void kbase_context_debugfs_term(struct kbase_context *const kctx);
*/
struct kbase_context *kbase_create_context(struct kbase_device *kbdev, bool is_compat,
base_context_create_flags const flags,
unsigned long api_version,
struct kbase_file *const kfile);
unsigned long api_version, struct file *filp);
/**
* kbase_destroy_context - Destroy a kernel base context.

4
drivers/gpu/arm/bifrost/csf/Kbuild
View File

@@ -1,6 +1,6 @@
# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
#
# (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved.
# (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
@@ -48,8 +48,10 @@ bifrost_kbase-y += \
ifeq ($(CONFIG_MALI_BIFROST_NO_MALI),y)
bifrost_kbase-y += csf/mali_kbase_csf_firmware_no_mali.o
bifrost_kbase-y += csf/mali_kbase_csf_fw_io_no_mali.o
else
bifrost_kbase-y += csf/mali_kbase_csf_firmware.o
bifrost_kbase-y += csf/mali_kbase_csf_fw_io.o
endif
bifrost_kbase-$(CONFIG_DEBUG_FS) += csf/mali_kbase_debug_csf_fault.o

5
drivers/gpu/arm/bifrost/csf/ipa_control/mali_kbase_csf_ipa_control.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -943,6 +943,8 @@ void kbase_ipa_control_protm_entered(struct kbase_device *kbdev)
struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control;
lockdep_assert_held(&kbdev->hwaccess_lock);
ipa_ctrl->protm_start = ktime_get_raw_ns();
}
@@ -955,6 +957,7 @@ void kbase_ipa_control_protm_exited(struct kbase_device *kbdev)
lockdep_assert_held(&kbdev->hwaccess_lock);
for (i = 0; i < KBASE_IPA_CONTROL_MAX_SESSIONS; i++) {
struct kbase_ipa_control_session *session = &ipa_ctrl->sessions[i];

545
drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -45,6 +45,9 @@
#define CS_RING_BUFFER_MAX_SIZE ((uint32_t)(1 << 31)) /* 2GiB */
#define CS_RING_BUFFER_MIN_SIZE ((uint32_t)4096)
/* 0.2 second assuming 600 MHz GPU clock, which is double of iterator disabling timeout */
#define MAX_PROGRESS_TIMEOUT_EVENT_DELAY ((u32)120000000)
#define PROTM_ALLOC_MAX_RETRIES ((u8)5)
const u8 kbasep_csf_queue_group_priority_to_relative[BASE_QUEUE_GROUP_PRIORITY_COUNT] = {
@@ -539,6 +542,8 @@ static int csf_queue_register_internal(struct kbase_context *kctx,
queue->blocked_reason = CS_STATUS_BLOCKED_REASON_REASON_UNBLOCKED;
queue->clear_faults = true;
INIT_LIST_HEAD(&queue->link);
atomic_set(&queue->pending_kick, 0);
INIT_LIST_HEAD(&queue->pending_kick_link);
@@ -589,11 +594,19 @@ int kbase_csf_queue_register_ex(struct kbase_context *kctx,
u32 const glb_version = iface->version;
u32 instr = iface->instr_features;
u8 max_size = GLB_INSTR_FEATURES_EVENT_SIZE_MAX_GET(instr);
u32 min_buf_size =
(1u << reg->ex_event_size) * GLB_INSTR_FEATURES_OFFSET_UPDATE_RATE_GET(instr);
const u8 event_size = reg->ex_event_size;
u64 min_buf_size;
/* If cs_trace_command not supported, the call fails */
if (glb_version < kbase_csf_interface_version(1, 1, 0))
return -EPERM;
/* Sanity check to avoid shift-out-of-bounds */
if (event_size >= 32)
return -EINVAL;
min_buf_size = ((u64)1 << event_size) * GLB_INSTR_FEATURES_OFFSET_UPDATE_RATE_GET(instr);
if (min_buf_size > UINT32_MAX)
return -EINVAL;
/* Validate the ring buffer configuration parameters */
@@ -605,8 +618,8 @@ int kbase_csf_queue_register_ex(struct kbase_context *kctx,
/* Validate the cs_trace configuration parameters */
if (reg->ex_buffer_size &&
((reg->ex_event_size > max_size) || (reg->ex_buffer_size & (reg->ex_buffer_size - 1)) ||
(reg->ex_buffer_size < min_buf_size)))
((event_size > max_size) || (reg->ex_buffer_size & (reg->ex_buffer_size - 1)) ||
(reg->ex_buffer_size < (u32)min_buf_size)))
return -EINVAL;
return csf_queue_register_internal(kctx, NULL, reg);
@@ -734,7 +747,7 @@ out:
}
/**
* get_bound_queue_group - Get the group to which a queue was bound
* get_bound_queue_group() - Get the group to which a queue was bound
*
* @queue: Pointer to the queue for this group
*
@@ -847,6 +860,47 @@ void kbase_csf_ring_cs_kernel_doorbell(struct kbase_device *kbdev, int csi_index
kbase_csf_ring_csg_doorbell(kbdev, csg_nr);
}
int kbase_csf_queue_group_clear_faults(struct kbase_context *kctx,
struct kbase_ioctl_queue_group_clear_faults *faults)
{
void __user *user_bufs = u64_to_user_ptr(faults->addr);
u32 i;
struct kbase_device *kbdev = kctx->kbdev;
const u32 nr_queues = faults->nr_queues;
if (unlikely(nr_queues > kbdev->csf.global_iface.groups[0].stream_num)) {
dev_warn(kbdev->dev, "Invalid nr_queues %u", nr_queues);
return -EINVAL;
}
for (i = 0; i < nr_queues; ++i) {
u64 buf_gpu_addr;
struct kbase_va_region *region;
if (copy_from_user(&buf_gpu_addr, user_bufs, sizeof(buf_gpu_addr)))
return -EFAULT;
mutex_lock(&kctx->csf.lock);
kbase_gpu_vm_lock(kctx);
region = kbase_region_tracker_find_region_enclosing_address(kctx, buf_gpu_addr);
if (likely(!kbase_is_region_invalid_or_free(region))) {
struct kbase_queue *queue = region->user_data;
queue->clear_faults = true;
} else {
dev_warn(kbdev->dev, "GPU queue %u without a valid command buffer region",
i);
kbase_gpu_vm_unlock(kctx);
mutex_unlock(&kctx->csf.lock);
return -EFAULT;
}
kbase_gpu_vm_unlock(kctx);
mutex_unlock(&kctx->csf.lock);
user_bufs = (void __user *)((uintptr_t)user_bufs + sizeof(buf_gpu_addr));
}
return 0;
}
int kbase_csf_queue_kick(struct kbase_context *kctx, struct kbase_ioctl_cs_queue_kick *kick)
{
struct kbase_device *kbdev = kctx->kbdev;
@@ -868,7 +922,7 @@ int kbase_csf_queue_kick(struct kbase_context *kctx, struct kbase_ioctl_cs_queue
struct kbase_queue *queue = region->user_data;
if (queue && (queue->bind_state == KBASE_CSF_QUEUE_BOUND)) {
spin_lock(&kbdev->csf.pending_gpuq_kicks_lock);
spin_lock(&kbdev->csf.pending_gpuq_kick_queues_lock);
if (list_empty(&queue->pending_kick_link)) {
/* Queue termination shall block until this
* kick has been handled.
@@ -876,10 +930,12 @@ int kbase_csf_queue_kick(struct kbase_context *kctx, struct kbase_ioctl_cs_queue
atomic_inc(&queue->pending_kick);
list_add_tail(
&queue->pending_kick_link,
&kbdev->csf.pending_gpuq_kicks[queue->group_priority]);
complete(&kbdev->csf.scheduler.kthread_signal);
&kbdev->csf.pending_gpuq_kick_queues[queue->group_priority]);
if (atomic_cmpxchg(&kbdev->csf.pending_gpuq_kicks, false, true) ==
false)
complete(&kbdev->csf.scheduler.kthread_signal);
}
spin_unlock(&kbdev->csf.pending_gpuq_kicks_lock);
spin_unlock(&kbdev->csf.pending_gpuq_kick_queues_lock);
}
} else {
dev_dbg(kbdev->dev,
@@ -1095,12 +1151,11 @@ static int create_normal_suspend_buffer(struct kbase_context *const kctx,
}
static void timer_event_worker(struct work_struct *data);
static void protm_event_worker(struct work_struct *data);
static void term_normal_suspend_buffer(struct kbase_context *const kctx,
struct kbase_normal_suspend_buffer *s_buf);
/**
* create_suspend_buffers - Setup normal and protected mode
* create_suspend_buffers() - Setup normal and protected mode
* suspend buffers.
*
* @kctx: Address of the kbase context within which the queue group
@@ -1199,6 +1254,8 @@ static int create_queue_group(struct kbase_context *const kctx,
group->deschedule_deferred_cnt = 0;
#endif
group->cs_fault_report_enable = create->in.cs_fault_report_enable;
group->group_uid = generate_group_uid();
create->out.group_uid = group->group_uid;
@@ -1206,7 +1263,9 @@ static int create_queue_group(struct kbase_context *const kctx,
INIT_LIST_HEAD(&group->link_to_schedule);
INIT_LIST_HEAD(&group->error_fatal.link);
INIT_WORK(&group->timer_event_work, timer_event_worker);
INIT_WORK(&group->protm_event_work, protm_event_worker);
INIT_LIST_HEAD(&group->protm_event_work);
group->progress_timer_state = 0;
atomic_set(&group->pending_protm_event_work, 0);
bitmap_zero(group->protm_pending_bitmap, MAX_SUPPORTED_STREAMS_PER_GROUP);
group->run_state = KBASE_CSF_GROUP_INACTIVE;
@@ -1251,14 +1310,6 @@ int kbase_csf_queue_group_create(struct kbase_context *const kctx,
const u32 tiler_count = hweight64(create->in.tiler_mask);
const u32 fragment_count = hweight64(create->in.fragment_mask);
const u32 compute_count = hweight64(create->in.compute_mask);
size_t i;
for (i = 0; i < ARRAY_SIZE(create->in.padding); i++) {
if (create->in.padding[i] != 0) {
dev_warn(kctx->kbdev->dev, "Invalid padding not 0 in queue group create\n");
return -EINVAL;
}
}
mutex_lock(&kctx->csf.lock);
@@ -1379,7 +1430,7 @@ void kbase_csf_term_descheduled_queue_group(struct kbase_queue_group *group)
}
/**
* term_queue_group - Terminate a GPU command queue group.
* term_queue_group() - Terminate a GPU command queue group.
*
* @group: Pointer to GPU command queue group data.
*
@@ -1407,8 +1458,8 @@ static void term_queue_group(struct kbase_queue_group *group)
}
/**
* wait_group_deferred_deschedule_completion - Wait for refcount of the group to
* become 0 that was taken when the group deschedule had to be deferred.
* wait_group_deferred_deschedule_completion() - Wait for refcount of the group
* to become 0 that was taken when the group deschedule had to be deferred.
*
* @group: Pointer to GPU command queue group that is being deleted.
*
@@ -1437,7 +1488,10 @@ static void wait_group_deferred_deschedule_completion(struct kbase_queue_group *
static void cancel_queue_group_events(struct kbase_queue_group *group)
{
cancel_work_sync(&group->timer_event_work);
cancel_work_sync(&group->protm_event_work);
/* Drain a pending protected mode request if any */
kbase_csf_scheduler_wait_for_kthread_pending_work(group->kctx->kbdev,
&group->pending_protm_event_work);
}
static void remove_pending_group_fatal_error(struct kbase_queue_group *group)
@@ -1592,6 +1646,7 @@ int kbase_csf_ctx_init(struct kbase_context *kctx)
INIT_LIST_HEAD(&kctx->csf.queue_list);
INIT_LIST_HEAD(&kctx->csf.link);
atomic_set(&kctx->csf.pending_sync_update, 0);
kbase_csf_event_init(kctx);
@@ -1827,7 +1882,7 @@ void kbase_csf_ctx_term(struct kbase_context *kctx)
}
/**
* handle_oom_event - Handle the OoM event generated by the firmware for the
* handle_oom_event() - Handle the OoM event generated by the firmware for the
* CSI.
*
* @group: Pointer to the CSG group the oom-event belongs to.
@@ -1902,7 +1957,7 @@ static int handle_oom_event(struct kbase_queue_group *const group,
}
/**
* report_tiler_oom_error - Report a CSG error due to a tiler heap OOM event
* report_tiler_oom_error() - Report a CSG error due to a tiler heap OOM event
*
* @group: Pointer to the GPU command queue group that encountered the error
*/
@@ -1945,7 +2000,7 @@ static void flush_gpu_cache_on_fatal_error(struct kbase_device *kbdev)
}
/**
* kbase_queue_oom_event - Handle tiler out-of-memory for a GPU command queue.
* kbase_queue_oom_event() - Handle tiler out-of-memory for a GPU command queue.
*
* @queue: Pointer to queue for which out-of-memory event was received.
*
@@ -2033,7 +2088,7 @@ unlock:
}
/**
* oom_event_worker - Tiler out-of-memory handler called from a workqueue.
* oom_event_worker() - Tiler out-of-memory handler called from a workqueue.
*
* @data: Pointer to a work_struct embedded in GPU command queue data.
*
@@ -2061,7 +2116,8 @@ static void oom_event_worker(struct work_struct *data)
}
/**
* report_group_timeout_error - Report the timeout error for the group to userspace.
* report_group_timeout_error() - Report the timeout error for the group to
* userspace.
*
* @group: Pointer to the group for which timeout error occurred
*/
@@ -2085,7 +2141,7 @@ static void report_group_timeout_error(struct kbase_queue_group *const group)
}
/**
* timer_event_worker - Handle the progress timeout error for the group
* timer_event_worker() - Handle the progress timeout error for the group
*
* @data: Pointer to a work_struct embedded in GPU command queue group data.
*
@@ -2120,19 +2176,74 @@ static void timer_event_worker(struct work_struct *data)
}
/**
* handle_progress_timer_event - Progress timer timeout event handler.
* handle_progress_timer_events() - Progress timer timeout events handler.
*
* @group: Pointer to GPU queue group for which the timeout event is received.
* @kbdev: Instance of a GPU platform device that implements a CSF interface.
* @slot_mask: Bitmap reflecting the slots on which progress timer timeouts happen.
*
* Notify a waiting user space client of the timeout.
* Enqueue a work item to terminate the group and notify the event notification
* thread of progress timeout fault for the GPU command queue group.
* Ignore fragment timeout if it is following a compute timeout.
*/
static void handle_progress_timer_event(struct kbase_queue_group *const group)
static void handle_progress_timer_events(struct kbase_device *const kbdev, unsigned long *slot_mask)
{
kbase_debug_csf_fault_notify(group->kctx->kbdev, group->kctx, DF_PROGRESS_TIMER_TIMEOUT);
u32 max_csg_slots = kbdev->csf.global_iface.group_num;
u32 csg_nr;
struct kbase_queue_group *group = NULL;
struct kbase_csf_cmd_stream_group_info *ginfo;
queue_work(group->kctx->csf.wq, &group->timer_event_work);
kbase_csf_scheduler_spin_lock_assert_held(kbdev);
if (likely(bitmap_empty(slot_mask, MAX_SUPPORTED_CSGS)))
return;
/* Log each timeout and Update timestamp of compute progress timeout */
for_each_set_bit(csg_nr, slot_mask, max_csg_slots) {
group = kbdev->csf.scheduler.csg_slots[csg_nr].resident_group;
ginfo = &kbdev->csf.global_iface.groups[csg_nr];
group->progress_timer_state =
kbase_csf_firmware_csg_output(ginfo, CSG_PROGRESS_TIMER_STATE);
dev_info(
kbdev->dev,
"[%llu] Iterator PROGRESS_TIMER timeout notification received for group %u of ctx %d_%d on slot %u with state %x",
kbase_backend_get_cycle_cnt(kbdev), group->handle, group->kctx->tgid,
group->kctx->id, csg_nr, group->progress_timer_state);
if (CSG_PROGRESS_TIMER_STATE_GET(group->progress_timer_state) ==
CSG_PROGRESS_TIMER_STATE_COMPUTE)
kbdev->csf.compute_progress_timeout_cc = kbase_backend_get_cycle_cnt(kbdev);
}
/* Ignore fragment timeout if it is following a compute timeout.
* Otherwise, terminate the command stream group.
*/
for_each_set_bit(csg_nr, slot_mask, max_csg_slots) {
group = kbdev->csf.scheduler.csg_slots[csg_nr].resident_group;
/* Check if it is a fragment timeout right after another compute timeout.
* In such case, kill compute CSG and give fragment CSG a second chance
*/
if (CSG_PROGRESS_TIMER_STATE_GET(group->progress_timer_state) ==
CSG_PROGRESS_TIMER_STATE_FRAGMENT) {
u64 cycle_counter = kbase_backend_get_cycle_cnt(kbdev);
u64 compute_progress_timeout_cc = kbdev->csf.compute_progress_timeout_cc;
if (compute_progress_timeout_cc <= cycle_counter &&
cycle_counter <= compute_progress_timeout_cc +
MAX_PROGRESS_TIMEOUT_EVENT_DELAY) {
dev_info(
kbdev->dev,
"Ignored Fragment iterator timeout for group %d on slot %d",
group->handle, group->csg_nr);
continue;
}
}
kbase_debug_csf_fault_notify(group->kctx->kbdev, group->kctx,
DF_PROGRESS_TIMER_TIMEOUT);
queue_work(group->kctx->csf.wq, &group->timer_event_work);
}
}
/**
@@ -2211,41 +2322,7 @@ static void report_group_fatal_error(struct kbase_queue_group *const group)
}
/**
* protm_event_worker - Protected mode switch request event handler
* called from a workqueue.
*
* @data: Pointer to a work_struct embedded in GPU command queue group data.
*
* Request to switch to protected mode.
*/
static void protm_event_worker(struct work_struct *data)
{
struct kbase_queue_group *const group =
container_of(data, struct kbase_queue_group, protm_event_work);
struct kbase_protected_suspend_buffer *sbuf = &group->protected_suspend_buf;
int err = 0;
KBASE_KTRACE_ADD_CSF_GRP(group->kctx->kbdev, PROTM_EVENT_WORKER_START, group, 0u);
err = alloc_grp_protected_suspend_buffer_pages(group);
if (!err) {
kbase_csf_scheduler_group_protm_enter(group);
} else if (err == -ENOMEM && sbuf->alloc_retries <= PROTM_ALLOC_MAX_RETRIES) {
sbuf->alloc_retries++;
/* try again to allocate pages */
queue_work(group->kctx->csf.wq, &group->protm_event_work);
} else if (sbuf->alloc_retries >= PROTM_ALLOC_MAX_RETRIES || err != -ENOMEM) {
dev_err(group->kctx->kbdev->dev,
"Failed to allocate physical pages for Protected mode suspend buffer for the group %d of context %d_%d",
group->handle, group->kctx->tgid, group->kctx->id);
report_group_fatal_error(group);
}
KBASE_KTRACE_ADD_CSF_GRP(group->kctx->kbdev, PROTM_EVENT_WORKER_END, group, 0u);
}
/**
* handle_fault_event - Handler for CS fault.
* handle_fault_event() - Handler for CS fault.
*
* @queue: Pointer to queue for which fault event was received.
* @cs_ack: Value of the CS_ACK register in the CS kernel input page used for
@@ -2267,14 +2344,14 @@ static void handle_fault_event(struct kbase_queue *const queue, const u32 cs_ack
const u8 cs_fault_exception_type = CS_FAULT_EXCEPTION_TYPE_GET(cs_fault);
const u32 cs_fault_exception_data = CS_FAULT_EXCEPTION_DATA_GET(cs_fault);
const u64 cs_fault_info_exception_data = CS_FAULT_INFO_EXCEPTION_DATA_GET(cs_fault_info);
bool use_old_log_format = true;
bool has_trace_info = false;
bool skip_fault_report = kbase_ctx_flag(queue->kctx, KCTX_PAGE_FAULT_REPORT_SKIP);
kbase_csf_scheduler_spin_lock_assert_held(kbdev);
if (use_old_log_format && !skip_fault_report)
if (!has_trace_info && !skip_fault_report)
dev_warn(kbdev->dev,
"Ctx %d_%d Group %d CSG %d CSI: %d\n"
"CS_FAULT.EXCEPTION_TYPE: 0x%x (%s)\n"
@@ -2286,47 +2363,32 @@ static void handle_fault_event(struct kbase_queue *const queue, const u32 cs_ack
cs_fault_info_exception_data);
#if IS_ENABLED(CONFIG_DEBUG_FS)
/* CS_RESOURCE_TERMINATED type fault event can be ignored from the
* standpoint of dump on error. It is used to report fault for the CSIs
* that are associated with the same CSG as the CSI for which the actual
* fault was reported by the Iterator.
* Dumping would be triggered when the actual fault is reported.
/* If dump-on-fault daemon is waiting for a fault, wake up the daemon.
* Acknowledging the fault is deferred to the bottom-half until the wait
* of the dump completion is done.
*
* CS_INHERIT_FAULT can also be ignored. It could happen due to the error
* in other types of queues (cpu/kcpu). If a fault had occurred in some
* other GPU queue then the dump would have been performed anyways when
* that fault was reported.
* Otherwise acknowledge the fault and ring the doorbell for the faulty queue
* to enter into recoverable state.
*/
if ((cs_fault_exception_type != CS_FAULT_EXCEPTION_TYPE_CS_INHERIT_FAULT) &&
(cs_fault_exception_type != CS_FAULT_EXCEPTION_TYPE_CS_RESOURCE_TERMINATED)) {
if (unlikely(kbase_debug_csf_fault_notify(kbdev, queue->kctx, DF_CS_FAULT))) {
queue->cs_error = cs_fault;
queue->cs_error_info = cs_fault_info;
queue->cs_error_fatal = false;
queue_work(queue->kctx->csf.wq, &queue->cs_error_work);
return;
}
}
#endif
if (likely(!kbase_debug_csf_fault_notify(kbdev, queue->kctx, DF_CS_FAULT))) {
kbase_csf_firmware_cs_input_mask(stream, CS_REQ, cs_ack, CS_REQ_FAULT_MASK);
kbase_csf_ring_cs_kernel_doorbell(kbdev, queue->csi_index, queue->group->csg_nr,
true);
queue->cs_error_acked = true;
} else
queue->cs_error_acked = false;
kbase_csf_firmware_cs_input_mask(stream, CS_REQ, cs_ack, CS_REQ_FAULT_MASK);
kbase_csf_ring_cs_kernel_doorbell(kbdev, queue->csi_index, queue->group->csg_nr, true);
queue->cs_error = cs_fault;
queue->cs_error_info = cs_fault_info;
queue->cs_error_fatal = false;
if (!queue_work(queue->kctx->csf.wq, &queue->cs_error_work))
dev_warn(kbdev->dev, "%s: failed to enqueue a work", __func__);
}
static void report_queue_fatal_error(struct kbase_queue *const queue, u32 cs_fatal,
u64 cs_fatal_info, struct kbase_queue_group *group)
static void report_queue_error(struct kbase_queue *const queue, u32 cs_error, u64 cs_error_info,
struct kbase_queue_group *group, bool fatal)
{
struct base_csf_notification
error = { .type = BASE_CSF_NOTIFICATION_GPU_QUEUE_GROUP_ERROR,
.payload = {
.csg_error = {
.error = { .error_type =
BASE_GPU_QUEUE_GROUP_QUEUE_ERROR_FATAL,
.payload = { .fatal_queue = {
.sideband = cs_fatal_info,
.status = cs_fatal,
} } } } } };
struct base_csf_notification error = { .type = BASE_CSF_NOTIFICATION_GPU_QUEUE_GROUP_ERROR };
if (!queue)
return;
@@ -2335,17 +2397,30 @@ static void report_queue_fatal_error(struct kbase_queue *const queue, u32 cs_fat
return;
error.payload.csg_error.handle = group->handle;
error.payload.csg_error.error.payload.fatal_queue.csi_index = (__u8)queue->csi_index;
if (fatal) {
error.payload.csg_error.error.error_type = BASE_GPU_QUEUE_GROUP_QUEUE_ERROR_FATAL;
error.payload.csg_error.error.payload.fatal_queue.sideband = cs_error_info;
error.payload.csg_error.error.payload.fatal_queue.status = cs_error;
error.payload.csg_error.error.payload.fatal_queue.csi_index = queue->csi_index;
} else {
error.payload.csg_error.error.error_type = BASE_GPU_QUEUE_GROUP_QUEUE_ERROR_FAULT;
error.payload.csg_error.error.payload.fault_queue.sideband = cs_error_info;
error.payload.csg_error.error.payload.fault_queue.status = cs_error;
error.payload.csg_error.error.payload.fault_queue.csi_index = queue->csi_index;
}
kbase_csf_event_add_error(queue->kctx, &group->error_fatal, &error);
kbase_event_wakeup(queue->kctx);
if (!fatal)
queue->clear_faults = false;
}
/**
* cs_error_worker - Handle the CS_FATAL/CS_FAULT error for the GPU queue
* cs_error_worker() - Handle the CS_FATAL/CS_FAULT error for the GPU queue
*
* @data: Pointer to a work_struct embedded in GPU command queue.
*
* Terminate the CSG and report the error to userspace.
* Terminate the CSG for CS_FATAL and report the error to userspace.
*/
static void cs_error_worker(struct work_struct *const data)
{
@@ -2356,6 +2431,7 @@ static void cs_error_worker(struct work_struct *const data)
struct kbase_queue_group *group;
bool reset_prevented = false;
int err;
const bool cs_fatal = queue->cs_error_fatal;
kbase_debug_csf_fault_wait_completion(kbdev);
err = kbase_reset_gpu_prevent_and_wait(kbdev);
@@ -2371,45 +2447,57 @@ static void cs_error_worker(struct work_struct *const data)
group = get_bound_queue_group(queue);
if (!group) {
dev_warn(kbdev->dev, "queue not bound when handling fatal event");
dev_warn(kbdev->dev, "queue not bound when handling an error event");
goto unlock;
}
#if IS_ENABLED(CONFIG_DEBUG_FS)
if (!queue->cs_error_fatal) {
unsigned long flags;
int slot_num;
if (!cs_fatal) {
if (group->cs_fault_report_enable && queue->clear_faults)
report_queue_error(queue, queue->cs_error, queue->cs_error_info, group,
false);
if (unlikely(!queue->cs_error_acked)) {
unsigned long flags;
int slot_num;
kbase_csf_scheduler_spin_lock(kbdev, &flags);
slot_num = kbase_csf_scheduler_group_get_slot_locked(group);
if (slot_num >= 0) {
struct kbase_csf_cmd_stream_group_info const *ginfo =
&kbdev->csf.global_iface.groups[slot_num];
struct kbase_csf_cmd_stream_info const *stream =
&ginfo->streams[queue->csi_index];
u32 const cs_ack = kbase_csf_firmware_cs_output(stream, CS_ACK);
kbase_csf_scheduler_spin_lock(kbdev, &flags);
slot_num = kbase_csf_scheduler_group_get_slot_locked(group);
if (likely(slot_num >= 0)) {
struct kbase_csf_cmd_stream_group_info const *ginfo =
&kbdev->csf.global_iface.groups[slot_num];
struct kbase_csf_cmd_stream_info const *stream =
&ginfo->streams[queue->csi_index];
u32 const cs_ack = kbase_csf_firmware_cs_output(stream, CS_ACK);
u32 const cs_req = kbase_csf_firmware_cs_input_read(stream, CS_REQ);
kbase_csf_firmware_cs_input_mask(stream, CS_REQ, cs_ack, CS_REQ_FAULT_MASK);
kbase_csf_ring_cs_kernel_doorbell(kbdev, queue->csi_index, slot_num, true);
/* Acknowledge the fault and ring the doorbell for the queue
* if it hasn't yet done.
*/
if ((cs_ack & CS_ACK_FAULT_MASK) != (cs_req & CS_REQ_FAULT_MASK)) {
kbase_csf_firmware_cs_input_mask(stream, CS_REQ, cs_ack,
CS_REQ_FAULT_MASK);
kbase_csf_ring_cs_kernel_doorbell(kbdev, queue->csi_index,
slot_num, true);
}
}
kbase_csf_scheduler_spin_unlock(kbdev, flags);
}
kbase_csf_scheduler_spin_unlock(kbdev, flags);
goto unlock;
}
#endif
term_queue_group(group);
flush_gpu_cache_on_fatal_error(kbdev);
/* For an invalid GPU page fault, CS_BUS_FAULT fatal error is expected after the
* page fault handler disables the AS of faulty context. Need to skip reporting the
* CS_BUS_FAULT fatal error to the Userspace as it doesn't have the full fault info.
* Page fault handler will report the fatal error with full page fault info.
*/
if ((cs_fatal_exception_type == CS_FATAL_EXCEPTION_TYPE_CS_BUS_FAULT) && group->faulted) {
dev_dbg(kbdev->dev,
"Skipped reporting CS_BUS_FAULT for queue %d of group %d of ctx %d_%d",
queue->csi_index, group->handle, kctx->tgid, kctx->id);
} else {
report_queue_fatal_error(queue, queue->cs_error, queue->cs_error_info, group);
term_queue_group(group);
flush_gpu_cache_on_fatal_error(kbdev);
/* For an invalid GPU page fault, CS_BUS_FAULT fatal error is expected after the
* page fault handler disables the AS of faulty context. Need to skip reporting the
* CS_BUS_FAULT fatal error to the Userspace as it doesn't have the full fault info.
* Page fault handler will report the fatal error with full page fault info.
*/
if ((cs_fatal_exception_type == CS_FATAL_EXCEPTION_TYPE_CS_BUS_FAULT) &&
group->faulted) {
dev_dbg(kbdev->dev,
"Skipped reporting CS_BUS_FAULT for queue %d of group %d of ctx %d_%d",
queue->csi_index, group->handle, kctx->tgid, kctx->id);
} else {
report_queue_error(queue, queue->cs_error, queue->cs_error_info, group,
true);
}
}
unlock:
@@ -2419,7 +2507,7 @@ unlock:
}
/**
* handle_fatal_event - Handler for CS fatal.
* handle_fatal_event() - Handler for CS fatal.
*
* @queue: Pointer to queue for which fatal event was received.
* @stream: Pointer to the structure containing info provided by the
@@ -2443,13 +2531,13 @@ static void handle_fatal_event(struct kbase_queue *const queue,
const u32 cs_fatal_exception_type = CS_FATAL_EXCEPTION_TYPE_GET(cs_fatal);
const u32 cs_fatal_exception_data = CS_FATAL_EXCEPTION_DATA_GET(cs_fatal);
const u64 cs_fatal_info_exception_data = CS_FATAL_INFO_EXCEPTION_DATA_GET(cs_fatal_info);
bool use_old_log_format = true;
bool has_trace_info = false;
bool skip_fault_report = kbase_ctx_flag(queue->kctx, KCTX_PAGE_FAULT_REPORT_SKIP);
kbase_csf_scheduler_spin_lock_assert_held(kbdev);
if (use_old_log_format && !skip_fault_report)
if (!has_trace_info && !skip_fault_report)
dev_warn(kbdev->dev,
"Ctx %d_%d Group %d CSG %d CSI: %d\n"
"CS_FATAL.EXCEPTION_TYPE: 0x%x (%s)\n"
@@ -2481,7 +2569,7 @@ static void handle_fatal_event(struct kbase_queue *const queue,
}
/**
* process_cs_interrupts - Process interrupts for a CS.
* process_cs_interrupts() - Process interrupts for a CS.
*
* @group: Pointer to GPU command queue group data.
* @ginfo: The CSG interface provided by the firmware.
@@ -2595,7 +2683,7 @@ static void process_cs_interrupts(struct kbase_queue_group *const group,
}
if (!group->protected_suspend_buf.pma)
queue_work(group->kctx->csf.wq, &group->protm_event_work);
kbase_csf_scheduler_enqueue_protm_event_work(group);
if (test_bit(group->csg_nr, scheduler->csg_slots_idle_mask)) {
clear_bit(group->csg_nr, scheduler->csg_slots_idle_mask);
@@ -2608,12 +2696,14 @@ static void process_cs_interrupts(struct kbase_queue_group *const group,
}
/**
* process_csg_interrupts - Process interrupts for a CSG.
* process_csg_interrupts() - Process interrupts for a CSG.
*
* @kbdev: Instance of a GPU platform device that implements a CSF interface.
* @csg_nr: CSG number.
* @track: Pointer that tracks the highest idle CSG and the newly possible viable
* protected mode requesting group, in current IRQ context.
* @progress_timeout_slot_mask: slot mask to indicate on which slot progress timeout
* happens.
*
* Handles interrupts for a CSG and for CSs within it.
*
@@ -2625,7 +2715,8 @@ static void process_cs_interrupts(struct kbase_queue_group *const group,
* See process_cs_interrupts() for details of per-stream interrupt handling.
*/
static void process_csg_interrupts(struct kbase_device *const kbdev, u32 const csg_nr,
struct irq_idle_and_protm_track *track)
struct irq_idle_and_protm_track *track,
unsigned long *progress_timeout_slot_mask)
{
struct kbase_csf_cmd_stream_group_info *ginfo;
struct kbase_queue_group *group = NULL;
@@ -2712,13 +2803,9 @@ static void process_csg_interrupts(struct kbase_device *const kbdev, u32 const c
KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_INTERRUPT_PROGRESS_TIMER_EVENT, group,
req ^ ack);
dev_info(
kbdev->dev,
"[%llu] Iterator PROGRESS_TIMER timeout notification received for group %u of ctx %d_%d on slot %u\n",
kbase_backend_get_cycle_cnt(kbdev), group->handle, group->kctx->tgid,
group->kctx->id, csg_nr);
handle_progress_timer_event(group);
set_bit(csg_nr, progress_timeout_slot_mask);
}
process_cs_interrupts(group, ginfo, irqreq, irqack, track);
@@ -2728,7 +2815,7 @@ static void process_csg_interrupts(struct kbase_device *const kbdev, u32 const c
}
/**
* process_prfcnt_interrupts - Process performance counter interrupts.
* process_prfcnt_interrupts() - Process performance counter interrupts.
*
* @kbdev: Instance of a GPU platform device that implements a CSF interface.
* @glb_req: Global request register value.
@@ -2800,7 +2887,7 @@ static void process_prfcnt_interrupts(struct kbase_device *kbdev, u32 glb_req, u
}
/**
* check_protm_enter_req_complete - Check if PROTM_ENTER request completed
* check_protm_enter_req_complete() - Check if PROTM_ENTER request completed
*
* @kbdev: Instance of a GPU platform device that implements a CSF interface.
* @glb_req: Global request register value.
@@ -2828,13 +2915,14 @@ static inline void check_protm_enter_req_complete(struct kbase_device *kbdev, u3
dev_dbg(kbdev->dev, "Protected mode entry interrupt received");
kbdev->protected_mode = true;
kbase_ipa_protection_mode_switch_event(kbdev);
kbase_ipa_control_protm_entered(kbdev);
kbase_hwcnt_backend_csf_protm_entered(&kbdev->hwcnt_gpu_iface);
}
/**
* process_protm_exit - Handle the protected mode exit interrupt
* process_protm_exit() - Handle the protected mode exit interrupt
*
* @kbdev: Instance of a GPU platform device that implements a CSF interface.
* @glb_ack: Global acknowledge register value.
@@ -2923,7 +3011,7 @@ static inline void process_tracked_info_for_protm(struct kbase_device *kbdev,
if (!tock_triggered) {
dev_dbg(kbdev->dev, "Group-%d on slot-%d start protm work\n", group->handle,
group->csg_nr);
queue_work(group->kctx->csf.wq, &group->protm_event_work);
kbase_csf_scheduler_enqueue_protm_event_work(group);
}
}
}
@@ -2952,6 +3040,46 @@ static void order_job_irq_clear_with_iface_mem_read(void)
dmb(osh);
}
static const char *const glb_fatal_status_errors[GLB_FATAL_STATUS_VALUE_COUNT] = {
[GLB_FATAL_STATUS_VALUE_OK] = "OK",
[GLB_FATAL_STATUS_VALUE_ASSERT] = "Firmware assert triggered",
[GLB_FATAL_STATUS_VALUE_UNEXPECTED_EXCEPTION] =
"Hardware raised an exception firmware did not expect",
[GLB_FATAL_STATUS_VALUE_HANG] = "Firmware hangs and watchdog timer expired",
};
/**
* handle_glb_fatal_event() - Handle the GLB fatal event
*
* @kbdev: Instance of GPU device.
* @global_iface: CSF global interface
*/
static void handle_glb_fatal_event(struct kbase_device *kbdev,
const struct kbase_csf_global_iface *const global_iface)
{
const char *error_string = NULL;
const u32 fatal_status = kbase_csf_firmware_global_output(global_iface, GLB_FATAL_STATUS);
lockdep_assert_held(&kbdev->hwaccess_lock);
kbase_csf_scheduler_spin_lock_assert_held(kbdev);
dev_warn(kbdev->dev, "MCU encountered unrecoverable error");
if (fatal_status < GLB_FATAL_STATUS_VALUE_COUNT)
error_string = glb_fatal_status_errors[fatal_status];
else {
dev_err(kbdev->dev, "Invalid GLB_FATAL_STATUS (%u)", fatal_status);
return;
}
if (fatal_status == GLB_FATAL_STATUS_VALUE_OK)
dev_err(kbdev->dev, "GLB_FATAL_STATUS(OK) must be set with proper reason");
else {
dev_warn(kbdev->dev, "GLB_FATAL_STATUS: %s", error_string);
if (kbase_prepare_to_reset_gpu_locked(kbdev, RESET_FLAGS_NONE))
kbase_reset_gpu_locked(kbdev);
}
}
void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val)
{
bool deferred_handling_glb_idle_irq = false;
@@ -2972,18 +3100,25 @@ void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val)
struct irq_idle_and_protm_track track = { .protm_grp = NULL,
.idle_seq = U32_MAX,
.idle_slot = S8_MAX };
DECLARE_BITMAP(progress_timeout_csgs, MAX_SUPPORTED_CSGS) = { 0 };
kbase_csf_scheduler_spin_lock(kbdev, &flags);
/* Looping through and track the highest idle and protm groups */
/* Looping through and track the highest idle and protm groups.
* Also track the groups for which progress timer timeout happened.
*/
while (csg_interrupts != 0) {
u32 const csg_nr = (u32)ffs((int)csg_interrupts) - 1;
process_csg_interrupts(kbdev, csg_nr, &track);
process_csg_interrupts(kbdev, csg_nr, &track,
progress_timeout_csgs);
csg_interrupts &= ~(1U << csg_nr);
}
/* Handle protm from the tracked information */
process_tracked_info_for_protm(kbdev, &track);
/* Handle pending progress timeout(s) */
handle_progress_timer_events(kbdev, progress_timeout_csgs);
kbase_csf_scheduler_spin_unlock(kbdev, flags);
}
@@ -3012,11 +3147,28 @@ void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val)
/* Handle IDLE Hysteresis notification event */
if ((glb_req ^ glb_ack) & GLB_REQ_IDLE_EVENT_MASK) {
u32 const glb_idle_timer_cfg =
kbase_csf_firmware_global_input_read(
global_iface, GLB_IDLE_TIMER_CONFIG);
dev_dbg(kbdev->dev, "Idle-hysteresis event flagged");
kbase_csf_firmware_global_input_mask(
global_iface, GLB_REQ, glb_ack,
GLB_REQ_IDLE_EVENT_MASK);
if (glb_idle_timer_cfg &
GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_MASK) {
/* The FW is going to sleep, we shall:
* - Enable fast GPU idle handling to avoid
* confirming CSGs status in gpu_idle_worker().
* - Enable doorbell mirroring to minimise the
* chance of KBase raising kernel doorbells which
* would cause the FW to be woken up.
*/
kbdev->csf.scheduler.fast_gpu_idle_handling = true;
kbase_pm_enable_db_mirror_interrupt(kbdev);
}
glb_idle_irq_received = true;
/* Defer handling this IRQ to account for a race condition
* where the idle worker could be executed before we have
@@ -3026,6 +3178,9 @@ void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val)
deferred_handling_glb_idle_irq = true;
}
if (glb_ack & GLB_ACK_FATAL_MASK)
handle_glb_fatal_event(kbdev, global_iface);
process_prfcnt_interrupts(kbdev, glb_req, glb_ack);
kbase_csf_scheduler_spin_unlock(kbdev, flags);
@@ -3050,13 +3205,10 @@ void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val)
if (deferred_handling_glb_idle_irq) {
unsigned long flags;
bool invoke_pm_state_machine;
kbase_csf_scheduler_spin_lock(kbdev, &flags);
invoke_pm_state_machine = kbase_csf_scheduler_process_gpu_idle_event(kbdev);
kbase_csf_scheduler_process_gpu_idle_event(kbdev);
kbase_csf_scheduler_spin_unlock(kbdev, flags);
if (unlikely(invoke_pm_state_machine))
kbase_pm_update_state(kbdev);
}
wake_up_all(&kbdev->csf.event_wait);
@@ -3087,6 +3239,11 @@ void kbase_csf_doorbell_mapping_term(struct kbase_device *kbdev)
if (kbdev->csf.db_filp) {
struct page *page = as_page(kbdev->csf.dummy_db_page);
/* This is a shared dummy sink page for avoiding potential segmentation fault
* to user-side library when a csi is off slot. Additionally, the call is on
* module unload path, so the page can be left uncleared before returning it
* back to kbdev memory pool.
*/
kbase_mem_pool_free(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], page, false);
fput(kbdev->csf.db_filp);
@@ -3118,26 +3275,27 @@ int kbase_csf_doorbell_mapping_init(struct kbase_device *kbdev)
return 0;
}
void kbase_csf_pending_gpuq_kicks_init(struct kbase_device *kbdev)
void kbase_csf_pending_gpuq_kick_queues_init(struct kbase_device *kbdev)
{
size_t i;
for (i = 0; i != ARRAY_SIZE(kbdev->csf.pending_gpuq_kicks); ++i)
INIT_LIST_HEAD(&kbdev->csf.pending_gpuq_kicks[i]);
spin_lock_init(&kbdev->csf.pending_gpuq_kicks_lock);
atomic_set(&kbdev->csf.pending_gpuq_kicks, false);
for (i = 0; i != ARRAY_SIZE(kbdev->csf.pending_gpuq_kick_queues); ++i)
INIT_LIST_HEAD(&kbdev->csf.pending_gpuq_kick_queues[i]);
spin_lock_init(&kbdev->csf.pending_gpuq_kick_queues_lock);
}
void kbase_csf_pending_gpuq_kicks_term(struct kbase_device *kbdev)
void kbase_csf_pending_gpuq_kick_queues_term(struct kbase_device *kbdev)
{
size_t i;
spin_lock(&kbdev->csf.pending_gpuq_kicks_lock);
for (i = 0; i != ARRAY_SIZE(kbdev->csf.pending_gpuq_kicks); ++i) {
if (!list_empty(&kbdev->csf.pending_gpuq_kicks[i]))
spin_lock(&kbdev->csf.pending_gpuq_kick_queues_lock);
for (i = 0; i != ARRAY_SIZE(kbdev->csf.pending_gpuq_kick_queues); ++i) {
if (!list_empty(&kbdev->csf.pending_gpuq_kick_queues[i]))
dev_warn(kbdev->dev,
"Some GPU queue kicks for priority %zu were not handled", i);
}
spin_unlock(&kbdev->csf.pending_gpuq_kicks_lock);
spin_unlock(&kbdev->csf.pending_gpuq_kick_queues_lock);
}
void kbase_csf_free_dummy_user_reg_page(struct kbase_device *kbdev)
@@ -3145,6 +3303,11 @@ void kbase_csf_free_dummy_user_reg_page(struct kbase_device *kbdev)
if (kbdev->csf.user_reg.filp) {
struct page *page = as_page(kbdev->csf.user_reg.dummy_page);
/* This is a shared dummy page in place of the real USER Register page just
* before the GPU is powered down. Additionally, the call is on module unload
* path, so the page can be left uncleared before returning it back to kbdev
* memory pool.
*/
kbase_mem_pool_free(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], page, false);
fput(kbdev->csf.user_reg.filp);
}
@@ -3227,17 +3390,17 @@ void kbase_csf_process_queue_kick(struct kbase_queue *queue)
if (err == -EBUSY) {
retry_kick = true;
spin_lock(&kbdev->csf.pending_gpuq_kicks_lock);
spin_lock(&kbdev->csf.pending_gpuq_kick_queues_lock);
if (list_empty(&queue->pending_kick_link)) {
/* A failed queue kick shall be pushed to the
* back of the queue to avoid potential abuse.
*/
list_add_tail(
&queue->pending_kick_link,
&kbdev->csf.pending_gpuq_kicks[queue->group_priority]);
spin_unlock(&kbdev->csf.pending_gpuq_kicks_lock);
&kbdev->csf.pending_gpuq_kick_queues[queue->group_priority]);
spin_unlock(&kbdev->csf.pending_gpuq_kick_queues_lock);
} else {
spin_unlock(&kbdev->csf.pending_gpuq_kicks_lock);
spin_unlock(&kbdev->csf.pending_gpuq_kick_queues_lock);
WARN_ON(atomic_read(&queue->pending_kick) == 0);
}
@@ -3260,3 +3423,27 @@ out_release_queue:
WARN_ON(atomic_read(&queue->pending_kick) == 0);
atomic_dec(&queue->pending_kick);
}
void kbase_csf_process_protm_event_request(struct kbase_queue_group *group)
{
struct kbase_protected_suspend_buffer *sbuf = &group->protected_suspend_buf;
int err = 0;
KBASE_KTRACE_ADD_CSF_GRP(group->kctx->kbdev, PROTM_EVENT_WORKER_START, group, 0u);
err = alloc_grp_protected_suspend_buffer_pages(group);
if (!err) {
kbase_csf_scheduler_group_protm_enter(group);
} else if (err == -ENOMEM && sbuf->alloc_retries <= PROTM_ALLOC_MAX_RETRIES) {
sbuf->alloc_retries++;
/* try again to allocate pages */
kbase_csf_scheduler_enqueue_protm_event_work(group);
} else if (sbuf->alloc_retries >= PROTM_ALLOC_MAX_RETRIES || err != -ENOMEM) {
dev_err(group->kctx->kbdev->dev,
"Failed to allocate physical pages for Protected mode suspend buffer for the group %d of context %d_%d",
group->handle, group->kctx->tgid, group->kctx->id);
report_group_fatal_error(group);
}
KBASE_KTRACE_ADD_CSF_GRP(group->kctx->kbdev, PROTM_EVENT_WORKER_END, group, 0u);
}

34
drivers/gpu/arm/bifrost/csf/mali_kbase_csf.h
View File

@@ -243,6 +243,19 @@ struct kbase_queue_group *kbase_csf_find_queue_group(struct kbase_context *kctx,
*/
int kbase_csf_queue_group_handle_is_valid(struct kbase_context *kctx, u8 group_handle);
/**
* kbase_csf_queue_group_clear_faults - Re-enable CS Fault reporting.
*
* @kctx: Pointer to the kbase context within which the
* CS Faults for the queues has to be re-enabled.
* @clear_faults: Pointer to the structure which contains details of the
* queues for which the CS Fault reporting has to be re-enabled.
*
* Return: 0 on success, or negative on failure.
*/
int kbase_csf_queue_group_clear_faults(struct kbase_context *kctx,
struct kbase_ioctl_queue_group_clear_faults *clear_faults);
/**
* kbase_csf_queue_group_create - Create a GPU command queue group.
*
@@ -379,20 +392,20 @@ int kbase_csf_setup_dummy_user_reg_page(struct kbase_device *kbdev);
void kbase_csf_free_dummy_user_reg_page(struct kbase_device *kbdev);
/**
* kbase_csf_pending_gpuq_kicks_init - Initialize the data used for handling
* GPU queue kicks.
* kbase_csf_pending_gpuq_kick_queues_init - Initialize the data used for handling
* GPU queue kicks.
*
* @kbdev: Instance of a GPU platform device that implements a CSF interface.
*/
void kbase_csf_pending_gpuq_kicks_init(struct kbase_device *kbdev);
void kbase_csf_pending_gpuq_kick_queues_init(struct kbase_device *kbdev);
/**
* kbase_csf_pending_gpuq_kicks_term - De-initialize the data used for handling
* GPU queue kicks.
* kbase_csf_pending_gpuq_kick_queues_term - De-initialize the data used for handling
* GPU queue kicks.
*
* @kbdev: Instance of a GPU platform device that implements a CSF interface.
*/
void kbase_csf_pending_gpuq_kicks_term(struct kbase_device *kbdev);
void kbase_csf_pending_gpuq_kick_queues_term(struct kbase_device *kbdev);
/**
* kbase_csf_ring_csg_doorbell - ring the doorbell for a CSG interface.
@@ -546,4 +559,13 @@ static inline u64 kbase_csf_ktrace_gpu_cycle_cnt(struct kbase_device *kbdev)
*/
void kbase_csf_process_queue_kick(struct kbase_queue *queue);
/**
* kbase_csf_process_protm_event_request - Handle protected mode switch request
*
* @group: The group to handle protected mode request
*
* Request to switch to protected mode.
*/
void kbase_csf_process_protm_event_request(struct kbase_queue_group *group);
#endif /* _KBASE_CSF_H_ */

87
drivers/gpu/arm/bifrost/csf/mali_kbase_csf_csg_debugfs.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -28,6 +28,8 @@
#include <mali_kbase.h>
#include <linux/seq_file.h>
#include <linux/version_compat_defs.h>
#include <mali_kbase_reset_gpu.h>
#include <mali_kbase_config_defaults.h>
#define MAX_SCHED_STATE_STRING_LEN (16)
/**
@@ -268,6 +270,87 @@ static const struct file_operations kbasep_csf_debugfs_scheduler_state_fops = {
.open = simple_open,
.llseek = default_llseek,
};
static int kbasep_csf_debugfs_eviction_timeout_get(void *data, u64 *val)
{
struct kbase_device *const kbdev = data;
unsigned long flags;
kbase_csf_scheduler_spin_lock(kbdev, &flags);
*val = kbdev->csf.csg_suspend_timeout_ms - CSG_SUSPEND_TIMEOUT_HOST_ADDED_MS;
kbase_csf_scheduler_spin_unlock(kbdev, flags);
return 0;
}
static int kbasep_csf_debugfs_eviction_timeout_set(void *data, u64 val)
{
struct kbase_device *const kbdev = data;
unsigned long flags_schd, flags_hw;
u64 dur_ms = val;
int ret = 0;
if (unlikely(dur_ms < CSG_SUSPEND_TIMEOUT_FIRMWARE_MS_MIN ||
dur_ms > CSG_SUSPEND_TIMEOUT_FIRMWARE_MS_MAX)) {
dev_err(kbdev->dev, "Invalid CSG suspend timeout input (%llu)", dur_ms);
return -EFAULT;
}
dur_ms = dur_ms + CSG_SUSPEND_TIMEOUT_HOST_ADDED_MS;
/* The 'fw_load_lock' is taken to synchronize against the deferred
* loading of FW, update will take effect after firmware gets loaded.
*/
mutex_lock(&kbdev->fw_load_lock);
if (unlikely(!kbdev->csf.firmware_inited)) {
kbase_csf_scheduler_spin_lock(kbdev, &flags_schd);
kbdev->csf.csg_suspend_timeout_ms = (unsigned int)dur_ms;
kbase_csf_scheduler_spin_unlock(kbdev, flags_schd);
mutex_unlock(&kbdev->fw_load_lock);
dev_info(kbdev->dev, "CSF set csg suspend timeout deferred till fw is loaded");
goto end;
}
mutex_unlock(&kbdev->fw_load_lock);
/* Firmware reloading is triggered by silent reset, and then update will take effect.
*/
kbase_csf_scheduler_pm_active(kbdev);
if (kbase_csf_scheduler_killable_wait_mcu_active(kbdev)) {
dev_err(kbdev->dev,
"Unable to activate the MCU, the csg suspend timeout value shall remain unchanged");
kbase_csf_scheduler_pm_idle(kbdev);
ret = -EFAULT;
goto exit;
}
spin_lock_irqsave(&kbdev->hwaccess_lock, flags_hw);
if (kbase_reset_gpu_silent(kbdev)) {
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags_hw);
dev_err(kbdev->dev, "CSF set csg suspend timeout pending reset, try again");
kbase_csf_scheduler_pm_idle(kbdev);
ret = -EFAULT;
goto exit;
}
/* GPU reset is placed and it will take place only after hwaccess_lock is released,
* update on host side should be done after GPU reset is placed and before it takes place.
*/
kbase_csf_scheduler_spin_lock(kbdev, &flags_schd);
kbdev->csf.csg_suspend_timeout_ms = (unsigned int)dur_ms;
kbase_csf_scheduler_spin_unlock(kbdev, flags_schd);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags_hw);
/* Keep PM active until reset finished to allow FW reloading to take place,
* and then update request will be sent to FW during initialization.
*/
kbase_reset_gpu_wait(kbdev);
kbase_csf_scheduler_pm_idle(kbdev);
end:
dev_info(kbdev->dev, "CSF set csg suspend timeout: %u ms", (unsigned int)dur_ms);
exit:
return ret;
}
DEFINE_DEBUGFS_ATTRIBUTE(kbasep_csf_debugfs_eviction_timeout_fops,
&kbasep_csf_debugfs_eviction_timeout_get,
&kbasep_csf_debugfs_eviction_timeout_set, "%llu\n");
void kbase_csf_debugfs_init(struct kbase_device *kbdev)
{
@@ -280,6 +363,8 @@ void kbase_csf_debugfs_init(struct kbase_device *kbdev)
&kbasep_csf_debugfs_scheduling_timer_kick_fops);
debugfs_create_file("scheduler_state", 0644, kbdev->mali_debugfs_directory, kbdev,
&kbasep_csf_debugfs_scheduler_state_fops);
debugfs_create_file("eviction_timeout_ms", 0644, kbdev->mali_debugfs_directory, kbdev,
&kbasep_csf_debugfs_eviction_timeout_fops);
kbase_csf_tl_reader_debugfs_init(kbdev);
}

160
drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h
View File

@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/*
*
* (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -30,6 +30,7 @@
#include "mali_kbase_csf_firmware.h"
#include "mali_kbase_csf_event.h"
#include <uapi/gpu/arm/bifrost/csf/mali_kbase_csf_errors_dumpfault.h>
#include "mali_kbase_csf_fw_io.h"
#include <linux/version_compat_defs.h>
@@ -267,7 +268,7 @@ enum kbase_queue_group_priority {
* @CSF_PM_TIMEOUT: Timeout for GPU Power Management to reach the desired
* Shader, L2 and MCU state.
* @CSF_GPU_RESET_TIMEOUT: Waiting timeout for GPU reset to complete.
* @CSF_CSG_SUSPEND_TIMEOUT: Timeout given for a CSG to be suspended.
* @CSF_CSG_TERM_TIMEOUT: Timeout given for a CSG to be terminated.
* @CSF_FIRMWARE_BOOT_TIMEOUT: Maximum time to wait for firmware to boot.
* @CSF_FIRMWARE_PING_TIMEOUT: Maximum time to wait for firmware to respond
* to a ping from KBase.
@@ -289,7 +290,7 @@ enum kbase_timeout_selector {
CSF_FIRMWARE_TIMEOUT,
CSF_PM_TIMEOUT,
CSF_GPU_RESET_TIMEOUT,
CSF_CSG_SUSPEND_TIMEOUT,
CSF_CSG_TERM_TIMEOUT,
CSF_FIRMWARE_BOOT_TIMEOUT,
CSF_FIRMWARE_PING_TIMEOUT,
CSF_SCHED_PROTM_PROGRESS_TIMEOUT,
@@ -398,6 +399,10 @@ struct kbase_csf_notification {
* @cs_error: Records information about the CS fatal event or
* about CS fault event if dump on fault is enabled.
* @cs_error_fatal: Flag to track if the CS fault or CS fatal event occurred.
* @cs_error_acked: Flag to indicate that acknowledging the fault has been done
* at top-half of fault handler.
* @clear_faults: Flag to track if the CS fault reporting is enabled for this queue.
* It's protected by &kbase_context.csf.lock.
* @extract_ofs: The current EXTRACT offset, this is only updated when handling
* the GLB IDLE IRQ if the idle timeout value is non-0 in order
* to help detect a queue's true idle status.
@@ -441,6 +446,8 @@ struct kbase_queue {
u64 cs_error_info;
u32 cs_error;
bool cs_error_fatal;
bool cs_error_acked;
bool clear_faults;
u64 extract_ofs;
u64 saved_cmd_ptr;
};
@@ -501,6 +508,8 @@ struct kbase_protected_suspend_buffer {
* @compute_max: Maximum number of compute endpoints the group is
* allowed to use.
* @csi_handlers: Requested CSI exception handler flags for the group.
* @cs_fault_report_enable: Indicated if reporting of CS_FAULTs to
* userspace is enabled.
* @tiler_mask: Mask of tiler endpoints the group is allowed to use.
* @fragment_mask: Mask of fragment endpoints the group is allowed to use.
* @compute_mask: Mask of compute endpoints the group is allowed to use.
@@ -531,14 +540,21 @@ struct kbase_protected_suspend_buffer {
* @bound_queues: Array of registered queues bound to this queue group.
* @doorbell_nr: Index of the hardware doorbell page assigned to the
* group.
* @protm_event_work: Work item corresponding to the protected mode entry
* event for this queue.
* @protm_event_work: List item corresponding to the protected mode entry
* event for this queue. This would be handled by
* kbase_csf_scheduler_kthread().
* @pending_protm_event_work: Indicates that kbase_csf_scheduler_kthread() should
* handle PROTM request for this group. This would
* be set to false when the work is done. This is used
* mainly for synchronisation with group termination.
* @protm_pending_bitmap: Bit array to keep a track of CSs that
* have pending protected mode entry requests.
* @error_fatal: An error of type BASE_GPU_QUEUE_GROUP_ERROR_FATAL to be
* returned to userspace if such an error has occurred.
* @timer_event_work: Work item to handle the progress timeout fatal event
* for the group.
* @progress_timer_state: Value of CSG_PROGRESS_TIMER_STATE register when progress
* timer timeout is reported for the group.
* @deschedule_deferred_cnt: Counter keeping a track of the number of threads
* that tried to deschedule the group and had to defer
* the descheduling due to the dump on fault.
@@ -569,7 +585,7 @@ struct kbase_queue_group {
u8 compute_max;
u8 csi_handlers;
__u8 cs_fault_report_enable;
u64 tiler_mask;
u64 fragment_mask;
u64 compute_mask;
@@ -588,12 +604,14 @@ struct kbase_queue_group {
struct kbase_queue *bound_queues[MAX_SUPPORTED_STREAMS_PER_GROUP];
int doorbell_nr;
struct work_struct protm_event_work;
struct list_head protm_event_work;
atomic_t pending_protm_event_work;
DECLARE_BITMAP(protm_pending_bitmap, MAX_SUPPORTED_STREAMS_PER_GROUP);
struct kbase_csf_notification error_fatal;
struct work_struct timer_event_work;
u32 progress_timer_state;
/**
* @dvs_buf: Address and size of scratch memory.
@@ -625,6 +643,9 @@ struct kbase_queue_group {
* @cmd_seq_num: The sequence number assigned to an enqueued command,
* in incrementing order (older commands shall have a
* smaller number).
* @kcpu_wq: Work queue to process KCPU commands for all queues in this
* context. This would be used if the context is not prioritised,
* otherwise it would be handled by kbase_csf_scheduler_kthread().
* @jit_lock: Lock to serialise JIT operations.
* @jit_cmds_head: A list of the just-in-time memory commands, both
* allocate & free, in submission order, protected
@@ -640,6 +661,8 @@ struct kbase_csf_kcpu_queue_context {
DECLARE_BITMAP(in_use, KBASEP_MAX_KCPU_QUEUES);
atomic64_t cmd_seq_num;
struct workqueue_struct *kcpu_wq;
struct mutex jit_lock;
struct list_head jit_cmds_head;
struct list_head jit_blocked_queues;
@@ -747,15 +770,7 @@ struct kbase_csf_ctx_heap_reclaim_info {
* GPU command queues are idle and at least one of them
* is blocked on a sync wait operation.
* @num_idle_wait_grps: Length of the @idle_wait_groups list.
* @sync_update_wq_high_prio: high-priority work queue to process the
* SYNC_UPDATE events by sync_set / sync_add
* instruction execution on command streams bound to
* groups of @idle_wait_groups list. This WQ would
* be used if the context is prioritised.
* @sync_update_wq_normal_prio: similar to sync_update_wq_high_prio, but this
* WQ would be used if the context is not
* prioritised.
* @sync_update_work: Work item to process the SYNC_UPDATE events.
* @sync_update_work: List item to process the SYNC_UPDATE event.
* @ngrp_to_schedule: Number of groups added for the context to the
* 'groups_to_schedule' list of scheduler instance.
* @heap_info: Heap reclaim information data of the kctx. As the
@@ -768,9 +783,7 @@ struct kbase_csf_scheduler_context {
u32 num_runnable_grps;
struct list_head idle_wait_groups;
u32 num_idle_wait_grps;
struct workqueue_struct *sync_update_wq_high_prio;
struct workqueue_struct *sync_update_wq_normal_prio;
struct work_struct sync_update_work;
struct list_head sync_update_work;
u32 ngrp_to_schedule;
struct kbase_csf_ctx_heap_reclaim_info heap_info;
};
@@ -865,17 +878,16 @@ struct kbase_csf_user_reg_context {
* @wq: Dedicated workqueue to process work items corresponding
* to the OoM events raised for chunked tiler heaps being
* used by GPU command queues, and progress timeout events.
* @kcpu_wq_high_prio: High-priority work queue to process KCPU commands for
* all queues in this context. This WQ would be used if
* the context is prioritised.
* @kcpu_wq_normal_prio: Similar to kcpu_wq_high_prio, but this WQ would be
* used if the context is not prioritised.
* @link: Link to this csf context in the 'runnable_kctxs' list of
* the scheduler instance
* @sched: Object representing the scheduler's context
* @cpu_queue: CPU queue information. Only be available when DEBUG_FS
* is enabled.
* @user_reg: Collective information to support mapping to USER Register page.
* @pending_sync_update: Indicates that kbase_csf_scheduler_kthread() should
* handle SYNC_UPDATE event for this context. This would
* be set to false when the work is done. This is used
* mainly for synchronisation with context termination.
*/
struct kbase_csf_context {
struct list_head event_pages_head;
@@ -888,12 +900,11 @@ struct kbase_csf_context {
struct kbase_csf_event event;
struct kbase_csf_tiler_heap_context tiler_heaps;
struct workqueue_struct *wq;
struct workqueue_struct *kcpu_wq_high_prio;
struct workqueue_struct *kcpu_wq_normal_prio;
struct list_head link;
struct kbase_csf_scheduler_context sched;
struct kbase_csf_cpu_queue_context cpu_queue;
struct kbase_csf_user_reg_context user_reg;
atomic_t pending_sync_update;
};
/**
@@ -922,13 +933,11 @@ struct kbase_csf_reset_gpu {
* of CSG slots.
* @resident_group: pointer to the queue group that is resident on the CSG slot.
* @state: state of the slot as per enum @kbase_csf_csg_slot_state.
* @trigger_jiffies: value of jiffies when change in slot state is recorded.
* @priority: dynamic priority assigned to CSG slot.
*/
struct kbase_csf_csg_slot {
struct kbase_queue_group *resident_group;
atomic_t state;
unsigned long trigger_jiffies;
u8 priority;
};
@@ -936,14 +945,15 @@ struct kbase_csf_csg_slot {
* struct kbase_csf_sched_heap_reclaim_mgr - Object for managing tiler heap reclaim
* kctx lists inside the CSF device's scheduler.
*
* @heap_reclaim: Tiler heap reclaim shrinker object.
* @heap_reclaim: Defines Tiler heap reclaim shrinker object.
* @ctx_lists: Array of kctx lists, size matching CSG defined priorities. The
* lists track the kctxs attached to the reclaim manager.
* @unused_pages: Estimated number of unused pages from the @ctxlist array. The
* number is indicative for use with reclaim shrinker's count method.
*/
struct kbase_csf_sched_heap_reclaim_mgr {
struct shrinker heap_reclaim;
DEFINE_KBASE_SHRINKER heap_reclaim;
struct list_head ctx_lists[KBASE_QUEUE_GROUP_PRIORITY_COUNT];
atomic_t unused_pages;
};
@@ -1042,10 +1052,29 @@ struct kbase_csf_mcu_shared_regions {
* workqueue items (kernel-provided delayed_work
* items do not use hrtimer and for some reason do
* not provide sufficiently reliable periodicity).
* @pending_tick_work: Indicates that kbase_csf_scheduler_kthread() should perform
* a scheduling tick.
* @pending_tock_work: Indicates that kbase_csf_scheduler_kthread() should perform
* a scheduling tock.
* @pending_sync_update_works: Indicates that kbase_csf_scheduler_kthread()
* should handle SYNC_UPDATE events.
* @sync_update_work_ctxs_lock: Lock protecting the list of contexts that
* require handling SYNC_UPDATE events.
* @sync_update_work_ctxs: The list of contexts that require handling
* SYNC_UPDATE events.
* @pending_protm_event_works: Indicates that kbase_csf_scheduler_kthread()
* should handle PROTM requests.
* @protm_event_work_grps_lock: Lock protecting the list of groups that
* have requested protected mode.
* @protm_event_work_grps: The list of groups that have requested
* protected mode.
* @pending_kcpuq_works: Indicates that kbase_csf_scheduler_kthread()
* should process pending KCPU queue works.
* @kcpuq_work_queues_lock: Lock protecting the list of KCPU queues that
* need to be processed.
* @kcpuq_work_queues: The list of KCPU queue that need to be processed
* @pending_tick_work: Indicates that kbase_csf_scheduler_kthread() should
* perform a scheduling tick.
* @pending_tock_work: Indicates that kbase_csf_scheduler_kthread() should
* perform a scheduling tock.
* @pending_gpu_idle_work: Indicates that kbase_csf_scheduler_kthread() should
* handle the GPU IDLE event.
* @ping_work: Work item that would ping the firmware at regular
* intervals, only if there is a single active CSG
* slot, to check if firmware is alive and would
@@ -1063,10 +1092,6 @@ struct kbase_csf_mcu_shared_regions {
* This pointer being set doesn't necessarily indicates
* that GPU is in protected mode, kbdev->protected_mode
* needs to be checked for that.
* @idle_wq: Workqueue for executing GPU idle notification
* handler.
* @gpu_idle_work: Work item for facilitating the scheduler to bring
* the GPU to a low-power mode on becoming idle.
* @fast_gpu_idle_handling: Indicates whether to relax many of the checks
* normally done in the GPU idle worker. This is
* set to true when handling the GLB IDLE IRQ if the
@@ -1109,8 +1134,11 @@ struct kbase_csf_mcu_shared_regions {
* thread when a queue needs attention.
* @kthread_running: Whether the GPU queue submission thread should keep
* executing.
* @gpuq_kthread: High-priority thread used to handle GPU queue
* @gpuq_kthread: Dedicated thread primarily used to handle
* latency-sensitive tasks such as GPU queue
* submissions.
* @gpu_idle_timer_enabled: Tracks whether the GPU idle timer is enabled or disabled.
* @fw_soi_enabled: True if FW Sleep-on-Idle is currently enabled.
*/
struct kbase_csf_scheduler {
struct mutex lock;
@@ -1134,14 +1162,22 @@ struct kbase_csf_scheduler {
unsigned long last_schedule;
atomic_t timer_enabled;
struct hrtimer tick_timer;
atomic_t pending_sync_update_works;
spinlock_t sync_update_work_ctxs_lock;
struct list_head sync_update_work_ctxs;
atomic_t pending_protm_event_works;
spinlock_t protm_event_work_grps_lock;
struct list_head protm_event_work_grps;
atomic_t pending_kcpuq_works;
spinlock_t kcpuq_work_queues_lock;
struct list_head kcpuq_work_queues;
atomic_t pending_tick_work;
atomic_t pending_tock_work;
atomic_t pending_gpu_idle_work;
struct delayed_work ping_work;
struct kbase_context *top_kctx;
struct kbase_queue_group *top_grp;
struct kbase_queue_group *active_protm_grp;
struct workqueue_struct *idle_wq;
struct work_struct gpu_idle_work;
bool fast_gpu_idle_handling;
atomic_t gpu_no_longer_idle;
atomic_t non_idle_offslot_grps;
@@ -1180,6 +1216,8 @@ struct kbase_csf_scheduler {
*/
spinlock_t gpu_metrics_lock;
#endif /* CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD */
atomic_t gpu_idle_timer_enabled;
atomic_t fw_soi_enabled;
};
/*
@@ -1643,6 +1681,7 @@ struct kbase_csf_user_reg {
* @gpu_idle_dur_count_no_modifier: Update csffw_glb_req_idle_enable to make the shr(10)
* modifier conditional on the new flag
* in GLB_IDLE_TIMER_CONFIG.
* @csg_suspend_timeout_ms: Timeout given for a CSG to be suspended.
* for any request sent to the firmware.
* @hwcnt: Contain members required for handling the dump of
* HW counters.
@@ -1653,12 +1692,29 @@ struct kbase_csf_user_reg {
* @dof: Structure for dump on fault.
* @user_reg: Collective information to support the mapping to
* USER Register page for user processes.
* @pending_gpuq_kicks: Lists of GPU queue that have been kicked but not
* yet processed, categorised by queue group's priority.
* @pending_gpuq_kicks_lock: Protect @pending_gpu_kicks and
* kbase_queue.pending_kick_link.
* @pending_gpuq_kicks: Indicates that kbase_csf_scheduler_kthread()
* should handle GPU queue kicks.
* @pending_gpuq_kick_queues: Lists of GPU queued that have been kicked but not
* yet processed, categorised by queue group's priority.
* @pending_gpuq_kick_queues_lock: Protect @pending_gpuq_kick_queues and
* kbase_queue.pending_kick_link.
* @quirks_ext: Pointer to an allocated buffer containing the firmware
* workarounds configuration.
* @mmu_sync_sem: RW Semaphore to defer MMU operations till the P.Mode entrance
* or DCS request has been completed.
* @pmode_sync_sem: RW Semaphore to prevent MMU operations during P.Mode entrance.
* @page_fault_cnt_ptr_address: GPU VA of the location in FW data memory, extracted from the
* FW image header, that will store the GPU VA of FW visible
* memory location where the @page_fault_cnt value will be written to.
* @page_fault_cnt_ptr: CPU VA of the FW visible memory location where the @page_fault_cnt
* value will be written to.
* @page_fault_cnt: Counter that is incremented on every GPU page fault, just before the
* MMU is unblocked to retry the memory transaction that caused the GPU
* page fault. The access to counter is serialized appropriately.
* @mcu_halted: Flag to inform MCU FSM that the MCU has already halted.
* @fw_io: Firmware I/O interface.
* @compute_progress_timeout_cc: Value of GPU cycle count register when progress
* timer timeout is reported for the compute iterator.
*/
struct kbase_csf_device {
struct kbase_mmu_table mcu_mmu;
@@ -1696,6 +1752,7 @@ struct kbase_csf_device {
u64 gpu_idle_hysteresis_ns;
u32 gpu_idle_dur_count;
u32 gpu_idle_dur_count_no_modifier;
u32 csg_suspend_timeout_ms;
struct kbase_csf_hwcnt hwcnt;
struct kbase_csf_mcu_fw fw;
struct kbase_csf_firmware_log fw_log;
@@ -1710,9 +1767,18 @@ struct kbase_csf_device {
struct kbase_debug_coresight_device coresight;
#endif /* IS_ENABLED(CONFIG_MALI_CORESIGHT) */
struct kbase_csf_user_reg user_reg;
struct list_head pending_gpuq_kicks[KBASE_QUEUE_GROUP_PRIORITY_COUNT];
spinlock_t pending_gpuq_kicks_lock;
atomic_t pending_gpuq_kicks;
struct list_head pending_gpuq_kick_queues[KBASE_QUEUE_GROUP_PRIORITY_COUNT];
spinlock_t pending_gpuq_kick_queues_lock;
u32 *quirks_ext;
struct rw_semaphore mmu_sync_sem;
struct rw_semaphore pmode_sync_sem;
u32 page_fault_cnt_ptr_address;
u32 *page_fault_cnt_ptr;
u32 page_fault_cnt;
bool mcu_halted;
struct kbase_csf_fw_io fw_io;
u64 compute_progress_timeout_cc;
};
/**

544
drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -40,6 +40,7 @@
#include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h"
#include <csf/ipa_control/mali_kbase_csf_ipa_control.h>
#include <csf/mali_kbase_csf_registers.h>
#include <csf/mali_kbase_csf_fw_io.h>
#include <linux/list.h>
#include <linux/slab.h>
#include <linux/firmware.h>
@@ -55,6 +56,7 @@
#include <linux/delay.h>
#include <linux/version_compat_defs.h>
#include <mali_kbase_config_defaults.h>
#define MALI_MAX_DEFAULT_FIRMWARE_NAME_LEN ((size_t)64)
#define DEFAULT_FW_NAME MALI_RELEASE_NAME".mali_csffw.bin"
@@ -68,6 +70,7 @@ static unsigned int csf_firmware_boot_timeout_ms;
module_param(csf_firmware_boot_timeout_ms, uint, 0444);
MODULE_PARM_DESC(csf_firmware_boot_timeout_ms, "Maximum time to wait for firmware to boot.");
static bool kbase_iter_trace_enable;
#ifdef CONFIG_MALI_BIFROST_DEBUG
/* Makes Driver wait indefinitely for an acknowledgment for the different
@@ -97,6 +100,7 @@ MODULE_PARM_DESC(fw_debug, "Enables effective use of a debugger for debugging fi
#define CSF_FIRMWARE_ENTRY_TYPE_TIMELINE_METADATA (4)
#define CSF_FIRMWARE_ENTRY_TYPE_BUILD_INFO_METADATA (6)
#define CSF_FIRMWARE_ENTRY_TYPE_FUNC_CALL_LIST (7)
#define CSF_FIRMWARE_ENTRY_TYPE_PAGE_FAULT_CNT (8)
#define CSF_FIRMWARE_ENTRY_TYPE_CORE_DUMP (9)
#define CSF_FIRMWARE_CACHE_MODE_NONE (0ul << 3)
@@ -115,7 +119,8 @@ MODULE_PARM_DESC(fw_debug, "Enables effective use of a debugger for debugging fi
#define CSF_GLB_REQ_CFG_MASK \
(GLB_REQ_CFG_ALLOC_EN_MASK | GLB_REQ_CFG_PROGRESS_TIMER_MASK | \
GLB_REQ_CFG_PWROFF_TIMER_MASK | GLB_REQ_IDLE_ENABLE_MASK)
GLB_REQ_CFG_PWROFF_TIMER_MASK | GLB_REQ_IDLE_ENABLE_MASK | \
GLB_REQ_CFG_EVICTION_TIMER_MASK | GLB_REQ_ITER_TRACE_ENABLE_MASK)
static inline u32 input_page_read(const u32 *const input, const u32 offset)
{
@@ -179,6 +184,92 @@ struct firmware_timeline_metadata {
size_t size;
};
static void reinit_page_fault_cnt_firmware_memory(struct kbase_device *kbdev)
{
if (!kbdev->csf.page_fault_cnt_ptr)
return;
/* Store the GPU address of shared memory location, where the page fault counter
* value will be written, inside the FW data memory.
*/
kbase_csf_update_firmware_memory(
kbdev, kbdev->csf.page_fault_cnt_ptr_address,
(u32)((kbdev->csf.firmware_trace_buffers.mcu_rw.va_reg->start_pfn << PAGE_SHIFT) +
PAGE_SIZE - sizeof(u32)));
*kbdev->csf.page_fault_cnt_ptr = kbdev->csf.page_fault_cnt = 0;
}
static void init_page_fault_cnt_firmware_memory(struct kbase_device *kbdev)
{
if (!kbdev->csf.page_fault_cnt_ptr_address)
return;
if (WARN_ON_ONCE(!kbdev->csf.firmware_trace_buffers.mcu_rw.va_reg))
return;
/* Save the CPU address of shared memory location where the page fault counter
* value will be written.
* The shared memory location comes from the last 4 bytes of the page that
* is allocated to maintain the extract offset value for different trace
* buffers. Only the first 4 bytes of every cacheline is used for the extract offset
* value.
*/
kbdev->csf.page_fault_cnt_ptr =
(u32 *)((u8 *)kbdev->csf.firmware_trace_buffers.mcu_rw.cpu_addr + PAGE_SIZE -
sizeof(u32));
reinit_page_fault_cnt_firmware_memory(kbdev);
}
/**
* set_iterator_trace_enable - Set the value for 'kbase_iter_trace_enable' global variable
* according to the value of GLB_FEATURES.ITER_TRACE_SUPPORTED bit,
* and the corresponding device tree entry.
* @kbdev: Kernel base device pointer
*/
static void set_iterator_trace_enable(struct kbase_device *kbdev)
{
const struct kbase_csf_global_iface *iface = &kbdev->csf.global_iface;
bool dev_support_iter_trace = iface->features & GLB_FEATURES_ITER_TRACE_SUPPORTED_MASK;
const void *dt_iter_trace_param;
unsigned int val;
if (!dev_support_iter_trace) {
kbase_iter_trace_enable = false;
return;
}
/* check device tree for iterator trace enable property and
* fallback to "iter_trace_enable" if not found and try again
*/
dt_iter_trace_param = of_get_property(kbdev->dev->of_node, "iter-trace-enable", NULL);
if (!dt_iter_trace_param)
dt_iter_trace_param =
of_get_property(kbdev->dev->of_node, "iter_trace_enable", NULL);
val = (dt_iter_trace_param) ? be32_to_cpup(dt_iter_trace_param) : 0;
dev_dbg(kbdev->dev, "Iterator trace enable device-tree config value: %u", val);
kbase_iter_trace_enable = val ? true : false;
}
static void iterator_trace_reinit(struct kbase_device *kbdev)
{
if (kbase_iter_trace_enable) {
kbase_csf_firmware_global_input_mask(&kbdev->csf.global_iface, GLB_REQ,
GLB_REQ_ITER_TRACE_ENABLE_MASK,
GLB_REQ_ITER_TRACE_ENABLE_MASK);
}
}
static void iterator_trace_init(struct kbase_device *kbdev)
{
set_iterator_trace_enable(kbdev);
iterator_trace_reinit(kbdev);
}
/* The shared interface area, used for communicating with firmware, is managed
* like a virtual memory zone. Reserve the virtual space from that zone
* corresponding to shared interface entry parsed from the firmware image.
@@ -217,7 +308,7 @@ void kbase_csf_firmware_disable_mcu(struct kbase_device *kbdev)
kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(MCU_CONTROL), MCU_CONTROL_REQ_DISABLE);
}
static void wait_for_firmware_stop(struct kbase_device *kbdev)
void kbase_csf_firmware_disable_mcu_wait(struct kbase_device *kbdev)
{
u32 val;
const u32 timeout_us =
@@ -232,17 +323,12 @@ static void wait_for_firmware_stop(struct kbase_device *kbdev)
KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_OFF(kbdev, kbase_backend_get_cycle_cnt(kbdev));
}
void kbase_csf_firmware_disable_mcu_wait(struct kbase_device *kbdev)
{
wait_for_firmware_stop(kbdev);
}
static void stop_csf_firmware(struct kbase_device *kbdev)
void kbase_csf_stop_firmware_and_wait(struct kbase_device *kbdev)
{
/* Stop the MCU firmware */
kbase_csf_firmware_disable_mcu(kbdev);
wait_for_firmware_stop(kbdev);
kbase_csf_firmware_disable_mcu_wait(kbdev);
}
static void wait_for_firmware_boot(struct kbase_device *kbdev)
@@ -261,7 +347,6 @@ static void wait_for_firmware_boot(struct kbase_device *kbdev)
*/
remaining = wait_event_timeout(kbdev->csf.event_wait, kbdev->csf.interrupt_received == true,
wait_timeout);
if (!remaining)
dev_err(kbdev->dev, "Timed out waiting for fw boot completion");
@@ -485,6 +570,8 @@ static int reload_fw_image(struct kbase_device *kbdev)
kbdev->csf.firmware_full_reload_needed = false;
kbase_csf_firmware_reload_trace_buffers_data(kbdev);
reinit_page_fault_cnt_firmware_memory(kbdev);
iterator_trace_reinit(kbdev);
out:
return ret;
}
@@ -1043,6 +1130,14 @@ static int load_firmware_entry(struct kbase_device *kbdev, const struct kbase_cs
}
kbase_csf_firmware_log_parse_logging_call_list_entry(kbdev, entry);
return 0;
case CSF_FIRMWARE_ENTRY_TYPE_PAGE_FAULT_CNT:
/* Entry about the location of page fault counter */
if (size < sizeof(*entry)) {
dev_err(kbdev->dev, "Page fault counter entry too short (size=%u)", size);
return -EINVAL;
}
kbdev->csf.page_fault_cnt_ptr_address = *entry;
return 0;
case CSF_FIRMWARE_ENTRY_TYPE_CORE_DUMP:
/* Core Dump section */
if (size < CORE_DUMP_ENTRY_START_ADDR_OFFSET + sizeof(*entry)) {
@@ -1552,7 +1647,6 @@ static bool global_request_complete(struct kbase_device *const kbdev, u32 const
unsigned long flags;
kbase_csf_scheduler_spin_lock(kbdev, &flags);
if ((kbase_csf_firmware_global_output(global_iface, GLB_ACK) & req_mask) ==
(kbase_csf_firmware_global_input_read(global_iface, GLB_REQ) & req_mask))
complete = true;
@@ -1644,9 +1738,27 @@ static void set_timeout_global(const struct kbase_csf_global_iface *const global
set_global_request(global_iface, GLB_REQ_CFG_PROGRESS_TIMER_MASK);
}
static inline void set_gpu_idle_timer_glb_req(struct kbase_device *const kbdev, bool set)
{
struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface;
kbase_csf_scheduler_spin_lock_assert_held(kbdev);
if (set) {
kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, GLB_REQ_REQ_IDLE_ENABLE,
GLB_REQ_IDLE_ENABLE_MASK);
} else {
kbase_csf_firmware_global_input_mask(
global_iface, GLB_REQ, GLB_REQ_REQ_IDLE_DISABLE, GLB_REQ_IDLE_DISABLE_MASK);
}
atomic_set(&kbdev->csf.scheduler.gpu_idle_timer_enabled, set);
}
static void enable_gpu_idle_timer(struct kbase_device *const kbdev)
{
struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface;
bool const fw_soi_allowed = kbase_pm_fw_sleep_on_idle_allowed(kbdev);
kbase_csf_scheduler_spin_lock_assert_held(kbdev);
@@ -1654,15 +1766,114 @@ static void enable_gpu_idle_timer(struct kbase_device *const kbdev)
kbdev->csf.gpu_idle_dur_count);
kbase_csf_firmware_global_input_mask(global_iface, GLB_IDLE_TIMER_CONFIG,
kbdev->csf.gpu_idle_dur_count_no_modifier,
kbdev->csf.gpu_idle_dur_count_no_modifier
<< GLB_IDLE_TIMER_CONFIG_NO_MODIFIER_SHIFT,
GLB_IDLE_TIMER_CONFIG_NO_MODIFIER_MASK);
kbase_csf_firmware_global_input_mask(global_iface, GLB_IDLE_TIMER_CONFIG,
fw_soi_allowed
<< GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_SHIFT,
GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_MASK);
kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, GLB_REQ_REQ_IDLE_ENABLE,
GLB_REQ_IDLE_ENABLE_MASK);
set_gpu_idle_timer_glb_req(kbdev, true);
atomic_set(&kbdev->csf.scheduler.fw_soi_enabled, fw_soi_allowed);
dev_dbg(kbdev->dev, "Enabling GPU idle timer with count-value: 0x%.8x",
kbdev->csf.gpu_idle_dur_count);
}
/**
* convert_dur_to_suspend_count() - Convert CSG suspend timeout from ms to cycle count
* @kbdev: Instance of a GPU platform device that implements a CSF interface
* @dur_ms: Timeout value in ms
* @no_modifier: Indicate whether bit-shift is applied, 0 when applied, 1 otherwise
*
* Convert CSG suspend timeout from ms to cycle count, then generate a register value
* combining cycle count and timer source
*
* Return: Register value which will be stored into register GLB_EVICTION_TIMER.
*/
static u32 convert_dur_to_suspend_count(struct kbase_device *kbdev, const u64 dur_ms,
u32 *no_modifier)
{
/* Get the cntfreq_el0 value, which drives the SYSTEM_TIMESTAMP */
u64 freq = kbase_arch_timer_get_cntfrq(kbdev);
u64 dur_val = dur_ms;
u32 cnt_val_u32, reg_val_u32;
const bool src_system_timestamp = freq > 0;
const u8 SUSPEND_VAL_UNIT_SHIFT = 10;
if (!src_system_timestamp) {
/* Get the cycle_counter source alternative */
spin_lock(&kbdev->pm.clk_rtm.lock);
if (kbdev->pm.clk_rtm.clks[0])
freq = kbdev->pm.clk_rtm.clks[0]->clock_val;
else
dev_err(kbdev->dev, "No GPU clock, unexpected intregration issue!");
spin_unlock(&kbdev->pm.clk_rtm.lock);
dev_info(kbdev->dev,
"No timestamp frequency, use cycle counter for csg suspend timeout!");
}
/* Formula for dur_val = (dur/1e3) * freq_HZ) */
dur_val = dur_val * freq;
dur_val = div_u64(dur_val, MSEC_PER_SEC);
if (dur_val < S32_MAX) {
*no_modifier = 1;
} else {
dur_val = dur_val >> SUSPEND_VAL_UNIT_SHIFT;
*no_modifier = 0;
}
/* Interface limits the value field to S32_MAX */
cnt_val_u32 = (dur_val > S32_MAX) ? S32_MAX : (u32)dur_val;
reg_val_u32 = GLB_EVICTION_TIMER_TIMEOUT_SET(0, cnt_val_u32);
/* add the source flag */
reg_val_u32 = GLB_EVICTION_TIMER_TIMER_SOURCE_SET(
reg_val_u32,
(src_system_timestamp ? GLB_EVICTION_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP :
GLB_EVICTION_TIMER_TIMER_SOURCE_GPU_COUNTER));
return reg_val_u32;
}
/**
* set_csg_suspend_timeout() - Update CSG suspend timeout setting on FW side
*
* @kbdev: Instance of a GPU platform device that implements a CSF interface
*/
static void set_csg_suspend_timeout(struct kbase_device *const kbdev)
{
u32 dur_ms, dur_val;
u32 no_modifier = 0;
struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface;
kbase_csf_scheduler_spin_lock_assert_held(kbdev);
dur_ms = kbdev->csf.csg_suspend_timeout_ms;
if (unlikely(dur_ms < CSG_SUSPEND_TIMEOUT_FIRMWARE_MS_MIN +
CSG_SUSPEND_TIMEOUT_HOST_ADDED_MS ||
dur_ms > CSG_SUSPEND_TIMEOUT_FIRMWARE_MS_MAX +
CSG_SUSPEND_TIMEOUT_HOST_ADDED_MS)) {
dev_err(kbdev->dev, "Unexpected CSG suspend timeout: %ums, default to: %ums",
dur_ms, CSG_SUSPEND_TIMEOUT_MS);
kbdev->csf.csg_suspend_timeout_ms = CSG_SUSPEND_TIMEOUT_MS;
dur_ms = CSG_SUSPEND_TIMEOUT_MS;
}
dur_ms = dur_ms - CSG_SUSPEND_TIMEOUT_HOST_ADDED_MS;
dur_val = convert_dur_to_suspend_count(kbdev, dur_ms, &no_modifier);
kbase_csf_firmware_global_input(global_iface, GLB_EVICTION_TIMER, dur_val);
kbase_csf_firmware_global_input_mask(global_iface, GLB_EVICTION_TIMER_CONFIG, no_modifier,
GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_MASK);
set_global_request(global_iface, GLB_REQ_CFG_EVICTION_TIMER_MASK);
dev_dbg(kbdev->dev, "Updating CSG suspend timeout with count-value: 0x%.8x", dur_val);
}
static bool global_debug_request_complete(struct kbase_device *const kbdev, u32 const req_mask)
{
struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface;
@@ -1751,7 +1962,8 @@ static void global_init(struct kbase_device *const kbdev, u64 core_mask)
GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_MASK | GLB_ACK_IRQ_MASK_PROTM_ENTER_MASK |
GLB_ACK_IRQ_MASK_PROTM_EXIT_MASK | GLB_ACK_IRQ_MASK_FIRMWARE_CONFIG_UPDATE_MASK |
GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_MASK | GLB_ACK_IRQ_MASK_IDLE_EVENT_MASK |
GLB_REQ_DEBUG_CSF_REQ_MASK | GLB_ACK_IRQ_MASK_IDLE_ENABLE_MASK;
GLB_REQ_DEBUG_CSF_REQ_MASK | GLB_ACK_IRQ_MASK_IDLE_ENABLE_MASK |
GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_MASK | GLB_ACK_IRQ_MASK_ITER_TRACE_ENABLE_MASK;
const struct kbase_csf_global_iface *const global_iface = &kbdev->csf.global_iface;
unsigned long flags;
@@ -1766,11 +1978,10 @@ static void global_init(struct kbase_device *const kbdev, u64 core_mask)
set_timeout_global(global_iface, kbase_csf_timeout_get(kbdev));
/* The GPU idle timer is always enabled for simplicity. Checks will be
* done before scheduling the GPU idle worker to see if it is
* appropriate for the current power policy.
/* The csg suspend timeout is always enabled so customer has the flexibility to update it
* at any time.
*/
enable_gpu_idle_timer(kbdev);
set_csg_suspend_timeout(kbdev);
/* Unmask the interrupts */
kbase_csf_firmware_global_input(global_iface, GLB_ACK_IRQ_MASK, ack_irq_mask);
@@ -1890,6 +2101,7 @@ static void kbase_csf_firmware_reload_worker(struct work_struct *work)
{
struct kbase_device *kbdev =
container_of(work, struct kbase_device, csf.firmware_reload_work);
unsigned long flags;
int err;
dev_info(kbdev->dev, "reloading firmware");
@@ -1908,7 +2120,9 @@ static void kbase_csf_firmware_reload_worker(struct work_struct *work)
return;
/* Reboot the firmware */
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
kbase_csf_firmware_enable_mcu(kbdev);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
}
void kbase_csf_firmware_trigger_reload(struct kbase_device *kbdev)
@@ -1945,6 +2159,7 @@ void kbase_csf_firmware_reload_completed(struct kbase_device *kbdev)
KBASE_KTRACE_ADD(kbdev, CSF_FIRMWARE_REBOOT, NULL, 0u);
/* Tell MCU state machine to transit to next state */
kbdev->csf.firmware_reloaded = true;
kbase_pm_update_state(kbdev);
@@ -2045,29 +2260,44 @@ u32 kbase_csf_firmware_set_gpu_idle_hysteresis_time(struct kbase_device *kbdev,
return kbdev->csf.gpu_idle_dur_count;
}
/* The 'reg_lock' is also taken and is held till the update is not
/* The scheduler lock is also taken and is held till the update is not
* complete, to ensure the update of idle timer value by multiple Users
* gets serialized.
*/
mutex_lock(&kbdev->csf.reg_lock);
/* The firmware only reads the new idle timer value when the timer is
* disabled.
*/
kbase_csf_scheduler_lock(kbdev);
while (atomic_read(&kbdev->csf.scheduler.pending_gpu_idle_work) > 0) {
kbase_csf_scheduler_unlock(kbdev);
kbase_csf_scheduler_wait_for_kthread_pending_work(
kbdev, &kbdev->csf.scheduler.pending_gpu_idle_work);
kbase_csf_scheduler_lock(kbdev);
}
kbase_csf_scheduler_spin_lock(kbdev, &flags);
kbase_csf_firmware_disable_gpu_idle_timer(kbdev);
kbase_csf_scheduler_spin_unlock(kbdev, flags);
/* Ensure that the request has taken effect */
wait_for_global_request(kbdev, GLB_REQ_IDLE_DISABLE_MASK);
kbase_csf_scheduler_spin_lock(kbdev, &flags);
kbdev->csf.gpu_idle_hysteresis_ns = dur_ns;
kbdev->csf.gpu_idle_dur_count = hysteresis_val;
kbdev->csf.gpu_idle_dur_count_no_modifier = no_modifier;
kbase_csf_firmware_enable_gpu_idle_timer(kbdev);
kbase_csf_scheduler_spin_unlock(kbdev, flags);
wait_for_global_request(kbdev, GLB_REQ_IDLE_ENABLE_MASK);
mutex_unlock(&kbdev->csf.reg_lock);
if (atomic_read(&kbdev->csf.scheduler.gpu_idle_timer_enabled)) {
/* Timer is already enabled. Disable the timer as FW only reads
* the new idle timer value when timer is re-enabled.
*/
kbase_csf_firmware_disable_gpu_idle_timer(kbdev);
kbase_csf_scheduler_spin_unlock(kbdev, flags);
/* Ensure that the request has taken effect */
if (wait_for_global_request(kbdev, GLB_REQ_IDLE_DISABLE_MASK))
dev_err(kbdev->dev,
"Failed to disable GLB_IDLE timer when setting a new idle hysteresis timeout");
kbase_csf_scheduler_spin_lock(kbdev, &flags);
kbase_csf_firmware_enable_gpu_idle_timer(kbdev);
kbase_csf_scheduler_spin_unlock(kbdev, flags);
if (wait_for_global_request(kbdev, GLB_REQ_IDLE_ENABLE_MASK))
dev_err(kbdev->dev,
"Failed to re-enable GLB_IDLE timer when setting a new idle hysteresis timeout");
} else {
kbase_csf_scheduler_spin_unlock(kbdev, flags);
}
kbase_csf_scheduler_unlock(kbdev);
kbase_csf_scheduler_pm_idle(kbdev);
kbase_reset_gpu_allow(kbdev);
end:
@@ -2168,78 +2398,6 @@ u32 kbase_csf_firmware_reset_mcu_core_pwroff_time(struct kbase_device *kbdev)
return kbase_csf_firmware_set_mcu_core_pwroff_time(kbdev, DEFAULT_GLB_PWROFF_TIMEOUT_NS);
}
/**
* kbase_csf_get_iterator_trace_enable - Parsing the iterator_trace enable firstly from
* the module parameter, and then from device-tree.
* @kbdev: Kernel base device pointer
*
* Return: true on enabled, otherwise false.
*/
static bool kbase_csf_get_iterator_trace_enable(struct kbase_device *kbdev)
{
const void *dt_iter_trace_param;
unsigned int val;
/* check device tree for iterator trace enable property and
* fallback to "iter_trace_enable" if not found and try again
*/
dt_iter_trace_param = of_get_property(kbdev->dev->of_node, "iter-trace-enable", NULL);
if (!dt_iter_trace_param)
dt_iter_trace_param =
of_get_property(kbdev->dev->of_node, "iter_trace_enable", NULL);
val = (dt_iter_trace_param) ? be32_to_cpup(dt_iter_trace_param) : 0;
dev_dbg(kbdev->dev, "Iterator trace enable device-tree config value: %u", val);
return (val != 0);
}
/**
* kbase_device_csf_iterator_trace_init - Send request to enable iterator
* trace port.
* @kbdev: Kernel base device pointer
*
* Return: 0 on success (or if enable request is not sent), or error
* code -EINVAL on failure of GPU to acknowledge enable request.
*/
static int kbase_device_csf_iterator_trace_init(struct kbase_device *kbdev)
{
/* Enable the iterator trace port if supported by the GPU and is
* configured to do so. The FW must advertise this feature in GLB_FEATURES.
*/
if (kbdev->pm.backend.gpu_powered) {
const struct kbase_csf_global_iface *iface = &kbdev->csf.global_iface;
bool dev_support_iter_trace = iface->features &
GLB_FEATURES_ITER_TRACE_SUPPORTED_MASK;
dev_dbg(kbdev->dev, "Device supporting iterator trace: %s\n",
dev_support_iter_trace ? "true" : "false");
if (dev_support_iter_trace && kbase_csf_get_iterator_trace_enable(kbdev)) {
long ack_timeout = kbase_csf_timeout_in_jiffies(
kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT));
/* write enable request to global input */
kbase_csf_firmware_global_input_mask(iface, GLB_REQ,
GLB_REQ_ITER_TRACE_ENABLE_MASK,
GLB_REQ_ITER_TRACE_ENABLE_MASK);
/* Ring global doorbell */
kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR);
ack_timeout = wait_event_timeout(
kbdev->csf.event_wait,
!((kbase_csf_firmware_global_input_read(iface, GLB_REQ) ^
kbase_csf_firmware_global_output(iface, GLB_ACK)) &
GLB_REQ_ITER_TRACE_ENABLE_MASK),
ack_timeout);
return ack_timeout ? 0 : -EINVAL;
}
}
return 0;
}
int kbase_csf_firmware_early_init(struct kbase_device *kbdev)
{
init_waitqueue_head(&kbdev->csf.event_wait);
@@ -2253,10 +2411,9 @@ int kbase_csf_firmware_early_init(struct kbase_device *kbdev)
INIT_WORK(&kbdev->csf.firmware_reload_work, kbase_csf_firmware_reload_worker);
INIT_WORK(&kbdev->csf.fw_error_work, firmware_error_worker);
kbdev->csf.glb_init_request_pending = true;
init_rwsem(&kbdev->csf.mmu_sync_sem);
mutex_init(&kbdev->csf.reg_lock);
kbase_csf_pending_gpuq_kicks_init(kbdev);
kbase_csf_pending_gpuq_kick_queues_init(kbdev);
kbdev->csf.fw = (struct kbase_csf_mcu_fw){ .data = NULL };
@@ -2265,7 +2422,7 @@ int kbase_csf_firmware_early_init(struct kbase_device *kbdev)
void kbase_csf_firmware_early_term(struct kbase_device *kbdev)
{
kbase_csf_pending_gpuq_kicks_term(kbdev);
kbase_csf_pending_gpuq_kick_queues_term(kbdev);
mutex_destroy(&kbdev->csf.reg_lock);
}
@@ -2284,6 +2441,8 @@ int kbase_csf_firmware_late_init(struct kbase_device *kbdev)
convert_dur_to_idle_count(kbdev, kbdev->csf.gpu_idle_hysteresis_ns, &no_modifier);
kbdev->csf.gpu_idle_dur_count_no_modifier = no_modifier;
kbdev->csf.csg_suspend_timeout_ms = CSG_SUSPEND_TIMEOUT_MS;
return 0;
}
@@ -2472,6 +2631,8 @@ int kbase_csf_firmware_load_init(struct kbase_device *kbdev)
goto err_out;
}
init_page_fault_cnt_firmware_memory(kbdev);
ret = kbase_csf_firmware_cfg_fw_wa_init(kbdev);
if (ret != 0) {
dev_err(kbdev->dev, "Failed to initialize firmware workarounds");
@@ -2492,6 +2653,8 @@ int kbase_csf_firmware_load_init(struct kbase_device *kbdev)
if (ret != 0)
goto err_out;
iterator_trace_init(kbdev);
ret = kbase_csf_doorbell_mapping_init(kbdev);
if (ret != 0)
goto err_out;
@@ -2522,10 +2685,6 @@ int kbase_csf_firmware_load_init(struct kbase_device *kbdev)
if (ret != 0)
goto err_out;
ret = kbase_device_csf_iterator_trace_init(kbdev);
if (ret != 0)
goto err_out;
if (kbdev->csf.fw_core_dump.available)
kbase_csf_firmware_core_dump_init(kbdev);
@@ -2575,7 +2734,7 @@ void kbase_csf_firmware_unload_term(struct kbase_device *kbdev)
kbdev->csf.firmware_inited = false;
if (WARN_ON(kbdev->pm.backend.mcu_state != KBASE_MCU_OFF)) {
kbdev->pm.backend.mcu_state = KBASE_MCU_OFF;
stop_csf_firmware(kbdev);
kbase_csf_stop_firmware_and_wait(kbdev);
}
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
@@ -2731,7 +2890,7 @@ int kbase_csf_firmware_mcu_register_poll(struct kbase_device *const kbdev, u32 c
unsigned long remaining =
kbase_csf_timeout_in_jiffies(kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT)) +
jiffies;
u32 read_val;
u32 read_val = 0;
dev_dbg(kbdev->dev, "p: reg %08x val %08x mask %08x", reg_addr, reg_val, val_mask);
@@ -2778,12 +2937,10 @@ void kbase_csf_firmware_enable_gpu_idle_timer(struct kbase_device *kbdev)
void kbase_csf_firmware_disable_gpu_idle_timer(struct kbase_device *kbdev)
{
struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface;
kbase_csf_scheduler_spin_lock_assert_held(kbdev);
kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, GLB_REQ_REQ_IDLE_DISABLE,
GLB_REQ_IDLE_DISABLE_MASK);
set_gpu_idle_timer_glb_req(kbdev, false);
atomic_set(&kbdev->csf.scheduler.fw_soi_enabled, false);
dev_dbg(kbdev->dev, "Sending request to disable gpu idle timer");
kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR);
@@ -2807,6 +2964,7 @@ int kbase_csf_firmware_ping_wait(struct kbase_device *const kbdev, unsigned int
return wait_for_global_request_with_timeout(kbdev, GLB_REQ_PING_MASK, wait_timeout_ms);
}
int kbase_csf_firmware_set_timeout(struct kbase_device *const kbdev, u64 const timeout)
{
const struct kbase_csf_global_iface *const global_iface = &kbdev->csf.global_iface;
@@ -2845,8 +3003,6 @@ int kbase_csf_wait_protected_mode_enter(struct kbase_device *kbdev)
{
int err;
lockdep_assert_held(&kbdev->mmu_hw_mutex);
err = wait_for_global_request(kbdev, GLB_REQ_PROTM_ENTER_MASK);
if (!err) {
@@ -2912,6 +3068,7 @@ void kbase_csf_firmware_enable_mcu(struct kbase_device *kbdev)
{
struct kbase_csf_global_iface *iface = &kbdev->csf.global_iface;
lockdep_assert_held(&kbdev->hwaccess_lock);
/* Clear the HALT bit before triggering the boot of MCU firmware */
kbase_csf_firmware_global_input_mask(iface, GLB_REQ, 0, GLB_REQ_HALT_MASK);
@@ -2935,11 +3092,23 @@ void kbase_csf_firmware_trigger_mcu_sleep(struct kbase_device *kbdev)
bool kbase_csf_firmware_is_mcu_in_sleep(struct kbase_device *kbdev)
{
bool db_notif_disabled;
lockdep_assert_held(&kbdev->hwaccess_lock);
return (global_request_complete(kbdev, GLB_REQ_SLEEP_MASK) &&
kbase_csf_firmware_mcu_halted(kbdev));
db_notif_disabled = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(MCU_CONTROL)) &
MCU_CNTRL_DOORBELL_DISABLE_MASK;
if (!db_notif_disabled || !kbase_csf_firmware_mcu_halted(kbdev))
return false;
if (global_request_complete(kbdev, GLB_REQ_SLEEP_MASK))
return true;
kbase_pm_enable_mcu_db_notification(kbdev);
dev_dbg(kbdev->dev, "Enabled DB notification");
return false;
}
#endif
@@ -3191,6 +3360,9 @@ void kbase_csf_firmware_mcu_shared_mapping_term(struct kbase_device *kbdev,
}
if (csf_mapping->phys) {
/* This is on module unload path, so the pages can be left uncleared before
* returning them back to kbdev memory pool.
*/
kbase_mem_pool_free_pages(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW],
csf_mapping->num_pages, csf_mapping->phys, false, false);
}
@@ -3198,3 +3370,127 @@ void kbase_csf_firmware_mcu_shared_mapping_term(struct kbase_device *kbdev,
vunmap(csf_mapping->cpu_addr);
kfree(csf_mapping->phys);
}
#ifdef KBASE_PM_RUNTIME
void kbase_csf_firmware_soi_update(struct kbase_device *kbdev)
{
struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler;
unsigned long flags;
/* There are 3 possibilities:
* - Sleep-on-Idle allowed
* - Sleep-on-Idle not allowed, GLB_IDLE timer disabled
* - Sleep-on-Idle not allowed, GLB_IDLE timer enabled
*/
if (kbase_pm_fw_sleep_on_idle_allowed(kbdev)) {
if (likely(atomic_read(&kbdev->csf.scheduler.fw_soi_enabled)))
return;
} else {
if (test_bit(KBASE_GPU_NON_IDLE_OFF_SLOT_GROUPS_AVAILABLE,
&kbdev->pm.backend.gpu_sleep_allowed)) {
if (likely(!atomic_read(&kbdev->csf.scheduler.gpu_idle_timer_enabled)))
return;
} else if (likely(atomic_read(&kbdev->csf.scheduler.gpu_idle_timer_enabled))) {
return;
}
}
if (kbase_reset_gpu_try_prevent(kbdev))
return;
kbase_csf_scheduler_lock(kbdev);
if (atomic_read(&scheduler->pending_gpu_idle_work) > 0)
goto out_unlock_scheduler_lock;
if ((scheduler->state == SCHED_SUSPENDED) || (scheduler->state == SCHED_SLEEPING))
goto out_unlock_scheduler_lock;
if (kbdev->pm.backend.mcu_state != KBASE_MCU_ON)
goto out_unlock_scheduler_lock;
/* Ensure that an existing DISABLE request is completed before
* proceeding. They are made without waiting for them to complete such
* as when enabling the MCU.
*/
if (wait_for_global_request(kbdev, GLB_REQ_IDLE_DISABLE_MASK)) {
dev_err(kbdev->dev,
"Existing GLB_IDLE timer config change failed to complete in time (gpu_sleep_allowed:%lx)",
kbdev->pm.backend.gpu_sleep_allowed);
goto out_unlock_scheduler_lock;
}
/* Disable the GLB IDLE timer if it's currently enabled */
if (atomic_read(&kbdev->csf.scheduler.gpu_idle_timer_enabled)) {
kbase_csf_scheduler_spin_lock(kbdev, &flags);
kbase_csf_firmware_disable_gpu_idle_timer(kbdev);
kbase_csf_scheduler_spin_unlock(kbdev, flags);
if (wait_for_global_request(kbdev, GLB_REQ_IDLE_DISABLE_MASK)) {
dev_err(kbdev->dev,
"Failed to disable GLB_IDLE timer following FW Sleep-on-Idle config change (gpu_sleep_allowed:%lx)",
kbdev->pm.backend.gpu_sleep_allowed);
goto out_unlock_scheduler_lock;
}
}
/* The GLB IDLE timer and, consequently, FW Sleep-on-Idle could remain
* disabled in certain cases. Otherwise, we shall re-enable GLB IDLE
* timer with the new FW Sleep-on-Idle configuration.
*/
if (!test_bit(KBASE_GPU_NON_IDLE_OFF_SLOT_GROUPS_AVAILABLE,
&kbdev->pm.backend.gpu_sleep_allowed)) {
kbase_csf_scheduler_spin_lock(kbdev, &flags);
kbase_csf_firmware_enable_gpu_idle_timer(kbdev);
kbase_csf_scheduler_spin_unlock(kbdev, flags);
if (wait_for_global_request(kbdev, GLB_REQ_IDLE_ENABLE_MASK)) {
dev_err(kbdev->dev,
"Failed to re-enable GLB_IDLE timer following FW Sleep-on-Idle config change (gpu_sleep_allowed:%lx)",
kbdev->pm.backend.gpu_sleep_allowed);
goto out_unlock_scheduler_lock;
}
}
if (atomic_read(&scheduler->fw_soi_enabled)) {
dev_dbg(kbdev->dev, "FW Sleep-on-Idle was enabled");
KBASE_KTRACE_ADD(kbdev, FIRMWARE_SLEEP_ON_IDLE_CHANGED, NULL, true);
} else {
dev_dbg(kbdev->dev, "FW Sleep-on-Idle was disabled");
KBASE_KTRACE_ADD(kbdev, FIRMWARE_SLEEP_ON_IDLE_CHANGED, NULL, false);
}
out_unlock_scheduler_lock:
kbase_csf_scheduler_unlock(kbdev);
kbase_reset_gpu_allow(kbdev);
}
int kbase_csf_firmware_soi_disable_on_scheduler_suspend(struct kbase_device *kbdev)
{
struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler;
unsigned long flags;
lockdep_assert_held(&scheduler->lock);
if (WARN_ON_ONCE(scheduler->state != SCHED_INACTIVE))
return 0;
if (!atomic_read(&kbdev->csf.scheduler.fw_soi_enabled))
return 0;
kbase_csf_scheduler_spin_lock(kbdev, &flags);
if (atomic_read(&scheduler->fw_soi_enabled)) {
kbase_csf_firmware_disable_gpu_idle_timer(kbdev);
kbase_csf_scheduler_spin_unlock(kbdev, flags);
if (wait_for_global_request(kbdev, GLB_REQ_IDLE_DISABLE_MASK)) {
dev_err(kbdev->dev, "Failed to disable Sleep-on-Idle config");
return -ETIMEDOUT;
}
KBASE_KTRACE_ADD(kbdev, FIRMWARE_SLEEP_ON_IDLE_CHANGED, NULL, false);
} else {
kbase_csf_scheduler_spin_unlock(kbdev, flags);
}
return 0;
}
#endif /* KBASE_PM_RUNTIME */

37
drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.h
View File

@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/*
*
* (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -591,13 +591,20 @@ void kbase_csf_firmware_enable_mcu(struct kbase_device *kbdev);
void kbase_csf_firmware_disable_mcu(struct kbase_device *kbdev);
/**
* kbase_csf_firmware_disable_mcu_wait - Wait for the MCU to reach disabled
* status.
* kbase_csf_firmware_disable_mcu_wait - Wait for the MCU to reach disabled status.
*
* @kbdev: Instance of a GPU platform device that implements a CSF interface.
*/
void kbase_csf_firmware_disable_mcu_wait(struct kbase_device *kbdev);
/**
* kbase_csf_stop_firmware_and_wait - Disable firmware and wait for the MCU to reach
* disabled status.
*
* @kbdev: Instance of a GPU platform device that implements a CSF interface.
*/
void kbase_csf_stop_firmware_and_wait(struct kbase_device *kbdev);
#ifdef KBASE_PM_RUNTIME
/**
* kbase_csf_firmware_trigger_mcu_sleep - Send the command to put MCU in sleep
@@ -618,6 +625,7 @@ void kbase_csf_firmware_trigger_mcu_sleep(struct kbase_device *kbdev);
bool kbase_csf_firmware_is_mcu_in_sleep(struct kbase_device *kbdev);
#endif
/**
* kbase_csf_firmware_trigger_reload() - Trigger the reboot of MCU firmware, for
* the cold boot case firmware image would
@@ -926,4 +934,27 @@ int kbase_csf_trigger_firmware_config_update(struct kbase_device *kbdev);
*/
int kbase_csf_firmware_req_core_dump(struct kbase_device *const kbdev);
#ifdef KBASE_PM_RUNTIME
/**
* kbase_csf_firmware_soi_update - Update FW Sleep-on-Idle config
*
* @kbdev: Device pointer
*
* This function reconfigures the FW Sleep-on-Idle configuration if necessary.
*/
void kbase_csf_firmware_soi_update(struct kbase_device *kbdev);
/**
* kbase_csf_firmware_soi_disable_on_scheduler_suspend - Disable FW Sleep-on-Idle config
* on scheduler suspension
*
* @kbdev: Device pointer
*
* Return: 0 on success, otherwise failure
*/
int kbase_csf_firmware_soi_disable_on_scheduler_suspend(struct kbase_device *kbdev);
#endif /* KBASE_PM_RUNTIME */
#endif

4
drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_cfg.c
View File

@@ -367,10 +367,10 @@ int kbase_csf_firmware_cfg_fw_wa_init(struct kbase_device *kbdev)
*/
entry_count = of_property_count_u32_elems(kbdev->dev->of_node, "quirks-ext");
if (entry_count == -EINVAL)
if (entry_count < 0)
entry_count = of_property_count_u32_elems(kbdev->dev->of_node, "quirks_ext");
if (entry_count == -EINVAL || entry_count == -ENODATA)
if (entry_count < 0)
return 0;
entry_bytes = (size_t)entry_count * sizeof(u32);

113
drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_no_mali.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -670,6 +670,23 @@ static void set_timeout_global(const struct kbase_csf_global_iface *const global
set_global_request(global_iface, GLB_REQ_CFG_PROGRESS_TIMER_MASK);
}
static inline void set_gpu_idle_timer_glb_req(struct kbase_device *const kbdev, bool set)
{
struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface;
kbase_csf_scheduler_spin_lock_assert_held(kbdev);
if (set) {
kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, GLB_REQ_REQ_IDLE_ENABLE,
GLB_REQ_IDLE_ENABLE_MASK);
} else {
kbase_csf_firmware_global_input_mask(
global_iface, GLB_REQ, GLB_REQ_REQ_IDLE_DISABLE, GLB_REQ_IDLE_DISABLE_MASK);
}
atomic_set(&kbdev->csf.scheduler.gpu_idle_timer_enabled, set);
}
static void enable_gpu_idle_timer(struct kbase_device *const kbdev)
{
struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface;
@@ -678,8 +695,11 @@ static void enable_gpu_idle_timer(struct kbase_device *const kbdev)
kbase_csf_firmware_global_input(global_iface, GLB_IDLE_TIMER,
kbdev->csf.gpu_idle_dur_count);
kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, GLB_REQ_REQ_IDLE_ENABLE,
GLB_REQ_IDLE_ENABLE_MASK);
kbase_csf_firmware_global_input_mask(global_iface, GLB_IDLE_TIMER_CONFIG,
kbdev->csf.gpu_idle_dur_count_no_modifier,
GLB_IDLE_TIMER_CONFIG_NO_MODIFIER_MASK);
set_gpu_idle_timer_glb_req(kbdev, true);
dev_dbg(kbdev->dev, "Enabling GPU idle timer with count-value: 0x%.8x",
kbdev->csf.gpu_idle_dur_count);
}
@@ -768,12 +788,6 @@ static void global_init(struct kbase_device *const kbdev, u64 core_mask)
set_timeout_global(global_iface, kbase_csf_timeout_get(kbdev));
/* The GPU idle timer is always enabled for simplicity. Checks will be
* done before scheduling the GPU idle worker to see if it is
* appropriate for the current power policy.
*/
enable_gpu_idle_timer(kbdev);
/* Unmask the interrupts */
kbase_csf_firmware_global_input(global_iface, GLB_ACK_IRQ_MASK, ack_irq_mask);
@@ -857,11 +871,11 @@ static void kbase_csf_firmware_reload_worker(struct work_struct *work)
container_of(work, struct kbase_device, csf.firmware_reload_work);
unsigned long flags;
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
/* Reboot the firmware */
kbase_csf_firmware_enable_mcu(kbdev);
/* Tell MCU state machine to transit to next state */
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
kbdev->csf.firmware_reloaded = true;
kbase_pm_update_state(kbdev);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
@@ -881,6 +895,7 @@ void kbase_csf_firmware_trigger_reload(struct kbase_device *kbdev)
kbdev->csf.firmware_reloaded = true;
}
}
KBASE_EXPORT_TEST_API(kbase_csf_firmware_trigger_reload);
void kbase_csf_firmware_reload_completed(struct kbase_device *kbdev)
{
@@ -889,6 +904,7 @@ void kbase_csf_firmware_reload_completed(struct kbase_device *kbdev)
if (unlikely(!kbdev->csf.firmware_inited))
return;
/* Tell MCU state machine to transit to next state */
kbdev->csf.firmware_reloaded = true;
kbase_pm_update_state(kbdev);
@@ -900,7 +916,7 @@ static u32 convert_dur_to_idle_count(struct kbase_device *kbdev, const u32 dur_n
/* Get the cntfreq_el0 value, which drives the SYSTEM_TIMESTAMP */
u64 freq = kbase_arch_timer_get_cntfrq(kbdev);
u64 dur_val = dur_ns;
u32 cnt_val_u32, reg_val_u32;
u32 cnt_val_u32, reg_val_u32, timer_src;
bool src_system_timestamp = freq > 0;
if (!src_system_timestamp) {
@@ -932,9 +948,9 @@ static u32 convert_dur_to_idle_count(struct kbase_device *kbdev, const u32 dur_n
reg_val_u32 = GLB_IDLE_TIMER_TIMEOUT_SET(0, cnt_val_u32);
/* add the source flag */
reg_val_u32 = GLB_IDLE_TIMER_TIMER_SOURCE_SET(
reg_val_u32, (src_system_timestamp ? GLB_IDLE_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP :
GLB_IDLE_TIMER_TIMER_SOURCE_GPU_COUNTER));
timer_src = src_system_timestamp ? GLB_IDLE_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP :
GLB_IDLE_TIMER_TIMER_SOURCE_GPU_COUNTER;
reg_val_u32 = GLB_IDLE_TIMER_TIMER_SOURCE_SET(reg_val_u32, timer_src);
return reg_val_u32;
}
@@ -989,29 +1005,33 @@ u32 kbase_csf_firmware_set_gpu_idle_hysteresis_time(struct kbase_device *kbdev,
return kbdev->csf.gpu_idle_dur_count;
}
/* The 'reg_lock' is also taken and is held till the update is not
/* The scheduler lock is also taken and is held till the update is not
* complete, to ensure the update of idle timer value by multiple Users
* gets serialized.
*/
mutex_lock(&kbdev->csf.reg_lock);
/* The firmware only reads the new idle timer value when the timer is
* disabled.
*/
kbase_csf_scheduler_spin_lock(kbdev, &flags);
kbase_csf_firmware_disable_gpu_idle_timer(kbdev);
kbase_csf_scheduler_spin_unlock(kbdev, flags);
/* Ensure that the request has taken effect */
wait_for_global_request(kbdev, GLB_REQ_IDLE_DISABLE_MASK);
kbase_csf_scheduler_lock(kbdev);
kbase_csf_scheduler_spin_lock(kbdev, &flags);
kbdev->csf.gpu_idle_hysteresis_ns = dur_ns;
kbdev->csf.gpu_idle_dur_count = hysteresis_val;
kbdev->csf.gpu_idle_dur_count_no_modifier = no_modifier;
kbase_csf_firmware_enable_gpu_idle_timer(kbdev);
kbase_csf_scheduler_spin_unlock(kbdev, flags);
wait_for_global_request(kbdev, GLB_REQ_IDLE_ENABLE_MASK);
mutex_unlock(&kbdev->csf.reg_lock);
if (atomic_read(&kbdev->csf.scheduler.gpu_idle_timer_enabled)) {
/* Timer is already enabled. Disable the timer as FW only reads
* the new idle timer value when timer is re-enabled.
*/
kbase_csf_firmware_disable_gpu_idle_timer(kbdev);
kbase_csf_scheduler_spin_unlock(kbdev, flags);
/* Ensure that the request has taken effect */
wait_for_global_request(kbdev, GLB_REQ_IDLE_DISABLE_MASK);
kbase_csf_scheduler_spin_lock(kbdev, &flags);
kbase_csf_firmware_enable_gpu_idle_timer(kbdev);
kbase_csf_scheduler_spin_unlock(kbdev, flags);
wait_for_global_request(kbdev, GLB_REQ_IDLE_ENABLE_MASK);
} else {
kbase_csf_scheduler_spin_unlock(kbdev, flags);
}
kbase_csf_scheduler_unlock(kbdev);
kbase_csf_scheduler_pm_idle(kbdev);
kbase_reset_gpu_allow(kbdev);
end:
@@ -1118,15 +1138,16 @@ int kbase_csf_firmware_early_init(struct kbase_device *kbdev)
INIT_WORK(&kbdev->csf.firmware_reload_work, kbase_csf_firmware_reload_worker);
INIT_WORK(&kbdev->csf.fw_error_work, firmware_error_worker);
init_rwsem(&kbdev->csf.mmu_sync_sem);
mutex_init(&kbdev->csf.reg_lock);
kbase_csf_pending_gpuq_kicks_init(kbdev);
kbase_csf_pending_gpuq_kick_queues_init(kbdev);
return 0;
}
void kbase_csf_firmware_early_term(struct kbase_device *kbdev)
{
kbase_csf_pending_gpuq_kicks_term(kbdev);
kbase_csf_pending_gpuq_kick_queues_term(kbdev);
mutex_destroy(&kbdev->csf.reg_lock);
}
@@ -1185,6 +1206,7 @@ int kbase_csf_firmware_load_init(struct kbase_device *kbdev)
/* NO_MALI: Don't load the MMU tables or boot CSF firmware */
ret = invent_capabilities(kbdev);
if (ret != 0)
goto error;
@@ -1278,13 +1300,9 @@ void kbase_csf_firmware_enable_gpu_idle_timer(struct kbase_device *kbdev)
void kbase_csf_firmware_disable_gpu_idle_timer(struct kbase_device *kbdev)
{
struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface;
kbase_csf_scheduler_spin_lock_assert_held(kbdev);
kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, GLB_REQ_REQ_IDLE_DISABLE,
GLB_REQ_IDLE_DISABLE_MASK);
set_gpu_idle_timer_glb_req(kbdev, false);
dev_dbg(kbdev->dev, "Sending request to disable gpu idle timer");
kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR);
@@ -1308,6 +1326,7 @@ int kbase_csf_firmware_ping_wait(struct kbase_device *const kbdev, unsigned int
return wait_for_global_request(kbdev, GLB_REQ_PING_MASK);
}
int kbase_csf_firmware_set_timeout(struct kbase_device *const kbdev, u64 const timeout)
{
const struct kbase_csf_global_iface *const global_iface = &kbdev->csf.global_iface;
@@ -1370,6 +1389,8 @@ void kbase_csf_firmware_trigger_mcu_halt(struct kbase_device *kbdev)
void kbase_csf_firmware_enable_mcu(struct kbase_device *kbdev)
{
lockdep_assert_held(&kbdev->hwaccess_lock);
/* Trigger the boot of MCU firmware, Use the AUTO mode as
* otherwise on fast reset, to exit protected mode, MCU will
* not reboot by itself to enter normal mode.
@@ -1384,6 +1405,7 @@ void kbase_csf_firmware_trigger_mcu_sleep(struct kbase_device *kbdev)
unsigned long flags;
kbase_csf_scheduler_spin_lock(kbdev, &flags);
set_gpu_idle_timer_glb_req(kbdev, false);
set_global_request(global_iface, GLB_REQ_SLEEP_MASK);
dev_dbg(kbdev->dev, "Sending sleep request to MCU");
kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR);
@@ -1515,6 +1537,12 @@ void kbase_csf_firmware_disable_mcu(struct kbase_device *kbdev)
kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(MCU_CONTROL), MCU_CONTROL_REQ_DISABLE);
}
void kbase_csf_stop_firmware_and_wait(struct kbase_device *kbdev)
{
/* Stop the MCU firmware, no wait required on NO_MALI instance */
kbase_csf_firmware_disable_mcu(kbdev);
}
void kbase_csf_firmware_disable_mcu_wait(struct kbase_device *kbdev)
{
/* NO_MALI: Nothing to do here */
@@ -1637,3 +1665,16 @@ void kbase_csf_firmware_mcu_shared_mapping_term(struct kbase_device *kbdev,
vunmap(csf_mapping->cpu_addr);
kfree(csf_mapping->phys);
}
#ifdef KBASE_PM_RUNTIME
void kbase_csf_firmware_soi_update(struct kbase_device *kbdev)
{
}
int kbase_csf_firmware_soi_disable_on_scheduler_suspend(struct kbase_device *kbdev)
{
return 0;
}
#endif /* KBASE_PM_RUNTIME */

251
drivers/gpu/arm/bifrost/csf/mali_kbase_csf_fw_io.c Normal file
View File

@@ -0,0 +1,251 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU license.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
*/
#include "mali_kbase.h"
#include "mali_kbase_csf_fw_io.h"
#include <mali_kbase_linux.h>
#include <linux/mutex.h>
static inline u32 input_page_read(const u32 *const input, const u32 offset)
{
WARN_ON(offset % sizeof(u32));
return input[offset / sizeof(u32)];
}
static inline void input_page_write(u32 *const input, const u32 offset, const u32 value)
{
WARN_ON(offset % sizeof(u32));
input[offset / sizeof(u32)] = value;
}
static inline void input_page_partial_write(u32 *const input, const u32 offset, u32 value, u32 mask)
{
WARN_ON(offset % sizeof(u32));
input[offset / sizeof(u32)] = (input_page_read(input, offset) & ~mask) | (value & mask);
}
static inline u32 output_page_read(const u32 *const output, const u32 offset)
{
WARN_ON(offset % sizeof(u32));
return output[offset / sizeof(u32)];
}
void kbase_csf_fw_io_init(struct kbase_csf_fw_io *fw_io)
{
spin_lock_init(&fw_io->lock);
bitmap_zero(fw_io->status, KBASE_FW_IO_STATUS_NUM_BITS);
}
KBASE_EXPORT_TEST_API(kbase_csf_fw_io_init);
void kbase_csf_fw_io_term(struct kbase_csf_fw_io *fw_io)
{
/* Nothing to do. */
}
KBASE_EXPORT_TEST_API(kbase_csf_fw_io_term);
void kbase_csf_fw_io_global_write(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_global_iface *iface, u32 offset, u32 value)
{
const struct kbase_device *const kbdev = iface->kbdev;
lockdep_assert_held(&fw_io->lock);
dev_dbg(kbdev->dev, "glob input w: reg %08x val %08x\n", offset, value);
input_page_write(iface->input, offset, value);
}
KBASE_EXPORT_TEST_API(kbase_csf_fw_io_global_write);
void kbase_csf_fw_io_global_write_mask(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_global_iface *iface, u32 offset,
u32 value, u32 mask)
{
const struct kbase_device *const kbdev = iface->kbdev;
lockdep_assert_held(&fw_io->lock);
dev_dbg(kbdev->dev, "glob input w: reg %08x val %08x mask %08x\n", offset, value, mask);
input_page_partial_write(iface->input, offset, value, mask);
}
KBASE_EXPORT_TEST_API(kbase_csf_fw_io_global_write_mask);
u32 kbase_csf_fw_io_global_input_read(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_global_iface *iface, u32 offset)
{
const struct kbase_device *const kbdev = iface->kbdev;
u32 val;
lockdep_assert_held(&fw_io->lock);
val = input_page_read(iface->input, offset);
dev_dbg(kbdev->dev, "glob input r: reg %08x val %08x\n", offset, val);
return val;
}
KBASE_EXPORT_TEST_API(kbase_csf_fw_io_global_input_read);
u32 kbase_csf_fw_io_global_read(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_global_iface *iface, u32 offset)
{
const struct kbase_device *const kbdev = iface->kbdev;
u32 val;
lockdep_assert_held(&fw_io->lock);
val = output_page_read(iface->output, offset);
dev_dbg(kbdev->dev, "glob output r: reg %08x val %08x\n", offset, val);
return val;
}
void kbase_csf_fw_io_group_write(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_cmd_stream_group_info *info, u32 offset,
u32 value)
{
const struct kbase_device *const kbdev = info->kbdev;
lockdep_assert_held(&fw_io->lock);
dev_dbg(kbdev->dev, "csg input w: reg %08x val %08x\n", offset, value);
input_page_write(info->input, offset, value);
}
KBASE_EXPORT_TEST_API(kbase_csf_fw_io_group_write);
void kbase_csf_fw_io_group_write_mask(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_cmd_stream_group_info *info,
u32 offset, u32 value, u32 mask)
{
const struct kbase_device *const kbdev = info->kbdev;
lockdep_assert_held(&fw_io->lock);
dev_dbg(kbdev->dev, "csg input w: reg %08x val %08x mask %08x\n", offset, value, mask);
input_page_partial_write(info->input, offset, value, mask);
}
KBASE_EXPORT_TEST_API(kbase_csf_fw_io_group_write_mask);
u32 kbase_csf_fw_io_group_input_read(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_cmd_stream_group_info *info, u32 offset)
{
const struct kbase_device *const kbdev = info->kbdev;
u32 val;
lockdep_assert_held(&fw_io->lock);
val = input_page_read(info->input, offset);
dev_dbg(kbdev->dev, "csg input r: reg %08x val %08x\n", offset, val);
return val;
}
KBASE_EXPORT_TEST_API(kbase_csf_fw_io_group_input_read);
u32 kbase_csf_fw_io_group_read(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_cmd_stream_group_info *info, u32 offset)
{
const struct kbase_device *const kbdev = info->kbdev;
u32 val;
lockdep_assert_held(&fw_io->lock);
val = output_page_read(info->output, offset);
dev_dbg(kbdev->dev, "csg output r: reg %08x val %08x\n", offset, val);
return val;
}
void kbase_csf_fw_io_stream_write(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_cmd_stream_info *info, u32 offset,
u32 value)
{
const struct kbase_device *const kbdev = info->kbdev;
lockdep_assert_held(&fw_io->lock);
dev_dbg(kbdev->dev, "cs input w: reg %08x val %08x\n", offset, value);
input_page_write(info->input, offset, value);
}
KBASE_EXPORT_TEST_API(kbase_csf_fw_io_stream_write);
void kbase_csf_fw_io_stream_write_mask(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_cmd_stream_info *info, u32 offset,
u32 value, u32 mask)
{
const struct kbase_device *const kbdev = info->kbdev;
lockdep_assert_held(&fw_io->lock);
dev_dbg(kbdev->dev, "cs input w: reg %08x val %08x mask %08x\n", offset, value, mask);
input_page_partial_write(info->input, offset, value, mask);
}
KBASE_EXPORT_TEST_API(kbase_csf_fw_io_stream_write_mask);
u32 kbase_csf_fw_io_stream_input_read(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_cmd_stream_info *info, u32 offset)
{
const struct kbase_device *const kbdev = info->kbdev;
u32 val;
lockdep_assert_held(&fw_io->lock);
val = input_page_read(info->input, offset);
dev_dbg(kbdev->dev, "cs input r: reg %08x val %08x\n", offset, val);
return val;
}
KBASE_EXPORT_TEST_API(kbase_csf_fw_io_stream_input_read);
u32 kbase_csf_fw_io_stream_read(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_cmd_stream_info *info, u32 offset)
{
const struct kbase_device *const kbdev = info->kbdev;
u32 val;
lockdep_assert_held(&fw_io->lock);
val = output_page_read(info->output, offset);
dev_dbg(kbdev->dev, "cs output r: reg %08x val %08x\n", offset, val);
return val;
}
void kbase_csf_fw_io_set_status(struct kbase_csf_fw_io *fw_io,
enum kbase_csf_fw_io_status_bits status_bit)
{
set_bit(status_bit, fw_io->status);
}
KBASE_EXPORT_TEST_API(kbase_csf_fw_io_set_status);
void kbase_csf_fw_io_clear_status(struct kbase_csf_fw_io *fw_io,
enum kbase_csf_fw_io_status_bits status_bit)
{
clear_bit(status_bit, fw_io->status);
}
bool kbase_csf_fw_io_test_status(struct kbase_csf_fw_io *fw_io,
enum kbase_csf_fw_io_status_bits status_bit)
{
return test_bit(status_bit, fw_io->status);
}
KBASE_EXPORT_TEST_API(kbase_csf_fw_io_test_status);

362
drivers/gpu/arm/bifrost/csf/mali_kbase_csf_fw_io.h Normal file
View File

@@ -0,0 +1,362 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/*
*
* (C) COPYRIGHT 2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU license.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
*/
#ifndef _KBASE_CSF_FW_IO_H_
#define _KBASE_CSF_FW_IO_H_
#include <linux/sched.h>
#include <linux/types.h>
#include <linux/wait.h>
#include <linux/spinlock.h>
/** The wait completed because the GPU was lost. */
#define KBASE_CSF_FW_IO_WAIT_GPU_LOST 1
/** The wait was aborted because of an unexpected event. */
#define KBASE_CSF_FW_IO_WAIT_UNSUPPORTED 255
/**
* enum kbase_csf_fw_io_status_bits - Status bits for firmware I/O interface.
*
* @KBASE_FW_IO_STATUS_GPU_SUSPENDED: The GPU is suspended.
* @KBASE_FW_IO_STATUS_NUM_BITS: Number of bits used to encode the state.
*/
enum kbase_csf_fw_io_status_bits {
KBASE_FW_IO_STATUS_GPU_SUSPENDED = 0,
KBASE_FW_IO_STATUS_NUM_BITS,
};
/**
* struct kbase_csf_fw_io - Manager of firmware input/output interface.
*
* @lock: Mutex to serialize access to the interface.
* @status: Internal status of the MCU interface.
*/
struct kbase_csf_fw_io {
spinlock_t lock;
DECLARE_BITMAP(status, KBASE_FW_IO_STATUS_NUM_BITS);
};
struct kbase_csf_global_iface;
struct kbase_csf_cmd_stream_group_info;
struct kbase_csf_cmd_stream_info;
/**
* kbase_csf_fw_io_init() - Initialize manager of firmware input/output interface.
*
* @fw_io: Firmware I/O interface to initialize.
*/
void kbase_csf_fw_io_init(struct kbase_csf_fw_io *fw_io);
/**
* kbase_csf_fw_io_term() - Terminate manager of firmware input/output interface.
*
* @fw_io: Firmware I/O interface to terminate.
*/
void kbase_csf_fw_io_term(struct kbase_csf_fw_io *fw_io);
/**
* kbase_csf_fw_io_open() - Start a transaction with the firmware input/output interface.
*
* @fw_io: Firmware I/O interface to open.
*
* Return: 0 on success, otherwise an error code reflecting the status of the
* interface.
*/
static inline int kbase_csf_fw_io_open(struct kbase_csf_fw_io *fw_io)
{
if (test_bit(KBASE_FW_IO_STATUS_GPU_SUSPENDED, fw_io->status))
return -KBASE_CSF_FW_IO_WAIT_GPU_LOST;
spin_lock(&fw_io->lock);
return 0;
}
/**
* kbase_csf_fw_io_open_force() - Force a transaction with the firmware input/output interface.
*
* @fw_io: Firmware I/O interface to open.
*
* This function forces the start of a transaction regardless of the status
* of the interface.
*/
static inline void kbase_csf_fw_io_open_force(struct kbase_csf_fw_io *fw_io)
{
spin_lock(&fw_io->lock);
}
/**
* kbase_csf_fw_io_close() - End a transaction with the firmware input/output interface.
*
* @fw_io: Firmware I/O interface to close.
*/
static inline void kbase_csf_fw_io_close(struct kbase_csf_fw_io *fw_io)
{
spin_unlock(&fw_io->lock);
}
/**
* kbase_csf_fw_io_assert_opened() - Assert if a transaction with the firmware input/output
* interface has started.
*
* @fw_io: Firmware I/O interface.
*/
static inline void kbase_csf_fw_io_assert_opened(struct kbase_csf_fw_io *fw_io)
{
lockdep_assert_held(&fw_io->lock);
}
/**
* kbase_csf_fw_io_global_write() - Write a word in the global input page.
*
* @fw_io: Firmware I/O manager.
* @iface: CSF interface provided by the firmware.
* @offset: Offset of the word to write, in bytes.
* @value: Value to be written.
*/
void kbase_csf_fw_io_global_write(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_global_iface *iface, u32 offset,
u32 value);
/**
* kbase_csf_fw_io_global_write_mask() - Write part of a word in the global input page.
*
* @fw_io: Firmware I/O manager.
* @iface: CSF interface provided by the firmware.
* @offset: Offset of the word to write, in bytes.
* @value: Value to be written.
* @mask: Bitmask with the bits to be modified set.
*/
void kbase_csf_fw_io_global_write_mask(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_global_iface *iface, u32 offset,
u32 value, u32 mask);
/**
* kbase_csf_fw_io_global_input_read() - Read a word in the global input page.
*
* @fw_io: Firmware I/O manager.
* @iface: CSF interface provided by the firmware.
* @offset: Offset of the word to be read, in bytes.
*
* Return: Value of the word read from the global input page.
*/
u32 kbase_csf_fw_io_global_input_read(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_global_iface *iface, u32 offset);
/**
* kbase_csf_fw_io_global_read() - Read a word in the global output page.
*
* @fw_io: Firmware I/O manager.
* @iface: CSF interface provided by the firmware.
* @offset: Offset of the word to be read, in bytes.
*
* Return: Value of the word read from the global output page.
*/
u32 kbase_csf_fw_io_global_read(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_global_iface *iface, u32 offset);
/**
* kbase_csf_fw_io_group_write() - Write a word in a CSG's input page.
*
* @fw_io: Firmware I/O manager.
* @info: CSG interface provided by the firmware.
* @offset: Offset of the word to write, in bytes.
* @value: Value to be written.
*/
void kbase_csf_fw_io_group_write(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_cmd_stream_group_info *info, u32 offset,
u32 value);
/**
* kbase_csf_fw_io_group_write_mask() - Write part of a word in a CSG's input page.
*
* @fw_io: Firmware I/O manager.
* @info: CSG interface provided by the firmware.
* @offset: Offset of the word to write, in bytes.
* @value: Value to be written.
* @mask: Bitmask with the bits to be modified set.
*/
void kbase_csf_fw_io_group_write_mask(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_cmd_stream_group_info *info,
u32 offset, u32 value, u32 mask);
/**
* kbase_csf_fw_io_group_input_read() - Read a word in a CSG's input page.
*
* @fw_io: Firmware I/O manager.
* @info: CSG interface provided by the firmware.
* @offset: Offset of the word to be read, in bytes.
*
* Return: Value of the word read from a CSG's input page.
*/
u32 kbase_csf_fw_io_group_input_read(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_cmd_stream_group_info *info,
u32 offset);
/**
* kbase_csf_fw_io_group_read() - Read a word in a CSG's output page.
*
* @fw_io: Firmware I/O manager.
* @info: CSG interface provided by the firmware.
* @offset: Offset of the word to be read, in bytes.
*
* Return: Value of the word read from the CSG's output page.
*/
u32 kbase_csf_fw_io_group_read(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_cmd_stream_group_info *info, u32 offset);
/**
* kbase_csf_fw_io_stream_write() - Write a word in a CS's input page.
*
* @fw_io: Firmware I/O manager.
* @info: CSI interface provided by the firmware.
* @offset: Offset of the word to write, in bytes.
* @value: Value to be written.
*/
void kbase_csf_fw_io_stream_write(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_cmd_stream_info *info, u32 offset,
u32 value);
/**
* kbase_csf_fw_io_stream_write_mask() - Write part of a word in a CS's input page.
*
* @fw_io: Firmware I/O manager.
* @info: CSI interface provided by the firmware.
* @offset: Offset of the word to write, in bytes.
* @value: Value to be written.
* @mask: Bitmask with the bits to be modified set.
*/
void kbase_csf_fw_io_stream_write_mask(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_cmd_stream_info *info, u32 offset,
u32 value, u32 mask);
/**
* kbase_csf_fw_io_stream_input_read() - Read a word in a CS's input page.
*
* @fw_io: Firmware I/O manager.
* @info: CSI interface provided by the firmware.
* @offset: Offset of the word to be read, in bytes.
*
* Return: Value of the word read from a CS's input page.
*/
u32 kbase_csf_fw_io_stream_input_read(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_cmd_stream_info *info, u32 offset);
/**
* kbase_csf_fw_io_stream_read() - Read a word in a CS's output page.
*
* @fw_io: Firmware I/O manager.
* @info: CSI interface provided by the firmware.
* @offset: Offset of the word to be read, in bytes.
*
* Return: Value of the word read from the CS's output page.
*/
u32 kbase_csf_fw_io_stream_read(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_cmd_stream_info *info, u32 offset);
/**
* kbase_csf_fw_io_set_status() - Set a FW I/O status bit.
*
* @fw_io: Firmware I/O manager.
* @status_bit: Status bit to set.
*/
void kbase_csf_fw_io_set_status(struct kbase_csf_fw_io *fw_io,
enum kbase_csf_fw_io_status_bits status_bit);
/**
* kbase_csf_fw_io_clear_status() - Clear a FW I/O status bit.
*
* @fw_io: Firmware I/O manager.
* @status_bit: Status bit to clear.
*/
void kbase_csf_fw_io_clear_status(struct kbase_csf_fw_io *fw_io,
enum kbase_csf_fw_io_status_bits status_bit);
/**
* kbase_csf_fw_io_test_status() - Test a FW I/O status bit.
*
* @fw_io: Firmware I/O manager.
* @status_bit: Status bit to test.
*
* Return: Value of the tested status bit.
*/
bool kbase_csf_fw_io_test_status(struct kbase_csf_fw_io *fw_io,
enum kbase_csf_fw_io_status_bits status_bit);
/**
* kbase_csf_fw_io_wait_event_timeout() - Wait until condition gets true, timeout
* occurs or a FW I/O status bit is set. The rest of the functionalities is equal
* to wait_event_timeout().
*
* @fw_io: Firmware I/O manager.
* @wq_head: The waitqueue to wait on.
* @condition: C expression for the event to wait for
* @timeout: Timeout, in jiffies
*
* Return: Remaining jiffies (at least 1) on success,
* 0 on timeout,
* negative KBASE_CSF_FW_IO_WAIT_* error codes otherwise.
*/
#define kbase_csf_fw_io_wait_event_timeout(fw_io, wq_head, condition, timeout) \
({ \
int __ret; \
int __wait_remaining = wait_event_timeout( \
wq_head, condition || kbasep_csf_fw_io_check_status(fw_io), timeout); \
__ret = kbasep_csf_fw_io_handle_wait_result(fw_io, __wait_remaining); \
__ret; \
})
/**
* kbasep_csf_fw_io_check_status() - Private function to check if any FW I/O status bit is set.
*
* @fw_io: Firmware I/O manager.
*
* Return: True if any FW I/O status bit is set, false otherwise.
*/
static inline bool kbasep_csf_fw_io_check_status(struct kbase_csf_fw_io *fw_io)
{
return !bitmap_empty(fw_io->status, KBASE_FW_IO_STATUS_NUM_BITS);
}
/**
* kbasep_csf_fw_io_handle_wait_result() - Private function to handle the wait_event_timeout()
* result.
*
* @fw_io: Firmware I/O manager
* @wait_remaining: Remaining jiffies returned by wait_event_timeout()
*
* Return: Remaining jiffies (at least 1) on success,
* 0 on timeout,
* negative KBASE_CSF_FW_IO_WAIT_* error codes otherwise.
*/
static inline int kbasep_csf_fw_io_handle_wait_result(struct kbase_csf_fw_io *fw_io,
int wait_remaining)
{
/* Check for any FW IO status bit set */
if (!bitmap_empty(fw_io->status, KBASE_FW_IO_STATUS_NUM_BITS))
return (test_bit(KBASE_FW_IO_STATUS_GPU_SUSPENDED, fw_io->status)) ?
-KBASE_CSF_FW_IO_WAIT_GPU_LOST :
-KBASE_CSF_FW_IO_WAIT_UNSUPPORTED;
return wait_remaining;
}
#endif

294
drivers/gpu/arm/bifrost/csf/mali_kbase_csf_fw_io_no_mali.c Normal file
View File

@@ -0,0 +1,294 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU license.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
*/
#include "mali_kbase.h"
#include "mali_kbase_csf_fw_io.h"
#include <mali_kbase_linux.h>
#include <linux/mutex.h>
static inline u32 input_page_read(const u32 *const input, const u32 offset)
{
WARN_ON(offset % sizeof(u32));
return input[offset / sizeof(u32)];
}
static inline void input_page_write(u32 *const input, const u32 offset, const u32 value)
{
WARN_ON(offset % sizeof(u32));
input[offset / sizeof(u32)] = value;
}
static inline void input_page_partial_write(u32 *const input, const u32 offset, u32 value, u32 mask)
{
WARN_ON(offset % sizeof(u32));
input[offset / sizeof(u32)] = (input_page_read(input, offset) & ~mask) | (value & mask);
}
static inline u32 output_page_read(const u32 *const output, const u32 offset)
{
WARN_ON(offset % sizeof(u32));
return output[offset / sizeof(u32)];
}
static inline void output_page_write(u32 *const output, const u32 offset, const u32 value)
{
WARN_ON(offset % sizeof(u32));
output[offset / sizeof(u32)] = value;
}
void kbase_csf_fw_io_init(struct kbase_csf_fw_io *fw_io)
{
spin_lock_init(&fw_io->lock);
bitmap_zero(fw_io->status, KBASE_FW_IO_STATUS_NUM_BITS);
}
KBASE_EXPORT_TEST_API(kbase_csf_fw_io_init);
void kbase_csf_fw_io_term(struct kbase_csf_fw_io *fw_io)
{
/* Nothing to do. */
}
KBASE_EXPORT_TEST_API(kbase_csf_fw_io_term);
void kbase_csf_fw_io_global_write(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_global_iface *iface, u32 offset, u32 value)
{
const struct kbase_device *const kbdev = iface->kbdev;
lockdep_assert_held(&fw_io->lock);
dev_dbg(kbdev->dev, "glob input w: reg %08x val %08x\n", offset, value);
input_page_write(iface->input, offset, value);
if (offset == GLB_REQ) {
/* NO_MALI: Immediately acknowledge requests - except for PRFCNT_ENABLE
* and PRFCNT_SAMPLE. These will be processed along with the
* corresponding performance counter registers when the global doorbell
* is rung in order to emulate the performance counter sampling behavior
* of the real firmware.
*/
const u32 ack = output_page_read(iface->output, GLB_ACK);
const u32 req_mask = ~(GLB_REQ_PRFCNT_ENABLE_MASK | GLB_REQ_PRFCNT_SAMPLE_MASK);
const u32 toggled = (value ^ ack) & req_mask;
output_page_write(iface->output, GLB_ACK, ack ^ toggled);
}
}
KBASE_EXPORT_TEST_API(kbase_csf_fw_io_global_write);
void kbase_csf_fw_io_global_write_mask(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_global_iface *iface, u32 offset,
u32 value, u32 mask)
{
const struct kbase_device *const kbdev = iface->kbdev;
lockdep_assert_held(&fw_io->lock);
dev_dbg(kbdev->dev, "glob input w: reg %08x val %08x mask %08x\n", offset, value, mask);
/* NO_MALI: Go through existing function to capture writes */
kbase_csf_fw_io_global_write(fw_io, iface, offset,
(input_page_read(iface->input, offset) & ~mask) |
(value & mask));
}
KBASE_EXPORT_TEST_API(kbase_csf_fw_io_global_write_mask);
u32 kbase_csf_fw_io_global_input_read(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_global_iface *iface, u32 offset)
{
const struct kbase_device *const kbdev = iface->kbdev;
u32 val;
lockdep_assert_held(&fw_io->lock);
val = input_page_read(iface->input, offset);
dev_dbg(kbdev->dev, "glob input r: reg %08x val %08x\n", offset, val);
return val;
}
KBASE_EXPORT_TEST_API(kbase_csf_fw_io_global_input_read);
u32 kbase_csf_fw_io_global_read(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_global_iface *iface, u32 offset)
{
const struct kbase_device *const kbdev = iface->kbdev;
u32 val;
lockdep_assert_held(&fw_io->lock);
val = output_page_read(iface->output, offset);
dev_dbg(kbdev->dev, "glob output r: reg %08x val %08x\n", offset, val);
return val;
}
void kbase_csf_fw_io_group_write(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_cmd_stream_group_info *info, u32 offset,
u32 value)
{
const struct kbase_device *const kbdev = info->kbdev;
lockdep_assert_held(&fw_io->lock);
dev_dbg(kbdev->dev, "csg input w: reg %08x val %08x\n", offset, value);
input_page_write(info->input, offset, value);
if (offset == CSG_REQ) {
/* NO_MALI: Immediately acknowledge requests */
output_page_write(info->output, CSG_ACK, value);
}
}
KBASE_EXPORT_TEST_API(kbase_csf_fw_io_group_write);
void kbase_csf_fw_io_group_write_mask(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_cmd_stream_group_info *info,
u32 offset, u32 value, u32 mask)
{
const struct kbase_device *const kbdev = info->kbdev;
lockdep_assert_held(&fw_io->lock);
dev_dbg(kbdev->dev, "csg input w: reg %08x val %08x mask %08x\n", offset, value, mask);
/* NO_MALI: Go through existing function to capture writes */
kbase_csf_fw_io_group_write(fw_io, info, offset,
(input_page_read(info->input, offset) & ~mask) |
(value & mask));
}
KBASE_EXPORT_TEST_API(kbase_csf_fw_io_group_write_mask);
u32 kbase_csf_fw_io_group_input_read(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_cmd_stream_group_info *info, u32 offset)
{
const struct kbase_device *const kbdev = info->kbdev;
u32 val;
lockdep_assert_held(&fw_io->lock);
val = input_page_read(info->input, offset);
dev_dbg(kbdev->dev, "csg input r: reg %08x val %08x\n", offset, val);
return val;
}
KBASE_EXPORT_TEST_API(kbase_csf_fw_io_group_input_read);
u32 kbase_csf_fw_io_group_read(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_cmd_stream_group_info *info, u32 offset)
{
const struct kbase_device *const kbdev = info->kbdev;
u32 val;
lockdep_assert_held(&fw_io->lock);
val = output_page_read(info->output, offset);
dev_dbg(kbdev->dev, "csg output r: reg %08x val %08x\n", offset, val);
return val;
}
void kbase_csf_fw_io_stream_write(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_cmd_stream_info *info, u32 offset,
u32 value)
{
const struct kbase_device *const kbdev = info->kbdev;
lockdep_assert_held(&fw_io->lock);
dev_dbg(kbdev->dev, "cs input w: reg %08x val %08x\n", offset, value);
input_page_write(info->input, offset, value);
if (offset == CS_REQ) {
/* NO_MALI: Immediately acknowledge requests */
output_page_write(info->output, CS_ACK, value);
}
}
KBASE_EXPORT_TEST_API(kbase_csf_fw_io_stream_write);
void kbase_csf_fw_io_stream_write_mask(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_cmd_stream_info *info, u32 offset,
u32 value, u32 mask)
{
const struct kbase_device *const kbdev = info->kbdev;
lockdep_assert_held(&fw_io->lock);
dev_dbg(kbdev->dev, "cs input w: reg %08x val %08x mask %08x\n", offset, value, mask);
/* NO_MALI: Go through existing function to capture writes */
kbase_csf_fw_io_stream_write(fw_io, info, offset,
(input_page_read(info->input, offset) & ~mask) |
(value & mask));
}
KBASE_EXPORT_TEST_API(kbase_csf_fw_io_stream_write_mask);
u32 kbase_csf_fw_io_stream_input_read(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_cmd_stream_info *info, u32 offset)
{
const struct kbase_device *const kbdev = info->kbdev;
u32 val;
lockdep_assert_held(&fw_io->lock);
val = input_page_read(info->input, offset);
dev_dbg(kbdev->dev, "cs input r: reg %08x val %08x\n", offset, val);
return val;
}
KBASE_EXPORT_TEST_API(kbase_csf_fw_io_stream_input_read);
u32 kbase_csf_fw_io_stream_read(struct kbase_csf_fw_io *fw_io,
const struct kbase_csf_cmd_stream_info *info, u32 offset)
{
const struct kbase_device *const kbdev = info->kbdev;
u32 val;
lockdep_assert_held(&fw_io->lock);
val = output_page_read(info->output, offset);
dev_dbg(kbdev->dev, "cs output r: reg %08x val %08x\n", offset, val);
return val;
}
void kbase_csf_fw_io_set_status(struct kbase_csf_fw_io *fw_io,
enum kbase_csf_fw_io_status_bits status_bit)
{
set_bit(status_bit, fw_io->status);
}
KBASE_EXPORT_TEST_API(kbase_csf_fw_io_set_status);
void kbase_csf_fw_io_clear_status(struct kbase_csf_fw_io *fw_io,
enum kbase_csf_fw_io_status_bits status_bit)
{
clear_bit(status_bit, fw_io->status);
}
bool kbase_csf_fw_io_test_status(struct kbase_csf_fw_io *fw_io,
enum kbase_csf_fw_io_status_bits status_bit)
{
return test_bit(status_bit, fw_io->status);
}
KBASE_EXPORT_TEST_API(kbase_csf_fw_io_test_status);

7
drivers/gpu/arm/bifrost/csf/mali_kbase_csf_heap_context_alloc.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -180,8 +180,9 @@ void kbase_csf_heap_context_allocator_term(struct kbase_csf_heap_context_allocat
u64 kbase_csf_heap_context_allocator_alloc(struct kbase_csf_heap_context_allocator *const ctx_alloc)
{
struct kbase_context *const kctx = ctx_alloc->kctx;
u64 flags = BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR | BASE_MEM_PROT_CPU_WR |
BASEP_MEM_NO_USER_FREE | BASE_MEM_PROT_CPU_RD;
base_mem_alloc_flags flags = BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR |
BASE_MEM_PROT_CPU_WR | BASEP_MEM_NO_USER_FREE |
BASE_MEM_PROT_CPU_RD;
u64 nr_pages = PFN_UP(MAX_TILER_HEAPS * ctx_alloc->heap_context_size_aligned);
u64 heap_gpu_va = 0;

232
drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -39,13 +39,7 @@
static DEFINE_SPINLOCK(kbase_csf_fence_lock);
#endif
#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG
#define FENCE_WAIT_TIMEOUT_MS 3000
#endif
static void kcpu_queue_process(struct kbase_kcpu_command_queue *kcpu_queue, bool drain_queue);
static void kcpu_queue_process_worker(struct work_struct *data);
static int kbase_kcpu_map_import_prepare(struct kbase_kcpu_command_queue *kcpu_queue,
struct base_kcpu_command_import_info *import_info,
@@ -377,7 +371,7 @@ static int kbase_kcpu_jit_allocate_prepare(struct kbase_kcpu_command_queue *kcpu
goto out;
}
if (copy_from_user(info, data, sizeof(*info) * count) != 0) {
if (copy_from_user(info, data, size_mul(sizeof(*info), count)) != 0) {
ret = -EINVAL;
goto out_free;
}
@@ -445,6 +439,16 @@ static void kbase_kcpu_jit_allocate_finish(struct kbase_kcpu_command_queue *queu
kfree(cmd->info.jit_alloc.info);
}
static void enqueue_kcpuq_work(struct kbase_kcpu_command_queue *queue)
{
struct kbase_context *const kctx = queue->kctx;
if (!atomic_read(&kctx->prioritized))
queue_work(kctx->csf.kcpu_queues.kcpu_wq, &queue->work);
else
kbase_csf_scheduler_enqueue_kcpuq_work(queue);
}
/**
* kbase_kcpu_jit_retry_pending_allocs() - Retry blocked JIT_ALLOC commands
*
@@ -464,9 +468,7 @@ static void kbase_kcpu_jit_retry_pending_allocs(struct kbase_context *kctx)
* kbase_csf_kcpu_queue_context.jit_lock .
*/
list_for_each_entry(blocked_queue, &kctx->csf.kcpu_queues.jit_blocked_queues, jit_blocked)
queue_work(atomic_read(&kctx->prioritized) ? kctx->csf.kcpu_wq_high_prio :
kctx->csf.kcpu_wq_normal_prio,
&blocked_queue->work);
enqueue_kcpuq_work(blocked_queue);
}
static int kbase_kcpu_jit_free_process(struct kbase_kcpu_command_queue *queue,
@@ -561,7 +563,7 @@ static int kbase_kcpu_jit_free_prepare(struct kbase_kcpu_command_queue *kcpu_que
goto out_free;
}
if (copy_from_user(ids, data, sizeof(*ids) * count)) {
if (copy_from_user(ids, data, size_mul(sizeof(*ids), count))) {
ret = -EINVAL;
goto out_free;
}
@@ -717,11 +719,8 @@ static int kbase_csf_queue_group_suspend_process(struct kbase_context *kctx,
static enum kbase_csf_event_callback_action event_cqs_callback(void *param)
{
struct kbase_kcpu_command_queue *kcpu_queue = (struct kbase_kcpu_command_queue *)param;
struct kbase_context *kctx = kcpu_queue->kctx;
queue_work(atomic_read(&kctx->prioritized) ? kctx->csf.kcpu_wq_high_prio :
kctx->csf.kcpu_wq_normal_prio,
&kcpu_queue->work);
enqueue_kcpuq_work(kcpu_queue);
return KBASE_CSF_EVENT_CALLBACK_KEEP;
}
@@ -853,7 +852,8 @@ static int kbase_kcpu_cqs_wait_prepare(struct kbase_kcpu_command_queue *queue,
if (!objs)
return -ENOMEM;
if (copy_from_user(objs, u64_to_user_ptr(cqs_wait_info->objs), nr_objs * sizeof(*objs))) {
if (copy_from_user(objs, u64_to_user_ptr(cqs_wait_info->objs),
size_mul(nr_objs, sizeof(*objs)))) {
kfree(objs);
return -ENOMEM;
}
@@ -958,7 +958,8 @@ static int kbase_kcpu_cqs_set_prepare(struct kbase_kcpu_command_queue *kcpu_queu
if (!objs)
return -ENOMEM;
if (copy_from_user(objs, u64_to_user_ptr(cqs_set_info->objs), nr_objs * sizeof(*objs))) {
if (copy_from_user(objs, u64_to_user_ptr(cqs_set_info->objs),
size_mul(nr_objs, sizeof(*objs)))) {
kfree(objs);
return -ENOMEM;
}
@@ -1116,7 +1117,7 @@ static int kbase_kcpu_cqs_wait_operation_prepare(
return -ENOMEM;
if (copy_from_user(objs, u64_to_user_ptr(cqs_wait_operation_info->objs),
nr_objs * sizeof(*objs))) {
size_mul(nr_objs, sizeof(*objs)))) {
kfree(objs);
return -ENOMEM;
}
@@ -1281,7 +1282,7 @@ static int kbase_kcpu_cqs_set_operation_prepare(
return -ENOMEM;
if (copy_from_user(objs, u64_to_user_ptr(cqs_set_operation_info->objs),
nr_objs * sizeof(*objs))) {
size_mul(nr_objs, sizeof(*objs)))) {
kfree(objs);
return -ENOMEM;
}
@@ -1322,9 +1323,7 @@ static void kbase_csf_fence_wait_callback(struct dma_fence *fence, struct dma_fe
fence->seqno);
/* Resume kcpu command queue processing. */
queue_work(atomic_read(&kctx->prioritized) ? kctx->csf.kcpu_wq_high_prio :
kctx->csf.kcpu_wq_normal_prio,
&kcpu_queue->work);
enqueue_kcpuq_work(kcpu_queue);
}
static void kbasep_kcpu_fence_wait_cancel(struct kbase_kcpu_command_queue *kcpu_queue,
@@ -1360,7 +1359,6 @@ static void kbasep_kcpu_fence_wait_cancel(struct kbase_kcpu_command_queue *kcpu_
fence_info->fence = NULL;
}
#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG
/**
* fence_timeout_callback() - Timeout callback function for fence-wait
*
@@ -1399,9 +1397,7 @@ static void fence_timeout_callback(struct timer_list *timer)
kbase_sync_fence_info_get(fence, &info);
if (info.status == 1) {
queue_work(atomic_read(&kctx->prioritized) ? kctx->csf.kcpu_wq_high_prio :
kctx->csf.kcpu_wq_normal_prio,
&kcpu_queue->work);
enqueue_kcpuq_work(kcpu_queue);
} else if (info.status == 0) {
dev_warn(kctx->kbdev->dev, "fence has not yet signalled in %ums",
FENCE_WAIT_TIMEOUT_MS);
@@ -1430,7 +1426,6 @@ static void fence_wait_timeout_start(struct kbase_kcpu_command_queue *cmd)
{
mod_timer(&cmd->fence_timeout, jiffies + msecs_to_jiffies(FENCE_WAIT_TIMEOUT_MS));
}
#endif
/**
* kbase_kcpu_fence_wait_process() - Process the kcpu fence wait command
@@ -1469,9 +1464,8 @@ static int kbase_kcpu_fence_wait_process(struct kbase_kcpu_command_queue *kcpu_q
fence_status = cb_err;
if (cb_err == 0) {
kcpu_queue->fence_wait_processed = true;
#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG
fence_wait_timeout_start(kcpu_queue);
#endif
if (IS_ENABLED(CONFIG_MALI_BIFROST_FENCE_DEBUG))
fence_wait_timeout_start(kcpu_queue);
} else if (cb_err == -ENOENT) {
fence_status = dma_fence_get_status(fence);
if (!fence_status) {
@@ -1692,9 +1686,7 @@ static void fence_signal_timeout_cb(struct timer_list *timer)
if (atomic_read(&kcpu_queue->fence_signal_pending_cnt) > 1)
fence_signal_timeout_start(kcpu_queue);
queue_work(atomic_read(&kctx->prioritized) ? kctx->csf.kcpu_wq_high_prio :
kctx->csf.kcpu_wq_normal_prio,
&kcpu_queue->timeout_work);
queue_work(kctx->csf.kcpu_queues.kcpu_wq, &kcpu_queue->timeout_work);
}
}
@@ -1973,7 +1965,7 @@ static void kcpu_queue_process_worker(struct work_struct *data)
container_of(data, struct kbase_kcpu_command_queue, work);
mutex_lock(&queue->lock);
kcpu_queue_process(queue, false);
kbase_csf_kcpu_queue_process(queue, false);
mutex_unlock(&queue->lock);
}
@@ -2006,7 +1998,7 @@ static int delete_queue(struct kbase_context *kctx, u32 id)
/* Drain the remaining work for this queue first and go past
* all the waits.
*/
kcpu_queue_process(queue, true);
kbase_csf_kcpu_queue_process(queue, true);
/* All commands should have been processed */
WARN_ON(queue->num_pending_cmds);
@@ -2022,11 +2014,20 @@ static int delete_queue(struct kbase_context *kctx, u32 id)
mutex_unlock(&queue->lock);
cancel_work_sync(&queue->timeout_work);
/*
* Drain a pending request to process this queue in
* kbase_csf_scheduler_kthread() if any. By this point the
* queue would be empty so this would be a no-op.
*/
kbase_csf_scheduler_wait_for_kthread_pending_work(kctx->kbdev,
&queue->pending_kick);
cancel_work_sync(&queue->work);
mutex_destroy(&queue->lock);
kfree(queue);
vfree(queue);
} else {
dev_dbg(kctx->kbdev->dev, "Attempt to delete a non-existent KCPU queue");
mutex_unlock(&kctx->csf.kcpu_queues.lock);
@@ -2079,7 +2080,7 @@ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_END(struct kbase_device *kbde
KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END(kbdev, queue);
}
static void kcpu_queue_process(struct kbase_kcpu_command_queue *queue, bool drain_queue)
void kbase_csf_kcpu_queue_process(struct kbase_kcpu_command_queue *queue, bool drain_queue)
{
struct kbase_device *kbdev = queue->kctx->kbdev;
bool process_next = true;
@@ -2199,10 +2200,10 @@ static void kcpu_queue_process(struct kbase_kcpu_command_queue *queue, bool drai
KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START(kbdev,
queue);
kbase_gpu_vm_lock(queue->kctx);
kbase_gpu_vm_lock_with_pmode_sync(queue->kctx);
meta = kbase_sticky_resource_acquire(queue->kctx,
cmd->info.import.gpu_va);
kbase_gpu_vm_unlock(queue->kctx);
cmd->info.import.gpu_va, NULL);
kbase_gpu_vm_unlock_with_pmode_sync(queue->kctx);
if (meta == NULL) {
queue->has_error = true;
@@ -2219,10 +2220,10 @@ static void kcpu_queue_process(struct kbase_kcpu_command_queue *queue, bool drai
KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START(kbdev, queue);
kbase_gpu_vm_lock(queue->kctx);
kbase_gpu_vm_lock_with_pmode_sync(queue->kctx);
ret = kbase_sticky_resource_release(queue->kctx, NULL,
cmd->info.import.gpu_va);
kbase_gpu_vm_unlock(queue->kctx);
kbase_gpu_vm_unlock_with_pmode_sync(queue->kctx);
if (!ret) {
queue->has_error = true;
@@ -2240,10 +2241,10 @@ static void kcpu_queue_process(struct kbase_kcpu_command_queue *queue, bool drai
KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START(kbdev,
queue);
kbase_gpu_vm_lock(queue->kctx);
kbase_gpu_vm_lock_with_pmode_sync(queue->kctx);
ret = kbase_sticky_resource_release_force(queue->kctx, NULL,
cmd->info.import.gpu_va);
kbase_gpu_vm_unlock(queue->kctx);
kbase_gpu_vm_unlock_with_pmode_sync(queue->kctx);
if (!ret) {
queue->has_error = true;
@@ -2642,7 +2643,7 @@ int kbase_csf_kcpu_queue_enqueue(struct kbase_context *kctx,
}
queue->num_pending_cmds += enq->nr_commands;
kcpu_queue_process(queue, false);
kbase_csf_kcpu_queue_process(queue, false);
}
out:
@@ -2653,23 +2654,14 @@ out:
int kbase_csf_kcpu_queue_context_init(struct kbase_context *kctx)
{
kctx->csf.kcpu_wq_high_prio = alloc_workqueue("mali_kcpu_wq_%i_high_prio",
WQ_UNBOUND | WQ_HIGHPRI, 0, kctx->tgid);
if (kctx->csf.kcpu_wq_high_prio == NULL) {
kctx->csf.kcpu_queues.kcpu_wq =
alloc_workqueue("mali_kcpu_wq_%i_%i", 0, 0, kctx->tgid, kctx->id);
if (kctx->csf.kcpu_queues.kcpu_wq == NULL) {
dev_err(kctx->kbdev->dev,
"Failed to initialize KCPU queue high-priority workqueue");
return -ENOMEM;
}
kctx->csf.kcpu_wq_normal_prio =
alloc_workqueue("mali_kcpu_wq_%i_normal_prio", 0, 0, kctx->tgid);
if (kctx->csf.kcpu_wq_normal_prio == NULL) {
dev_err(kctx->kbdev->dev,
"Failed to initialize KCPU queue normal-priority workqueue");
destroy_workqueue(kctx->csf.kcpu_wq_high_prio);
return -ENOMEM;
}
mutex_init(&kctx->csf.kcpu_queues.lock);
return 0;
@@ -2688,8 +2680,7 @@ void kbase_csf_kcpu_queue_context_term(struct kbase_context *kctx)
mutex_destroy(&kctx->csf.kcpu_queues.lock);
destroy_workqueue(kctx->csf.kcpu_wq_normal_prio);
destroy_workqueue(kctx->csf.kcpu_wq_high_prio);
destroy_workqueue(kctx->csf.kcpu_queues.kcpu_wq);
}
KBASE_EXPORT_TEST_API(kbase_csf_kcpu_queue_context_term);
@@ -2699,15 +2690,42 @@ int kbase_csf_kcpu_queue_delete(struct kbase_context *kctx,
return delete_queue(kctx, (u32)del->id);
}
static struct kbase_kcpu_dma_fence_meta *
kbase_csf_kcpu_queue_metadata_new(struct kbase_context *kctx, u64 fence_context)
{
int n;
struct kbase_kcpu_dma_fence_meta *metadata = kzalloc(sizeof(*metadata), GFP_KERNEL);
if (!metadata)
goto early_ret;
*metadata = (struct kbase_kcpu_dma_fence_meta){
.kbdev = kctx->kbdev,
.kctx_id = kctx->id,
};
/* Please update MAX_TIMELINE_NAME macro when making changes to the string. */
n = scnprintf(metadata->timeline_name, MAX_TIMELINE_NAME, "%u-%d_%u-%llu-kcpu",
kctx->kbdev->id, kctx->tgid, kctx->id, fence_context);
if (WARN_ON(n >= MAX_TIMELINE_NAME)) {
kfree(metadata);
metadata = NULL;
goto early_ret;
}
kbase_refcount_set(&metadata->refcount, 1);
early_ret:
return metadata;
}
KBASE_ALLOW_ERROR_INJECTION_TEST_API(kbase_csf_kcpu_queue_metadata_new, ERRNO_NULL);
int kbase_csf_kcpu_queue_new(struct kbase_context *kctx, struct kbase_ioctl_kcpu_queue_new *newq)
{
struct kbase_kcpu_command_queue *queue;
int idx;
int n;
int ret = 0;
#if IS_ENABLED(CONFIG_SYNC_FILE)
struct kbase_kcpu_dma_fence_meta *metadata;
#endif
int idx;
int ret = 0;
/* The queue id is of u8 type and we use the index of the kcpu_queues
* array as an id, so the number of elements in the array can't be
* more than 256.
@@ -2727,54 +2745,48 @@ int kbase_csf_kcpu_queue_new(struct kbase_context *kctx, struct kbase_ioctl_kcpu
goto out;
}
queue = kzalloc(sizeof(*queue), GFP_KERNEL);
queue = vzalloc(sizeof(*queue));
if (!queue) {
ret = -ENOMEM;
goto out;
}
*queue = (struct kbase_kcpu_command_queue)
{
.kctx = kctx, .start_offset = 0, .num_pending_cmds = 0, .enqueue_failed = false,
.command_started = false, .has_error = false, .id = idx,
#if IS_ENABLED(CONFIG_SYNC_FILE)
.fence_context = dma_fence_context_alloc(1), .fence_seqno = 0,
.fence_wait_processed = false,
#endif /* IS_ENABLED(CONFIG_SYNC_FILE) */
};
mutex_init(&queue->lock);
INIT_WORK(&queue->work, kcpu_queue_process_worker);
INIT_LIST_HEAD(&queue->high_prio_work);
atomic_set(&queue->pending_kick, 0);
INIT_WORK(&queue->timeout_work, kcpu_queue_timeout_worker);
INIT_LIST_HEAD(&queue->jit_blocked);
if (IS_ENABLED(CONFIG_SYNC_FILE)) {
metadata = kbase_csf_kcpu_queue_metadata_new(kctx, queue->fence_context);
if (!metadata) {
vfree(queue);
ret = -ENOMEM;
goto out;
}
queue->metadata = metadata;
atomic_inc(&kctx->kbdev->live_fence_metadata);
atomic_set(&queue->fence_signal_pending_cnt, 0);
kbase_timer_setup(&queue->fence_signal_timeout, fence_signal_timeout_cb);
}
if (IS_ENABLED(CONFIG_MALI_BIFROST_FENCE_DEBUG))
kbase_timer_setup(&queue->fence_timeout, fence_timeout_callback);
bitmap_set(kctx->csf.kcpu_queues.in_use, (unsigned int)idx, 1);
kctx->csf.kcpu_queues.array[idx] = queue;
mutex_init(&queue->lock);
queue->kctx = kctx;
queue->start_offset = 0;
queue->num_pending_cmds = 0;
#if IS_ENABLED(CONFIG_SYNC_FILE)
queue->fence_context = dma_fence_context_alloc(1);
queue->fence_seqno = 0;
queue->fence_wait_processed = false;
metadata = kzalloc(sizeof(*metadata), GFP_KERNEL);
if (!metadata) {
kfree(queue);
ret = -ENOMEM;
goto out;
}
metadata->kbdev = kctx->kbdev;
metadata->kctx_id = kctx->id;
n = snprintf(metadata->timeline_name, MAX_TIMELINE_NAME, "%u-%d_%u-%llu-kcpu",
kctx->kbdev->id, kctx->tgid, kctx->id, queue->fence_context);
if (WARN_ON(n >= MAX_TIMELINE_NAME)) {
kfree(queue);
kfree(metadata);
ret = -EINVAL;
goto out;
}
kbase_refcount_set(&metadata->refcount, 1);
queue->metadata = metadata;
atomic_inc(&kctx->kbdev->live_fence_metadata);
#endif /* CONFIG_SYNC_FILE */
queue->enqueue_failed = false;
queue->command_started = false;
INIT_LIST_HEAD(&queue->jit_blocked);
queue->has_error = false;
INIT_WORK(&queue->work, kcpu_queue_process_worker);
INIT_WORK(&queue->timeout_work, kcpu_queue_timeout_worker);
queue->id = idx;
newq->id = idx;
/* Fire the tracepoint with the mutex held to enforce correct ordering
@@ -2784,14 +2796,6 @@ int kbase_csf_kcpu_queue_new(struct kbase_context *kctx, struct kbase_ioctl_kcpu
queue->num_pending_cmds);
KBASE_KTRACE_ADD_CSF_KCPU(kctx->kbdev, KCPU_QUEUE_CREATE, queue, queue->fence_context, 0);
#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG
kbase_timer_setup(&queue->fence_timeout, fence_timeout_callback);
#endif
#if IS_ENABLED(CONFIG_SYNC_FILE)
atomic_set(&queue->fence_signal_pending_cnt, 0);
kbase_timer_setup(&queue->fence_signal_timeout, fence_signal_timeout_cb);
#endif
out:
mutex_unlock(&kctx->csf.kcpu_queues.lock);

30
drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.h
View File

@@ -243,7 +243,19 @@ struct kbase_kcpu_command {
* @work: struct work_struct which contains a pointer to
* the function which handles processing of kcpu
* commands enqueued into a kcpu command queue;
* part of kernel API for processing workqueues
* part of kernel API for processing workqueues.
* This would be used if the context is not
* prioritised, otherwise it would be handled by
* kbase_csf_scheduler_kthread().
* @high_prio_work: A counterpart to @work, this queue would be
* added to a list to be processed by
* kbase_csf_scheduler_kthread() if it is
* prioritised.
* @pending_kick: Indicates that kbase_csf_scheduler_kthread()
* should re-evaluate pending commands for this
* queue. This would be set to false when the work
* is done. This is used mainly for
* synchronisation with queue termination.
* @timeout_work: struct work_struct which contains a pointer to the
* function which handles post-timeout actions
* queue when a fence signal timeout occurs.
@@ -287,6 +299,8 @@ struct kbase_kcpu_command_queue {
struct kbase_context *kctx;
struct kbase_kcpu_command commands[KBASEP_KCPU_QUEUE_SIZE];
struct work_struct work;
struct list_head high_prio_work;
atomic_t pending_kick;
struct work_struct timeout_work;
u8 start_offset;
u8 id;
@@ -299,9 +313,7 @@ struct kbase_kcpu_command_queue {
bool command_started;
struct list_head jit_blocked;
bool has_error;
#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG
struct timer_list fence_timeout;
#endif /* CONFIG_MALI_BIFROST_FENCE_DEBUG */
#if IS_ENABLED(CONFIG_SYNC_FILE)
struct kbase_kcpu_dma_fence_meta *metadata;
#endif /* CONFIG_SYNC_FILE */
@@ -334,6 +346,18 @@ int kbase_csf_kcpu_queue_new(struct kbase_context *kctx, struct kbase_ioctl_kcpu
int kbase_csf_kcpu_queue_delete(struct kbase_context *kctx,
struct kbase_ioctl_kcpu_queue_delete *del);
/**
* kbase_csf_kcpu_queue_process - Proces pending KCPU queue commands
*
* @queue: The queue to process pending commands for
* @drain_queue: Whether to skip all blocking commands in the queue.
* This is expected to be set to true on queue
* termination.
*
* Return: 0 if successful or a negative error code on failure.
*/
void kbase_csf_kcpu_queue_process(struct kbase_kcpu_command_queue *queue, bool drain_queue);
/**
* kbase_csf_kcpu_queue_enqueue - Enqueue a KCPU command into a KCPU command
* queue.

110
drivers/gpu/arm/bifrost/csf/mali_kbase_csf_registers.h
View File

@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/*
*
* (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -172,6 +172,11 @@
#define CSG_STATUS_EP_CURRENT 0x0010 /* () Endpoint allocation status register */
#define CSG_STATUS_EP_REQ 0x0014 /* () Endpoint request status register */
#define CSG_RESOURCE_DEP 0x001C /* () Current resource dependencies */
/* TODO: GPUCORE-xxxx: Remove after spec alignment, use 0x1C as CSG_RESOURCE_DEP is deprecated*/
/* CSG_OUTPUT_BLOCK register offsets */
#ifndef CSG_PROGRESS_TIMER_STATE
#define CSG_PROGRESS_TIMER_STATE 0x001C /* () Current resource status */
#endif
/* GLB_CONTROL_BLOCK register offsets */
#define GLB_VERSION 0x0000 /* () Global interface version */
@@ -250,7 +255,7 @@
#define GLB_ACK 0x0000 /* () Global acknowledge */
#define GLB_DB_ACK 0x0008 /* () Global doorbell acknowledge */
#define GLB_HALT_STATUS 0x0010 /* () Global halt status */
#define GLB_FATAL_STATUS 0x0010 /* () Global fatal error status */
#define GLB_PRFCNT_STATUS 0x0014 /* () Performance counter status */
#define GLB_PRFCNT_INSERT 0x0018 /* () Performance counter buffer insert index */
#define GLB_DEBUG_FWUTF_RESULT GLB_DEBUG_ARG_OUT0 /* () Firmware debug test result */
@@ -1243,6 +1248,21 @@
CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_MASK))
/* CSG_PROGRESS_TIMER_STATE register */
#ifndef CSG_PROGRESS_TIMER_STATE_GET
#define CSG_PROGRESS_TIMER_STATE_SHIFT 0
#define CSG_PROGRESS_TIMER_STATE_MASK ((u32)0xFFFFFFFF << CSG_PROGRESS_TIMER_STATE_SHIFT)
#define CSG_PROGRESS_TIMER_STATE_GET(reg_val) \
(((reg_val)&CSG_PROGRESS_TIMER_STATE_MASK) >> CSG_PROGRESS_TIMER_STATE_SHIFT)
#define CSG_PROGRESS_TIMER_STATE_SET(reg_val, value) \
(((reg_val) & ~CSG_PROGRESS_TIMER_STATE_MASK) | \
(((value) << CSG_PROGRESS_TIMER_STATE_SHIFT) & CSG_PROGRESS_TIMER_STATE_MASK))
/* CSG_PROGRESS_TIMER_STATE values */
#define CSG_PROGRESS_TIMER_STATE_COMPUTE 0x0
#define CSG_PROGRESS_TIMER_STATE_FRAGMENT 0x1
#define CSG_PROGRESS_TIMER_STATE_TILER 0x2
#define CSG_PROGRESS_TIMER_STATE_NEURAL 0x3
#endif
/* End of CSG_OUTPUT_BLOCK register set definitions */
/* STREAM_CONTROL_BLOCK register set definitions */
@@ -1380,6 +1400,13 @@
#define GLB_REQ_SLEEP_SET(reg_val, value) \
(((reg_val) & ~GLB_REQ_SLEEP_MASK) | \
(((value) << GLB_REQ_SLEEP_SHIFT) & GLB_REQ_SLEEP_MASK))
#define GLB_REQ_CFG_EVICTION_TIMER_SHIFT 16
#define GLB_REQ_CFG_EVICTION_TIMER_MASK (0x1 << GLB_REQ_CFG_EVICTION_TIMER_SHIFT)
#define GLB_REQ_CFG_EVICTION_TIMER_GET(reg_val) \
(((reg_val)&GLB_REQ_CFG_EVICTION_TIMER_MASK) >> GLB_REQ_CFG_EVICTION_TIMER_SHIFT)
#define GLB_REQ_CFG_EVICTION_TIMER_SET(reg_val, value) \
(((reg_val) & ~GLB_REQ_CFG_EVICTION_TIMER_MASK) | \
(((value) << GLB_REQ_CFG_EVICTION_TIMER_SHIFT) & GLB_REQ_CFG_EVICTION_TIMER_MASK))
#define GLB_REQ_INACTIVE_COMPUTE_SHIFT 20
#define GLB_REQ_INACTIVE_COMPUTE_MASK (0x1 << GLB_REQ_INACTIVE_COMPUTE_SHIFT)
#define GLB_REQ_INACTIVE_COMPUTE_GET(reg_val) \
@@ -1422,6 +1449,12 @@
#define GLB_REQ_PRFCNT_OVERFLOW_SET(reg_val, value) \
(((reg_val) & ~GLB_REQ_PRFCNT_OVERFLOW_MASK) | \
(((value) << GLB_REQ_PRFCNT_OVERFLOW_SHIFT) & GLB_REQ_PRFCNT_OVERFLOW_MASK))
#define GLB_ACK_FATAL_SHIFT GPU_U(27)
#define GLB_ACK_FATAL_MASK (GPU_U(0x1) << GLB_ACK_FATAL_SHIFT)
#define GLB_ACK_FATAL_GET(reg_val) (((reg_val)&GLB_ACK_FATAL_MASK) >> GLB_ACK_FATAL_SHIFT)
#define GLB_ACK_FATAL_SET(reg_val, value) \
(~(~(reg_val) | GLB_ACK_FATAL_MASK) | \
(((value) << GLB_ACK_FATAL_SHIFT) & GLB_ACK_FATAL_MASK))
#define GLB_REQ_DEBUG_CSF_REQ_SHIFT 30
#define GLB_REQ_DEBUG_CSF_REQ_MASK (0x1 << GLB_REQ_DEBUG_CSF_REQ_SHIFT)
#define GLB_REQ_DEBUG_CSF_REQ_GET(reg_val) \
@@ -1518,6 +1551,17 @@
(((reg_val) & ~GLB_ACK_IRQ_MASK_FIRMWARE_CONFIG_UPDATE_MASK) | \
(((value) << GLB_ACK_IRQ_MASK_FIRMWARE_CONFIG_UPDATE_SHIFT) & \
GLB_ACK_IRQ_MASK_FIRMWARE_CONFIG_UPDATE_MASK))
#define GLB_ACK_IRQ_MASK_ITER_TRACE_ENABLE_SHIFT 11
#define GLB_ACK_IRQ_MASK_ITER_TRACE_ENABLE_MASK (0x1 << GLB_ACK_IRQ_MASK_ITER_TRACE_ENABLE_SHIFT)
#define GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_SHIFT 16
#define GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_MASK (0x1 << GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_SHIFT)
#define GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_GET(reg_val) \
(((reg_val)&GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_MASK) >> \
GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_SHIFT)
#define GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_SET(reg_val, value) \
(((reg_val) & ~GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_MASK) | \
(((value) << GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_SHIFT) & \
GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_MASK))
#define GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_SHIFT 20
#define GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_MASK (0x1 << GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_SHIFT)
#define GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_GET(reg_val) \
@@ -1629,6 +1673,45 @@
GLB_PWROFF_TIMER_CONFIG_NO_MODIFIER_MASK))
#endif /* End of GLB_PWROFF_TIMER_CONFIG values */
/* GLB_EVICTION_TIMER register */
#ifndef GLB_EVICTION_TIMER
#define GLB_EVICTION_TIMER 0x0090
#define GLB_EVICTION_TIMER_TIMEOUT_SHIFT (0)
#define GLB_EVICTION_TIMER_TIMEOUT_MASK ((0x7FFFFFFF) << GLB_EVICTION_TIMER_TIMEOUT_SHIFT)
#define GLB_EVICTION_TIMER_TIMEOUT_GET(reg_val) \
(((reg_val)&GLB_EVICTION_TIMER_TIMEOUT_MASK) >> GLB_EVICTION_TIMER_TIMEOUT_SHIFT)
#define GLB_EVICTION_TIMER_TIMEOUT_SET(reg_val, value) \
(((reg_val) & ~GLB_EVICTION_TIMER_TIMEOUT_MASK) | \
(((value) << GLB_EVICTION_TIMER_TIMEOUT_SHIFT) & GLB_EVICTION_TIMER_TIMEOUT_MASK))
#define GLB_EVICTION_TIMER_TIMER_SOURCE_SHIFT (31)
#define GLB_EVICTION_TIMER_TIMER_SOURCE_MASK ((0x1) << GLB_EVICTION_TIMER_TIMER_SOURCE_SHIFT)
#define GLB_EVICTION_TIMER_TIMER_SOURCE_GET(reg_val) \
(((reg_val)&GLB_EVICTION_TIMER_TIMER_SOURCE_MASK) >> GLB_EVICTION_TIMER_TIMER_SOURCE_SHIFT)
#define GLB_EVICTION_TIMER_TIMER_SOURCE_SET(reg_val, value) \
(((reg_val) & ~GLB_EVICTION_TIMER_TIMER_SOURCE_MASK) | \
(((value) << GLB_EVICTION_TIMER_TIMER_SOURCE_SHIFT) & \
GLB_EVICTION_TIMER_TIMER_SOURCE_MASK))
/* GLB_EVICTION_TIMER_TIMER_SOURCE values */
#define GLB_EVICTION_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP 0x0U
#define GLB_EVICTION_TIMER_TIMER_SOURCE_GPU_COUNTER 0x1U
/* End of GLB_EVICTION_TIMER_TIMER_SOURCE values */
#endif /* End of GLB_EVICTION_TIMER */
/* GLB_EVICTION_TIMER_CONFIG register */
#ifndef GLB_EVICTION_TIMER_CONFIG
#define GLB_EVICTION_TIMER_CONFIG 0x0094 /* () Configuration fields for GLB_EVICTION_TIMER */
#define GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_SHIFT 0
#define GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_MASK \
(0x1 << GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_SHIFT)
#define GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_GET(reg_val) \
(((reg_val)&GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_MASK) >> \
GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_SHIFT)
#define GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_SET(reg_val, value) \
(((reg_val) & ~GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_MASK) | \
(((value) << GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_SHIFT) & \
GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_MASK))
#endif /* End of GLB_EVICTION_TIMER_CONFIG values */
/* GLB_ALLOC_EN register */
#define GLB_ALLOC_EN_MASK_SHIFT 0
#define GLB_ALLOC_EN_MASK_MASK (GPU_ULL(0xFFFFFFFFFFFFFFFF) << GLB_ALLOC_EN_MASK_SHIFT)
@@ -1711,6 +1794,15 @@
(((reg_val) & ~GLB_IDLE_TIMER_CONFIG_NO_MODIFIER_MASK) | \
(((value) << GLB_IDLE_TIMER_CONFIG_NO_MODIFIER_SHIFT) & \
GLB_IDLE_TIMER_CONFIG_NO_MODIFIER_MASK))
#define GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_SHIFT 9
#define GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_MASK (0x1 << GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_SHIFT)
#define GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_GET(reg_val) \
(((reg_val)&GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_MASK) >> \
GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_SHIFT)
#define GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_SET(reg_val, value) \
(((reg_val) & ~GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_MASK) | \
(((value) << GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_SHIFT) & \
GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_MASK))
#endif /* End of GLB_IDLE_TIMER_CONFIG values */
/* GLB_INSTR_FEATURES register */
@@ -1822,6 +1914,20 @@
(((reg_val) & ~GLB_DEBUG_REQ_RUN_MODE_MASK) | \
(((value) << GLB_DEBUG_REQ_RUN_MODE_SHIFT) & GLB_DEBUG_REQ_RUN_MODE_MASK))
/* GLB_FATAL_STATUS register */
#define GLB_FATAL_STATUS_VALUE_SHIFT GPU_U(0)
#define GLB_FATAL_STATUS_VALUE_MASK (GPU_U(0xFFFFFFFF) << GLB_FATAL_STATUS_VALUE_SHIFT)
#define GLB_FATAL_STATUS_VALUE_GET(reg_val) \
(((reg_val)&GLB_FATAL_STATUS_VALUE_MASK) >> GLB_FATAL_STATUS_VALUE_SHIFT)
enum glb_fatal_status {
GLB_FATAL_STATUS_VALUE_OK,
GLB_FATAL_STATUS_VALUE_ASSERT,
GLB_FATAL_STATUS_VALUE_UNEXPECTED_EXCEPTION,
GLB_FATAL_STATUS_VALUE_HANG,
GLB_FATAL_STATUS_VALUE_COUNT
};
/* GLB_DEBUG_ACK register */
#define GLB_DEBUG_ACK_DEBUG_RUN_SHIFT GPU_U(23)
#define GLB_DEBUG_ACK_DEBUG_RUN_MASK (GPU_U(0x1) << GLB_DEBUG_ACK_DEBUG_RUN_SHIFT)

13
drivers/gpu/arm/bifrost/csf/mali_kbase_csf_reset_gpu.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -224,8 +224,11 @@ static void kbase_csf_reset_end_hw_access(struct kbase_device *kbdev, int err_du
static void kbase_csf_debug_dump_registers(struct kbase_device *kbdev)
{
unsigned long flags;
kbase_io_history_dump(kbdev);
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
dev_err(kbdev->dev, "Register state:");
dev_err(kbdev->dev, " GPU_IRQ_RAWSTAT=0x%08x GPU_STATUS=0x%08x MCU_STATUS=0x%08x",
kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_RAWSTAT)),
@@ -251,6 +254,7 @@ static void kbase_csf_debug_dump_registers(struct kbase_device *kbdev)
kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(TILER_CONFIG)));
}
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
}
/**
@@ -396,6 +400,7 @@ static int kbase_csf_reset_gpu_now(struct kbase_device *kbdev, bool firmware_ini
*/
if (likely(firmware_inited))
kbase_csf_scheduler_reset(kbdev);
cancel_work_sync(&kbdev->csf.firmware_reload_work);
dev_dbg(kbdev->dev, "Disable GPU hardware counters.\n");
@@ -403,6 +408,7 @@ static int kbase_csf_reset_gpu_now(struct kbase_device *kbdev, bool firmware_ini
kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx);
ret = kbase_csf_reset_gpu_once(kbdev, firmware_inited, silent);
if (ret == SOFT_RESET_FAILED) {
dev_err(kbdev->dev, "Soft-reset failed");
goto err;
@@ -490,6 +496,11 @@ static void kbase_csf_reset_gpu_worker(struct work_struct *data)
bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev, unsigned int flags)
{
if (kbase_pm_is_gpu_lost(kbdev)) {
/* GPU access has been removed, reset will be done by Arbiter instead */
return false;
}
if (flags & RESET_FLAGS_HWC_UNRECOVERABLE_ERROR)
kbase_hwcnt_backend_csf_on_unrecoverable_error(&kbdev->hwcnt_gpu_iface);

696
drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.c
View File

File diff suppressed because it is too large Load Diff

54
drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.h
View File

@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/*
*
* (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -235,7 +235,8 @@ void kbase_csf_scheduler_early_term(struct kbase_device *kbdev);
* No explicit re-initialization is done for CSG & CS interface I/O pages;
* instead, that happens implicitly on firmware reload.
*
* Should be called only after initiating the GPU reset.
* Should be called either after initiating the GPU reset or when MCU reset is
* expected to follow such as GPU_LOST case.
*/
void kbase_csf_scheduler_reset(struct kbase_device *kbdev);
@@ -487,6 +488,48 @@ static inline bool kbase_csf_scheduler_all_csgs_idle(struct kbase_device *kbdev)
kbdev->csf.global_iface.group_num);
}
/**
* kbase_csf_scheduler_enqueue_sync_update_work() - Add a context to the list
* of contexts to handle
* SYNC_UPDATE events.
*
* @kctx: The context to handle SYNC_UPDATE event
*
* This function wakes up kbase_csf_scheduler_kthread() to handle pending
* SYNC_UPDATE events for all contexts.
*/
void kbase_csf_scheduler_enqueue_sync_update_work(struct kbase_context *kctx);
/**
* kbase_csf_scheduler_enqueue_protm_event_work() - Add a group to the list
* of groups to handle
* PROTM requests.
*
* @group: The group to handle protected mode request
*
* This function wakes up kbase_csf_scheduler_kthread() to handle pending
* protected mode requests for all groups.
*/
void kbase_csf_scheduler_enqueue_protm_event_work(struct kbase_queue_group *group);
/**
* kbase_csf_scheduler_enqueue_kcpuq_work() - Wake up kbase_csf_scheduler_kthread() to process
* pending commands for a KCPU queue.
*
* @queue: The queue to process pending commands for
*/
void kbase_csf_scheduler_enqueue_kcpuq_work(struct kbase_kcpu_command_queue *queue);
/**
* kbase_csf_scheduler_wait_for_kthread_pending_work - Wait until a pending work has completed in
* kbase_csf_scheduler_kthread().
*
* @kbdev: Instance of a GPU platform device that implements a CSF interface
* @pending: The work to wait for
*/
void kbase_csf_scheduler_wait_for_kthread_pending_work(struct kbase_device *kbdev,
atomic_t *pending);
/**
* kbase_csf_scheduler_invoke_tick() - Invoke the scheduling tick
*
@@ -591,11 +634,8 @@ int kbase_csf_scheduler_handle_runtime_suspend(struct kbase_device *kbdev);
* @kbdev: Pointer to the device
*
* This function is called when a GPU idle IRQ has been raised.
*
* Return: true if the PM state machine needs to be invoked after the processing
* of GPU idle irq, otherwise false.
*/
bool kbase_csf_scheduler_process_gpu_idle_event(struct kbase_device *kbdev);
void kbase_csf_scheduler_process_gpu_idle_event(struct kbase_device *kbdev);
/**
* kbase_csf_scheduler_get_nr_active_csgs() - Get the number of active CSGs
@@ -653,4 +693,6 @@ void kbase_csf_scheduler_force_wakeup(struct kbase_device *kbdev);
void kbase_csf_scheduler_force_sleep(struct kbase_device *kbdev);
#endif
bool is_gpu_level_suspend_supported(struct kbase_device *const kbdev);
#endif /* _KBASE_CSF_SCHEDULER_H_ */

90
drivers/gpu/arm/bifrost/csf/mali_kbase_csf_sync.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -117,13 +117,13 @@ static void kbasep_csf_sync_print_kcpu_fence_wait_or_signal(char *buffer, int *l
timeline_name = fence->ops->get_timeline_name(fence);
is_signaled = info.status > 0;
*length += snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"cmd:%s obj:0x%pK live_value:0x%.8x | ", cmd_name, fence, is_signaled);
*length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"cmd:%s obj:0x%pK live_value:0x%.8x | ", cmd_name, fence, is_signaled);
/* Note: fence->seqno was u32 until 5.1 kernel, then u64 */
*length += snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"timeline_name:%s timeline_context:0x%.16llx fence_seqno:0x%.16llx",
timeline_name, fence->context, (u64)fence->seqno);
*length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"timeline_name:%s timeline_context:0x%.16llx fence_seqno:0x%.16llx",
timeline_name, fence->context, (u64)fence->seqno);
kbase_fence_put(fence);
}
@@ -149,19 +149,19 @@ static void kbasep_csf_sync_print_kcpu_cqs_wait(struct kbase_context *kctx, char
int ret = kbasep_csf_sync_get_cqs_live_u32(kctx, cqs_obj->addr, &live_val);
bool live_val_valid = (ret >= 0);
*length +=
snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"cmd:CQS_WAIT_OPERATION obj:0x%.16llx live_value:", cqs_obj->addr);
*length += scnprintf(
buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"cmd:CQS_WAIT_OPERATION obj:0x%.16llx live_value:", cqs_obj->addr);
if (live_val_valid)
*length += snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"0x%.16llx", (u64)live_val);
*length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"0x%.16llx", (u64)live_val);
else
*length += snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
CQS_UNREADABLE_LIVE_VALUE);
*length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
CQS_UNREADABLE_LIVE_VALUE);
*length += snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
" | op:gt arg_value:0x%.8x", cqs_obj->val);
*length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
" | op:gt arg_value:0x%.8x", cqs_obj->val);
}
}
@@ -187,18 +187,18 @@ static void kbasep_csf_sync_print_kcpu_cqs_set(struct kbase_context *kctx, char
bool live_val_valid = (ret >= 0);
*length +=
snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"cmd:CQS_SET_OPERATION obj:0x%.16llx live_value:", cqs_obj->addr);
scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"cmd:CQS_SET_OPERATION obj:0x%.16llx live_value:", cqs_obj->addr);
if (live_val_valid)
*length += snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"0x%.16llx", (u64)live_val);
*length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"0x%.16llx", (u64)live_val);
else
*length += snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
CQS_UNREADABLE_LIVE_VALUE);
*length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
CQS_UNREADABLE_LIVE_VALUE);
*length += snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
" | op:add arg_value:0x%.8x", 1);
*length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
" | op:add arg_value:0x%.8x", 1);
}
}
@@ -277,19 +277,19 @@ static void kbasep_csf_sync_print_kcpu_cqs_wait_op(struct kbase_context *kctx, c
bool live_val_valid = (ret >= 0);
*length +=
snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"cmd:CQS_WAIT_OPERATION obj:0x%.16llx live_value:", wait_op->addr);
*length += scnprintf(
buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"cmd:CQS_WAIT_OPERATION obj:0x%.16llx live_value:", wait_op->addr);
if (live_val_valid)
*length += snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"0x%.16llx", live_val);
*length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"0x%.16llx", live_val);
else
*length += snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
CQS_UNREADABLE_LIVE_VALUE);
*length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
CQS_UNREADABLE_LIVE_VALUE);
*length += snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
" | op:%s arg_value:0x%.16llx", op_name, wait_op->val);
*length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
" | op:%s arg_value:0x%.16llx", op_name, wait_op->val);
}
}
@@ -319,18 +319,18 @@ static void kbasep_csf_sync_print_kcpu_cqs_set_op(struct kbase_context *kctx, ch
bool live_val_valid = (ret >= 0);
*length +=
snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"cmd:CQS_SET_OPERATION obj:0x%.16llx live_value:", set_op->addr);
scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"cmd:CQS_SET_OPERATION obj:0x%.16llx live_value:", set_op->addr);
if (live_val_valid)
*length += snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"0x%.16llx", live_val);
*length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"0x%.16llx", live_val);
else
*length += snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
CQS_UNREADABLE_LIVE_VALUE);
*length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
CQS_UNREADABLE_LIVE_VALUE);
*length += snprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
" | op:%s arg_value:0x%.16llx", op_name, set_op->val);
*length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
" | op:%s arg_value:0x%.16llx", op_name, set_op->val);
}
}
@@ -360,8 +360,8 @@ static void kbasep_csf_sync_kcpu_print_queue(struct kbase_context *kctx,
int length = 0;
started_or_pending = ((i == 0) && queue->command_started) ? 'S' : 'P';
length += snprintf(buffer, CSF_SYNC_DUMP_SIZE, "queue:KCPU-%d-%d exec:%c ",
kctx->id, queue->id, started_or_pending);
length += scnprintf(buffer, CSF_SYNC_DUMP_SIZE, "queue:KCPU-%d-%d exec:%c ",
kctx->id, queue->id, started_or_pending);
cmd = &queue->commands[(u8)(queue->start_offset + i)];
switch (cmd->type) {
@@ -388,12 +388,12 @@ static void kbasep_csf_sync_kcpu_print_queue(struct kbase_context *kctx,
kbasep_csf_sync_print_kcpu_cqs_set_op(kctx, buffer, &length, cmd);
break;
default:
length += snprintf(buffer + length, CSF_SYNC_DUMP_SIZE - length,
", U, Unknown blocking command");
length += scnprintf(buffer + length, CSF_SYNC_DUMP_SIZE - length,
", U, Unknown blocking command");
break;
}
length += snprintf(buffer + length, CSF_SYNC_DUMP_SIZE - length, "\n");
length += scnprintf(buffer + length, CSF_SYNC_DUMP_SIZE - length, "\n");
kbasep_print(kbpr, buffer);
}

21
drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -218,7 +218,7 @@ static void remove_unlinked_chunk(struct kbase_context *kctx,
if (WARN_ON(!list_empty(&chunk->link)))
return;
kbase_gpu_vm_lock(kctx);
kbase_gpu_vm_lock_with_pmode_sync(kctx);
kbase_vunmap(kctx, &chunk->map);
/* KBASE_REG_DONT_NEED regions will be confused with ephemeral regions (inc freed JIT
* regions), and so we must clear that flag too before freeing.
@@ -231,7 +231,7 @@ static void remove_unlinked_chunk(struct kbase_context *kctx,
chunk->region->flags &= ~KBASE_REG_DONT_NEED;
#endif
kbase_mem_free_region(kctx, chunk->region);
kbase_gpu_vm_unlock(kctx);
kbase_gpu_vm_unlock_with_pmode_sync(kctx);
kfree(chunk);
}
@@ -260,8 +260,9 @@ static struct kbase_csf_tiler_heap_chunk *alloc_new_chunk(struct kbase_context *
u64 chunk_size)
{
u64 nr_pages = PFN_UP(chunk_size);
u64 flags = BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR | BASE_MEM_PROT_CPU_WR |
BASEP_MEM_NO_USER_FREE | BASE_MEM_COHERENT_LOCAL | BASE_MEM_PROT_CPU_RD;
base_mem_alloc_flags flags = BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR |
BASE_MEM_PROT_CPU_WR | BASEP_MEM_NO_USER_FREE |
BASE_MEM_COHERENT_LOCAL | BASE_MEM_PROT_CPU_RD;
struct kbase_csf_tiler_heap_chunk *chunk = NULL;
/* The chunk kernel mapping needs to be large enough to:
* - initially zero the CHUNK_HDR_SIZE area
@@ -350,13 +351,14 @@ static struct kbase_csf_tiler_heap_chunk *alloc_new_chunk(struct kbase_context *
}
remove_external_chunk_mappings(kctx, chunk);
kbase_gpu_vm_unlock(kctx);
/* If page migration is enabled, we don't want to migrate tiler heap pages.
* This does not change if the constituent pages are already marked as isolated.
*/
if (kbase_is_page_migration_enabled())
kbase_set_phy_alloc_page_status(chunk->region->gpu_alloc, NOT_MOVABLE);
kbase_set_phy_alloc_page_status(kctx, chunk->region->gpu_alloc, NOT_MOVABLE);
kbase_gpu_vm_unlock(kctx);
return chunk;
@@ -640,7 +642,7 @@ static bool kbasep_is_buffer_descriptor_region_suitable(struct kbase_context *co
if (!(reg->flags & KBASE_REG_CPU_RD) || kbase_is_region_shrinkable(reg) ||
(reg->flags & KBASE_REG_PF_GROW)) {
dev_err(kctx->kbdev->dev, "Region has invalid flags: 0x%lX!\n", reg->flags);
dev_err(kctx->kbdev->dev, "Region has invalid flags: 0x%llX!\n", reg->flags);
return false;
}
@@ -737,7 +739,7 @@ int kbase_csf_tiler_heap_init(struct kbase_context *const kctx, u32 const chunk_
KBASE_VMAP_FLAG_PERMANENT_MAP_ACCOUNTING);
if (kbase_is_page_migration_enabled())
kbase_set_phy_alloc_page_status(buf_desc_reg->gpu_alloc, NOT_MOVABLE);
kbase_set_phy_alloc_page_status(kctx, buf_desc_reg->gpu_alloc, NOT_MOVABLE);
kbase_gpu_vm_unlock(kctx);
@@ -1058,6 +1060,7 @@ static bool delete_chunk_physical_pages(struct kbase_csf_tiler_heap *heap, u64 c
struct kbase_csf_tiler_heap_chunk *chunk = NULL;
lockdep_assert_held(&heap->kctx->csf.tiler_heaps.lock);
lockdep_assert_held(&kctx->kbdev->csf.scheduler.lock);
chunk = find_chunk(heap, chunk_gpu_va);
if (unlikely(!chunk)) {

26
drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_reclaim.c
View File

@@ -331,8 +331,8 @@ static unsigned long kbase_csf_tiler_heap_reclaim_scan_free_pages(struct kbase_d
static unsigned long kbase_csf_tiler_heap_reclaim_count_objects(struct shrinker *s,
struct shrink_control *sc)
{
struct kbase_device *kbdev =
container_of(s, struct kbase_device, csf.scheduler.reclaim_mgr.heap_reclaim);
struct kbase_device *kbdev = KBASE_GET_KBASE_DATA_FROM_SHRINKER(
s, struct kbase_device, csf.scheduler.reclaim_mgr.heap_reclaim);
return kbase_csf_tiler_heap_reclaim_count_free_pages(kbdev, sc);
}
@@ -340,8 +340,8 @@ static unsigned long kbase_csf_tiler_heap_reclaim_count_objects(struct shrinker
static unsigned long kbase_csf_tiler_heap_reclaim_scan_objects(struct shrinker *s,
struct shrink_control *sc)
{
struct kbase_device *kbdev =
container_of(s, struct kbase_device, csf.scheduler.reclaim_mgr.heap_reclaim);
struct kbase_device *kbdev = KBASE_GET_KBASE_DATA_FROM_SHRINKER(
s, struct kbase_device, csf.scheduler.reclaim_mgr.heap_reclaim);
return kbase_csf_tiler_heap_reclaim_scan_free_pages(kbdev, sc);
}
@@ -352,11 +352,17 @@ void kbase_csf_tiler_heap_reclaim_ctx_init(struct kbase_context *kctx)
INIT_LIST_HEAD(&kctx->csf.sched.heap_info.mgr_link);
}
void kbase_csf_tiler_heap_reclaim_mgr_init(struct kbase_device *kbdev)
int kbase_csf_tiler_heap_reclaim_mgr_init(struct kbase_device *kbdev)
{
struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler;
struct shrinker *reclaim = &scheduler->reclaim_mgr.heap_reclaim;
u8 prio;
struct shrinker *reclaim;
reclaim =
KBASE_INIT_RECLAIM(&(scheduler->reclaim_mgr), heap_reclaim, "mali-csf-tiler-heap");
if (!reclaim)
return -ENOMEM;
KBASE_SET_RECLAIM(&(scheduler->reclaim_mgr), heap_reclaim, reclaim);
for (prio = KBASE_QUEUE_GROUP_PRIORITY_REALTIME; prio < KBASE_QUEUE_GROUP_PRIORITY_COUNT;
prio++)
@@ -366,6 +372,11 @@ void kbase_csf_tiler_heap_reclaim_mgr_init(struct kbase_device *kbdev)
reclaim->scan_objects = kbase_csf_tiler_heap_reclaim_scan_objects;
reclaim->seeks = HEAP_SHRINKER_SEEKS;
reclaim->batch = HEAP_SHRINKER_BATCH;
if (!IS_ENABLED(CONFIG_MALI_VECTOR_DUMP))
KBASE_REGISTER_SHRINKER(reclaim, "mali-csf-tiler-heap", kbdev);
return 0;
}
void kbase_csf_tiler_heap_reclaim_mgr_term(struct kbase_device *kbdev)
@@ -373,6 +384,9 @@ void kbase_csf_tiler_heap_reclaim_mgr_term(struct kbase_device *kbdev)
struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler;
u8 prio;
if (!IS_ENABLED(CONFIG_MALI_VECTOR_DUMP))
KBASE_UNREGISTER_SHRINKER(scheduler->reclaim_mgr.heap_reclaim);
for (prio = KBASE_QUEUE_GROUP_PRIORITY_REALTIME; prio < KBASE_QUEUE_GROUP_PRIORITY_COUNT;
prio++)
WARN_ON(!list_empty(&scheduler->reclaim_mgr.ctx_lists[prio]));

4
drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_reclaim.h
View File

@@ -66,8 +66,10 @@ void kbase_csf_tiler_heap_reclaim_ctx_init(struct kbase_context *kctx);
* @kbdev: Pointer to the device.
*
* This function must be called only when a kbase device is initialized.
*
* Return: 0 if issuing reclaim_mgr init was successful, otherwise an error code.
*/
void kbase_csf_tiler_heap_reclaim_mgr_init(struct kbase_device *kbdev);
int kbase_csf_tiler_heap_reclaim_mgr_init(struct kbase_device *kbdev);
/**
* kbase_csf_tiler_heap_reclaim_mgr_term - Termination call for the tiler heap reclaim manger.

37
drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tl_reader.c
View File

@@ -151,13 +151,22 @@ static bool tl_reader_overflow_check(struct kbase_csf_tl_reader *self, u16 event
*
* Reset the reader to the default state, i.e. set all the
* mutable fields to zero.
*
* NOTE: this function expects the irq spinlock to be held.
*/
static void tl_reader_reset(struct kbase_csf_tl_reader *self)
{
lockdep_assert_held(&self->read_lock);
self->got_first_event = false;
self->is_active = false;
self->expected_event_id = 0;
self->tl_header.btc = 0;
/* There might be data left in the trace buffer from the previous
* tracing session. We don't want it to leak into this session.
*/
kbase_csf_firmware_trace_buffer_discard_all(self->trace_buffer);
}
int kbase_csf_tl_reader_flush_buffer(struct kbase_csf_tl_reader *self)
@@ -324,21 +333,16 @@ static int tl_reader_update_enable_bit(struct kbase_csf_tl_reader *self, bool va
void kbase_csf_tl_reader_init(struct kbase_csf_tl_reader *self, struct kbase_tlstream *stream)
{
self->timer_interval = KBASE_CSF_TL_READ_INTERVAL_DEFAULT;
*self = (struct kbase_csf_tl_reader){
.timer_interval = KBASE_CSF_TL_READ_INTERVAL_DEFAULT,
.stream = stream,
.kbdev = NULL, /* This will be initialized by tl_reader_init_late() */
.is_active = false,
};
kbase_timer_setup(&self->read_timer, kbasep_csf_tl_reader_read_callback);
self->stream = stream;
/* This will be initialized by tl_reader_init_late() */
self->kbdev = NULL;
self->trace_buffer = NULL;
self->tl_header.data = NULL;
self->tl_header.size = 0;
spin_lock_init(&self->read_lock);
tl_reader_reset(self);
}
void kbase_csf_tl_reader_term(struct kbase_csf_tl_reader *self)
@@ -348,13 +352,19 @@ void kbase_csf_tl_reader_term(struct kbase_csf_tl_reader *self)
int kbase_csf_tl_reader_start(struct kbase_csf_tl_reader *self, struct kbase_device *kbdev)
{
unsigned long flags;
int rcode;
spin_lock_irqsave(&self->read_lock, flags);
/* If already running, early exit. */
if (self->is_active)
if (self->is_active) {
spin_unlock_irqrestore(&self->read_lock, flags);
return 0;
}
if (tl_reader_init_late(self, kbdev)) {
spin_unlock_irqrestore(&self->read_lock, flags);
#if IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI)
dev_warn(kbdev->dev, "CSFFW timeline is not available for MALI_BIFROST_NO_MALI builds!");
return 0;
@@ -366,6 +376,9 @@ int kbase_csf_tl_reader_start(struct kbase_csf_tl_reader *self, struct kbase_dev
tl_reader_reset(self);
self->is_active = true;
spin_unlock_irqrestore(&self->read_lock, flags);
/* Set bytes to copy to the header size. This is to trigger copying
* of the header to the user space.
*/

15
drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -469,14 +469,15 @@ unsigned int kbase_csf_firmware_trace_buffer_read_data(struct firmware_trace_buf
} else {
unsigned int bytes_copied_head, bytes_copied_tail;
bytes_copied_tail = min_t(unsigned int, num_bytes, (buffer_size - extract_offset));
bytes_copied_tail =
min_t(unsigned int, num_bytes, size_sub(buffer_size, extract_offset));
memcpy(data, &data_cpu_va[extract_offset], bytes_copied_tail);
bytes_copied_head =
min_t(unsigned int, (num_bytes - bytes_copied_tail), insert_offset);
memcpy(&data[bytes_copied_tail], data_cpu_va, bytes_copied_head);
bytes_copied = bytes_copied_head + bytes_copied_tail;
bytes_copied = size_add(bytes_copied_head, bytes_copied_tail);
extract_offset += bytes_copied;
if (extract_offset >= buffer_size)
extract_offset = bytes_copied_head;
@@ -519,6 +520,14 @@ void kbase_csf_firmware_trace_buffer_discard(struct firmware_trace_buffer *trace
}
EXPORT_SYMBOL(kbase_csf_firmware_trace_buffer_discard);
void kbase_csf_firmware_trace_buffer_discard_all(struct firmware_trace_buffer *trace_buffer)
{
if (WARN_ON(!trace_buffer))
return;
*(trace_buffer->cpu_va.extract_cpu_va) = *(trace_buffer->cpu_va.insert_cpu_va);
}
static void update_trace_buffer_active_mask64(struct firmware_trace_buffer *tb, u64 mask)
{
unsigned int i;

9
drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.h
View File

@@ -179,6 +179,15 @@ unsigned int kbase_csf_firmware_trace_buffer_read_data(struct firmware_trace_buf
*/
void kbase_csf_firmware_trace_buffer_discard(struct firmware_trace_buffer *trace_buffer);
/**
* kbase_csf_firmware_trace_buffer_discard_all - Discard all data from a trace buffer
*
* @trace_buffer: Trace buffer handle
*
* Discard all the data in the trace buffer to make it empty.
*/
void kbase_csf_firmware_trace_buffer_discard_all(struct firmware_trace_buffer *trace_buffer);
/**
* kbase_csf_firmware_trace_buffer_get_active_mask64 - Get trace buffer active mask
*

6
drivers/gpu/arm/bifrost/csf/mali_kbase_csf_util.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -115,7 +115,7 @@ struct kbasep_printer *kbasep_printer_buffer_init(struct kbase_device *kbdev,
if (kbpr) {
if (kfifo_alloc(&kbpr->fifo, KBASEP_PRINTER_BUFFER_MAX_SIZE, GFP_KERNEL)) {
kfree(kbpr);
vfree(kbpr);
return NULL;
}
kbpr->kbdev = kbdev;
@@ -224,7 +224,7 @@ __attribute__((format(__printf__, 2, 3))) void kbasep_print(struct kbasep_printe
va_list arglist;
va_start(arglist, fmt);
len = vsnprintf(buffer, KBASEP_PRINT_FORMAT_BUFFER_MAX_SIZE, fmt, arglist);
len = vscnprintf(buffer, KBASEP_PRINT_FORMAT_BUFFER_MAX_SIZE, fmt, arglist);
if (len <= 0) {
pr_err("message write to the buffer failed");
goto exit;

4
drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_coresight_csf.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2022-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2022-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -443,7 +443,7 @@ kbase_debug_coresight_csf_config_create(void *client_data,
}
config = kzalloc(sizeof(struct kbase_debug_coresight_csf_config), GFP_KERNEL);
if (WARN_ON(!client))
if (WARN_ON(!config))
return NULL;
config->client = client;

5
drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_codes_csf.h
View File

@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/*
*
* (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -212,6 +212,9 @@ KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_EVICT_CTX_SLOTS_START),
KBASE_KTRACE_CODE_MAKE_CODE(SCHED_BUSY), KBASE_KTRACE_CODE_MAKE_CODE(SCHED_INACTIVE),
KBASE_KTRACE_CODE_MAKE_CODE(SCHED_SUSPENDED), KBASE_KTRACE_CODE_MAKE_CODE(SCHED_SLEEPING),
/* info_val == true if FW Sleep-on-Idle is enabled, false otherwise */
KBASE_KTRACE_CODE_MAKE_CODE(FIRMWARE_SLEEP_ON_IDLE_CHANGED),
/* info_val = mcu state */
#define KBASEP_MCU_STATE(n) KBASE_KTRACE_CODE_MAKE_CODE(PM_MCU_##n),
#include "backend/gpu/mali_kbase_pm_mcu_states.h"

31
drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_csf.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -27,8 +27,8 @@
void kbasep_ktrace_backend_format_header(char *buffer, int sz, s32 *written)
{
*written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0),
"group,slot,prio,csi,kcpu"),
*written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0),
"group,slot,prio,csi,kcpu"),
0);
}
@@ -44,38 +44,39 @@ void kbasep_ktrace_backend_format_msg(struct kbase_ktrace_msg *trace_msg, char *
if (be_msg->gpu.flags & KBASE_KTRACE_FLAG_CSF_GROUP) {
const s8 slot = be_msg->gpu.csg_nr;
/* group,slot, */
*written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), "%u,%d,",
be_msg->gpu.group_handle, slot),
*written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0),
"%u,%d,", be_msg->gpu.group_handle, slot),
0);
/* prio */
if (slot >= 0)
*written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0),
"%u", be_msg->gpu.slot_prio),
*written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0),
"%u", be_msg->gpu.slot_prio),
0);
/* , */
*written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), ","), 0);
*written +=
MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), ","), 0);
} else {
/* No group,slot,prio fields, but ensure ending with "," */
*written +=
MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), ",,,"), 0);
MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), ",,,"), 0);
}
/* queue parts: csi */
if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_CSF_QUEUE)
*written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), "%d",
be_msg->gpu.csi_index),
*written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), "%d",
be_msg->gpu.csi_index),
0);
/* , */
*written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), ","), 0);
*written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), ","), 0);
if (be_msg->gpu.flags & KBASE_KTRACE_FLAG_CSF_KCPU) {
/* kcpu data */
*written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0),
"kcpu %d (0x%llx)", be_msg->kcpu.id,
be_msg->kcpu.extra_info_val),
*written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0),
"kcpu %d (0x%llx)", be_msg->kcpu.id,
be_msg->kcpu.extra_info_val),
0);
}

32
drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_jm.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -27,8 +27,8 @@
void kbasep_ktrace_backend_format_header(char *buffer, int sz, s32 *written)
{
*written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0),
"katom,gpu_addr,jobslot,refcount"),
*written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0),
"katom,gpu_addr,jobslot,refcount"),
0);
}
@@ -37,34 +37,34 @@ void kbasep_ktrace_backend_format_msg(struct kbase_ktrace_msg *trace_msg, char *
{
/* katom */
if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_JM_ATOM)
*written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0),
"atom %u (ud: 0x%llx 0x%llx)",
trace_msg->backend.gpu.atom_number,
trace_msg->backend.gpu.atom_udata[0],
trace_msg->backend.gpu.atom_udata[1]),
*written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0),
"atom %u (ud: 0x%llx 0x%llx)",
trace_msg->backend.gpu.atom_number,
trace_msg->backend.gpu.atom_udata[0],
trace_msg->backend.gpu.atom_udata[1]),
0);
/* gpu_addr */
if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_BACKEND)
*written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0),
",%.8llx,", trace_msg->backend.gpu.gpu_addr),
*written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0),
",%.8llx,", trace_msg->backend.gpu.gpu_addr),
0);
else
*written +=
MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), ",,"), 0);
MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), ",,"), 0);
/* jobslot */
if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_JM_JOBSLOT)
*written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), "%d",
trace_msg->backend.gpu.jobslot),
*written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), "%d",
trace_msg->backend.gpu.jobslot),
0);
*written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), ","), 0);
*written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), ","), 0);
/* refcount */
if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_JM_REFCOUNT)
*written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), "%d",
trace_msg->backend.gpu.refcount),
*written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), "%d",
trace_msg->backend.gpu.refcount),
0);
}

3
drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_linux_ktrace_csf.h
View File

@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/*
*
* (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -69,6 +69,7 @@ DEFINE_MALI_ADD_EVENT(SCHED_BUSY);
DEFINE_MALI_ADD_EVENT(SCHED_INACTIVE);
DEFINE_MALI_ADD_EVENT(SCHED_SUSPENDED);
DEFINE_MALI_ADD_EVENT(SCHED_SLEEPING);
DEFINE_MALI_ADD_EVENT(FIRMWARE_SLEEP_ON_IDLE_CHANGED);
#define KBASEP_MCU_STATE(n) DEFINE_MALI_ADD_EVENT(PM_MCU_##n);
#include "backend/gpu/mali_kbase_pm_mcu_states.h"
#undef KBASEP_MCU_STATE

32
drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -71,15 +71,15 @@ static const char *const kbasep_ktrace_code_string[] = {
static void kbasep_ktrace_format_header(char *buffer, int sz, s32 written)
{
written += MAX(snprintf(buffer + written, (size_t)MAX(sz - written, 0),
"secs,thread_id,cpu,code,kctx,"),
written += MAX(scnprintf(buffer + written, (size_t)MAX(sz - written, 0),
"secs,thread_id,cpu,code,kctx,"),
0);
kbasep_ktrace_backend_format_header(buffer, sz, &written);
written += MAX(snprintf(buffer + written, (size_t)MAX(sz - written, 0),
",info_val,ktrace_version=%u.%u", KBASE_KTRACE_VERSION_MAJOR,
KBASE_KTRACE_VERSION_MINOR),
written += MAX(scnprintf(buffer + written, (size_t)MAX(sz - written, 0),
",info_val,ktrace_version=%u.%u", KBASE_KTRACE_VERSION_MAJOR,
KBASE_KTRACE_VERSION_MINOR),
0);
buffer[sz - 1] = 0;
@@ -93,21 +93,21 @@ static void kbasep_ktrace_format_msg(struct kbase_ktrace_msg *trace_msg, char *b
*
* secs,thread_id,cpu,code,
*/
written += MAX(snprintf(buffer + written, (size_t)MAX(sz - written, 0), "%d.%.6d,%d,%d,%s,",
(int)trace_msg->timestamp.tv_sec,
(int)(trace_msg->timestamp.tv_nsec / 1000), trace_msg->thread_id,
trace_msg->cpu,
kbasep_ktrace_code_string[trace_msg->backend.gpu.code]),
written += MAX(scnprintf(buffer + written, (size_t)MAX(sz - written, 0),
"%d.%.6d,%d,%d,%s,", (int)trace_msg->timestamp.tv_sec,
(int)(trace_msg->timestamp.tv_nsec / 1000), trace_msg->thread_id,
trace_msg->cpu,
kbasep_ktrace_code_string[trace_msg->backend.gpu.code]),
0);
/* kctx part: */
if (trace_msg->kctx_tgid) {
written += MAX(snprintf(buffer + written, (size_t)MAX(sz - written, 0), "%d_%u",
trace_msg->kctx_tgid, trace_msg->kctx_id),
written += MAX(scnprintf(buffer + written, (size_t)MAX(sz - written, 0), "%d_%u",
trace_msg->kctx_tgid, trace_msg->kctx_id),
0);
}
/* Trailing comma */
written += MAX(snprintf(buffer + written, (size_t)MAX(sz - written, 0), ","), 0);
written += MAX(scnprintf(buffer + written, (size_t)MAX(sz - written, 0), ","), 0);
/* Backend parts */
kbasep_ktrace_backend_format_msg(trace_msg, buffer, sz, &written);
@@ -119,8 +119,8 @@ static void kbasep_ktrace_format_msg(struct kbase_ktrace_msg *trace_msg, char *b
* Note that the last column is empty, it's simply to hold the ktrace
* version in the header
*/
written += MAX(snprintf(buffer + written, (size_t)MAX(sz - written, 0), ",0x%.16llx",
(unsigned long long)trace_msg->info_val),
written += MAX(scnprintf(buffer + written, (size_t)MAX(sz - written, 0), ",0x%.16llx",
(unsigned long long)trace_msg->info_val),
0);
buffer[sz - 1] = 0;
}

12
drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_codes.h
View File

@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/*
*
* (C) COPYRIGHT 2011-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2011-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -149,13 +149,17 @@ KBASE_KTRACE_CODE_MAKE_CODE(CORE_CTX_DESTROY),
KBASE_KTRACE_CODE_MAKE_CODE(SCHED_RETAIN_CTX_NOLOCK),
/* info_val == kctx->refcount */
KBASE_KTRACE_CODE_MAKE_CODE(SCHED_RELEASE_CTX),
#ifdef CONFIG_MALI_ARBITER_SUPPORT
/*
* Arbitration events
*/
KBASE_KTRACE_CODE_MAKE_CODE(ARB_GPU_LOST), KBASE_KTRACE_CODE_MAKE_CODE(ARB_VM_STATE),
KBASE_KTRACE_CODE_MAKE_CODE(ARB_VM_STATE),
KBASE_KTRACE_CODE_MAKE_CODE(ARB_VM_EVT),
#endif
KBASE_KTRACE_CODE_MAKE_CODE(ARB_GPU_GRANTED),
KBASE_KTRACE_CODE_MAKE_CODE(ARB_GPU_LOST),
KBASE_KTRACE_CODE_MAKE_CODE(ARB_GPU_STARTED),
KBASE_KTRACE_CODE_MAKE_CODE(ARB_GPU_STOP_REQUESTED),
KBASE_KTRACE_CODE_MAKE_CODE(ARB_GPU_STOPPED),
KBASE_KTRACE_CODE_MAKE_CODE(ARB_GPU_REQUESTED),
#if MALI_USE_CSF
#include "debug/backend/mali_kbase_debug_ktrace_codes_csf.h"

11
drivers/gpu/arm/bifrost/debug/mali_kbase_debug_linux_ktrace.h
View File

@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/*
*
* (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -95,13 +95,16 @@ DEFINE_MALI_ADD_EVENT(PM_RUNTIME_RESUME_CALLBACK);
#undef KBASEP_L2_STATE
DEFINE_MALI_ADD_EVENT(SCHED_RETAIN_CTX_NOLOCK);
DEFINE_MALI_ADD_EVENT(SCHED_RELEASE_CTX);
#ifdef CONFIG_MALI_ARBITER_SUPPORT
DEFINE_MALI_ADD_EVENT(ARB_GPU_LOST);
DEFINE_MALI_ADD_EVENT(ARB_VM_STATE);
DEFINE_MALI_ADD_EVENT(ARB_VM_EVT);
DEFINE_MALI_ADD_EVENT(ARB_GPU_GRANTED);
DEFINE_MALI_ADD_EVENT(ARB_GPU_LOST);
DEFINE_MALI_ADD_EVENT(ARB_GPU_STARTED);
DEFINE_MALI_ADD_EVENT(ARB_GPU_STOP_REQUESTED);
DEFINE_MALI_ADD_EVENT(ARB_GPU_STOPPED);
DEFINE_MALI_ADD_EVENT(ARB_GPU_REQUESTED);
#endif
#if MALI_USE_CSF
#include "backend/mali_kbase_debug_linux_ktrace_csf.h"
#else

26
drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_csf.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -132,11 +132,15 @@ static int kbase_backend_late_init(struct kbase_device *kbdev)
fail_update_l2_features:
kbase_backend_devfreq_term(kbdev);
fail_devfreq_init:
kbasep_pm_metrics_term(kbdev);
fail_pm_metrics_init:
kbase_ipa_control_term(kbdev);
fail_devfreq_init:
{
kbasep_pm_metrics_term(kbdev);
}
fail_pm_metrics_init:
{
kbase_ipa_control_term(kbdev);
}
#ifdef CONFIG_MALI_BIFROST_DEBUG
#if IS_ENABLED(CONFIG_MALI_REAL_HW)
fail_interrupt_test:
@@ -159,9 +163,11 @@ fail_reset_gpu_init:
*/
static void kbase_backend_late_term(struct kbase_device *kbdev)
{
kbase_backend_devfreq_term(kbdev);
kbasep_pm_metrics_term(kbdev);
kbase_ipa_control_term(kbdev);
{
kbase_backend_devfreq_term(kbdev);
kbasep_pm_metrics_term(kbdev);
kbase_ipa_control_term(kbdev);
}
kbase_hwaccess_pm_halt(kbdev);
kbase_reset_gpu_term(kbdev);
kbase_hwaccess_pm_term(kbdev);
@@ -279,10 +285,8 @@ static const struct kbase_device_init dev_init[] = {
{ kbase_gpu_device_create, kbase_gpu_device_destroy, "Dummy model initialization failed" },
#else /* !IS_ENABLED(CONFIG_MALI_REAL_HW) */
{ kbase_get_irqs, NULL, "IRQ search failed" },
#endif /* !IS_ENABLED(CONFIG_MALI_REAL_HW) */
#if !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI)
{ registers_map, registers_unmap, "Register map failed" },
#endif /* !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) */
#endif /* !IS_ENABLED(CONFIG_MALI_REAL_HW) */
#if IS_ENABLED(CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD)
{ kbase_gpu_metrics_init, kbase_gpu_metrics_term, "GPU metrics initialization failed" },
#endif /* IS_ENABLED(CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD) */

46
drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_csf.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -28,6 +28,16 @@
#include <mali_kbase_reset_gpu.h>
#include <mmu/mali_kbase_mmu.h>
#include <mali_kbase_ctx_sched.h>
#include <mmu/mali_kbase_mmu_faults_decoder.h>
bool kbase_is_gpu_removed(struct kbase_device *kbdev)
{
if (!kbase_has_arbiter(kbdev))
return false;
return (KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(GPU_ID)) == 0);
}
/**
* kbase_report_gpu_fault - Report a GPU fault of the device.
@@ -78,6 +88,7 @@ static void kbase_gpu_fault_interrupt(struct kbase_device *kbdev)
void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val)
{
u32 power_changed_mask = (POWER_CHANGED_ALL | MCU_STATUS_GPU_IRQ);
struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler;
KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ, NULL, val);
@@ -85,7 +96,6 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val)
kbase_gpu_fault_interrupt(kbdev);
if (val & GPU_PROTECTED_FAULT) {
struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler;
unsigned long flags;
dev_err_ratelimited(kbdev->dev, "GPU fault in protected mode");
@@ -139,10 +149,33 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val)
unsigned long flags;
dev_dbg(kbdev->dev, "Doorbell mirror interrupt received");
/* Assume that the doorbell comes from userspace which
* presents new works in order to invalidate a possible GPU
* idle event.
* If the doorbell was raised by KBase then the FW would handle
* the pending doorbell then raise a 2nd GBL_IDLE IRQ which
* would allow us to put the GPU to sleep.
*/
atomic_set(&scheduler->gpu_no_longer_idle, true);
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
kbase_pm_disable_db_mirror_interrupt(kbdev);
kbdev->pm.backend.exit_gpu_sleep_mode = true;
kbase_csf_scheduler_invoke_tick(kbdev);
if (likely(kbdev->pm.backend.mcu_state == KBASE_MCU_IN_SLEEP)) {
kbdev->pm.backend.exit_gpu_sleep_mode = true;
kbase_csf_scheduler_invoke_tick(kbdev);
} else if (likely(test_bit(KBASE_GPU_SUPPORTS_FW_SLEEP_ON_IDLE,
&kbdev->pm.backend.gpu_sleep_allowed)) &&
(kbdev->pm.backend.mcu_state != KBASE_MCU_ON_PEND_SLEEP)) {
/* The firmware is going to sleep on its own but new
* doorbells were rung before we manage to handle
* the GLB_IDLE IRQ in the bottom half. We shall enable
* DB notification to allow the DB to be handled by FW.
*/
dev_dbg(kbdev->dev, "Re-enabling MCU immediately following DB_MIRROR IRQ");
kbase_pm_enable_mcu_db_notification(kbdev);
}
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
}
#endif
@@ -169,10 +202,13 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val)
* cores.
*/
if (kbdev->pm.backend.l2_always_on ||
kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_921))
kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TTRX_921))
kbase_pm_power_changed(kbdev);
}
if (val & MCU_STATUS_GPU_IRQ)
wake_up_all(&kbdev->csf.event_wait);
KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_DONE, NULL, val);
}
KBASE_EXPORT_TEST_API(kbase_gpu_interrupt);

12
drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_jm.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -28,6 +28,14 @@
#include <mali_kbase_reset_gpu.h>
#include <mmu/mali_kbase_mmu.h>
bool kbase_is_gpu_removed(struct kbase_device *kbdev)
{
if (!kbase_has_arbiter(kbdev))
return false;
return (KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(GPU_ID)) == 0);
}
/**
* kbase_report_gpu_fault - Report a GPU fault.
* @kbdev: Kbase device pointer
@@ -95,7 +103,7 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val)
* cores.
*/
if (kbdev->pm.backend.l2_always_on ||
kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_921))
kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TTRX_921))
kbase_pm_power_changed(kbdev);
}

12
drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_jm.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -32,10 +32,6 @@
#include <hwcnt/backend/mali_kbase_hwcnt_backend_jm_watchdog.h>
#include <backend/gpu/mali_kbase_model_linux.h>
#ifdef CONFIG_MALI_ARBITER_SUPPORT
#include <arbiter/mali_kbase_arbiter_pm.h>
#endif
#include <mali_kbase.h>
#include <backend/gpu/mali_kbase_irq_internal.h>
#include <backend/gpu/mali_kbase_jm_internal.h>
@@ -217,16 +213,14 @@ static const struct kbase_device_init dev_init[] = {
{ kbase_gpu_device_create, kbase_gpu_device_destroy, "Dummy model initialization failed" },
#else /* !IS_ENABLED(CONFIG_MALI_REAL_HW) */
{ kbase_get_irqs, NULL, "IRQ search failed" },
#endif /* !IS_ENABLED(CONFIG_MALI_REAL_HW) */
#if !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI)
{ registers_map, registers_unmap, "Register map failed" },
#endif /* !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) */
#endif /* !IS_ENABLED(CONFIG_MALI_REAL_HW) */
#if IS_ENABLED(CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD)
{ kbase_gpu_metrics_init, kbase_gpu_metrics_term, "GPU metrics initialization failed" },
#endif /* IS_ENABLED(CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD) */
{ power_control_init, power_control_term, "Power control initialization failed" },
{ kbase_device_io_history_init, kbase_device_io_history_term,
"Register access history initialization failed" },
{ kbase_device_pm_init, kbase_device_pm_term, "Power management initialization failed" },
{ kbase_device_early_init, kbase_device_early_term, "Early device initialization failed" },
{ kbase_backend_time_init, NULL, "Time backend initialization failed" },
{ kbase_device_misc_init, kbase_device_misc_term,

78
drivers/gpu/arm/bifrost/device/mali_kbase_device.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -51,10 +51,7 @@
#include "backend/gpu/mali_kbase_irq_internal.h"
#include "mali_kbase_regs_history_debugfs.h"
#include "mali_kbase_pbha.h"
#ifdef CONFIG_MALI_ARBITER_SUPPORT
#include "arbiter/mali_kbase_arbiter_pm.h"
#endif /* CONFIG_MALI_ARBITER_SUPPORT */
#if defined(CONFIG_DEBUG_FS) && !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI)
@@ -69,6 +66,22 @@ static DEFINE_MUTEX(kbase_dev_list_lock);
static LIST_HEAD(kbase_dev_list);
static unsigned int kbase_dev_nr;
static unsigned int mma_wa_id;
static int set_mma_wa_id(const char *val, const struct kernel_param *kp)
{
return kbase_param_set_uint_minmax(val, kp, 1, 15);
}
static const struct kernel_param_ops mma_wa_id_ops = {
.set = set_mma_wa_id,
.get = param_get_uint,
};
module_param_cb(mma_wa_id, &mma_wa_id_ops, &mma_wa_id, 0444);
__MODULE_PARM_TYPE(mma_wa_id, "uint");
MODULE_PARM_DESC(mma_wa_id, "PBHA ID for MMA workaround. Valid range is from 1 to 15.");
struct kbase_device *kbase_device_alloc(void)
{
return vzalloc(sizeof(struct kbase_device));
@@ -320,6 +333,10 @@ int kbase_device_misc_init(struct kbase_device *const kbdev)
if (err)
goto dma_set_mask_failed;
/* Set mma_wa_id if it has been passed in as a module parameter */
if ((kbdev->gpu_props.gpu_id.arch_id >= GPU_ID_ARCH_MAKE(14, 8, 0)) && mma_wa_id != 0)
kbdev->mma_wa_id = mma_wa_id;
err = kbase_pbha_read_dtb(kbdev);
if (err)
goto term_as;
@@ -556,14 +573,27 @@ int kbase_device_early_init(struct kbase_device *kbdev)
/* Ensure we can access the GPU registers */
kbase_pm_register_access_enable(kbdev);
/* Initialize GPU_ID props */
kbase_gpuprops_parse_gpu_id(&kbdev->gpu_props.gpu_id, kbase_reg_get_gpu_id(kbdev));
/* Initialize register mapping LUTs */
err = kbase_regmap_init(kbdev);
if (err)
/*
* If -EPERM is returned, it means the device backend is not supported, but
* device initialization can continue.
*/
err = kbase_device_backend_init(kbdev);
if (err != 0 && err != -EPERM)
goto pm_runtime_term;
/*
* Initialize register mapping LUTs. This would have been initialized on HW
* Arbitration but not on PV or non-arbitration devices.
*/
if (!kbase_reg_is_init(kbdev)) {
/* Initialize GPU_ID props */
kbase_gpuprops_parse_gpu_id(&kbdev->gpu_props.gpu_id, kbase_reg_get_gpu_id(kbdev));
err = kbase_regmap_init(kbdev);
if (err)
goto backend_term;
}
/* Set the list of features available on the current HW
* (identified by the GPU_ID register)
*/
@@ -572,7 +602,7 @@ int kbase_device_early_init(struct kbase_device *kbdev)
/* Find out GPU properties based on the GPU feature registers. */
err = kbase_gpuprops_init(kbdev);
if (err)
goto regmap_term;
goto backend_term;
/* Get the list of workarounds for issues on the current HW
* (identified by the GPU_ID register and impl_tech in THREAD_FEATURES)
@@ -584,14 +614,12 @@ int kbase_device_early_init(struct kbase_device *kbdev)
/* We're done accessing the GPU registers for now. */
kbase_pm_register_access_disable(kbdev);
#ifdef CONFIG_MALI_ARBITER_SUPPORT
if (kbdev->arb.arb_if)
err = kbase_arbiter_pm_install_interrupts(kbdev);
else
if (kbase_has_arbiter(kbdev)) {
if (kbdev->pm.arb_vm_state)
err = kbase_arbiter_pm_install_interrupts(kbdev);
} else {
err = kbase_install_interrupts(kbdev);
#else
err = kbase_install_interrupts(kbdev);
#endif
}
if (err)
goto gpuprops_term;
@@ -599,9 +627,13 @@ int kbase_device_early_init(struct kbase_device *kbdev)
gpuprops_term:
kbase_gpuprops_term(kbdev);
regmap_term:
backend_term:
kbase_device_backend_term(kbdev);
kbase_regmap_term(kbdev);
pm_runtime_term:
if (kbdev->pm.backend.gpu_powered)
kbase_pm_register_access_disable(kbdev);
kbase_pm_runtime_term(kbdev);
platform_device_term:
kbasep_platform_device_term(kbdev);
@@ -613,15 +645,13 @@ ktrace_term:
void kbase_device_early_term(struct kbase_device *kbdev)
{
#ifdef CONFIG_MALI_ARBITER_SUPPORT
if (kbdev->arb.arb_if)
if (kbase_has_arbiter(kbdev))
kbase_arbiter_pm_release_interrupts(kbdev);
else
kbase_release_interrupts(kbdev);
#else
kbase_release_interrupts(kbdev);
#endif /* CONFIG_MALI_ARBITER_SUPPORT */
kbase_gpuprops_term(kbdev);
kbase_device_backend_term(kbdev);
kbase_regmap_term(kbdev);
kbase_pm_runtime_term(kbdev);
kbasep_platform_device_term(kbdev);
kbase_ktrace_term(kbdev);

5
drivers/gpu/arm/bifrost/device/mali_kbase_device.h
View File

@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/*
*
* (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -58,6 +58,9 @@ void kbase_increment_device_id(void);
* When a device file is opened for the first time,
* load firmware and initialize hardware counter components.
*
* It is safe for this function to be called multiple times without ill
* effects. Only the first call would be effective.
*
* Return: 0 on success. An error code on failure.
*/
int kbase_device_firmware_init_once(struct kbase_device *kbdev);

10
drivers/gpu/arm/bifrost/device/mali_kbase_device_hw.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -27,14 +27,6 @@
#include <mali_kbase_reset_gpu.h>
#include <mmu/mali_kbase_mmu.h>
bool kbase_is_gpu_removed(struct kbase_device *kbdev)
{
if (!IS_ENABLED(CONFIG_MALI_ARBITER_SUPPORT))
return false;
return (kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_ID)) == 0);
}
/**
* busy_wait_cache_operation - Wait for a pending cache flush to complete
*

12
drivers/gpu/arm/bifrost/hw_access/backend/mali_kbase_hw_access_model_linux.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -46,7 +46,7 @@ u32 kbase_reg_read32(struct kbase_device *kbdev, u32 reg_enum)
u32 val = 0;
u32 offset;
if (WARN_ON(!kbdev->pm.backend.gpu_powered))
if (WARN_ON(!kbase_reg_is_powered_access_allowed(kbdev, reg_enum)))
return 0;
if (unlikely(!kbase_reg_is_accessible(kbdev, reg_enum,
KBASE_REGMAP_PERM_READ | KBASE_REGMAP_WIDTH_32_BIT)))
@@ -68,7 +68,7 @@ u64 kbase_reg_read64(struct kbase_device *kbdev, u32 reg_enum)
u32 val32[2] = { 0 };
u32 offset;
if (WARN_ON(!kbdev->pm.backend.gpu_powered))
if (WARN_ON(!kbase_reg_is_powered_access_allowed(kbdev, reg_enum)))
return 0;
if (unlikely(!kbase_reg_is_accessible(kbdev, reg_enum,
KBASE_REGMAP_PERM_READ | KBASE_REGMAP_WIDTH_64_BIT)))
@@ -91,7 +91,7 @@ u64 kbase_reg_read64_coherent(struct kbase_device *kbdev, u32 reg_enum)
u32 hi1 = 0, hi2 = 0, lo = 0;
u32 offset;
if (WARN_ON(!kbdev->pm.backend.gpu_powered))
if (WARN_ON(!kbase_reg_is_powered_access_allowed(kbdev, reg_enum)))
return 0;
if (unlikely(!kbase_reg_is_accessible(kbdev, reg_enum,
KBASE_REGMAP_PERM_READ | KBASE_REGMAP_WIDTH_64_BIT)))
@@ -116,7 +116,7 @@ void kbase_reg_write32(struct kbase_device *kbdev, u32 reg_enum, u32 value)
unsigned long flags;
u32 offset;
if (WARN_ON(!kbdev->pm.backend.gpu_powered))
if (WARN_ON(!kbase_reg_is_powered_access_allowed(kbdev, reg_enum)))
return;
if (unlikely(!kbase_reg_is_accessible(kbdev, reg_enum,
KBASE_REGMAP_PERM_WRITE | KBASE_REGMAP_WIDTH_32_BIT)))
@@ -135,7 +135,7 @@ void kbase_reg_write64(struct kbase_device *kbdev, u32 reg_enum, u64 value)
unsigned long flags;
u32 offset;
if (WARN_ON(!kbdev->pm.backend.gpu_powered))
if (WARN_ON(!kbase_reg_is_powered_access_allowed(kbdev, reg_enum)))
return;
if (unlikely(!kbase_reg_is_accessible(kbdev, reg_enum,
KBASE_REGMAP_PERM_WRITE | KBASE_REGMAP_WIDTH_64_BIT)))

36
drivers/gpu/arm/bifrost/hw_access/backend/mali_kbase_hw_access_real_hw.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -24,12 +24,13 @@
#include <mali_kbase.h>
#include <hw_access/mali_kbase_hw_access.h>
#include <linux/mali_hw_access.h>
u64 kbase_reg_get_gpu_id(struct kbase_device *kbdev)
{
u32 val[2] = { 0 };
val[0] = readl(kbdev->reg);
val[0] = mali_readl(kbdev->reg);
return (u64)val[0] | ((u64)val[1] << 32);
@@ -39,13 +40,13 @@ u32 kbase_reg_read32(struct kbase_device *kbdev, u32 reg_enum)
{
u32 val;
if (WARN_ON(!kbdev->pm.backend.gpu_powered))
if (WARN_ON(!kbase_reg_is_powered_access_allowed(kbdev, reg_enum)))
return 0;
if (unlikely(!kbase_reg_is_accessible(kbdev, reg_enum,
KBASE_REGMAP_PERM_READ | KBASE_REGMAP_WIDTH_32_BIT)))
return 0;
val = readl(kbdev->regmap.regs[reg_enum]);
val = mali_readl(kbdev->regmap.regs[reg_enum]);
#if IS_ENABLED(CONFIG_DEBUG_FS)
if (unlikely(kbdev->io_history.enabled))
@@ -63,14 +64,13 @@ u64 kbase_reg_read64(struct kbase_device *kbdev, u32 reg_enum)
{
u64 val;
if (WARN_ON(!kbdev->pm.backend.gpu_powered))
if (WARN_ON(!kbase_reg_is_powered_access_allowed(kbdev, reg_enum)))
return 0;
if (unlikely(!kbase_reg_is_accessible(kbdev, reg_enum,
KBASE_REGMAP_PERM_READ | KBASE_REGMAP_WIDTH_64_BIT)))
return 0;
val = (u64)readl(kbdev->regmap.regs[reg_enum]) |
((u64)readl(kbdev->regmap.regs[reg_enum] + 4) << 32);
val = mali_readq(kbdev->regmap.regs[reg_enum]);
#if IS_ENABLED(CONFIG_DEBUG_FS)
if (unlikely(kbdev->io_history.enabled)) {
@@ -90,23 +90,14 @@ KBASE_EXPORT_TEST_API(kbase_reg_read64);
u64 kbase_reg_read64_coherent(struct kbase_device *kbdev, u32 reg_enum)
{
u64 val;
#if !IS_ENABLED(CONFIG_MALI_64BIT_HW_ACCESS)
u32 hi1, hi2, lo;
#endif
if (WARN_ON(!kbdev->pm.backend.gpu_powered))
if (WARN_ON(!kbase_reg_is_powered_access_allowed(kbdev, reg_enum)))
return 0;
if (unlikely(!kbase_reg_is_accessible(kbdev, reg_enum,
KBASE_REGMAP_PERM_READ | KBASE_REGMAP_WIDTH_64_BIT)))
return 0;
do {
hi1 = readl(kbdev->regmap.regs[reg_enum] + 4);
lo = readl(kbdev->regmap.regs[reg_enum]);
hi2 = readl(kbdev->regmap.regs[reg_enum] + 4);
} while (hi1 != hi2);
val = lo | (((u64)hi1) << 32);
val = mali_readq_coherent(kbdev->regmap.regs[reg_enum]);
#if IS_ENABLED(CONFIG_DEBUG_FS)
if (unlikely(kbdev->io_history.enabled)) {
@@ -125,13 +116,13 @@ KBASE_EXPORT_TEST_API(kbase_reg_read64_coherent);
void kbase_reg_write32(struct kbase_device *kbdev, u32 reg_enum, u32 value)
{
if (WARN_ON(!kbdev->pm.backend.gpu_powered))
if (WARN_ON(!kbase_reg_is_powered_access_allowed(kbdev, reg_enum)))
return;
if (unlikely(!kbase_reg_is_accessible(kbdev, reg_enum,
KBASE_REGMAP_PERM_WRITE | KBASE_REGMAP_WIDTH_32_BIT)))
return;
writel(value, kbdev->regmap.regs[reg_enum]);
mali_writel(value, kbdev->regmap.regs[reg_enum]);
#if IS_ENABLED(CONFIG_DEBUG_FS)
if (unlikely(kbdev->io_history.enabled))
@@ -145,14 +136,13 @@ KBASE_EXPORT_TEST_API(kbase_reg_write32);
void kbase_reg_write64(struct kbase_device *kbdev, u32 reg_enum, u64 value)
{
if (WARN_ON(!kbdev->pm.backend.gpu_powered))
if (WARN_ON(!kbase_reg_is_powered_access_allowed(kbdev, reg_enum)))
return;
if (unlikely(!kbase_reg_is_accessible(kbdev, reg_enum,
KBASE_REGMAP_PERM_WRITE | KBASE_REGMAP_WIDTH_64_BIT)))
return;
writel(value & 0xFFFFFFFF, kbdev->regmap.regs[reg_enum]);
writel(value >> 32, kbdev->regmap.regs[reg_enum] + 4);
mali_writeq(value, kbdev->regmap.regs[reg_enum]);
#if IS_ENABLED(CONFIG_DEBUG_FS)
if (unlikely(kbdev->io_history.enabled)) {

60
drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -24,9 +24,56 @@
#include <mali_kbase.h>
#include "mali_kbase_hw_access.h"
#include "mali_kbase_hw_access_regmap.h"
#include <uapi/gpu/arm/bifrost/gpu/mali_kbase_gpu_id.h>
#define KBASE_REGMAP_ACCESS_ALWAYS_POWERED (1U << 16)
static u32 always_powered_regs[] = {
#if !MALI_USE_CSF
PTM_AW_IRQ_CLEAR,
PTM_AW_IRQ_INJECTION,
PTM_AW_IRQ_MASK,
PTM_AW_IRQ_RAWSTAT,
PTM_AW_IRQ_STATUS,
PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE0,
PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE1,
PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE0,
PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE1,
PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE_STATUS,
PTM_ID,
#endif /* !MALI_USE_CSF */
};
static void kbasep_reg_setup_always_powered_registers(struct kbase_device *kbdev)
{
u32 i;
#if !MALI_USE_CSF
if (kbdev->gpu_props.gpu_id.arch_id < GPU_ID_ARCH_MAKE(9, 14, 0))
return;
#endif /* MALI_USE_CSF */
for (i = 0; i < ARRAY_SIZE(always_powered_regs); i++) {
u32 reg_enum = always_powered_regs[i];
if (!kbase_reg_is_valid(kbdev, reg_enum))
continue;
kbdev->regmap.flags[reg_enum] |= KBASE_REGMAP_ACCESS_ALWAYS_POWERED;
}
}
bool kbase_reg_is_powered_access_allowed(struct kbase_device *kbdev, u32 reg_enum)
{
if (kbdev->regmap.flags[reg_enum] & KBASE_REGMAP_ACCESS_ALWAYS_POWERED)
return true;
return kbdev->pm.backend.gpu_powered;
}
bool kbase_reg_is_size64(struct kbase_device *kbdev, u32 reg_enum)
{
if (WARN_ON(reg_enum >= kbdev->regmap.size))
@@ -67,6 +114,11 @@ bool kbase_reg_is_accessible(struct kbase_device *kbdev, u32 reg_enum, u32 flags
return true;
}
bool kbase_reg_is_init(struct kbase_device *kbdev)
{
return (kbdev->regmap.regs != NULL) && (kbdev->regmap.flags != NULL);
}
int kbase_reg_get_offset(struct kbase_device *kbdev, u32 reg_enum, u32 *offset)
{
if (unlikely(!kbase_reg_is_accessible(kbdev, reg_enum, 0)))
@@ -108,12 +160,12 @@ int kbase_regmap_init(struct kbase_device *kbdev)
return -ENOMEM;
}
kbasep_reg_setup_always_powered_registers(kbdev);
dev_info(kbdev->dev, "Register LUT %08x initialized for GPU arch 0x%08x\n", lut_arch_id,
kbdev->gpu_props.gpu_id.arch_id);
#if IS_ENABLED(CONFIG_MALI_64BIT_HW_ACCESS) && IS_ENABLED(CONFIG_MALI_REAL_HW)
dev_info(kbdev->dev, "64-bit HW access enabled\n");
#endif
return 0;
}

21
drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access.h
View File

@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/*
*
* (C) COPYRIGHT 2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -128,6 +128,25 @@ bool kbase_reg_is_valid(struct kbase_device *kbdev, u32 reg_enum);
*/
bool kbase_reg_is_accessible(struct kbase_device *kbdev, u32 reg_enum, u32 flags);
/**
* kbase_reg_is_powered_access_allowed - check if registered is accessible given
* current power state
*
* @kbdev: Kbase device pointer
* @reg_enum: Register enum
*
* Return: boolean if register is accessible
*/
bool kbase_reg_is_powered_access_allowed(struct kbase_device *kbdev, u32 reg_enum);
/**
* kbase_reg_is_init - check if regmap is initialized
*
* @kbdev: Kbase device pointer
* Return: boolean if regmap is initialized
*/
bool kbase_reg_is_init(struct kbase_device *kbdev);
/**
* kbase_reg_get_offset - get register offset from enum
* @kbdev: Kbase device pointer

12
drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access_regmap.h
View File

@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/*
*
* (C) COPYRIGHT 2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -308,6 +308,16 @@
#define TC_CLOCK_GATE_OVERRIDE (1ul << 0)
/* End TILER_CONFIG register */
/* L2_FEATURES register */
#define L2_FEATURES_CACHE_SIZE_SHIFT GPU_U(16)
#define L2_FEATURES_CACHE_SIZE_MASK (GPU_U(0xFF) << L2_FEATURES_CACHE_SIZE_SHIFT)
#define L2_FEATURES_CACHE_SIZE_GET(reg_val) \
(((reg_val)&L2_FEATURES_CACHE_SIZE_MASK) >> L2_FEATURES_CACHE_SIZE_SHIFT)
#define L2_FEATURES_CACHE_SIZE_SET(reg_val, value) \
(~(~(reg_val) | L2_FEATURES_CACHE_SIZE_MASK) | \
(((value) << L2_FEATURES_CACHE_SIZE_SHIFT) & L2_FEATURES_CACHE_SIZE_MASK))
/* End L2_FEATURES register */
/* L2_CONFIG register */
#define L2_CONFIG_SIZE_SHIFT 16
#define L2_CONFIG_SIZE_MASK (0xFFul << L2_CONFIG_SIZE_SHIFT)

2
drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access_regmap_legacy.h
View File

@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/*
*
* (C) COPYRIGHT 2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software

3
drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_csf_macros.h
View File

@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/*
*
* (C) COPYRIGHT 2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -185,6 +185,7 @@
*/
#define AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_SHARED 0x0
/* CSF_CONFIG register */
#define CSF_CONFIG_FORCE_COHERENCY_FEATURES_SHIFT 2

65
drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_jm.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -2240,6 +2240,56 @@ static void kbase_regmap_v9_2_init(struct kbase_device *kbdev)
kbdev->regmap.regs[GPU_CONTROL__L2_CONFIG] = kbdev->reg + 0x48;
}
static void kbase_regmap_v9_14_init(struct kbase_device *kbdev)
{
if (kbdev->regmap.regs == NULL && kbdev->regmap.flags == NULL) {
kbdev->regmap.size = NR_V9_14_REGS;
kbdev->regmap.regs =
kcalloc(kbdev->regmap.size, sizeof(void __iomem *), GFP_KERNEL);
kbdev->regmap.flags = kcalloc(kbdev->regmap.size, sizeof(u32), GFP_KERNEL);
}
if (WARN_ON(kbdev->regmap.regs == NULL))
return;
if (WARN_ON(kbdev->regmap.flags == NULL))
return;
kbase_regmap_v9_2_init(kbdev);
kbdev->regmap.flags[PTM_AW_IRQ_CLEAR] = KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ |
KBASE_REGMAP_PERM_WRITE;
kbdev->regmap.flags[PTM_AW_IRQ_INJECTION] =
KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE;
kbdev->regmap.flags[PTM_AW_IRQ_MASK] = KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ |
KBASE_REGMAP_PERM_WRITE;
kbdev->regmap.flags[PTM_AW_IRQ_RAWSTAT] = KBASE_REGMAP_WIDTH_32_BIT |
KBASE_REGMAP_PERM_READ;
kbdev->regmap.flags[PTM_AW_IRQ_STATUS] = KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ;
kbdev->regmap.flags[PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE0] = KBASE_REGMAP_WIDTH_32_BIT |
KBASE_REGMAP_PERM_READ;
kbdev->regmap.flags[PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE1] = KBASE_REGMAP_WIDTH_32_BIT |
KBASE_REGMAP_PERM_READ;
kbdev->regmap.flags[PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE0] =
KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE;
kbdev->regmap.flags[PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE1] =
KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE;
kbdev->regmap.flags[PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE_STATUS] =
KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ;
kbdev->regmap.flags[PTM_ID] = KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ;
kbdev->regmap.regs[PTM_AW_IRQ_CLEAR] = kbdev->reg + 0x1ffc8;
kbdev->regmap.regs[PTM_AW_IRQ_INJECTION] = kbdev->reg + 0x1ffd4;
kbdev->regmap.regs[PTM_AW_IRQ_MASK] = kbdev->reg + 0x1ffcc;
kbdev->regmap.regs[PTM_AW_IRQ_RAWSTAT] = kbdev->reg + 0x1ffc4;
kbdev->regmap.regs[PTM_AW_IRQ_STATUS] = kbdev->reg + 0x1ffd0;
kbdev->regmap.regs[PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE0] = kbdev->reg + 0x1ffd8;
kbdev->regmap.regs[PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE1] = kbdev->reg + 0x1ffdc;
kbdev->regmap.regs[PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE0] = kbdev->reg + 0x1ffe4;
kbdev->regmap.regs[PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE1] = kbdev->reg + 0x1ffe8;
kbdev->regmap.regs[PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE_STATUS] = kbdev->reg + 0x1ffe0;
kbdev->regmap.regs[PTM_ID] = kbdev->reg + 0x1ffc0;
}
u32 kbase_regmap_backend_init(struct kbase_device *kbdev)
{
int i = 0;
@@ -2254,6 +2304,7 @@ u32 kbase_regmap_backend_init(struct kbase_device *kbdev)
{ GPU_ID_ARCH_MAKE(7, 2, 0), kbase_regmap_v7_2_init },
{ GPU_ID_ARCH_MAKE(9, 0, 0), kbase_regmap_v9_0_init },
{ GPU_ID_ARCH_MAKE(9, 2, 0), kbase_regmap_v9_2_init },
{ GPU_ID_ARCH_MAKE(9, 14, 0), kbase_regmap_v9_14_init },
};
for (i = 0; i < ARRAY_SIZE(init_array) - 1; i++) {
@@ -2967,6 +3018,18 @@ static char *enum_strings[] = {
[GPU_CONTROL__CORE_FEATURES] = "GPU_CONTROL__CORE_FEATURES",
[GPU_CONTROL__THREAD_TLS_ALLOC] = "GPU_CONTROL__THREAD_TLS_ALLOC",
[GPU_CONTROL__L2_CONFIG] = "GPU_CONTROL__L2_CONFIG",
[PTM_AW_IRQ_CLEAR] = "PTM_AW_IRQ_CLEAR",
[PTM_AW_IRQ_INJECTION] = "PTM_AW_IRQ_INJECTION",
[PTM_AW_IRQ_MASK] = "PTM_AW_IRQ_MASK",
[PTM_AW_IRQ_RAWSTAT] = "PTM_AW_IRQ_RAWSTAT",
[PTM_AW_IRQ_STATUS] = "PTM_AW_IRQ_STATUS",
[PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE0] = "PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE0",
[PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE1] = "PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE1",
[PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE0] = "PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE0",
[PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE1] = "PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE1",
[PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE_STATUS] =
"PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE_STATUS",
[PTM_ID] = "PTM_ID",
};
const char *kbase_reg_get_enum_string(u32 reg_enum)

17
drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_jm_enums.h
View File

@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/*
*
* (C) COPYRIGHT 2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -759,4 +759,19 @@ enum kbase_regmap_enum_v9_2 {
NR_V9_2_REGS,
};
enum kbase_regmap_enum_v9_14 {
PTM_AW_IRQ_CLEAR = NR_V9_2_REGS, /* (RW) 32-bit 0x1FFC8 */
PTM_AW_IRQ_INJECTION, /* (RW) 32-bit 0x1FFD4 */
PTM_AW_IRQ_MASK, /* (RW) 32-bit 0x1FFCC */
PTM_AW_IRQ_RAWSTAT, /* (RO) 32-bit 0x1FFC4 */
PTM_AW_IRQ_STATUS, /* (RO) 32-bit 0x1FFD0 */
PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE0, /* (RO) 32-bit 0x1FFD8 */
PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE1, /* (RO) 32-bit 0x1FFDC */
PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE0, /* (RW) 32-bit 0x1FFE4 */
PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE1, /* (RW) 32-bit 0x1FFE8 */
PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE_STATUS, /* (RO) 32-bit 0x1FFE0 */
PTM_ID, /* (RO) 32-bit 0x1FFC0 */
NR_V9_14_REGS,
};
#endif /* _MALI_KBASE_REGMAP_JM_ENUMS_H_ */

11
drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_jm_macros.h
View File

@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/*
*
* (C) COPYRIGHT 2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -47,6 +47,8 @@
#define MMU_AS_OFFSET(n, regname) ENUM_OFFSET(n, MMU_AS_ENUM(0, regname), MMU_AS_ENUM(1, regname))
#define MMU_AS_BASE_OFFSET(n) MMU_AS_OFFSET(n, TRANSTAB)
#define PTM_AW_MESSAGE_ENUM(regname) PTM_AW_MESSAGE__##regname
/* register value macros */
/* GPU_STATUS values */
#define GPU_STATUS_PRFCNT_ACTIVE (1 << 2) /* Set if the performance counters are active. */
@@ -295,4 +297,11 @@
(GPU_FAULT | MULTIPLE_GPU_FAULTS | RESET_COMPLETED | POWER_CHANGED_ALL | \
PRFCNT_SAMPLE_COMPLETED)
#define WINDOW_IRQ_MESSAGE (1U << 0)
#define WINDOW_IRQ_INVALID_ACCESS (1U << 1)
#define WINDOW_IRQ_GPU (1U << 2)
#define WINDOW_IRQ_JOB (1U << 3)
#define WINDOW_IRQ_MMU (1U << 4)
#define WINDOW_IRQ_EVENT (1U << 5)
#endif /* _MALI_KBASE_REGMAP_JM_MACROS_H_ */

20
drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend.h
View File

@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/*
*
* (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -78,6 +78,18 @@ typedef int kbase_hwcnt_backend_init_fn(const struct kbase_hwcnt_backend_info *i
*/
typedef void kbase_hwcnt_backend_term_fn(struct kbase_hwcnt_backend *backend);
/**
* typedef kbase_hwcnt_backend_acquire_fn - Enable counter collection.
* @backend: Non-NULL pointer to backend interface.
*/
typedef void kbase_hwcnt_backend_acquire_fn(const struct kbase_hwcnt_backend *backend);
/**
* typedef kbase_hwcnt_backend_release_fn - Disable counter collection.
* @backend: Non-NULL pointer to backend interface.
*/
typedef void kbase_hwcnt_backend_release_fn(const struct kbase_hwcnt_backend *backend);
/**
* typedef kbase_hwcnt_backend_timestamp_ns_fn - Get the current backend
* timestamp.
@@ -206,6 +218,10 @@ typedef int kbase_hwcnt_backend_dump_get_fn(struct kbase_hwcnt_backend *backend,
* metadata.
* @init: Function ptr to initialise an instance of the backend.
* @term: Function ptr to terminate an instance of the backend.
* @acquire: Callback to indicate that counter collection has
* been enabled.
* @release: Callback to indicate that counter collection has
* been disabled.
* @timestamp_ns: Function ptr to get the current backend timestamp.
* @dump_enable: Function ptr to enable dumping.
* @dump_enable_nolock: Function ptr to enable dumping while the
@@ -222,6 +238,8 @@ struct kbase_hwcnt_backend_interface {
kbase_hwcnt_backend_metadata_fn *metadata;
kbase_hwcnt_backend_init_fn *init;
kbase_hwcnt_backend_term_fn *term;
kbase_hwcnt_backend_acquire_fn *acquire;
kbase_hwcnt_backend_release_fn *release;
kbase_hwcnt_backend_timestamp_ns_fn *timestamp_ns;
kbase_hwcnt_backend_dump_enable_fn *dump_enable;
kbase_hwcnt_backend_dump_enable_nolock_fn *dump_enable_nolock;

156
drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2021-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2021-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -21,7 +21,6 @@
#include "hwcnt/backend/mali_kbase_hwcnt_backend_csf.h"
#include "hwcnt/mali_kbase_hwcnt_gpu.h"
#include "hwcnt/mali_kbase_hwcnt_types.h"
#include <linux/log2.h>
#include <linux/kernel.h>
@@ -31,6 +30,7 @@
#include <linux/wait.h>
#include <linux/workqueue.h>
#include <linux/completion.h>
#include <linux/version_compat_defs.h>
#ifndef BASE_MAX_NR_CLOCKS_REGULATORS
#define BASE_MAX_NR_CLOCKS_REGULATORS 4
@@ -255,7 +255,8 @@ struct kbase_hwcnt_csf_physical_layout {
* @hwc_threshold_work: Worker for consuming available samples when
* threshold interrupt raised.
* @num_l2_slices: Current number of L2 slices allocated to the GPU.
* @shader_present_bitmap: Current shader-present bitmap that is allocated to the GPU.
* @powered_shader_core_mask: The common mask between the debug_core_mask
* and the shader_present_bitmap.
*/
struct kbase_hwcnt_backend_csf {
struct kbase_hwcnt_backend_csf_info *info;
@@ -283,7 +284,7 @@ struct kbase_hwcnt_backend_csf {
struct work_struct hwc_dump_work;
struct work_struct hwc_threshold_work;
size_t num_l2_slices;
u64 shader_present_bitmap;
u64 powered_shader_core_mask;
};
static bool kbasep_hwcnt_backend_csf_backend_exists(struct kbase_hwcnt_backend_csf_info *csf_info)
@@ -296,9 +297,11 @@ static bool kbasep_hwcnt_backend_csf_backend_exists(struct kbase_hwcnt_backend_c
}
void kbase_hwcnt_backend_csf_set_hw_availability(struct kbase_hwcnt_backend_interface *iface,
size_t num_l2_slices, u64 shader_present_bitmap)
size_t num_l2_slices, u64 shader_present,
u64 power_core_mask)
{
struct kbase_hwcnt_backend_csf_info *csf_info;
u64 norm_shader_present = power_core_mask & shader_present;
if (!iface)
return;
@@ -309,16 +312,17 @@ void kbase_hwcnt_backend_csf_set_hw_availability(struct kbase_hwcnt_backend_inte
if (!csf_info || !csf_info->backend)
return;
if (WARN_ON(csf_info->backend->enable_state != KBASE_HWCNT_BACKEND_CSF_DISABLED))
return;
if (WARN_ON(num_l2_slices > csf_info->backend->phys_layout.mmu_l2_cnt) ||
WARN_ON((shader_present_bitmap & csf_info->backend->phys_layout.shader_avail_mask) !=
shader_present_bitmap))
WARN_ON((norm_shader_present & csf_info->backend->phys_layout.shader_avail_mask) !=
norm_shader_present))
return;
csf_info->backend->num_l2_slices = num_l2_slices;
csf_info->backend->shader_present_bitmap = shader_present_bitmap;
csf_info->backend->powered_shader_core_mask = norm_shader_present;
}
/**
@@ -424,7 +428,7 @@ static void kbasep_hwcnt_backend_csf_init_layout(
WARN_ON(!prfcnt_info);
WARN_ON(!phys_layout);
shader_core_cnt = (size_t)fls64(prfcnt_info->core_mask);
shader_core_cnt = (size_t)fls64(prfcnt_info->sc_core_mask);
values_per_block = prfcnt_info->prfcnt_block_size / KBASE_HWCNT_VALUE_HW_BYTES;
fw_block_cnt = div_u64(prfcnt_info->prfcnt_fw_size, prfcnt_info->prfcnt_block_size);
hw_block_cnt = div_u64(prfcnt_info->prfcnt_hw_size, prfcnt_info->prfcnt_block_size);
@@ -445,7 +449,7 @@ static void kbasep_hwcnt_backend_csf_init_layout(
.fw_block_cnt = fw_block_cnt,
.hw_block_cnt = hw_block_cnt,
.block_cnt = fw_block_cnt + hw_block_cnt,
.shader_avail_mask = prfcnt_info->core_mask,
.shader_avail_mask = prfcnt_info->sc_core_mask,
.headers_per_block = KBASE_HWCNT_V5_HEADERS_PER_BLOCK,
.values_per_block = values_per_block,
.counters_per_block = values_per_block - KBASE_HWCNT_V5_HEADERS_PER_BLOCK,
@@ -454,17 +458,20 @@ static void kbasep_hwcnt_backend_csf_init_layout(
}
static void
kbasep_hwcnt_backend_csf_reset_internal_buffers(struct kbase_hwcnt_backend_csf *backend_csf)
kbasep_hwcnt_backend_csf_reset_internal_buffers(struct kbase_hwcnt_backend_csf *backend_csf,
bool user_bufs)
{
size_t user_buf_bytes = backend_csf->info->metadata->dump_buf_bytes;
size_t block_state_bytes = backend_csf->phys_layout.block_cnt *
KBASE_HWCNT_BLOCK_STATE_BYTES * KBASE_HWCNT_BLOCK_STATE_STRIDE;
memset(backend_csf->to_user_buf, 0, user_buf_bytes);
memset(backend_csf->accum_buf, 0, user_buf_bytes);
memset(backend_csf->old_sample_buf, 0, backend_csf->info->prfcnt_info.dump_bytes);
memset(backend_csf->block_states, 0, block_state_bytes);
memset(backend_csf->to_user_block_states, 0, block_state_bytes);
if (user_bufs) {
memset(backend_csf->to_user_buf, 0, user_buf_bytes);
memset(backend_csf->to_user_block_states, 0, block_state_bytes);
}
}
static void
@@ -517,34 +524,21 @@ static void kbasep_hwcnt_backend_csf_update_user_sample(struct kbase_hwcnt_backe
memset(backend_csf->block_states, 0, block_state_bytes);
}
/**
* kbasep_hwcnt_backend_csf_update_block_state - Update block state of a block instance with
* information from a sample.
* @phys_layout: Physical memory layout information of HWC
* sample buffer.
* @enable_mask: Counter enable mask for the block whose state is being updated.
* @enable_state: The CSF backend internal enabled state.
* @exiting_protm: Whether or not the sample is taken when the GPU is exiting
* protected mode.
* @block_idx: Index of block within the ringbuffer.
* @block_state: Pointer to existing block state of the block whose state is being
* updated.
* @fw_in_protected_mode: Whether or not GPU is in protected mode during sampling.
*/
static void kbasep_hwcnt_backend_csf_update_block_state(
const struct kbase_hwcnt_csf_physical_layout *phys_layout, const u32 enable_mask,
enum kbase_hwcnt_backend_csf_enable_state enable_state, bool exiting_protm,
size_t block_idx, blk_stt_t *const block_state, bool fw_in_protected_mode)
void kbasep_hwcnt_backend_csf_update_block_state(struct kbase_hwcnt_backend_csf *backend,
const u32 enable_mask, bool exiting_protm,
size_t block_idx, blk_stt_t *const block_state,
bool fw_in_protected_mode)
{
const struct kbase_hwcnt_csf_physical_layout *phys_layout = &backend->phys_layout;
/* Offset of shader core blocks from the start of the HW blocks in the sample */
size_t shader_core_block_offset =
(size_t)(phys_layout->hw_block_cnt - phys_layout->shader_cnt);
(size_t)(phys_layout->block_cnt - phys_layout->shader_cnt);
bool is_shader_core_block;
is_shader_core_block = block_idx >= shader_core_block_offset;
is_shader_core_block = (block_idx >= shader_core_block_offset);
/* Set power bits for the block state for the block, for the sample */
switch (enable_state) {
switch (backend->enable_state) {
/* Disabled states */
case KBASE_HWCNT_BACKEND_CSF_DISABLED:
case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED:
@@ -592,21 +586,45 @@ static void kbasep_hwcnt_backend_csf_update_block_state(
KBASE_HWCNT_STATE_NORMAL);
else
kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_NORMAL);
/* powered_shader_core_mask stored in the backend is a combination of
* the shader present and the debug core mask, so explicit checking of the
* core mask is not required here.
*/
if (is_shader_core_block) {
u64 current_shader_core = 1ULL << (block_idx - shader_core_block_offset);
WARN_ON_ONCE(backend->phys_layout.shader_cnt > 64);
if (current_shader_core & backend->info->backend->powered_shader_core_mask)
kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_AVAILABLE);
else if (current_shader_core & ~backend->info->backend->powered_shader_core_mask)
kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_UNAVAILABLE);
else
WARN_ON_ONCE(true);
}
else
kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_AVAILABLE);
}
static void kbasep_hwcnt_backend_csf_accumulate_sample(
const struct kbase_hwcnt_csf_physical_layout *phys_layout, size_t dump_bytes,
u64 *accum_buf, const u32 *old_sample_buf, const u32 *new_sample_buf,
blk_stt_t *const block_states, bool clearing_samples,
enum kbase_hwcnt_backend_csf_enable_state enable_state, bool fw_in_protected_mode)
static void kbasep_hwcnt_backend_csf_accumulate_sample(struct kbase_hwcnt_backend_csf *backend,
const u32 *old_sample_buf,
const u32 *new_sample_buf)
{
const struct kbase_hwcnt_csf_physical_layout *phys_layout = &backend->phys_layout;
const size_t dump_bytes = backend->info->prfcnt_info.dump_bytes;
const size_t values_per_block = phys_layout->values_per_block;
blk_stt_t *const block_states = backend->block_states;
const bool fw_in_protected_mode = backend->info->fw_in_protected_mode;
const bool clearing_samples = backend->info->prfcnt_info.clearing_samples;
u64 *accum_buf = backend->accum_buf;
size_t block_idx;
const u32 *old_block = old_sample_buf;
const u32 *new_block = new_sample_buf;
u64 *acc_block = accum_buf;
/* Flag to indicate whether current sample is exiting protected mode. */
bool exiting_protm = false;
const size_t values_per_block = phys_layout->values_per_block;
/* The block pointers now point to the first HW block, which is always a CSHW/front-end
* block. The counter enable mask for this block can be checked to determine whether this
@@ -620,9 +638,8 @@ static void kbasep_hwcnt_backend_csf_accumulate_sample(
const u32 old_enable_mask = old_block[phys_layout->enable_mask_offset];
const u32 new_enable_mask = new_block[phys_layout->enable_mask_offset];
/* Update block state with information of the current sample */
kbasep_hwcnt_backend_csf_update_block_state(phys_layout, new_enable_mask,
enable_state, exiting_protm, block_idx,
&block_states[block_idx],
kbasep_hwcnt_backend_csf_update_block_state(backend, new_enable_mask, exiting_protm,
block_idx, &block_states[block_idx],
fw_in_protected_mode);
if (!(new_enable_mask & HWCNT_BLOCK_EMPTY_SAMPLE)) {
@@ -706,7 +723,6 @@ static void kbasep_hwcnt_backend_csf_accumulate_samples(struct kbase_hwcnt_backe
u8 *cpu_dump_base = (u8 *)backend_csf->ring_buf_cpu_base;
const size_t ring_buf_cnt = backend_csf->info->ring_buf_cnt;
const size_t buf_dump_bytes = backend_csf->info->prfcnt_info.dump_bytes;
bool clearing_samples = backend_csf->info->prfcnt_info.clearing_samples;
u32 *old_sample_buf = backend_csf->old_sample_buf;
u32 *new_sample_buf = old_sample_buf;
const struct kbase_hwcnt_csf_physical_layout *phys_layout = &backend_csf->phys_layout;
@@ -740,10 +756,8 @@ static void kbasep_hwcnt_backend_csf_accumulate_samples(struct kbase_hwcnt_backe
const u32 buf_idx = raw_idx & (ring_buf_cnt - 1);
new_sample_buf = (u32 *)&cpu_dump_base[buf_idx * buf_dump_bytes];
kbasep_hwcnt_backend_csf_accumulate_sample(
phys_layout, buf_dump_bytes, backend_csf->accum_buf, old_sample_buf,
new_sample_buf, backend_csf->block_states, clearing_samples,
backend_csf->enable_state, backend_csf->info->fw_in_protected_mode);
kbasep_hwcnt_backend_csf_accumulate_sample(backend_csf, old_sample_buf,
new_sample_buf);
old_sample_buf = new_sample_buf;
}
@@ -1215,11 +1229,6 @@ static void kbasep_hwcnt_backend_csf_dump_disable(struct kbase_hwcnt_backend *ba
backend_csf->ring_buf, 0,
backend_csf->info->ring_buf_cnt, false);
/* Reset accumulator, old_sample_buf and user_sample to all-0 to prepare
* for next enable.
*/
kbasep_hwcnt_backend_csf_reset_internal_buffers(backend_csf);
/* Disabling HWCNT is an indication that blocks have been powered off. This is important to
* know for L2, CSHW, and Tiler blocks, as this is currently the only way a backend can
* know if they are being powered off.
@@ -1255,6 +1264,12 @@ static void kbasep_hwcnt_backend_csf_dump_disable(struct kbase_hwcnt_backend *ba
kbase_hwcnt_block_state_set(&backend_csf->accum_all_blk_stt,
KBASE_HWCNT_STATE_UNKNOWN);
}
/* Reset accumulator, old_sample_buf and block_states to all-0 to prepare for next enable.
* Reset user buffers if ownership is transferred to the caller (i.e. dump_buffer
* is provided).
*/
kbasep_hwcnt_backend_csf_reset_internal_buffers(backend_csf, dump_buffer);
}
/* CSF backend implementation of kbase_hwcnt_backend_dump_request_fn */
@@ -1279,6 +1294,11 @@ static int kbasep_hwcnt_backend_csf_dump_request(struct kbase_hwcnt_backend *bac
backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_COMPLETED;
*dump_time_ns = kbasep_hwcnt_backend_csf_timestamp_ns(backend);
kbasep_hwcnt_backend_csf_cc_update(backend_csf);
/* There is a possibility that the transition to enabled state will remain
* during multiple dumps, hence append the OFF state.
*/
kbase_hwcnt_block_state_append(&backend_csf->accum_all_blk_stt,
KBASE_HWCNT_STATE_OFF);
backend_csf->user_requested = true;
backend_csf->info->csf_if->unlock(backend_csf->info->csf_if->ctx, flags);
return 0;
@@ -1457,7 +1477,7 @@ static int kbasep_hwcnt_backend_csf_dump_get(struct kbase_hwcnt_backend *backend
ret = kbase_hwcnt_csf_dump_get(dst, backend_csf->to_user_buf,
backend_csf->to_user_block_states, dst_enable_map,
backend_csf->num_l2_slices,
backend_csf->shader_present_bitmap, accumulate);
backend_csf->powered_shader_core_mask, accumulate);
/* If no error occurred (zero ret value), then update block state for all blocks in the
* accumulation with the current sample's block state.
@@ -1469,6 +1489,12 @@ static int kbasep_hwcnt_backend_csf_dump_get(struct kbase_hwcnt_backend *backend
KBASE_HWCNT_STATE_UNKNOWN);
}
/* Clear consumed user buffers. */
memset(backend_csf->to_user_buf, 0, backend_csf->info->metadata->dump_buf_bytes);
memset(backend_csf->to_user_block_states, 0,
backend_csf->phys_layout.block_cnt * KBASE_HWCNT_BLOCK_STATE_BYTES *
KBASE_HWCNT_BLOCK_STATE_STRIDE);
return ret;
}
@@ -1684,6 +1710,22 @@ static void kbasep_hwcnt_backend_csf_term(struct kbase_hwcnt_backend *backend)
kbasep_hwcnt_backend_csf_destroy(backend_csf);
}
static void kbasep_hwcnt_backend_csf_acquire(const struct kbase_hwcnt_backend *backend)
{
struct kbase_hwcnt_backend_csf *backend_csf = (struct kbase_hwcnt_backend_csf *)backend;
struct kbase_hwcnt_backend_csf_info *csf_info = backend_csf->info;
csf_info->csf_if->acquire(csf_info->csf_if->ctx);
}
static void kbasep_hwcnt_backend_csf_release(const struct kbase_hwcnt_backend *backend)
{
struct kbase_hwcnt_backend_csf *backend_csf = (struct kbase_hwcnt_backend_csf *)backend;
struct kbase_hwcnt_backend_csf_info *csf_info = backend_csf->info;
csf_info->csf_if->release(csf_info->csf_if->ctx);
}
/**
* kbasep_hwcnt_backend_csf_info_destroy() - Destroy a CSF backend info.
* @info: Pointer to info to destroy.
@@ -2098,7 +2140,7 @@ int kbase_hwcnt_backend_csf_metadata_init(struct kbase_hwcnt_backend_interface *
gpu_info.has_fw_counters = csf_info->prfcnt_info.prfcnt_fw_size > 0;
gpu_info.l2_count = csf_info->prfcnt_info.l2_count;
gpu_info.csg_cnt = csf_info->prfcnt_info.csg_count;
gpu_info.core_mask = csf_info->prfcnt_info.core_mask;
gpu_info.sc_core_mask = csf_info->prfcnt_info.sc_core_mask;
gpu_info.clk_cnt = csf_info->prfcnt_info.clk_cnt;
gpu_info.prfcnt_values_per_block =
csf_info->prfcnt_info.prfcnt_block_size / KBASE_HWCNT_VALUE_HW_BYTES;
@@ -2115,7 +2157,7 @@ void kbase_hwcnt_backend_csf_metadata_term(struct kbase_hwcnt_backend_interface
csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info;
if (csf_info->metadata) {
kbase_hwcnt_csf_metadata_destroy(csf_info->metadata);
kbase_hwcnt_metadata_destroy(csf_info->metadata);
csf_info->metadata = NULL;
}
}
@@ -2142,6 +2184,8 @@ int kbase_hwcnt_backend_csf_create(struct kbase_hwcnt_backend_csf_if *csf_if, u3
iface->metadata = kbasep_hwcnt_backend_csf_metadata;
iface->init = kbasep_hwcnt_backend_csf_init;
iface->term = kbasep_hwcnt_backend_csf_term;
iface->acquire = kbasep_hwcnt_backend_csf_acquire;
iface->release = kbasep_hwcnt_backend_csf_release;
iface->timestamp_ns = kbasep_hwcnt_backend_csf_timestamp_ns;
iface->dump_enable = kbasep_hwcnt_backend_csf_dump_enable;
iface->dump_enable_nolock = kbasep_hwcnt_backend_csf_dump_enable_nolock;

28
drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf.h
View File

@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/*
*
* (C) COPYRIGHT 2021-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2021-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -30,8 +30,10 @@
#include "hwcnt/backend/mali_kbase_hwcnt_backend.h"
#include "hwcnt/backend/mali_kbase_hwcnt_backend_csf_if.h"
#include "hwcnt/mali_kbase_hwcnt_watchdog_if.h"
#include "hwcnt/mali_kbase_hwcnt_types.h"
struct kbase_hwcnt_physical_enable_map;
struct kbase_hwcnt_backend_csf;
/**
* kbase_hwcnt_backend_csf_create() - Create a CSF hardware counter backend
@@ -123,11 +125,12 @@ void kbase_hwcnt_backend_csf_on_before_reset(struct kbase_hwcnt_backend_interfac
* this function is called.
* @iface: Non-NULL pointer to HWC backend interface.
* @num_l2_slices: Current number of L2 slices allocated to the GPU.
* @shader_present_bitmap: Current shader-present bitmap that is allocated to the GPU.
* @shader_present: Shader_present of the current configuration.
* @power_core_mask: Mask containing changed shader core power state.
*/
void kbase_hwcnt_backend_csf_set_hw_availability(struct kbase_hwcnt_backend_interface *iface,
size_t num_l2_slices,
uint64_t shader_present_bitmap);
size_t num_l2_slices, u64 shader_present,
u64 power_core_mask);
/** kbasep_hwcnt_backend_csf_process_enable_map() - Process the enable_map to
* guarantee headers are
@@ -174,4 +177,21 @@ void kbase_hwcnt_backend_csf_on_prfcnt_enable(struct kbase_hwcnt_backend_interfa
*/
void kbase_hwcnt_backend_csf_on_prfcnt_disable(struct kbase_hwcnt_backend_interface *iface);
/**
* kbasep_hwcnt_backend_csf_update_block_state - Update block state of a block instance with
* information from a sample.
* @backend: CSF hardware counter backend.
* @enable_mask: Counter enable mask for the block whose state is being updated.
* @exiting_protm: Whether or not the sample is taken when the GPU is exiting
* protected mode.
* @block_idx: Index of block within the ringbuffer.
* @block_state: Pointer to existing block state of the block whose state is being
* updated.
* @fw_in_protected_mode: Whether or not GPU is in protected mode during sampling.
*/
void kbasep_hwcnt_backend_csf_update_block_state(struct kbase_hwcnt_backend_csf *backend,
const u32 enable_mask, bool exiting_protm,
size_t block_idx, blk_stt_t *const block_state,
bool fw_in_protected_mode);
#endif /* _KBASE_HWCNT_BACKEND_CSF_H_ */

26
drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if.h
View File

@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/*
*
* (C) COPYRIGHT 2021-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2021-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -68,7 +68,7 @@ struct kbase_hwcnt_backend_csf_if_enable {
* @prfcnt_block_size: Bytes of each performance counter block.
* @l2_count: The MMU L2 cache count.
* @csg_count: The total number of CSGs in the system
* @core_mask: Shader core mask.
* @sc_core_mask: Shader core mask.
* @clk_cnt: Clock domain count in the system.
* @clearing_samples: Indicates whether counters are cleared after each sample
* is taken.
@@ -80,7 +80,7 @@ struct kbase_hwcnt_backend_csf_if_prfcnt_info {
size_t prfcnt_block_size;
size_t l2_count;
u32 csg_count;
u64 core_mask;
u64 sc_core_mask;
u8 clk_cnt;
bool clearing_samples;
};
@@ -114,6 +114,20 @@ typedef void (*kbase_hwcnt_backend_csf_if_lock_fn)(struct kbase_hwcnt_backend_cs
typedef void (*kbase_hwcnt_backend_csf_if_unlock_fn)(struct kbase_hwcnt_backend_csf_if_ctx *ctx,
unsigned long flags);
/**
* typedef kbase_hwcnt_backend_csf_if_acquire_fn - Enable counter collection.
*
* @ctx: Non-NULL pointer to a CSF context.
*/
typedef void (*kbase_hwcnt_backend_csf_if_acquire_fn)(struct kbase_hwcnt_backend_csf_if_ctx *ctx);
/**
* typedef kbase_hwcnt_backend_csf_if_release_fn - Disable counter collection.
*
* @ctx: Non-NULL pointer to a CSF context.
*/
typedef void (*kbase_hwcnt_backend_csf_if_release_fn)(struct kbase_hwcnt_backend_csf_if_ctx *ctx);
/**
* typedef kbase_hwcnt_backend_csf_if_get_prfcnt_info_fn - Get performance
* counter information.
@@ -272,6 +286,10 @@ typedef void (*kbase_hwcnt_backend_csf_if_get_gpu_cycle_count_fn)(
* @assert_lock_held: Function ptr to assert backend spinlock is held.
* @lock: Function ptr to acquire backend spinlock.
* @unlock: Function ptr to release backend spinlock.
* @acquire: Callback to indicate that counter collection has
* been enabled.
* @release: Callback to indicate that counter collection has
* been disabled.
* @get_prfcnt_info: Function ptr to get performance counter related
* information.
* @ring_buf_alloc: Function ptr to allocate ring buffer for CSF HWC.
@@ -292,6 +310,8 @@ struct kbase_hwcnt_backend_csf_if {
kbase_hwcnt_backend_csf_if_assert_lock_held_fn assert_lock_held;
kbase_hwcnt_backend_csf_if_lock_fn lock;
kbase_hwcnt_backend_csf_if_unlock_fn unlock;
kbase_hwcnt_backend_csf_if_acquire_fn acquire;
kbase_hwcnt_backend_csf_if_release_fn release;
kbase_hwcnt_backend_csf_if_get_prfcnt_info_fn get_prfcnt_info;
kbase_hwcnt_backend_csf_if_ring_buf_alloc_fn ring_buf_alloc;
kbase_hwcnt_backend_csf_if_ring_buf_sync_fn ring_buf_sync;

36
drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2021-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2021-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -131,6 +131,26 @@ static void kbasep_hwcnt_backend_csf_if_fw_unlock(struct kbase_hwcnt_backend_csf
kbase_csf_scheduler_spin_unlock(kbdev, flags);
}
static void kbasep_hwcnt_backend_csf_if_fw_acquire(struct kbase_hwcnt_backend_csf_if_ctx *ctx)
{
struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx =
(struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx;
/* Mark performance counters collection as enabled */
set_bit(KBASE_GPU_PERF_COUNTERS_COLLECTION_ENABLED,
&fw_ctx->kbdev->pm.backend.gpu_sleep_allowed);
}
static void kbasep_hwcnt_backend_csf_if_fw_release(struct kbase_hwcnt_backend_csf_if_ctx *ctx)
{
struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx =
(struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx;
/* Mark performance counters collection as disabled */
clear_bit(KBASE_GPU_PERF_COUNTERS_COLLECTION_ENABLED,
&fw_ctx->kbdev->pm.backend.gpu_sleep_allowed);
}
/**
* kbasep_hwcnt_backend_csf_if_fw_on_freq_change() - On freq change callback
*
@@ -229,7 +249,7 @@ static void kbasep_hwcnt_backend_csf_if_fw_get_prfcnt_info(
*prfcnt_info = (struct kbase_hwcnt_backend_csf_if_prfcnt_info){
.l2_count = KBASE_DUMMY_MODEL_MAX_MEMSYS_BLOCKS,
.core_mask = (1ull << KBASE_DUMMY_MODEL_MAX_SHADER_CORES) - 1,
.sc_core_mask = (1ull << KBASE_DUMMY_MODEL_MAX_SHADER_CORES) - 1,
.prfcnt_hw_size =
KBASE_DUMMY_MODEL_MAX_NUM_HARDWARE_BLOCKS * KBASE_DUMMY_MODEL_BLOCK_SIZE,
.prfcnt_fw_size =
@@ -290,12 +310,13 @@ static void kbasep_hwcnt_backend_csf_if_fw_get_prfcnt_info(
.dump_bytes = fw_ctx->buf_bytes,
.prfcnt_block_size = prfcnt_block_size,
.l2_count = kbdev->gpu_props.num_l2_slices,
.core_mask = kbasep_hwcnt_backend_csf_core_mask(&kbdev->gpu_props),
.sc_core_mask = kbasep_hwcnt_backend_csf_core_mask(&kbdev->gpu_props),
.csg_count = fw_block_count > 1 ? csg_count : 0,
.clk_cnt = fw_ctx->clk_cnt,
.clearing_samples = true,
};
/* Block size must be multiple of counter size. */
WARN_ON((prfcnt_info->prfcnt_block_size % KBASE_HWCNT_VALUE_HW_BYTES) != 0);
/* Total size must be multiple of block size. */
@@ -513,10 +534,15 @@ kbasep_hwcnt_backend_csf_if_fw_ring_buf_free(struct kbase_hwcnt_backend_csf_if_c
fw_ring_buf->phys, fw_ring_buf->num_pages, fw_ring_buf->num_pages,
MCU_AS_NR));
/* Clear the dump ring_buf content to zeros */
memset(fw_ring_buf->cpu_dump_base, 0, fw_ring_buf->num_pages * PAGE_SIZE);
vunmap(fw_ring_buf->cpu_dump_base);
/* After zeroing, the ring_buf pages are dirty so need to pass the 'dirty' flag
* as true when freeing the pages to the Global pool.
*/
kbase_mem_pool_free_pages(&fw_ctx->kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW],
fw_ring_buf->num_pages, fw_ring_buf->phys, false, false);
fw_ring_buf->num_pages, fw_ring_buf->phys, true, false);
kfree(fw_ring_buf->phys);
@@ -807,6 +833,8 @@ int kbase_hwcnt_backend_csf_if_fw_create(struct kbase_device *kbdev,
if_fw->assert_lock_held = kbasep_hwcnt_backend_csf_if_fw_assert_lock_held;
if_fw->lock = kbasep_hwcnt_backend_csf_if_fw_lock;
if_fw->unlock = kbasep_hwcnt_backend_csf_if_fw_unlock;
if_fw->acquire = kbasep_hwcnt_backend_csf_if_fw_acquire;
if_fw->release = kbasep_hwcnt_backend_csf_if_fw_release;
if_fw->get_prfcnt_info = kbasep_hwcnt_backend_csf_if_fw_get_prfcnt_info;
if_fw->ring_buf_alloc = kbasep_hwcnt_backend_csf_if_fw_ring_buf_alloc;
if_fw->ring_buf_sync = kbasep_hwcnt_backend_csf_if_fw_ring_buf_sync;

36
drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_jm.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -165,7 +165,7 @@ static int kbasep_hwcnt_backend_jm_gpu_info_init(struct kbase_device *kbdev,
#endif
info->l2_count = l2_count;
info->core_mask = core_mask;
info->sc_core_mask = core_mask;
info->prfcnt_values_per_block = KBASE_HWCNT_V5_DEFAULT_VALUES_PER_BLOCK;
/* Determine the number of available clock domains. */
@@ -186,7 +186,7 @@ static void kbasep_hwcnt_backend_jm_init_layout(const struct kbase_hwcnt_gpu_inf
WARN_ON(!gpu_info);
WARN_ON(!phys_layout);
shader_core_cnt = fls64(gpu_info->core_mask);
shader_core_cnt = fls64(gpu_info->sc_core_mask);
*phys_layout = (struct kbase_hwcnt_jm_physical_layout){
.fe_cnt = KBASE_HWCNT_V5_FE_BLOCK_COUNT,
@@ -195,7 +195,7 @@ static void kbasep_hwcnt_backend_jm_init_layout(const struct kbase_hwcnt_gpu_inf
.shader_cnt = shader_core_cnt,
.block_cnt = KBASE_HWCNT_V5_FE_BLOCK_COUNT + KBASE_HWCNT_V5_TILER_BLOCK_COUNT +
gpu_info->l2_count + shader_core_cnt,
.shader_avail_mask = gpu_info->core_mask,
.shader_avail_mask = gpu_info->sc_core_mask,
.headers_per_block = KBASE_HWCNT_V5_HEADERS_PER_BLOCK,
.values_per_block = gpu_info->prfcnt_values_per_block,
.counters_per_block =
@@ -384,14 +384,12 @@ kbasep_hwcnt_backend_jm_dump_enable_nolock(struct kbase_hwcnt_backend *backend,
enable = (struct kbase_instr_hwcnt_enable)
{
.fe_bm = phys_enable_map.fe_bm,
.shader_bm = phys_enable_map.shader_bm,
.tiler_bm = phys_enable_map.tiler_bm,
.mmu_l2_bm = phys_enable_map.mmu_l2_bm,
.fe_bm = phys_enable_map.fe_bm, .shader_bm = phys_enable_map.shader_bm,
.tiler_bm = phys_enable_map.tiler_bm, .mmu_l2_bm = phys_enable_map.mmu_l2_bm,
.counter_set = phys_counter_set,
#if IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI)
/* The dummy model needs the CPU mapping. */
.dump_buffer = (uintptr_t)backend_jm->cpu_dump_va,
.dump_buffer = (uintptr_t)backend_jm->cpu_dump_va,
#else
.dump_buffer = backend_jm->gpu_dump_va,
#endif /* CONFIG_MALI_BIFROST_NO_MALI */
@@ -411,7 +409,7 @@ kbasep_hwcnt_backend_jm_dump_enable_nolock(struct kbase_hwcnt_backend *backend,
backend_jm->debug_core_mask = kbase_pm_ca_get_debug_core_mask(kbdev);
backend_jm->max_l2_slices = backend_jm->info->hwcnt_gpu_info.l2_count;
backend_jm->max_core_mask = backend_jm->info->hwcnt_gpu_info.core_mask;
backend_jm->max_core_mask = backend_jm->info->hwcnt_gpu_info.sc_core_mask;
backend_jm->pm_core_mask = kbase_pm_ca_get_instr_core_mask(kbdev);
@@ -660,8 +658,8 @@ static int kbasep_hwcnt_backend_jm_dump_get(struct kbase_hwcnt_backend *backend,
#endif /* CONFIG_MALI_BIFROST_NO_MALI */
errcode = kbase_hwcnt_jm_dump_get(dst, backend_jm->to_user_buf, dst_enable_map,
backend_jm->pm_core_mask, backend_jm->debug_core_mask,
backend_jm->max_core_mask, backend_jm->max_l2_slices,
&backend_jm->curr_config, accumulate);
backend_jm->max_l2_slices, &backend_jm->curr_config,
accumulate);
if (errcode)
return errcode;
@@ -685,7 +683,7 @@ static int kbasep_hwcnt_backend_jm_dump_alloc(const struct kbase_hwcnt_backend_j
struct kbase_context *kctx, u64 *gpu_dump_va)
{
struct kbase_va_region *reg;
u64 flags;
base_mem_alloc_flags flags;
u64 nr_pages;
/* Calls to this function are inherently asynchronous, with respect to
@@ -853,6 +851,14 @@ static void kbasep_hwcnt_backend_jm_term(struct kbase_hwcnt_backend *backend)
kbasep_hwcnt_backend_jm_destroy((struct kbase_hwcnt_backend_jm *)backend);
}
static void kbasep_hwcnt_backend_jm_acquire(const struct kbase_hwcnt_backend *backend)
{
}
static void kbasep_hwcnt_backend_jm_release(const struct kbase_hwcnt_backend *backend)
{
}
/**
* kbasep_hwcnt_backend_jm_info_destroy() - Destroy a JM backend info.
* @info: Pointer to info to destroy.
@@ -864,7 +870,7 @@ static void kbasep_hwcnt_backend_jm_info_destroy(const struct kbase_hwcnt_backen
if (!info)
return;
kbase_hwcnt_jm_metadata_destroy(info->metadata);
kbase_hwcnt_metadata_destroy(info->metadata);
kfree(info);
}
@@ -934,6 +940,8 @@ int kbase_hwcnt_backend_jm_create(struct kbase_device *kbdev,
iface->metadata = kbasep_hwcnt_backend_jm_metadata;
iface->init = kbasep_hwcnt_backend_jm_init;
iface->term = kbasep_hwcnt_backend_jm_term;
iface->acquire = kbasep_hwcnt_backend_jm_acquire;
iface->release = kbasep_hwcnt_backend_jm_release;
iface->timestamp_ns = kbasep_hwcnt_backend_jm_timestamp_ns;
iface->dump_enable = kbasep_hwcnt_backend_jm_dump_enable;
iface->dump_enable_nolock = kbasep_hwcnt_backend_jm_dump_enable_nolock;

12
drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_jm_watchdog.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2021-2023 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2021-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -317,6 +317,14 @@ kbasep_hwcnt_backend_jm_watchdog_term_partial(struct kbase_hwcnt_backend_jm_watc
kfree(wd_backend);
}
static void kbasep_hwcnt_backend_jm_watchdog_acquire(const struct kbase_hwcnt_backend *backend)
{
}
static void kbasep_hwcnt_backend_jm_watchdog_release(const struct kbase_hwcnt_backend *backend)
{
}
/* Job manager watchdog backend, implementation of kbase_hwcnt_backend_term_fn
* Calling term does *not* destroy the interface
*/
@@ -807,6 +815,8 @@ int kbase_hwcnt_backend_jm_watchdog_create(struct kbase_hwcnt_backend_interface
.metadata = kbasep_hwcnt_backend_jm_watchdog_metadata,
.init = kbasep_hwcnt_backend_jm_watchdog_init,
.term = kbasep_hwcnt_backend_jm_watchdog_term,
.acquire = kbasep_hwcnt_backend_jm_watchdog_acquire,
.release = kbasep_hwcnt_backend_jm_watchdog_release,
.timestamp_ns = kbasep_hwcnt_backend_jm_watchdog_timestamp_ns,
.dump_enable = kbasep_hwcnt_backend_jm_watchdog_dump_enable,
.dump_enable_nolock = kbasep_hwcnt_backend_jm_watchdog_dump_enable_nolock,

Some files were not shown because too many files have changed in this diff Show More