diff --git a/Documentation/ABI/testing/sysfs-device-mali b/Documentation/ABI/testing/sysfs-device-mali index 12a1667feeb2..c6326b3ca9ad 100644 --- a/Documentation/ABI/testing/sysfs-device-mali +++ b/Documentation/ABI/testing/sysfs-device-mali @@ -1,15 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2017-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software - * Foundation) and any use by you of this program is subject to the terms - * of such GNU licence. + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. * - * A copy of the licence is included with the program) and can also be obtained - * from Free Software Foundation) Inc.) 51 Franklin Street) Fifth Floor) - * Boston) MA 02110-1301) USA. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. * */ @@ -34,16 +40,6 @@ Description: driver, On reading it provides the current DVFS sampling period, on writing a value we set the DVFS sampling period. -What: /sys/class/misc/mali%u/device/dummy_job_wa_info -Description: - This attribute is available only with platform device that - supports a Job Manager based GPU that requires a GPU workaround - to execute the dummy fragment job on all shader cores to - workaround a hang issue. - - Its a readonly attribute and on reading gives details on the - options used with the dummy workaround. - What: /sys/class/misc/mali%u/device/fw_timeout Description: This attribute is available only with mali platform @@ -78,111 +74,6 @@ Description: is supported or is powered down after suspending command stream groups. -What: /sys/class/misc/mali%u/device/js_ctx_scheduling_mode -Description: - This attribute is available only with platform device that - supports a Job Manager based GPU. This attribute is used to set - context scheduling priority for a job slot. - - On Reading it provides the currently set job slot context - priority. - - Writing 0 to this attribute sets it to the mode were - higher priority atoms will be scheduled first, regardless of - the context they belong to. Newly-runnable higher priority atoms - can preempt lower priority atoms currently running on the GPU, - even if they belong to a different context. - - Writing 1 to this attribute set it to the mode were the - highest-priority atom will be chosen from each context in turn - using a round-robin algorithm, so priority only has an effect - within the context an atom belongs to. Newly-runnable higher - priority atoms can preempt the lower priority atoms currently - running on the GPU, but only if they belong to the same context. - -What: /sys/class/misc/mali%u/device/js_scheduling_period -Description: - This attribute is available only with platform device that - supports a Job Manager based GPU. Used to set the job scheduler - tick period in nano-seconds. The Job Scheduler determines the - jobs that are run on the GPU, and for how long, Job Scheduler - makes decisions at a regular time interval determined by value - in js_scheduling_period. - -What: /sys/class/misc/mali%u/device/js_softstop_always -Description: - This attribute is available only with platform device that - supports a Job Manager based GPU. Soft-stops are disabled when - only a single context is present, this attribute is used to - enable soft-stop when only a single context is present can be - used for debug and unit-testing purposes. - -What: /sys/class/misc/mali%u/device/js_timeouts -Description: - This attribute is available only with platform device that - supports a Job Manager based GPU. It used to set the soft stop - and hard stop times for the job scheduler. - - Writing value 0 causes no change, or -1 to restore the - default timeout. - - The format used to set js_timeouts is - " - - " - - -What: /sys/class/misc/mali%u/device/lp_mem_pool_max_size -Description: - This attribute is used to set the maximum number of large pages - memory pools that the driver can contain. Large pages are of - size 2MB. On read it displays all the max size of all memory - pools and can be used to modify each individual pools as well. - -What: /sys/class/misc/mali%u/device/lp_mem_pool_size -Description: - This attribute is used to set the number of large memory pages - which should be populated, changing this value may cause - existing pages to be removed from the pool, or new pages to be - created and then added to the pool. On read it will provide - pool size for all available pools and we can modify individual - pool. - -What: /sys/class/misc/mali%u/device/mem_pool_max_size -Description: - This attribute is used to set the maximum number of small pages - for memory pools that the driver can contain. Here small pages - are of size 4KB. On read it will display the max size for all - available pools and allows us to set max size of - individual pools. - -What: /sys/class/misc/mali%u/device/mem_pool_size -Description: - This attribute is used to set the number of small memory pages - which should be populated, changing this value may cause - existing pages to be removed from the pool, or new pages to - be created and then added to the pool. On read it will provide - pool size for all available pools and we can modify individual - pool. - -What: /sys/class/misc/mali%u/device/device/mempool/ctx_default_max_size -Description: - This attribute is used to set maximum memory pool size for - all the memory pool so that the maximum amount of free memory - that each pool can hold is identical. - -What: /sys/class/misc/mali%u/device/device/mempool/lp_max_size -Description: - This attribute is used to set the maximum number of large pages - for all memory pools that the driver can contain. - Large pages are of size 2MB. - -What: /sys/class/misc/mali%u/device/device/mempool/max_size -Description: - This attribute is used to set the maximum number of small pages - for all the memory pools that the driver can contain. - Here small pages are of size 4KB. - What: /sys/class/misc/mali%u/device/pm_poweroff Description: This attribute contains the current values, represented as the diff --git a/Documentation/ABI/testing/sysfs-device-mali-coresight-source b/Documentation/ABI/testing/sysfs-device-mali-coresight-source index 58d9085b8bb6..c8453702bdd9 100644 --- a/Documentation/ABI/testing/sysfs-device-mali-coresight-source +++ b/Documentation/ABI/testing/sysfs-device-mali-coresight-source @@ -1,15 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2017-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software - * Foundation) and any use by you of this program is subject to the terms - * of such GNU licence. + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. * - * A copy of the licence is included with the program) and can also be obtained - * from Free Software Foundation) Inc.) 51 Franklin Street) Fifth Floor) - * Boston) MA 02110-1301) USA. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. * */ diff --git a/Documentation/devicetree/bindings/arm/mali-valhall.txt b/Documentation/devicetree/bindings/arm/mali-valhall.txt new file mode 100644 index 000000000000..18f6af21210d --- /dev/null +++ b/Documentation/devicetree/bindings/arm/mali-valhall.txt @@ -0,0 +1,257 @@ +# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note +# +# (C) COPYRIGHT 2013-2024 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU license. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# + +* ARM Mali Midgard / Bifrost devices + + +Required properties: + +- compatible : Should be mali, replacing digits with x from the back, +until malitxx, and it must end with one of: "arm,malit6xx" or +"arm,mali-midgard" or "arm,mali-bifrost" +- reg : Physical base address of the device and length of the register area. +- interrupts : Contains the three IRQ lines required by T-6xx devices +- interrupt-names : Contains the names of IRQ resources in the order they were +provided in the interrupts property. Must contain: "JOB, "MMU", "GPU". + +Optional: + +- clocks : One or more pairs of phandle to clock and clock specifier + for the Mali device. The order is important: the first clock + shall correspond to the "clk_mali" source. Other clocks are optional + and, if present, they shall correspond to domains like "shadercores", + which is available for all GPUs, or "coregroup" and "neuralengines" + which are available for newer GPUs. Also notice that the "neuralengines" + clock domain, if present, doesn't expect a corresponding regulator. +- clock-names : Shall be set to values like: "clk_mali", "shadercores", "coregroup", + "neuralengines". Only the first one is mandatory. Most GPUs + support only the first two entries. +- mali-supply : Phandle to the top level regulator for the Mali device. + Refer to +Documentation/devicetree/bindings/regulator/regulator.txt for details. +- mem-supply : Phandle to memory regulator for the Mali device. This is optional. +- operating-points-v2 : Refer to Documentation/devicetree/bindings/power/mali-opp.txt +for details. +- quirks-gpu : Used to write to the JM_CONFIG or CSF_CONFIG register. + Should be used with care. Options passed here are used to override + certain default behavior. Note: This will override 'idvs-group-size' + field in devicetree and module param 'corestack_driver_control', + therefore if 'quirks-gpu' is used then 'idvs-group-size' and + 'corestack_driver_control' value should be incorporated into 'quirks-gpu'. +- quirks-sc : Used to write to the SHADER_CONFIG register. + Should be used with care. Options passed here are used to override + certain default behavior. +- quirks-tiler : Used to write to the TILER_CONFIG register. + Should be used with care. Options passed here are used to + disable or override certain default behavior. +- quirks-mmu : Used to write to the L2_CONFIG register. + Should be used with care. Options passed here are used to + disable or override certain default behavior. +- power-model : Sets the power model parameters. Defined power models include: + "mali-simple-power-model", "mali-g51-power-model", "mali-g52-power-model", + "mali-g52_r1-power-model", "mali-g71-power-model", "mali-g72-power-model", + "mali-g76-power-model", "mali-g77-power-model", "mali-tnax-power-model", + "mali-tbex-power-model" and "mali-tbax-power-model". + - mali-simple-power-model: this model derives the GPU power usage based + on the GPU voltage scaled by the system temperature. Note: it was + designed for the Juno platform, and may not be suitable for others. + - compatible: Should be "arm,mali-simple-power-model" + - dynamic-coefficient: Coefficient, in pW/(Hz V^2), which is + multiplied by v^2*f to calculate the dynamic power consumption. + - static-coefficient: Coefficient, in uW/V^3, which is + multiplied by v^3 to calculate the static power consumption. + - ts: An array containing coefficients for the temperature + scaling factor. This is used to scale the static power by a + factor of tsf/1000000, + where tsf = ts[3]*T^3 + ts[2]*T^2 + ts[1]*T + ts[0], + and T = temperature in degrees. + - thermal-zone: A string identifying the thermal zone used for + the GPU + - temp-poll-interval-ms: the interval at which the system + temperature is polled + - mali-g*-power-model(s): unless being stated otherwise, these models derive + the GPU power usage based on performance counters, so they are more + accurate. + - compatible: Should be, as examples, "arm,mali-g51-power-model" / + "arm,mali-g72-power-model". + - scale: the dynamic power calculated by the power model is + multiplied by a factor of 'scale'. This value should be + chosen to match a particular implementation. + - min_sample_cycles: Fall back to the simple power model if the + number of GPU cycles for a given counter dump is less than + 'min_sample_cycles'. The default value of this should suffice. + * Note: when IPA is used, two separate power models (simple and counter-based) + are used at different points so care should be taken to configure + both power models in the device tree (specifically dynamic-coefficient, + static-coefficient and scale) to best match the platform. +- power-policy : Sets the GPU power policy at probe time. Available options are + "coarse_demand" and "always_on". If not set, then "coarse_demand" is used. +- system-coherency : Sets the coherency protocol to be used for coherent + accesses made from the GPU. + If not set then no coherency is used. + - 0 : ACE-Lite + - 1 : ACE + - 31 : No coherency +- ipa-model : Sets the IPA model to be used for power management. GPU probe will fail if the + model is not found in the registered models list. If no model is specified here, + a gpu-id based model is picked if available, otherwise the default model is used. + - mali-simple-power-model: Default model used on mali +- idvs-group-size : Override the IDVS group size value. Tasks are sent to + cores in groups of N + 1, so i.e. 0xF means 16 tasks. + Valid values are between 0 to 0x3F (including). +- l2-size : Override L2 cache size on GPU that supports it. Value should be larger than the minimum + size 1KiB and smaller than the maximum size. Maximum size is Hardware integration dependent. + The value passed should be of log2(Cache Size in Bytes). + For example for a 1KiB of cache size, 0xa should be passed. +- l2-hash : Override L2 hash function on GPU that supports it +- l2-hash-values : Override L2 hash function using provided hash values, on GPUs that supports it. + It is mutually exclusive with 'l2-hash'. Only one or the other must be + used in a supported GPU. +- arbiter-if : Phandle to the arbif platform device, used to provide KBASE with an interface + to the Arbiter. This is required when using arbitration; setting to a non-NULL + value will enable arbitration. + If arbitration is in use, then there should be no external GPU control. + When arbiter-if is in use then the following must not be: + - power-model (no IPA allowed with arbitration) + - #cooling-cells + - operating-points-v2 (no dvfs in kbase with arbitration) + - system-coherency with a value of 1 (no full coherency with arbitration) +- int-id-override: list of tuples defining the IDs needed to be + set and the setting coresponding to the SYSC_ALLOC register. +- propagate-bits: Used to write to L2_CONFIG.PBHA_HWU. This bitset establishes which + PBHA bits are propagated on the AXI bus. +- mma-wa-id: Sets a GPU MMU PBHA value to be used as NONCACHEABLE. GPU Page Table Entries + (PTEs) with this PBHA value set will have uncached transactions from the GPU. + Valid values are from 1 to 15. See mgm_update_gpu_pte() in + include/linux/memory_group_manager.h for the interface to configure the PTEs. Only + supported by arch 14.8.x and later GPUs. + + +Example for a Mali GPU with 1 clock and 1 regulator: + +gpu@0xfc010000 { + compatible = "arm,malit602", "arm,malit60x", "arm,malit6xx", "arm,mali-midgard"; + reg = <0xfc010000 0x4000>; + interrupts = <0 36 4>, <0 37 4>, <0 38 4>; + interrupt-names = "JOB", "MMU", "GPU"; + + clocks = <&pclk_mali>; + clock-names = "clk_mali"; + mali-supply = <&vdd_mali>; + operating-points-v2 = <&gpu_opp_table>; + power_model@0 { + compatible = "arm,mali-simple-power-model"; + static-coefficient = <2427750>; + dynamic-coefficient = <4687>; + ts = <20000 2000 (-20) 2>; + thermal-zone = "gpu"; + }; + power_model@1 { + compatible = "arm,mali-g71-power-model"; + scale = <5>; + }; + + idvs-group-size = <0x7>; + l2-size = /bits/ 8 <0x10>; + l2-hash = /bits/ 8 <0x04>; /* or l2-hash-values = <0x12345678 0x8765 0xAB>; */ +}; + +gpu_opp_table: opp_table0 { + compatible = "operating-points-v2"; + + opp@533000000 { + opp-hz = /bits/ 64 <533000000>; + opp-microvolt = <1250000>; + }; + opp@450000000 { + opp-hz = /bits/ 64 <450000000>; + opp-microvolt = <1150000>; + }; + opp@400000000 { + opp-hz = /bits/ 64 <400000000>; + opp-microvolt = <1125000>; + }; + opp@350000000 { + opp-hz = /bits/ 64 <350000000>; + opp-microvolt = <1075000>; + }; + opp@266000000 { + opp-hz = /bits/ 64 <266000000>; + opp-microvolt = <1025000>; + }; + opp@160000000 { + opp-hz = /bits/ 64 <160000000>; + opp-microvolt = <925000>; + }; + opp@100000000 { + opp-hz = /bits/ 64 <100000000>; + opp-microvolt = <912500>; + }; +}; + +Example for a Mali GPU with 2 clocks and 2 regulators: + +gpu: gpu@6e000000 { + compatible = "arm,mali-midgard"; + reg = <0x0 0x6e000000 0x0 0x200000>; + interrupts = <0 168 4>, <0 168 4>, <0 168 4>; + interrupt-names = "JOB", "MMU", "GPU"; + clocks = <&clk_mali 0>, <&clk_mali 1>; + clock-names = "clk_mali", "shadercores"; + mali-supply = <&supply0_3v3>; + mem-supply = <&supply1_3v3>; + system-coherency = <31>; + operating-points-v2 = <&gpu_opp_table>; +}; + +gpu_opp_table: opp_table0 { + compatible = "operating-points-v2", "operating-points-v2-mali"; + + opp@0 { + opp-hz = /bits/ 64 <50000000>; + opp-hz-real = /bits/ 64 <50000000>, /bits/ 64 <45000000>; + opp-microvolt = <820000>, <800000>; + opp-core-mask = /bits/ 64 <0xf>; + }; + opp@1 { + opp-hz = /bits/ 64 <40000000>; + opp-hz-real = /bits/ 64 <40000000>, /bits/ 64 <35000000>; + opp-microvolt = <720000>, <700000>; + opp-core-mask = /bits/ 64 <0x7>; + }; + opp@2 { + opp-hz = /bits/ 64 <30000000>; + opp-hz-real = /bits/ 64 <30000000>, /bits/ 64 <25000000>; + opp-microvolt = <620000>, <700000>; + opp-core-mask = /bits/ 64 <0x3>; + }; +}; + +Example for a Mali GPU supporting PBHA configuration via DTB (default): + +gpu@0xfc010000 { + ... + pbha { + int-id-override = <2 0x32>, <9 0x05>, <16 0x32>; + propagate-bits = /bits/ 8 <0x03>; + mma-wa-id = <2>; + }; + ... +}; diff --git a/arch/arm/boot/dts/Makefile b/arch/arm/boot/dts/Makefile index 2bfcd951e676..a4d0587fada1 100644 --- a/arch/arm/boot/dts/Makefile +++ b/arch/arm/boot/dts/Makefile @@ -1179,6 +1179,9 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += \ rv1126b-evb1-v10-bt-sco.dtb \ rv1126b-evb1-v10-dual-4k.dtb \ rv1126b-evb1-v10-dv.dtb \ + rv1126b-evb1-v10-fastboot-emmc.dtb \ + rv1126b-evb1-v10-fastboot-spi-nand.dtb \ + rv1126b-evb1-v10-fastboot-spi-nor.dtb \ rv1126b-evb1-v10-spi-nor.dtb \ rv1126b-evb2-v10.dtb \ rv1126b-evb2-v10-mcu-k350c4516t.dtb \ diff --git a/arch/arm/boot/dts/rv1126b-evb1-v10-fastboot-emmc.dts b/arch/arm/boot/dts/rv1126b-evb1-v10-fastboot-emmc.dts new file mode 100644 index 000000000000..ae214ef870ac --- /dev/null +++ b/arch/arm/boot/dts/rv1126b-evb1-v10-fastboot-emmc.dts @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2025 Rockchip Electronics Co., Ltd. + */ + +#include "arm64/rockchip/rv1126b-evb1-v10-fastboot-emmc.dts" + +/ { + model = "Rockchip RV1126B EVB1 V10 Fastboot Board"; + compatible = "rockchip,rv1126b-evb1-v10-fastboot", "rockchip,rv1126b"; + + chosen { + bootargs = "loglevel=0 initcall_debug=0 earlycon=uart8250,mmio32,0x20810000 console=ttyFIQ0 root=/dev/rd0 rootfstype=erofs rootflags=dax snd_soc_core.prealloc_buffer_size_kbytes=16 coherent_pool=32K"; + }; +}; + +&ramdisk_r { + reg = <0x48c40000 (40 * 0x00100000)>; +}; + +&ramdisk_c { + reg = <0x4b440000 (20 * 0x00100000)>; +}; diff --git a/arch/arm/boot/dts/rv1126b-evb1-v10-fastboot-spi-nand.dts b/arch/arm/boot/dts/rv1126b-evb1-v10-fastboot-spi-nand.dts new file mode 100644 index 000000000000..d96e9b3845bd --- /dev/null +++ b/arch/arm/boot/dts/rv1126b-evb1-v10-fastboot-spi-nand.dts @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2025 Rockchip Electronics Co., Ltd. + */ + +#include "arm64/rockchip/rv1126b-evb1-v10-fastboot-spi-nand.dts" + +/ { + model = "Rockchip RV1126B EVB1 V10 Board"; + compatible = "rockchip,rv1126b-evb1-v10-fastboot-spi-nand", "rockchip,rv1126b"; + + chosen { + bootargs = "loglevel=0 initcall_debug=0 earlycon=uart8250,mmio32,0x20810000 console=ttyFIQ0 root=/dev/rd0 rootfstype=erofs rootflags=dax snd_soc_core.prealloc_buffer_size_kbytes=16 coherent_pool=32K"; + }; +}; + +&ramdisk_r { + reg = <0x48c40000 (40 * 0x00100000)>; +}; + +&ramdisk_c { + reg = <0x4b440000 (20 * 0x00100000)>; +}; diff --git a/arch/arm/boot/dts/rv1126b-evb1-v10-fastboot-spi-nor.dts b/arch/arm/boot/dts/rv1126b-evb1-v10-fastboot-spi-nor.dts new file mode 100644 index 000000000000..43e000f8e094 --- /dev/null +++ b/arch/arm/boot/dts/rv1126b-evb1-v10-fastboot-spi-nor.dts @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2025 Rockchip Electronics Co., Ltd. + */ + +#include "arm64/rockchip/rv1126b-evb1-v10-fastboot-spi-nor.dts" + +/ { + model = "Rockchip RV1126B EVB1 V10 Board"; + compatible = "rockchip,rv1126b-evb1-v10-fastboot-spi-nand", "rockchip,rv1126b"; + + chosen { + bootargs = "loglevel=0 initcall_debug=0 earlycon=uart8250,mmio32,0x20810000 console=ttyFIQ0 root=/dev/rd0 rootfstype=erofs rootflags=dax snd_soc_core.prealloc_buffer_size_kbytes=16 coherent_pool=32K"; + }; +}; + +&ramdisk_r { + reg = <0x48c40000 (20 * 0x00100000)>; +}; + +&ramdisk_c { + reg = <0x4a040000 (10 * 0x00100000)>; +}; diff --git a/arch/arm/configs/rv1126b-fastboot.config b/arch/arm/configs/rv1126b-fastboot.config index 0a84152f88f2..3cb56be2c5e0 100644 --- a/arch/arm/configs/rv1126b-fastboot.config +++ b/arch/arm/configs/rv1126b-fastboot.config @@ -1,45 +1,54 @@ CONFIG_BLK_DEV_INITRD=y -CONFIG_CRC16=m +CONFIG_CRC16=y CONFIG_CRYPTO=y CONFIG_DAX=y +CONFIG_ELF_CORE=y CONFIG_EROFS_FS=y -# CONFIG_ETHERNET is not set CONFIG_EXT4_FS=m +CONFIG_EXTCON=y CONFIG_FILE_LOCKING=y +CONFIG_INPUT=y CONFIG_JFFS2_FS=y CONFIG_KERNEL_GZIP=y # CONFIG_KERNEL_XZ is not set CONFIG_LIBCRC32C=y -CONFIG_MAILBOX=y -# CONFIG_MDIO_DEVICE is not set CONFIG_MMC=y CONFIG_MSDOS_FS=m CONFIG_MSDOS_PARTITION=y CONFIG_MTD_BLOCK=m CONFIG_NLS_CODEPAGE_936=m -# CONFIG_PHYLIB is not set CONFIG_PHY_ROCKCHIP_CSI2_DPHY=y +CONFIG_RK_DMABUF_PROCFS=y CONFIG_ROCKCHIP_DVBM=y CONFIG_ROCKCHIP_HW_DECOMPRESS=y CONFIG_ROCKCHIP_MULTI_RGA=y +CONFIG_ROCKCHIP_OPP=y CONFIG_ROCKCHIP_RAMDISK=y CONFIG_ROCKCHIP_RGA_PROC_FS=y CONFIG_ROCKCHIP_THUNDER_BOOT=y CONFIG_ROCKCHIP_THUNDER_BOOT_DEFER_FREE_MEMBLOCK=y CONFIG_ROCKCHIP_VENDOR_STORAGE=m # CONFIG_SLUB_SYSFS is not set -CONFIG_SND_SIMPLE_CARD=m -CONFIG_SND_SIMPLE_CARD_UTILS=m +# CONFIG_SND_SIMPLE_CARD is not set +CONFIG_SND_SOC_DUMMY_CODEC=m +CONFIG_SND_SOC_RK_DSM=m CONFIG_SND_SOC_ROCKCHIP=m +CONFIG_SND_SOC_ROCKCHIP_ASRC=m +CONFIG_SND_SOC_ROCKCHIP_PDM_V2=m CONFIG_SND_SOC_ROCKCHIP_SAI=m -CONFIG_SND_SOC_RV1106=m CONFIG_SPI=y +CONFIG_SQUASHFS_LZ4=y CONFIG_VFAT_FS=m +# CONFIG_VIDEO_RK_IRCUT is not set +CONFIG_VIDEO_ROCKCHIP_AIISP=y +CONFIG_VIDEO_ROCKCHIP_AVSP=y CONFIG_VIDEO_ROCKCHIP_CIF=y +CONFIG_VIDEO_ROCKCHIP_FEC=y CONFIG_VIDEO_ROCKCHIP_ISP=y CONFIG_VIDEO_ROCKCHIP_VPSS=y CONFIG_VIDEO_SC200AI=y CONFIG_VIDEO_SC450AI=y +CONFIG_VIDEO_SC850SL=y # CONFIG_AD2S1200 is not set # CONFIG_AD2S1210 is not set # CONFIG_AD2S90 is not set @@ -109,16 +118,14 @@ CONFIG_VIDEO_SC450AI=y # CONFIG_ADXRS290 is not set # CONFIG_ADXRS450 is not set # CONFIG_AFE4403 is not set -# CONFIG_ALTERA_MBOX is not set -# CONFIG_ARM_MHU is not set -# CONFIG_ARM_MHU_V2 is not set -# CONFIG_ARM_SCPI_PROTOCOL is not set # CONFIG_AS3935 is not set # CONFIG_BMA220 is not set # CONFIG_BMC150_MAGN_SPI is not set # CONFIG_BMI088_ACCEL is not set # CONFIG_BMI160_SPI is not set # CONFIG_BSD_DISKLABEL is not set +# CONFIG_CHARGER_BQ24190 is not set +CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS=y # CONFIG_CRYPTO_842 is not set # CONFIG_CRYPTO_ADIANTUM is not set # CONFIG_CRYPTO_AEGIS128 is not set @@ -215,8 +222,11 @@ CONFIG_CRYPTO_MANAGER_DISABLE_TESTS=y # CONFIG_CRYPTO_XTS is not set # CONFIG_CRYPTO_XXHASH is not set # CONFIG_CRYPTO_ZSTD is not set +# CONFIG_DM9051 is not set # CONFIG_EEPROM_93XX46 is not set # CONFIG_EEPROM_AT25 is not set +# CONFIG_ENC28J60 is not set +# CONFIG_ENCX24J600 is not set # CONFIG_EROFS_FS_DEBUG is not set # CONFIG_EROFS_FS_XATTR is not set # CONFIG_EROFS_FS_ZIP is not set @@ -224,6 +234,14 @@ CONFIG_CRYPTO_MANAGER_DISABLE_TESTS=y # CONFIG_EXT4_FS_POSIX_ACL is not set # CONFIG_EXT4_FS_SECURITY is not set CONFIG_EXT4_USE_FOR_EXT2=y +# CONFIG_EXTCON_ADC_JACK is not set +# CONFIG_EXTCON_FSA9480 is not set +# CONFIG_EXTCON_GPIO is not set +# CONFIG_EXTCON_MAX3355 is not set +# CONFIG_EXTCON_PTN5150 is not set +# CONFIG_EXTCON_RT8973A is not set +# CONFIG_EXTCON_SM5502 is not set +# CONFIG_EXTCON_USB_GPIO is not set # CONFIG_EZX_PCAP is not set CONFIG_FAT_DEFAULT_CODEPAGE=936 CONFIG_FAT_DEFAULT_IOCHARSET="cp936" @@ -241,10 +259,27 @@ CONFIG_FS_MBCACHE=m # CONFIG_GPIO_PISOSR is not set # CONFIG_GPIO_XRA1403 is not set # CONFIG_HI8435 is not set +# CONFIG_HID is not set +# CONFIG_I2C_HID_OF is not set +# CONFIG_I2C_HID_OF_ELAN is not set +# CONFIG_I2C_HID_OF_GOODIX is not set # CONFIG_IIO_SSP_SENSORHUB is not set CONFIG_INITCALL_ASYNC=y # CONFIG_INITRAMFS_FORCE is not set CONFIG_INITRAMFS_SOURCE="" +# CONFIG_INPUT_EVBUG is not set +CONFIG_INPUT_EVDEV=y +# CONFIG_INPUT_FF_MEMLESS is not set +# CONFIG_INPUT_JOYDEV is not set +# CONFIG_INPUT_JOYSTICK is not set +CONFIG_INPUT_KEYBOARD=y +# CONFIG_INPUT_MATRIXKMAP is not set +# CONFIG_INPUT_MISC is not set +# CONFIG_INPUT_MOUSE is not set +# CONFIG_INPUT_MOUSEDEV is not set +# CONFIG_INPUT_SPARSEKMAP is not set +# CONFIG_INPUT_TABLET is not set +# CONFIG_INPUT_TOUCHSCREEN is not set # CONFIG_INV_ICM42600_SPI is not set # CONFIG_INV_ICM42670_SPI is not set # CONFIG_INV_MPU6050_SPI is not set @@ -264,6 +299,36 @@ CONFIG_JFFS2_FS_WRITEBUFFER=y # CONFIG_JFFS2_RUBIN is not set # CONFIG_JFFS2_SUMMARY is not set CONFIG_JFFS2_ZLIB=y +CONFIG_KEYBOARD_ADC=y +# CONFIG_KEYBOARD_ADP5588 is not set +# CONFIG_KEYBOARD_ADP5589 is not set +# CONFIG_KEYBOARD_ATKBD is not set +# CONFIG_KEYBOARD_BCM is not set +# CONFIG_KEYBOARD_CAP11XX is not set +# CONFIG_KEYBOARD_CYPRESS_SF is not set +# CONFIG_KEYBOARD_DLINK_DIR685 is not set +# CONFIG_KEYBOARD_GPIO is not set +# CONFIG_KEYBOARD_GPIO_POLLED is not set +# CONFIG_KEYBOARD_LKKBD is not set +# CONFIG_KEYBOARD_LM8333 is not set +# CONFIG_KEYBOARD_MATRIX is not set +# CONFIG_KEYBOARD_MAX7359 is not set +# CONFIG_KEYBOARD_MCS is not set +# CONFIG_KEYBOARD_MPR121 is not set +# CONFIG_KEYBOARD_NEWTON is not set +# CONFIG_KEYBOARD_OMAP4 is not set +# CONFIG_KEYBOARD_OPENCORES is not set +# CONFIG_KEYBOARD_PINEPHONE is not set +# CONFIG_KEYBOARD_QT1050 is not set +# CONFIG_KEYBOARD_QT1070 is not set +# CONFIG_KEYBOARD_QT2160 is not set +# CONFIG_KEYBOARD_SAMSUNG is not set +# CONFIG_KEYBOARD_STOWAWAY is not set +# CONFIG_KEYBOARD_SUNKBD is not set +# CONFIG_KEYBOARD_TCA6416 is not set +# CONFIG_KEYBOARD_TCA8418 is not set +# CONFIG_KEYBOARD_XTKBD is not set +# CONFIG_KS8851 is not set # CONFIG_LATTICE_ECP3_CONFIG is not set # CONFIG_LMK04832 is not set # CONFIG_LTC1660 is not set @@ -271,8 +336,7 @@ CONFIG_JFFS2_ZLIB=y # CONFIG_LTC2632 is not set # CONFIG_LTC2688 is not set # CONFIG_LTC2983 is not set -# CONFIG_MAILBOX_POLL_PERIOD_US is not set -# CONFIG_MAILBOX_TEST is not set +CONFIG_LZ4_DECOMPRESS=y # CONFIG_MAX1027 is not set # CONFIG_MAX11100 is not set # CONFIG_MAX1118 is not set @@ -322,6 +386,7 @@ CONFIG_MMC_QUEUE_DEPTH=1 # CONFIG_MMC_USDHI6ROL0 is not set # CONFIG_MOXTET is not set # CONFIG_MPL115_SPI is not set +# CONFIG_MSE102X is not set CONFIG_MTD_BLKDEVS=m # CONFIG_MTD_DATAFLASH is not set # CONFIG_MTD_MCHP23K256 is not set @@ -357,11 +422,12 @@ CONFIG_MTD_SPI_NOR_WINBOND=y # CONFIG_MTD_SPI_NOR_XMC is not set # CONFIG_MTD_SPI_NOR_XTX is not set # CONFIG_MTD_SST25L is not set +CONFIG_NET_VENDOR_ADI=y # CONFIG_PI433 is not set -# CONFIG_PL320_MBOX is not set -# CONFIG_PLATFORM_MHU is not set # CONFIG_PWRSEQ_EMMC is not set CONFIG_PWRSEQ_SIMPLE=y +# CONFIG_QCA7000_SPI is not set +# CONFIG_RC_CORE is not set # CONFIG_RD_BZIP2 is not set # CONFIG_RD_GZIP is not set # CONFIG_RD_LZ4 is not set @@ -371,16 +437,15 @@ CONFIG_PWRSEQ_SIMPLE=y # CONFIG_RD_ZSTD is not set CONFIG_REGMAP_SPI=y # CONFIG_REGULATOR_TPS6524X is not set +# CONFIG_RMI4_CORE is not set # CONFIG_ROCKCHIP_HW_DECOMPRESS_TEST is not set -# CONFIG_ROCKCHIP_MBOX is not set # CONFIG_ROCKCHIP_MMC_VENDOR_STORAGE is not set CONFIG_ROCKCHIP_MTD_VENDOR_STORAGE=m # CONFIG_ROCKCHIP_RAM_VENDOR_STORAGE is not set +# CONFIG_ROCKCHIP_REMOTECTL is not set CONFIG_ROCKCHIP_RGA_DEBUGGER=y CONFIG_ROCKCHIP_THUNDER_BOOT_MMC=y -# CONFIG_ROCKCHIP_THUNDER_BOOT_SERVICE is not set CONFIG_ROCKCHIP_THUNDER_BOOT_SFC=y -# CONFIG_RPMSG_QCOM_GLINK_RPM is not set # CONFIG_RTC_DRV_DS1302 is not set # CONFIG_RTC_DRV_DS1305 is not set # CONFIG_RTC_DRV_DS1343 is not set @@ -399,15 +464,21 @@ CONFIG_ROCKCHIP_THUNDER_BOOT_SFC=y # CONFIG_SCA3300 is not set # CONFIG_SDIO_UART is not set # CONFIG_SENSORS_HMC5843_SPI is not set +# CONFIG_SENSORS_LIS3_I2C is not set +# CONFIG_SENSORS_LIS3_SPI is not set # CONFIG_SENSORS_RM3100_SPI is not set +# CONFIG_SENSOR_DEVICE is not set # CONFIG_SERIAL_MAX3100 is not set # CONFIG_SERIAL_MAX310X is not set +CONFIG_SND_JACK_INPUT_DEV=y # CONFIG_SND_SOC_ADAU1372_SPI is not set # CONFIG_SND_SOC_ADAU1761_SPI is not set # CONFIG_SND_SOC_AK4104 is not set # CONFIG_SND_SOC_CS35L41_SPI is not set # CONFIG_SND_SOC_CS35L45_SPI is not set # CONFIG_SND_SOC_CS4271_SPI is not set +# CONFIG_SND_SOC_CS42L52 is not set +# CONFIG_SND_SOC_CS42L56 is not set # CONFIG_SND_SOC_ES8328_SPI is not set # CONFIG_SND_SOC_PCM179X_SPI is not set # CONFIG_SND_SOC_PCM186X_SPI is not set @@ -415,6 +486,7 @@ CONFIG_ROCKCHIP_THUNDER_BOOT_SFC=y # CONFIG_SND_SOC_PCM3168A_SPI is not set # CONFIG_SND_SOC_PCM512x_SPI is not set # CONFIG_SND_SOC_RK3399_GRU_SOUND is not set +CONFIG_SND_SOC_ROCKCHIP_MULTICODECS=m # CONFIG_SND_SOC_ROCKCHIP_SPI_CODEC is not set # CONFIG_SND_SOC_SSM2602_SPI is not set # CONFIG_SND_SOC_TLV320AIC23_SPI is not set @@ -423,11 +495,13 @@ CONFIG_ROCKCHIP_THUNDER_BOOT_SFC=y # CONFIG_SND_SOC_WM8731_SPI is not set # CONFIG_SND_SOC_WM8770 is not set # CONFIG_SND_SOC_WM8804_SPI is not set +# CONFIG_SND_SOC_WM8962 is not set # CONFIG_SND_SOC_ZL38060 is not set # CONFIG_SND_SPI is not set # CONFIG_SOLARIS_X86_PARTITION is not set # CONFIG_SPI_ALTERA is not set # CONFIG_SPI_AMD is not set +# CONFIG_SPI_AX88796C is not set # CONFIG_SPI_AXI_SPI_ENGINE is not set # CONFIG_SPI_BITBANG is not set # CONFIG_SPI_CADENCE is not set diff --git a/arch/arm/configs/rv1126b-nand.config b/arch/arm/configs/rv1126b-nand.config index c356b4f35bfd..9f4c2f0163c5 100644 --- a/arch/arm/configs/rv1126b-nand.config +++ b/arch/arm/configs/rv1126b-nand.config @@ -1,6 +1,81 @@ CONFIG_CRC16=y CONFIG_CRYPTO=y CONFIG_MTD_UBI=y +CONFIG_SPI=y +# CONFIG_AD2S1200 is not set +# CONFIG_AD2S1210 is not set +# CONFIG_AD2S90 is not set +# CONFIG_AD3552R is not set +# CONFIG_AD5360 is not set +# CONFIG_AD5421 is not set +# CONFIG_AD5449 is not set +# CONFIG_AD5504 is not set +# CONFIG_AD5592R is not set +# CONFIG_AD5624R_SPI is not set +# CONFIG_AD5686_SPI is not set +# CONFIG_AD5755 is not set +# CONFIG_AD5758 is not set +# CONFIG_AD5761 is not set +# CONFIG_AD5764 is not set +# CONFIG_AD5766 is not set +# CONFIG_AD5770R is not set +# CONFIG_AD5791 is not set +# CONFIG_AD7124 is not set +# CONFIG_AD7192 is not set +# CONFIG_AD7266 is not set +# CONFIG_AD7280 is not set +# CONFIG_AD7292 is not set +# CONFIG_AD7293 is not set +# CONFIG_AD7298 is not set +# CONFIG_AD7303 is not set +# CONFIG_AD74413R is not set +# CONFIG_AD7476 is not set +# CONFIG_AD7606_IFACE_SPI is not set +# CONFIG_AD7766 is not set +# CONFIG_AD7768_1 is not set +# CONFIG_AD7780 is not set +# CONFIG_AD7791 is not set +# CONFIG_AD7793 is not set +# CONFIG_AD7816 is not set +# CONFIG_AD7887 is not set +# CONFIG_AD7923 is not set +# CONFIG_AD7949 is not set +# CONFIG_AD8366 is not set +# CONFIG_AD8801 is not set +# CONFIG_AD9523 is not set +# CONFIG_AD9832 is not set +# CONFIG_AD9834 is not set +# CONFIG_ADA4250 is not set +# CONFIG_ADF4350 is not set +# CONFIG_ADF4371 is not set +# CONFIG_ADIS16080 is not set +# CONFIG_ADIS16130 is not set +# CONFIG_ADIS16136 is not set +# CONFIG_ADIS16201 is not set +# CONFIG_ADIS16203 is not set +# CONFIG_ADIS16209 is not set +# CONFIG_ADIS16240 is not set +# CONFIG_ADIS16260 is not set +# CONFIG_ADIS16400 is not set +# CONFIG_ADIS16460 is not set +# CONFIG_ADIS16475 is not set +# CONFIG_ADIS16480 is not set +# CONFIG_ADMV1013 is not set +# CONFIG_ADMV4420 is not set +# CONFIG_ADRF6780 is not set +# CONFIG_ADXL313_SPI is not set +# CONFIG_ADXL345_SPI is not set +# CONFIG_ADXL355_SPI is not set +# CONFIG_ADXL367_SPI is not set +# CONFIG_ADXL372_SPI is not set +# CONFIG_ADXRS290 is not set +# CONFIG_ADXRS450 is not set +# CONFIG_AFE4403 is not set +# CONFIG_AS3935 is not set +# CONFIG_BMA220 is not set +# CONFIG_BMC150_MAGN_SPI is not set +# CONFIG_BMI088_ACCEL is not set +# CONFIG_BMI160_SPI is not set # CONFIG_CRYPTO_842 is not set CONFIG_CRYPTO_ACOMP2=y # CONFIG_CRYPTO_ADIANTUM is not set @@ -103,14 +178,204 @@ CONFIG_CRYPTO_MANAGER_DISABLE_TESTS=y # CONFIG_CRYPTO_XTS is not set # CONFIG_CRYPTO_XXHASH is not set # CONFIG_CRYPTO_ZSTD is not set +# CONFIG_DM9051 is not set +# CONFIG_EEPROM_93XX46 is not set +# CONFIG_EEPROM_AT25 is not set +# CONFIG_ENC28J60 is not set +# CONFIG_ENCX24J600 is not set +# CONFIG_EZX_PCAP is not set +# CONFIG_FXLS8962AF_SPI is not set +# CONFIG_FXOS8700_SPI is not set +# CONFIG_GPIO_74X164 is not set +# CONFIG_GPIO_MAX3191X is not set +# CONFIG_GPIO_MAX7301 is not set +# CONFIG_GPIO_MC33880 is not set +# CONFIG_GPIO_PISOSR is not set +# CONFIG_GPIO_XRA1403 is not set +# CONFIG_HI8435 is not set +# CONFIG_IIO_SSP_SENSORHUB is not set +# CONFIG_INV_ICM42600_SPI is not set +# CONFIG_INV_ICM42670_SPI is not set +# CONFIG_INV_MPU6050_SPI is not set +# CONFIG_KS8851 is not set +# CONFIG_LATTICE_ECP3_CONFIG is not set +# CONFIG_LMK04832 is not set +# CONFIG_LTC1660 is not set +# CONFIG_LTC2496 is not set +# CONFIG_LTC2632 is not set +# CONFIG_LTC2688 is not set +# CONFIG_LTC2983 is not set CONFIG_LZO_COMPRESS=y CONFIG_LZO_DECOMPRESS=y +# CONFIG_MAX1027 is not set +# CONFIG_MAX11100 is not set +# CONFIG_MAX1118 is not set +# CONFIG_MAX11205 is not set +# CONFIG_MAX1241 is not set +# CONFIG_MAX31856 is not set +# CONFIG_MAX31865 is not set +# CONFIG_MAX5481 is not set +# CONFIG_MAX5487 is not set +# CONFIG_MAXIM_THERMOCOUPLE is not set +# CONFIG_MCP320X is not set +# CONFIG_MCP3911 is not set +# CONFIG_MCP41010 is not set +# CONFIG_MCP4131 is not set +# CONFIG_MCP4922 is not set +# CONFIG_MFD_ARIZONA_SPI is not set +# CONFIG_MFD_CPCAP is not set +# CONFIG_MFD_DA9052_SPI is not set +# CONFIG_MFD_INTEL_M10_BMC is not set +# CONFIG_MFD_MC13XXX_SPI is not set +# CONFIG_MFD_OCELOT is not set +# CONFIG_MFD_RK806_SPI is not set +# CONFIG_MFD_RSMU_SPI is not set +# CONFIG_MFD_TPS65912_SPI is not set +# CONFIG_MFD_WM831X_SPI is not set +# CONFIG_MICREL_KS8995MA is not set +# CONFIG_MMA7455_SPI is not set +# CONFIG_MOXTET is not set +# CONFIG_MPL115_SPI is not set +# CONFIG_MSE102X is not set +# CONFIG_MTD_DATAFLASH is not set +# CONFIG_MTD_MCHP23K256 is not set +# CONFIG_MTD_MCHP48L640 is not set +CONFIG_MTD_NAND_BBT_USING_FLASH=y +CONFIG_MTD_NAND_CORE=y +CONFIG_MTD_NAND_ECC=y +CONFIG_MTD_SPI_NAND=y +CONFIG_MTD_SPI_NAND_ATO=y +CONFIG_MTD_SPI_NAND_BIWIN=y +CONFIG_MTD_SPI_NAND_DEVICE_AUTOSELECT=y +CONFIG_MTD_SPI_NAND_DOSILICON=y +CONFIG_MTD_SPI_NAND_ESMT=y +CONFIG_MTD_SPI_NAND_ETRON=y +CONFIG_MTD_SPI_NAND_FMSH=y +CONFIG_MTD_SPI_NAND_FORESEE=y +CONFIG_MTD_SPI_NAND_GIGADEVICE=y +CONFIG_MTD_SPI_NAND_GSTO=y +CONFIG_MTD_SPI_NAND_HIKSEMI=y +CONFIG_MTD_SPI_NAND_HYF=y +CONFIG_MTD_SPI_NAND_JSC=y +CONFIG_MTD_SPI_NAND_MACRONIX=y +CONFIG_MTD_SPI_NAND_MICRON=y +CONFIG_MTD_SPI_NAND_PARAGON=y +CONFIG_MTD_SPI_NAND_SILICONGO=y +CONFIG_MTD_SPI_NAND_SKYHIGH=y +CONFIG_MTD_SPI_NAND_TOSHIBA=y +CONFIG_MTD_SPI_NAND_UNIM=y +CONFIG_MTD_SPI_NAND_WINBOND=y +CONFIG_MTD_SPI_NAND_XINCUN=y +CONFIG_MTD_SPI_NAND_XTX=y +CONFIG_MTD_SPI_NAND_ZBIT=y +# CONFIG_MTD_SPI_NOR is not set +# CONFIG_MTD_SST25L is not set CONFIG_MTD_UBI_BEB_LIMIT=20 CONFIG_MTD_UBI_BLOCK=y # CONFIG_MTD_UBI_FASTMAP is not set # CONFIG_MTD_UBI_GLUEBI is not set CONFIG_MTD_UBI_WL_THRESHOLD=4096 +# CONFIG_NET_VENDOR_ADI is not set +# CONFIG_PI433 is not set +# CONFIG_QCA7000_SPI is not set +CONFIG_REGMAP_SPI=y +# CONFIG_REGULATOR_TPS6524X is not set +# CONFIG_ROCKCHIP_MTD_VENDOR_STORAGE is not set +# CONFIG_RTC_DRV_DS1302 is not set +# CONFIG_RTC_DRV_DS1305 is not set +# CONFIG_RTC_DRV_DS1343 is not set +# CONFIG_RTC_DRV_DS1347 is not set +# CONFIG_RTC_DRV_DS1390 is not set +# CONFIG_RTC_DRV_M41T93 is not set +# CONFIG_RTC_DRV_M41T94 is not set +# CONFIG_RTC_DRV_MAX6902 is not set +# CONFIG_RTC_DRV_MAX6916 is not set +# CONFIG_RTC_DRV_MCP795 is not set +# CONFIG_RTC_DRV_PCF2123 is not set +# CONFIG_RTC_DRV_R9701 is not set +# CONFIG_RTC_DRV_RS5C348 is not set +# CONFIG_RTC_DRV_RX4581 is not set +# CONFIG_SCA3000 is not set +# CONFIG_SCA3300 is not set +# CONFIG_SENSORS_HMC5843_SPI is not set +# CONFIG_SENSORS_RM3100_SPI is not set +# CONFIG_SERIAL_MAX3100 is not set +# CONFIG_SERIAL_MAX310X is not set CONFIG_SGL_ALLOC=y +# CONFIG_SND_SOC_ADAU1372_SPI is not set +# CONFIG_SND_SOC_ADAU1761_SPI is not set +# CONFIG_SND_SOC_AK4104 is not set +# CONFIG_SND_SOC_CS35L41_SPI is not set +# CONFIG_SND_SOC_CS35L45_SPI is not set +# CONFIG_SND_SOC_CS4271_SPI is not set +# CONFIG_SND_SOC_ES8328_SPI is not set +# CONFIG_SND_SOC_PCM179X_SPI is not set +# CONFIG_SND_SOC_PCM186X_SPI is not set +# CONFIG_SND_SOC_PCM3060_SPI is not set +# CONFIG_SND_SOC_PCM3168A_SPI is not set +# CONFIG_SND_SOC_PCM512x_SPI is not set +# CONFIG_SND_SOC_RK3399_GRU_SOUND is not set +# CONFIG_SND_SOC_ROCKCHIP_SPI_CODEC is not set +# CONFIG_SND_SOC_SSM2602_SPI is not set +# CONFIG_SND_SOC_TLV320AIC23_SPI is not set +# CONFIG_SND_SOC_TLV320AIC32X4_SPI is not set +# CONFIG_SND_SOC_TLV320AIC3X_SPI is not set +# CONFIG_SND_SOC_WM8731_SPI is not set +# CONFIG_SND_SOC_WM8770 is not set +# CONFIG_SND_SOC_WM8804_SPI is not set +# CONFIG_SND_SOC_ZL38060 is not set +# CONFIG_SND_SPI is not set +# CONFIG_SPI_ALTERA is not set +# CONFIG_SPI_AMD is not set +# CONFIG_SPI_AX88796C is not set +# CONFIG_SPI_AXI_SPI_ENGINE is not set +# CONFIG_SPI_BITBANG is not set +# CONFIG_SPI_CADENCE is not set +# CONFIG_SPI_CADENCE_QUADSPI is not set +# CONFIG_SPI_CADENCE_XSPI is not set +# CONFIG_SPI_DEBUG is not set +# CONFIG_SPI_DESIGNWARE is not set +# CONFIG_SPI_FSL_SPI is not set +# CONFIG_SPI_GPIO is not set +# CONFIG_SPI_LOOPBACK_TEST is not set +CONFIG_SPI_MASTER=y +CONFIG_SPI_MEM=y +# CONFIG_SPI_MICROCHIP_CORE is not set +# CONFIG_SPI_MICROCHIP_CORE_QSPI is not set +# CONFIG_SPI_MUX is not set +# CONFIG_SPI_MXIC is not set +# CONFIG_SPI_NXP_FLEXSPI is not set +# CONFIG_SPI_OC_TINY is not set +# CONFIG_SPI_PL022 is not set +# CONFIG_SPI_ROCKCHIP is not set +# CONFIG_SPI_ROCKCHIP_FLEXBUS_FSPI is not set +# CONFIG_SPI_ROCKCHIP_FLEXBUS_SPI is not set +CONFIG_SPI_ROCKCHIP_SFC=y +# CONFIG_SPI_ROCKCHIP_SLAVE is not set +# CONFIG_SPI_SC18IS602 is not set +# CONFIG_SPI_SIFIVE is not set +# CONFIG_SPI_SLAVE is not set +# CONFIG_SPI_SPIDEV is not set +# CONFIG_SPI_TLE62X0 is not set +# CONFIG_SPI_XCOMM is not set +# CONFIG_SPI_XILINX is not set +# CONFIG_SPI_ZYNQMP_GQSPI is not set +# CONFIG_TI_ADC0832 is not set +# CONFIG_TI_ADC084S021 is not set +# CONFIG_TI_ADC108S102 is not set +# CONFIG_TI_ADC12138 is not set +# CONFIG_TI_ADC128S052 is not set +# CONFIG_TI_ADC161S626 is not set +# CONFIG_TI_ADS124S08 is not set +# CONFIG_TI_ADS131E08 is not set +# CONFIG_TI_ADS7950 is not set +# CONFIG_TI_ADS8344 is not set +# CONFIG_TI_ADS8688 is not set +# CONFIG_TI_DAC082S085 is not set +# CONFIG_TI_DAC7311 is not set +# CONFIG_TI_DAC7612 is not set +# CONFIG_TI_TLC4541 is not set +# CONFIG_TI_TSC2046 is not set # CONFIG_UBIFS_ATIME_SUPPORT is not set CONFIG_UBIFS_FS=y CONFIG_UBIFS_FS_ADVANCED_COMPR=y @@ -120,6 +385,9 @@ CONFIG_UBIFS_FS_SECURITY=y CONFIG_UBIFS_FS_XATTR=y CONFIG_UBIFS_FS_ZLIB=y # CONFIG_UBIFS_FS_ZSTD is not set +# CONFIG_VIDEO_GS1662 is not set +# CONFIG_VIDEO_ROCKCHIP_PREISP is not set +# CONFIG_VIDEO_S5C73M3 is not set CONFIG_ZLIB_DEFLATE=y CONFIG_ZLIB_INFLATE=y # CONFIG_ZRAM is not set diff --git a/arch/arm/configs/rv1126b-rndis.config b/arch/arm/configs/rv1126b-rndis.config index b3d39b93ea11..9c0c5e431e4b 100644 --- a/arch/arm/configs/rv1126b-rndis.config +++ b/arch/arm/configs/rv1126b-rndis.config @@ -1,5 +1,6 @@ -CONFIG_CONFIGFS_FS=y +CONFIG_CONFIGFS_FS=m CONFIG_EXTCON=y +CONFIG_PHY_ROCKCHIP_NANENG_COMBO_PHY=m CONFIG_USB_SUPPORT=y CONFIG_VIDEOBUF2_DMA_SG=m # CONFIG_APPLE_MFI_FASTCHARGE is not set @@ -12,6 +13,7 @@ CONFIG_VIDEOBUF2_DMA_SG=m # CONFIG_EXTCON_RT8973A is not set # CONFIG_EXTCON_SM5502 is not set # CONFIG_EXTCON_USB_GPIO is not set +# CONFIG_HISI_HIKEY_USB is not set # CONFIG_I2C_CP2615 is not set # CONFIG_I2C_DIOLAN_U2C is not set # CONFIG_I2C_ROBOTFUZZ_OSIF is not set @@ -55,13 +57,13 @@ CONFIG_USB_CONFIGFS=m # CONFIG_USB_CONFIGFS_ECM_SUBSET is not set # CONFIG_USB_CONFIGFS_EEM is not set CONFIG_USB_CONFIGFS_F_FS=y -# CONFIG_USB_CONFIGFS_F_HID is not set +CONFIG_USB_CONFIGFS_F_HID=y # CONFIG_USB_CONFIGFS_F_LB_SS is not set # CONFIG_USB_CONFIGFS_F_MIDI is not set # CONFIG_USB_CONFIGFS_F_PRINTER is not set -# CONFIG_USB_CONFIGFS_F_UAC1 is not set +CONFIG_USB_CONFIGFS_F_UAC1=y # CONFIG_USB_CONFIGFS_F_UAC1_LEGACY is not set -# CONFIG_USB_CONFIGFS_F_UAC2 is not set +CONFIG_USB_CONFIGFS_F_UAC2=y CONFIG_USB_CONFIGFS_F_UVC=y # CONFIG_USB_CONFIGFS_MASS_STORAGE is not set # CONFIG_USB_CONFIGFS_NCM is not set @@ -76,12 +78,16 @@ CONFIG_USB_DEFAULT_PERSIST=y # CONFIG_USB_DUMMY_HCD is not set # CONFIG_USB_DWC2 is not set CONFIG_USB_DWC3=m -# CONFIG_USB_DWC3_DUAL_ROLE is not set -CONFIG_USB_DWC3_GADGET=y +CONFIG_USB_DWC3_DUAL_ROLE=y +# CONFIG_USB_DWC3_GADGET is not set # CONFIG_USB_DWC3_HOST is not set CONFIG_USB_DWC3_OF_SIMPLE=m # CONFIG_USB_DYNAMIC_MINORS is not set -# CONFIG_USB_EHCI_HCD is not set +# CONFIG_USB_EHCI_FSL is not set +CONFIG_USB_EHCI_HCD=m +CONFIG_USB_EHCI_HCD_PLATFORM=m +# CONFIG_USB_EHCI_ROOT_HUB_TT is not set +CONFIG_USB_EHCI_TT_NEWSCHED=y # CONFIG_USB_EHSET_TEST_FIXTURE is not set # CONFIG_USB_EMI26 is not set # CONFIG_USB_EMI62 is not set @@ -94,7 +100,10 @@ CONFIG_USB_DWC3_OF_SIMPLE=m # CONFIG_USB_FUNCTIONFS is not set # CONFIG_USB_FUSB300 is not set CONFIG_USB_F_FS=m +CONFIG_USB_F_HID=m CONFIG_USB_F_RNDIS=m +CONFIG_USB_F_UAC1=m +CONFIG_USB_F_UAC2=m CONFIG_USB_F_UVC=m CONFIG_USB_GADGET=m # CONFIG_USB_GADGETFS is not set @@ -138,7 +147,8 @@ CONFIG_USB_LIBCOMPOSITE=m # CONFIG_USB_MV_UDC is not set # CONFIG_USB_NET2272 is not set # CONFIG_USB_NET_DRIVERS is not set -# CONFIG_USB_OHCI_HCD is not set +CONFIG_USB_OHCI_HCD=m +CONFIG_USB_OHCI_HCD_PLATFORM=m # CONFIG_USB_ONBOARD_HUB is not set # CONFIG_USB_OTG is not set # CONFIG_USB_OTG_DISABLE_EXTERNAL_HUB is not set @@ -149,9 +159,10 @@ CONFIG_USB_LIBCOMPOSITE=m # CONFIG_USB_R8A66597 is not set # CONFIG_USB_R8A66597_HCD is not set # CONFIG_USB_RAW_GADGET is not set -# CONFIG_USB_ROLE_SWITCH is not set +CONFIG_USB_ROLE_SWITCH=m # CONFIG_USB_SERIAL is not set # CONFIG_USB_SEVSEG is not set +# CONFIG_USB_SISUSBVGA is not set # CONFIG_USB_SL811_HCD is not set # CONFIG_USB_SNP_UDC_PLAT is not set # CONFIG_USB_TEST is not set @@ -159,9 +170,13 @@ CONFIG_USB_LIBCOMPOSITE=m # CONFIG_USB_TRANCEVIBRATOR is not set # CONFIG_USB_ULPI is not set # CONFIG_USB_ULPI_BUS is not set +CONFIG_USB_U_AUDIO=m CONFIG_USB_U_ETHER=m # CONFIG_USB_WDM is not set -# CONFIG_USB_XHCI_HCD is not set +# CONFIG_USB_XHCI_DBGCAP is not set +CONFIG_USB_XHCI_HCD=m +# CONFIG_USB_XHCI_PCI_RENESAS is not set +CONFIG_USB_XHCI_PLATFORM=m # CONFIG_USB_YUREX is not set # CONFIG_USB_ZERO is not set # CONFIG_XILLYUSB is not set diff --git a/arch/arm/configs/rv1126b-tb.config b/arch/arm/configs/rv1126b-tb.config new file mode 100644 index 000000000000..8672c83a7b26 --- /dev/null +++ b/arch/arm/configs/rv1126b-tb.config @@ -0,0 +1,459 @@ +CONFIG_BLK_DEV_INITRD=y +# CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND is not set +CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE=y +CONFIG_CRYPTO=y +CONFIG_DAX=y +CONFIG_EROFS_FS=y +# CONFIG_ETHERNET is not set +CONFIG_JFFS2_FS=y +CONFIG_KERNEL_GZIP=y +# CONFIG_KERNEL_XZ is not set +CONFIG_LIBCRC32C=y +CONFIG_MAILBOX=y +# CONFIG_MDIO_DEVICE is not set +CONFIG_MMC=y +CONFIG_MTD_BLOCK=y +# CONFIG_PHYLIB is not set +CONFIG_PHY_ROCKCHIP_CSI2_DPHY=y +CONFIG_ROCKCHIP_DVBM=y +CONFIG_ROCKCHIP_HW_DECOMPRESS=y +CONFIG_ROCKCHIP_MULTI_RGA=y +CONFIG_ROCKCHIP_RAMDISK=y +CONFIG_ROCKCHIP_RGA_PROC_FS=y +CONFIG_ROCKCHIP_THUNDER_BOOT=y +CONFIG_ROCKCHIP_THUNDER_BOOT_DEFER_FREE_MEMBLOCK=y +CONFIG_ROCKCHIP_VENDOR_STORAGE=y +# CONFIG_SLUB_SYSFS is not set +CONFIG_SPI=y +CONFIG_VIDEO_ROCKCHIP_AIISP=y +CONFIG_VIDEO_ROCKCHIP_AVSP=y +CONFIG_VIDEO_ROCKCHIP_CIF=y +CONFIG_VIDEO_ROCKCHIP_ISP=y +CONFIG_VIDEO_ROCKCHIP_VPSS=y +CONFIG_VIDEO_SC200AI=y +CONFIG_VIDEO_SC450AI=y +# CONFIG_AD2S1200 is not set +# CONFIG_AD2S1210 is not set +# CONFIG_AD2S90 is not set +# CONFIG_AD3552R is not set +# CONFIG_AD5360 is not set +# CONFIG_AD5421 is not set +# CONFIG_AD5449 is not set +# CONFIG_AD5504 is not set +# CONFIG_AD5592R is not set +# CONFIG_AD5624R_SPI is not set +# CONFIG_AD5686_SPI is not set +# CONFIG_AD5755 is not set +# CONFIG_AD5758 is not set +# CONFIG_AD5761 is not set +# CONFIG_AD5764 is not set +# CONFIG_AD5766 is not set +# CONFIG_AD5770R is not set +# CONFIG_AD5791 is not set +# CONFIG_AD7124 is not set +# CONFIG_AD7192 is not set +# CONFIG_AD7266 is not set +# CONFIG_AD7280 is not set +# CONFIG_AD7292 is not set +# CONFIG_AD7293 is not set +# CONFIG_AD7298 is not set +# CONFIG_AD7303 is not set +# CONFIG_AD74413R is not set +# CONFIG_AD7476 is not set +# CONFIG_AD7606_IFACE_SPI is not set +# CONFIG_AD7766 is not set +# CONFIG_AD7768_1 is not set +# CONFIG_AD7780 is not set +# CONFIG_AD7791 is not set +# CONFIG_AD7793 is not set +# CONFIG_AD7816 is not set +# CONFIG_AD7887 is not set +# CONFIG_AD7923 is not set +# CONFIG_AD7949 is not set +# CONFIG_AD8366 is not set +# CONFIG_AD8801 is not set +# CONFIG_AD9523 is not set +# CONFIG_AD9832 is not set +# CONFIG_AD9834 is not set +# CONFIG_ADA4250 is not set +# CONFIG_ADF4350 is not set +# CONFIG_ADF4371 is not set +# CONFIG_ADIS16080 is not set +# CONFIG_ADIS16130 is not set +# CONFIG_ADIS16136 is not set +# CONFIG_ADIS16201 is not set +# CONFIG_ADIS16203 is not set +# CONFIG_ADIS16209 is not set +# CONFIG_ADIS16240 is not set +# CONFIG_ADIS16260 is not set +# CONFIG_ADIS16400 is not set +# CONFIG_ADIS16460 is not set +# CONFIG_ADIS16475 is not set +# CONFIG_ADIS16480 is not set +# CONFIG_ADMV1013 is not set +# CONFIG_ADMV4420 is not set +# CONFIG_ADRF6780 is not set +# CONFIG_ADXL313_SPI is not set +# CONFIG_ADXL345_SPI is not set +# CONFIG_ADXL355_SPI is not set +# CONFIG_ADXL367_SPI is not set +# CONFIG_ADXL372_SPI is not set +# CONFIG_ADXRS290 is not set +# CONFIG_ADXRS450 is not set +# CONFIG_AFE4403 is not set +# CONFIG_ALTERA_MBOX is not set +# CONFIG_ARM_MHU is not set +# CONFIG_ARM_MHU_V2 is not set +# CONFIG_ARM_SCPI_PROTOCOL is not set +# CONFIG_AS3935 is not set +# CONFIG_BMA220 is not set +# CONFIG_BMC150_MAGN_SPI is not set +# CONFIG_BMI088_ACCEL is not set +# CONFIG_BMI160_SPI is not set +# CONFIG_CRYPTO_842 is not set +# CONFIG_CRYPTO_ADIANTUM is not set +# CONFIG_CRYPTO_AEGIS128 is not set +# CONFIG_CRYPTO_AES is not set +# CONFIG_CRYPTO_AES_ARM is not set +# CONFIG_CRYPTO_AES_TI is not set +CONFIG_CRYPTO_ALGAPI=y +CONFIG_CRYPTO_ALGAPI2=y +# CONFIG_CRYPTO_ANSI_CPRNG is not set +# CONFIG_CRYPTO_ARIA is not set +# CONFIG_CRYPTO_AUTHENC is not set +# CONFIG_CRYPTO_BLAKE2B is not set +# CONFIG_CRYPTO_BLAKE2S_ARM is not set +# CONFIG_CRYPTO_BLOWFISH is not set +# CONFIG_CRYPTO_CAMELLIA is not set +# CONFIG_CRYPTO_CAST5 is not set +# CONFIG_CRYPTO_CAST6 is not set +# CONFIG_CRYPTO_CBC is not set +# CONFIG_CRYPTO_CCM is not set +# CONFIG_CRYPTO_CFB is not set +# CONFIG_CRYPTO_CHACHA20 is not set +# CONFIG_CRYPTO_CHACHA20POLY1305 is not set +# CONFIG_CRYPTO_CHACHA20_NEON is not set +# CONFIG_CRYPTO_CMAC is not set +# CONFIG_CRYPTO_CRC32 is not set +CONFIG_CRYPTO_CRC32C=y +# CONFIG_CRYPTO_CRCT10DIF is not set +# CONFIG_CRYPTO_CRYPTD is not set +# CONFIG_CRYPTO_CTR is not set +# CONFIG_CRYPTO_CTS is not set +# CONFIG_CRYPTO_CURVE25519 is not set +# CONFIG_CRYPTO_DEFLATE is not set +# CONFIG_CRYPTO_DES is not set +# CONFIG_CRYPTO_DH is not set +# CONFIG_CRYPTO_DRBG_MENU is not set +# CONFIG_CRYPTO_ECB is not set +# CONFIG_CRYPTO_ECDH is not set +# CONFIG_CRYPTO_ECDSA is not set +# CONFIG_CRYPTO_ECHAINIV is not set +# CONFIG_CRYPTO_ECRDSA is not set +# CONFIG_CRYPTO_ESSIV is not set +# CONFIG_CRYPTO_FCRYPT is not set +# CONFIG_CRYPTO_GCM is not set +# CONFIG_CRYPTO_GHASH is not set +CONFIG_CRYPTO_HASH=y +CONFIG_CRYPTO_HASH2=y +# CONFIG_CRYPTO_HCTR2 is not set +# CONFIG_CRYPTO_HMAC is not set +# CONFIG_CRYPTO_HW is not set +# CONFIG_CRYPTO_JITTERENTROPY is not set +# CONFIG_CRYPTO_KEYWRAP is not set +# CONFIG_CRYPTO_LIB_CHACHA20POLY1305 is not set +CONFIG_CRYPTO_LIB_UTILS=y +# CONFIG_CRYPTO_LRW is not set +# CONFIG_CRYPTO_LZ4 is not set +# CONFIG_CRYPTO_LZ4HC is not set +# CONFIG_CRYPTO_LZO is not set +# CONFIG_CRYPTO_MANAGER is not set +CONFIG_CRYPTO_MANAGER_DISABLE_TESTS=y +# CONFIG_CRYPTO_MD4 is not set +# CONFIG_CRYPTO_MD5 is not set +# CONFIG_CRYPTO_MICHAEL_MIC is not set +# CONFIG_CRYPTO_NULL is not set +# CONFIG_CRYPTO_OFB is not set +# CONFIG_CRYPTO_PCBC is not set +# CONFIG_CRYPTO_PCRYPT is not set +# CONFIG_CRYPTO_POLY1305 is not set +# CONFIG_CRYPTO_POLY1305_ARM is not set +# CONFIG_CRYPTO_RMD160 is not set +# CONFIG_CRYPTO_RSA is not set +# CONFIG_CRYPTO_SEQIV is not set +# CONFIG_CRYPTO_SERPENT is not set +# CONFIG_CRYPTO_SHA1 is not set +# CONFIG_CRYPTO_SHA1_ARM is not set +# CONFIG_CRYPTO_SHA256 is not set +# CONFIG_CRYPTO_SHA256_ARM is not set +# CONFIG_CRYPTO_SHA3 is not set +# CONFIG_CRYPTO_SHA512 is not set +# CONFIG_CRYPTO_SHA512_ARM is not set +# CONFIG_CRYPTO_SM2 is not set +# CONFIG_CRYPTO_SM3_GENERIC is not set +# CONFIG_CRYPTO_SM4_GENERIC is not set +# CONFIG_CRYPTO_STREEBOG is not set +# CONFIG_CRYPTO_TEST is not set +# CONFIG_CRYPTO_TWOFISH is not set +# CONFIG_CRYPTO_USER is not set +# CONFIG_CRYPTO_USER_API_AEAD is not set +# CONFIG_CRYPTO_USER_API_HASH is not set +# CONFIG_CRYPTO_USER_API_RNG is not set +# CONFIG_CRYPTO_USER_API_SKCIPHER is not set +# CONFIG_CRYPTO_VMAC is not set +# CONFIG_CRYPTO_WP512 is not set +# CONFIG_CRYPTO_XCBC is not set +# CONFIG_CRYPTO_XTS is not set +# CONFIG_CRYPTO_XXHASH is not set +# CONFIG_CRYPTO_ZSTD is not set +# CONFIG_EEPROM_93XX46 is not set +# CONFIG_EEPROM_AT25 is not set +# CONFIG_EROFS_FS_DEBUG is not set +# CONFIG_EROFS_FS_XATTR is not set +# CONFIG_EROFS_FS_ZIP is not set +# CONFIG_EZX_PCAP is not set +CONFIG_FS_DAX=y +CONFIG_FS_IOMAP=y +# CONFIG_FXLS8962AF_SPI is not set +# CONFIG_FXOS8700_SPI is not set +# CONFIG_GPIO_74X164 is not set +# CONFIG_GPIO_MAX3191X is not set +# CONFIG_GPIO_MAX7301 is not set +# CONFIG_GPIO_MC33880 is not set +# CONFIG_GPIO_PISOSR is not set +# CONFIG_GPIO_XRA1403 is not set +# CONFIG_HI8435 is not set +# CONFIG_IIO_SSP_SENSORHUB is not set +CONFIG_INITCALL_ASYNC=y +# CONFIG_INITRAMFS_FORCE is not set +CONFIG_INITRAMFS_SOURCE="" +# CONFIG_INV_ICM42600_SPI is not set +# CONFIG_INV_ICM42670_SPI is not set +# CONFIG_INV_MPU6050_SPI is not set +# CONFIG_JFFS2_COMPRESSION_OPTIONS is not set +CONFIG_JFFS2_FS_DEBUG=0 +# CONFIG_JFFS2_FS_WBUF_VERIFY is not set +CONFIG_JFFS2_FS_WRITEBUFFER=y +# CONFIG_JFFS2_FS_XATTR is not set +CONFIG_JFFS2_RTIME=y +# CONFIG_JFFS2_SUMMARY is not set +CONFIG_JFFS2_ZLIB=y +# CONFIG_LATTICE_ECP3_CONFIG is not set +# CONFIG_LMK04832 is not set +# CONFIG_LTC1660 is not set +# CONFIG_LTC2496 is not set +# CONFIG_LTC2632 is not set +# CONFIG_LTC2688 is not set +# CONFIG_LTC2983 is not set +CONFIG_MAILBOX_POLL_PERIOD_US=y +# CONFIG_MAILBOX_TEST is not set +# CONFIG_MAX1027 is not set +# CONFIG_MAX11100 is not set +# CONFIG_MAX1118 is not set +# CONFIG_MAX11205 is not set +# CONFIG_MAX1241 is not set +# CONFIG_MAX31856 is not set +# CONFIG_MAX31865 is not set +# CONFIG_MAX5481 is not set +# CONFIG_MAX5487 is not set +# CONFIG_MAXIM_THERMOCOUPLE is not set +# CONFIG_MCP320X is not set +# CONFIG_MCP3911 is not set +# CONFIG_MCP41010 is not set +# CONFIG_MCP4131 is not set +# CONFIG_MCP4922 is not set +# CONFIG_MFD_ARIZONA_SPI is not set +# CONFIG_MFD_CPCAP is not set +# CONFIG_MFD_DA9052_SPI is not set +# CONFIG_MFD_INTEL_M10_BMC is not set +# CONFIG_MFD_MC13XXX_SPI is not set +# CONFIG_MFD_OCELOT is not set +# CONFIG_MFD_RK806_SPI is not set +# CONFIG_MFD_RSMU_SPI is not set +# CONFIG_MFD_TPS65912_SPI is not set +# CONFIG_MFD_WM831X_SPI is not set +# CONFIG_MICREL_KS8995MA is not set +# CONFIG_MMA7455_SPI is not set +# CONFIG_MMC_ARMMMCI is not set +CONFIG_MMC_BLOCK=y +CONFIG_MMC_BLOCK_MINORS=32 +# CONFIG_MMC_CQHCI is not set +# CONFIG_MMC_DEBUG is not set +CONFIG_MMC_DW=y +# CONFIG_MMC_DW_BLUEFIELD is not set +# CONFIG_MMC_DW_EXYNOS is not set +# CONFIG_MMC_DW_HI3798CV200 is not set +# CONFIG_MMC_DW_K3 is not set +CONFIG_MMC_DW_PLTFM=y +CONFIG_MMC_DW_ROCKCHIP=m +# CONFIG_MMC_HSQ is not set +# CONFIG_MMC_MTK is not set +CONFIG_MMC_QUEUE_DEPTH=1 +# CONFIG_MMC_SDHCI is not set +# CONFIG_MMC_SPI is not set +# CONFIG_MMC_TEST is not set +# CONFIG_MMC_USDHI6ROL0 is not set +# CONFIG_MOXTET is not set +# CONFIG_MPL115_SPI is not set +CONFIG_MTD_BLKDEVS=y +# CONFIG_MTD_DATAFLASH is not set +# CONFIG_MTD_MCHP23K256 is not set +# CONFIG_MTD_MCHP48L640 is not set +# CONFIG_MTD_SPI_NAND is not set +CONFIG_MTD_SPI_NOR=y +# CONFIG_MTD_SPI_NOR_ATMEL is not set +# CONFIG_MTD_SPI_NOR_BOYA is not set +# CONFIG_MTD_SPI_NOR_CATALYST is not set +# CONFIG_MTD_SPI_NOR_DEVICE_AUTOSELECT is not set +# CONFIG_MTD_SPI_NOR_DOSILICON is not set +# CONFIG_MTD_SPI_NOR_EON is not set +# CONFIG_MTD_SPI_NOR_ESMT is not set +# CONFIG_MTD_SPI_NOR_EVERSPIN is not set +# CONFIG_MTD_SPI_NOR_FMSH is not set +# CONFIG_MTD_SPI_NOR_FUJITSU is not set +CONFIG_MTD_SPI_NOR_GIGADEVICE=y +# CONFIG_MTD_SPI_NOR_INTEL is not set +# CONFIG_MTD_SPI_NOR_ISSI is not set +CONFIG_MTD_SPI_NOR_MACRONIX=y +CONFIG_MTD_SPI_NOR_MISC=y +# CONFIG_MTD_SPI_NOR_NORMEM is not set +# CONFIG_MTD_SPI_NOR_PUYA is not set +# CONFIG_MTD_SPI_NOR_SPANSION is not set +# CONFIG_MTD_SPI_NOR_SST is not set +# CONFIG_MTD_SPI_NOR_STMICRO is not set +# CONFIG_MTD_SPI_NOR_SWP_DISABLE is not set +CONFIG_MTD_SPI_NOR_SWP_DISABLE_ON_VOLATILE=y +# CONFIG_MTD_SPI_NOR_SWP_KEEP is not set +CONFIG_MTD_SPI_NOR_USE_4K_SECTORS=y +CONFIG_MTD_SPI_NOR_WINBOND=y +# CONFIG_MTD_SPI_NOR_XILINX is not set +# CONFIG_MTD_SPI_NOR_XMC is not set +CONFIG_MTD_SPI_NOR_XTX=y +# CONFIG_MTD_SST25L is not set +# CONFIG_PI433 is not set +# CONFIG_PL320_MBOX is not set +# CONFIG_PLATFORM_MHU is not set +# CONFIG_PWRSEQ_EMMC is not set +CONFIG_PWRSEQ_SIMPLE=y +# CONFIG_RD_BZIP2 is not set +# CONFIG_RD_GZIP is not set +# CONFIG_RD_LZ4 is not set +# CONFIG_RD_LZMA is not set +# CONFIG_RD_LZO is not set +# CONFIG_RD_XZ is not set +# CONFIG_RD_ZSTD is not set +CONFIG_REGMAP_SPI=y +# CONFIG_REGULATOR_TPS6524X is not set +# CONFIG_ROCKCHIP_HW_DECOMPRESS_TEST is not set +CONFIG_ROCKCHIP_MBOX=y +# CONFIG_ROCKCHIP_MBOX_DEMO is not set +# CONFIG_ROCKCHIP_MMC_VENDOR_STORAGE is not set +# CONFIG_ROCKCHIP_MTD_VENDOR_STORAGE is not set +# CONFIG_ROCKCHIP_RAM_VENDOR_STORAGE is not set +CONFIG_ROCKCHIP_RGA_DEBUGGER=y +CONFIG_ROCKCHIP_THUNDER_BOOT_MMC=y +CONFIG_ROCKCHIP_THUNDER_BOOT_SERVICE=y +CONFIG_ROCKCHIP_THUNDER_BOOT_SFC=y +# CONFIG_RPMSG_QCOM_GLINK_RPM is not set +# CONFIG_RPMSG_ROCKCHIP_MBOX is not set +# CONFIG_RTC_DRV_DS1302 is not set +# CONFIG_RTC_DRV_DS1305 is not set +# CONFIG_RTC_DRV_DS1343 is not set +# CONFIG_RTC_DRV_DS1347 is not set +# CONFIG_RTC_DRV_DS1390 is not set +# CONFIG_RTC_DRV_M41T93 is not set +# CONFIG_RTC_DRV_M41T94 is not set +# CONFIG_RTC_DRV_MAX6902 is not set +# CONFIG_RTC_DRV_MAX6916 is not set +# CONFIG_RTC_DRV_MCP795 is not set +# CONFIG_RTC_DRV_PCF2123 is not set +# CONFIG_RTC_DRV_R9701 is not set +# CONFIG_RTC_DRV_RS5C348 is not set +# CONFIG_RTC_DRV_RX4581 is not set +# CONFIG_SCA3000 is not set +# CONFIG_SCA3300 is not set +# CONFIG_SDIO_UART is not set +# CONFIG_SENSORS_HMC5843_SPI is not set +# CONFIG_SENSORS_RM3100_SPI is not set +# CONFIG_SERIAL_MAX3100 is not set +# CONFIG_SERIAL_MAX310X is not set +# CONFIG_SND_SOC_ADAU1372_SPI is not set +# CONFIG_SND_SOC_ADAU1761_SPI is not set +# CONFIG_SND_SOC_AK4104 is not set +# CONFIG_SND_SOC_CS35L41_SPI is not set +# CONFIG_SND_SOC_CS35L45_SPI is not set +# CONFIG_SND_SOC_CS4271_SPI is not set +# CONFIG_SND_SOC_ES8328_SPI is not set +# CONFIG_SND_SOC_PCM179X_SPI is not set +# CONFIG_SND_SOC_PCM186X_SPI is not set +# CONFIG_SND_SOC_PCM3060_SPI is not set +# CONFIG_SND_SOC_PCM3168A_SPI is not set +# CONFIG_SND_SOC_PCM512x_SPI is not set +# CONFIG_SND_SOC_RK3399_GRU_SOUND is not set +# CONFIG_SND_SOC_ROCKCHIP_SPI_CODEC is not set +# CONFIG_SND_SOC_SSM2602_SPI is not set +# CONFIG_SND_SOC_TLV320AIC23_SPI is not set +# CONFIG_SND_SOC_TLV320AIC32X4_SPI is not set +# CONFIG_SND_SOC_TLV320AIC3X_SPI is not set +# CONFIG_SND_SOC_WM8731_SPI is not set +# CONFIG_SND_SOC_WM8770 is not set +# CONFIG_SND_SOC_WM8804_SPI is not set +# CONFIG_SND_SOC_ZL38060 is not set +# CONFIG_SND_SPI is not set +# CONFIG_SPI_ALTERA is not set +# CONFIG_SPI_AMD is not set +# CONFIG_SPI_AXI_SPI_ENGINE is not set +# CONFIG_SPI_BITBANG is not set +# CONFIG_SPI_CADENCE is not set +# CONFIG_SPI_CADENCE_QUADSPI is not set +# CONFIG_SPI_CADENCE_XSPI is not set +# CONFIG_SPI_DEBUG is not set +# CONFIG_SPI_DESIGNWARE is not set +# CONFIG_SPI_FSL_SPI is not set +# CONFIG_SPI_GPIO is not set +# CONFIG_SPI_LOOPBACK_TEST is not set +CONFIG_SPI_MASTER=y +CONFIG_SPI_MEM=y +# CONFIG_SPI_MICROCHIP_CORE is not set +# CONFIG_SPI_MICROCHIP_CORE_QSPI is not set +# CONFIG_SPI_MUX is not set +# CONFIG_SPI_MXIC is not set +# CONFIG_SPI_NXP_FLEXSPI is not set +# CONFIG_SPI_OC_TINY is not set +# CONFIG_SPI_PL022 is not set +# CONFIG_SPI_ROCKCHIP is not set +# CONFIG_SPI_ROCKCHIP_FLEXBUS_FSPI is not set +# CONFIG_SPI_ROCKCHIP_FLEXBUS_SPI is not set +CONFIG_SPI_ROCKCHIP_SFC=y +# CONFIG_SPI_ROCKCHIP_SLAVE is not set +# CONFIG_SPI_SC18IS602 is not set +# CONFIG_SPI_SIFIVE is not set +# CONFIG_SPI_SLAVE is not set +# CONFIG_SPI_SPIDEV is not set +# CONFIG_SPI_TLE62X0 is not set +# CONFIG_SPI_XCOMM is not set +# CONFIG_SPI_XILINX is not set +# CONFIG_SPI_ZYNQMP_GQSPI is not set +# CONFIG_TI_ADC0832 is not set +# CONFIG_TI_ADC084S021 is not set +# CONFIG_TI_ADC108S102 is not set +# CONFIG_TI_ADC12138 is not set +# CONFIG_TI_ADC128S052 is not set +# CONFIG_TI_ADC161S626 is not set +# CONFIG_TI_ADS124S08 is not set +# CONFIG_TI_ADS131E08 is not set +# CONFIG_TI_ADS7950 is not set +# CONFIG_TI_ADS8344 is not set +# CONFIG_TI_ADS8688 is not set +# CONFIG_TI_DAC082S085 is not set +# CONFIG_TI_DAC7311 is not set +# CONFIG_TI_DAC7612 is not set +# CONFIG_TI_TLC4541 is not set +# CONFIG_TI_TSC2046 is not set +# CONFIG_VIDEO_GS1662 is not set +# CONFIG_VIDEO_ROCKCHIP_PREISP is not set +CONFIG_VIDEO_ROCKCHIP_THUNDER_BOOT_ISP=y +CONFIG_VIDEO_ROCKCHIP_THUNDER_BOOT_SETUP=y +# CONFIG_VIDEO_S5C73M3 is not set +CONFIG_ZLIB_DEFLATE=y +CONFIG_ZLIB_INFLATE=y diff --git a/arch/arm64/boot/dts/rockchip/Makefile b/arch/arm64/boot/dts/rockchip/Makefile index 861f281c62c1..ecaecd996374 100644 --- a/arch/arm64/boot/dts/rockchip/Makefile +++ b/arch/arm64/boot/dts/rockchip/Makefile @@ -293,12 +293,14 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-test2-v10.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-test2-v10-linux.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-test3-v10.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-test5-v10.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-toybrick-d0-linux.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-vehicle-evb-v10.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-vehicle-evb-v10-linux.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-vehicle-evb-v20.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-vehicle-evb-v20-amp.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-vehicle-evb-v20-linux.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-vehicle-evb-v20-ufs.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-vehicle-evb-v21.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576s-evb1-v10.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576s-evb1-v10-linux.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576s-tablet-v10.dtb @@ -377,6 +379,9 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb1-v10-amp.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb1-v10-bt-sco.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb1-v10-dual-4k.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb1-v10-dv.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb1-v10-fastboot-emmc.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb1-v10-fastboot-spi-nand.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb1-v10-fastboot-spi-nor.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb1-v10-spi-nor.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb2-v10.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb2-v10-mcu-k350c4516t.dtb diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-evb.dtsi index b7b36b5b9bab..f2da0b2e62ea 100644 --- a/arch/arm64/boot/dts/rockchip/rk3568-evb.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3568-evb.dtsi @@ -396,6 +396,7 @@ reset-delay-ms = <60>; enable-delay-ms = <60>; prepare-delay-ms = <60>; + init-delay-ms = <10>; unprepare-delay-ms = <60>; disable-delay-ms = <60>; width-mm = <68>; @@ -730,6 +731,7 @@ reset-delay-ms = <60>; enable-delay-ms = <60>; prepare-delay-ms = <60>; + init-delay-ms = <10>; unprepare-delay-ms = <60>; disable-delay-ms = <60>; width-mm = <68>; diff --git a/arch/arm64/boot/dts/rockchip/rk3576-evb.dtsi b/arch/arm64/boot/dts/rockchip/rk3576-evb.dtsi index 60fec7dc8f08..3ae8e1ddb9c2 100644 --- a/arch/arm64/boot/dts/rockchip/rk3576-evb.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3576-evb.dtsi @@ -270,6 +270,7 @@ reset-delay-ms = <10>; enable-delay-ms = <10>; prepare-delay-ms = <10>; + init-delay-ms = <10>; unprepare-delay-ms = <10>; disable-delay-ms = <60>; width-mm = <68>; diff --git a/arch/arm64/boot/dts/rockchip/rk3576-toybrick-cam-dcphy0.dtsi b/arch/arm64/boot/dts/rockchip/rk3576-toybrick-cam-dcphy0.dtsi new file mode 100644 index 000000000000..f1e4b25a20c9 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3576-toybrick-cam-dcphy0.dtsi @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2024 Rockchip Electronics Co., Ltd. + * + */ + +&csi2_dcphy0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam0: endpoint@1 { + reg = <1>; + remote-endpoint = <&imx415_out0>; + data-lanes = <1 2 3 4>; + }; + + mipi_in_ucam1: endpoint@2 { + reg = <2>; + remote-endpoint = <&ov50c40_out0>; + data-lanes = <1 2 3 4>; + }; + }; + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + csidcphy0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&mipi0_csi2_input>; + }; + }; + }; +}; + +&i2c9 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c9m2_xfer>; + + imx415_dcphy0: imx415@1a { + compatible = "sony,imx415"; + reg = <0x1a>; + clocks = <&cru CLK_MIPI_CAMERAOUT_M0>; + clock-names = "xvclk"; + pinctrl-names = "default"; + pinctrl-0 = <&cam_clk0m1_clk0>; + power-domains = <&power RK3576_PD_VI>; + power-gpios = <&gpio2 RK_PD0 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio2 RK_PD3 GPIO_ACTIVE_LOW>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "CMK-OT2022-PX1"; + rockchip,camera-module-lens-name = "IR0147-50IRC-8M-F20"; + port { + imx415_out0: endpoint { + remote-endpoint = <&mipi_in_ucam0>; + data-lanes = <1 2 3 4>; + }; + }; + }; + + aw8601_0: aw8601@c { + compatible = "awinic,aw8601"; + status = "okay"; + reg = <0x0c>; + rockchip,vcm-start-current = <56>; + rockchip,vcm-rated-current = <96>; + rockchip,vcm-step-mode = <4>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + }; + + otp_eeprom_0: otp_eeprom@50 { + compatible = "rk,otp_eeprom"; + status = "okay"; + reg = <0x50>; + }; + + ov50c40_dcph0: ov50c40@36 { + compatible = "ovti,ov50c40"; + reg = <0x36>; + clocks = <&cru CLK_MIPI_CAMERAOUT_M0>; + clock-names = "xvclk"; + power-domains = <&power RK3576_PD_VI>; + pinctrl-names = "default"; + pinctrl-0 = <&cam_clk0m1_clk0>; + pwdn-gpios = <&gpio2 RK_PD0 GPIO_ACTIVE_LOW>;// must be high at last + reset-gpios = <&gpio2 RK_PD3 GPIO_ACTIVE_LOW>;// must be high at last + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "HZGA06"; + rockchip,camera-module-lens-name = "ZE0082C1-RK3568"; + eeprom-ctrl = <&otp_eeprom_0>; + lens-focus = <&aw8601_0>; + port { + ov50c40_out0: endpoint { + remote-endpoint = <&mipi_in_ucam1>; + data-lanes = <1 2 3 4>; + }; + }; + }; +}; + +&mipi0_csi2 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi0_csi2_input: endpoint@1 { + reg = <1>; + remote-endpoint = <&csidcphy0_out>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + mipi0_csi2_output: endpoint@0 { + reg = <0>; + remote-endpoint = <&cif_mipi_in0>; + }; + }; + }; +}; + +&rkcif { + status = "okay"; +}; + +&rkcif_mipi_lvds { + status = "okay"; + + port { + cif_mipi_in0: endpoint { + remote-endpoint = <&mipi0_csi2_output>; + }; + }; +}; + +&rkcif_mipi_lvds_sditf { + status = "okay"; + + port { + mipi_lvds_sditf: endpoint { + remote-endpoint = <&isp_vir0>; + }; + }; +}; + +&rkcif_mmu { + status = "okay"; +}; + +&rkisp { + status = "okay"; +}; + +&rkisp_mmu { + status = "okay"; +}; + +&rkisp_vir0 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp_vir0: endpoint@0 { + reg = <0>; + remote-endpoint = <&mipi_lvds_sditf>; + }; + }; +}; + +&rkisp_vir0_sditf { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3576-toybrick-cam-dphy3.dtsi b/arch/arm64/boot/dts/rockchip/rk3576-toybrick-cam-dphy3.dtsi new file mode 100644 index 000000000000..031c6e160f44 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3576-toybrick-cam-dphy3.dtsi @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2024 Rockchip Electronics Co., Ltd. + * + */ + +&csi2_dphy0_hw { + status = "okay"; +}; + +&csi2_dphy1_hw { + status = "okay"; +}; + +&csi2_dphy3 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam30: endpoint@1 { + reg = <1>; + remote-endpoint = <&imx415_out1>; + data-lanes = <1 2 3 4>; + }; + + mipi_in_ucam31: endpoint@2 { + reg = <2>; + remote-endpoint = <&ov50c40_out1>; + data-lanes = <1 2 3 4>; + }; + }; + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + csidphy3_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&mipi3_csi2_input>; + }; + }; + }; +}; + +&i2c6 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c6m1_xfer>; + + imx415_dphy3: imx415@1a { + compatible = "sony,imx415"; + reg = <0x1a>; + clocks = <&cru CLK_MIPI_CAMERAOUT_M2>; + clock-names = "xvclk"; + pinctrl-names = "default"; + pinctrl-0 = <&cam_clk2m1_clk2>; + power-domains = <&power RK3576_PD_VI>; + power-gpios = <&gpio1 RK_PD4 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio2 RK_PD6 GPIO_ACTIVE_LOW>; + rockchip,camera-module-index = <1>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "CMK-OT2022-PX1"; + rockchip,camera-module-lens-name = "IR0147-50IRC-8M-F20"; + port { + imx415_out1: endpoint { + remote-endpoint = <&mipi_in_ucam30>; + data-lanes = <1 2 3 4>; + }; + }; + }; + + aw8601_1: aw8601@c { + compatible = "awinic,aw8601"; + status = "okay"; + reg = <0x0c>; + rockchip,vcm-start-current = <56>; + rockchip,vcm-rated-current = <96>; + rockchip,vcm-step-mode = <4>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + }; + + otp_eeprom_1: otp_eeprom@50 { + compatible = "rk,otp_eeprom"; + status = "okay"; + reg = <0x50>; + }; + + ov50c40_dphy3: ov50c40@36 { + compatible = "ovti,ov50c40"; + reg = <0x36>; + clocks = <&cru CLK_MIPI_CAMERAOUT_M2>; + clock-names = "xvclk"; + power-domains = <&power RK3576_PD_VI>; + pinctrl-names = "default"; + pinctrl-0 = <&cam_clk2m1_clk2>; + pwdn-gpios = <&gpio1 RK_PD4 GPIO_ACTIVE_LOW>;// must be high at last + reset-gpios = <&gpio2 RK_PD6 GPIO_ACTIVE_LOW>;// must be high at last + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "HZGA06"; + rockchip,camera-module-lens-name = "ZE0082C1-RK3568"; + eeprom-ctrl = <&otp_eeprom_1>; + lens-focus = <&aw8601_1>; + port { + ov50c40_out1: endpoint { + remote-endpoint = <&mipi_in_ucam31>; + data-lanes = <1 2 3 4>; + }; + }; + }; +}; + +&mipi3_csi2 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi3_csi2_input: endpoint@1 { + reg = <1>; + remote-endpoint = <&csidphy3_out>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + mipi3_csi2_output: endpoint@0 { + reg = <0>; + remote-endpoint = <&cif_mipi3_in>; + }; + }; + }; +}; + +&rkcif { + status = "okay"; +}; + +&rkcif_mipi_lvds3 { + status = "okay"; + + port { + cif_mipi3_in: endpoint { + remote-endpoint = <&mipi3_csi2_output>; + }; + }; +}; + +&rkcif_mipi_lvds3_sditf { + status = "okay"; + + port { + mipi_lvds3_sditf: endpoint { + remote-endpoint = <&isp_vir1>; + }; + }; +}; + +&rkcif_mmu { + status = "okay"; +}; + +&rkisp { + status = "okay"; +}; + +&rkisp_mmu { + status = "okay"; +}; + +&rkisp_vir1 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp_vir1: endpoint@0 { + reg = <0>; + remote-endpoint = <&mipi_lvds3_sditf>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3576-toybrick-d0-linux.dts b/arch/arm64/boot/dts/rockchip/rk3576-toybrick-d0-linux.dts new file mode 100644 index 000000000000..ca214d45edf3 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3576-toybrick-d0-linux.dts @@ -0,0 +1,1444 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2024 Rockchip Electronics Co., Ltd. + * + */ + +/dts-v1/; + +#include +#include +#include +#include +#include +#include +#include +#include +#include "rk3576.dtsi" +#include "rk3576-linux.dtsi" +#include "rk3576-pinctrl.dtsi" +#include "rk3576-toybrick-cam-dcphy0.dtsi" +#include "rk3576-toybrick-cam-dphy3.dtsi" + +/ { + model = "Rockchip RK3576 TOYBRICK D0 Board"; + compatible = "rockchip,rk3576-toybrick-d0", "rockchip,rk3576"; + + reserved_memory: reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + vendor_storage_rm: vendor-storage-rm@00000000 { + compatible = "rockchip,vendor-storage-rm"; + reg = <0x0 0x0 0x0 0x0>; + }; + }; + vendor_storage: vendor-storage { + compatible = "rockchip,ram-vendor-storage"; + memory-region = <&vendor_storage_rm>; + status = "okay"; + }; + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm1_6ch_1 0 25000 0>; + brightness-levels = < + 0 20 20 21 21 22 22 23 + 23 24 24 25 25 26 26 27 + 27 28 28 29 29 30 30 31 + 31 32 32 33 33 34 34 35 + 35 36 36 37 37 38 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255 + >; + default-brightness-level = <200>; + }; + + fan1 { + compatible = "pwm-fan"; + pwms = <&pwm1_6ch_4 0 40000 0>; + cooling-levels = <0 80 170 255>; + }; + + // leds: gpio-leds { + // compatible = "gpio-leds"; + // pinctrl-names = "default"; + // pinctrl-0 =<&leds_gpio>; + + // led@1 { + // gpios = <&gpio3 16 GPIO_ACTIVE_HIGH>; + // label = "heartbeat"; + // linux,default-trigger = "heartbeat"; + // }; + // }; + + leds: leds { + compatible = "gpio-leds"; + work_led: work { + gpios = <&gpio3 16 GPIO_ACTIVE_HIGH>; + label = "heartbeat"; + linux,default-trigger = "heartbeat"; + }; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&hym8563>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_poweren_gpio>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + post-power-on-delay-ms = <200>; + reset-gpios = <&gpio2 RK_PC0 GPIO_ACTIVE_LOW>; + }; + + vbus_typec_power: vbus-typec-power { + compatible = "regulator-fixed"; + regulator-name = "vbus_typec_power"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + }; + + vcc_5v0_sys_s5: vcc-5v0-sys-s5 { + compatible = "regulator-fixed"; + regulator-name = "vcc_5v0_sys_s5"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + vin-supply = <&vbus_typec_power>; + }; + + vcc_2v0_pldo_s3: vcc-2v0-pldo-s3 { + compatible = "regulator-fixed"; + regulator-name = "vcc_2v0_pldo_s3"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <2000000>; + regulator-max-microvolt = <2000000>; + vin-supply = <&vcc_5v0_sys_s5>; + }; + + vcc_1v1_nldo_s3: vcc-1v1-nldo-s3 { + compatible = "regulator-fixed"; + regulator-name = "vcc_1v1_nldo_s3"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <1100000>; + regulator-max-microvolt = <1100000>; + vin-supply = <&vcc_5v0_sys_s5>; + }; + + vcc5v0_usb30_host1: vcc5v0-usb30-host1 { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_usb30_host1"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + vin-supply = <&vcc_5v0_sys_s5>; + enable-active-high; + gpio = <&gpio0 RK_PD3 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&usb_host_pwren>; + }; + + // vcc5v0_usb_otg0: vcc5v0-usb-otg0 { + // compatible = "regulator-fixed"; + // regulator-name = "vcc5v0_usb_otg0"; + // regulator-boot-on; + // regulator-always-on; + // regulator-min-microvolt = <5000000>; + // regulator-max-microvolt = <5000000>; + // vin-supply = <&vcc_5v0_sys_s5>; + // enable-active-high; + // gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; + // pinctrl-names = "default"; + // pinctrl-0 = <&usb_otg_pwren>; + // }; + + vcc_3v3_sd: vcc-3v3-sd { + compatible = "regulator-fixed"; + regulator-name = "vcc_3v3_sd"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc_3v3_s3>; + enable-active-high; + gpio = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&sd_pwren>; + }; + + vcc_3v3_pcie: vcc-3v3-pcie { + compatible = "regulator-fixed"; + regulator-name = "vcc_3v3_pcie"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc_5v0_sys_s5>; + enable-active-high; + gpio = <&gpio2 RK_PC7 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&pcie_pwren>; + }; + + vcc_3v3_s0: vcc-3v3-s0 { + compatible = "regulator-fixed"; + regulator-name = "vcc_3v3_s0"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc_3v3_s3>; + }; + + vcc3v3_lcd_n: vcc3v3-lcd0-n { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_lcd0_n"; + regulator-boot-on; + enable-active-high; + gpio = <&gpio0 RK_PC4 GPIO_ACTIVE_HIGH>; + vin-supply = <&vcc_3v3_s0>; + }; + + wireless_bluetooth: wireless-bluetooth { + compatible = "bluetooth-platdata"; + uart_rts_gpios = <&gpio2 RK_PB5 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart7m0_rtsn>, <&bt_reset_gpio>, <&bt_host_wake_irq>, <&bt_wake_gpio>; + pinctrl-1 = <&uart7_gpios>; + BT,reset_gpio = <&gpio2 RK_PC1 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio2 RK_PC6 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio0 RK_PB1 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless_wlan: wireless-wlan { + compatible = "wlan-platdata"; + wifi_chip_type = "ap6275p"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_host_wake_irq>; + WIFI,host_wake_irq = <&gpio0 RK_PB0 GPIO_ACTIVE_HIGH>; + // WIFI,poweren_gpio = <&gpio2 RK_PC0 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&combphy0_ps { + status = "okay"; +}; + +&combphy1_psu { + status = "okay"; +}; + +&cpu_l0 { + cpu-supply = <&vdd_cpu_lit_s0>; +}; + +&cpu_b0 { + cpu-supply = <&vdd_cpu_big_s0>; +}; + +&display_subsystem { + clocks = <&hdptxphy_hdmi>; + clock-names = "hdmi0_phy_pll"; +}; + +&dsi { + status = "disabled"; + //rockchip,lane-rate = <1000>; + dsi_panel: panel@0 { + status = "okay"; + compatible = "simple-panel-dsi"; + reg = <0>; + power-supply = <&vcc3v3_lcd_n>; + backlight = <&backlight>; + reset-delay-ms = <10>; + enable-delay-ms = <10>; + prepare-delay-ms = <10>; + unprepare-delay-ms = <10>; + disable-delay-ms = <60>; + width-mm = <68>; + height-mm = <121>; + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_NO_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + panel-init-sequence = [ + 23 00 02 FE 21 + 23 00 02 04 00 + 23 00 02 00 64 + 23 00 02 2A 00 + 23 00 02 26 64 + 23 00 02 54 00 + 23 00 02 50 64 + 23 00 02 7B 00 + 23 00 02 77 64 + 23 00 02 A2 00 + 23 00 02 9D 64 + 23 00 02 C9 00 + 23 00 02 C5 64 + 23 00 02 01 71 + 23 00 02 27 71 + 23 00 02 51 71 + 23 00 02 78 71 + 23 00 02 9E 71 + 23 00 02 C6 71 + 23 00 02 02 89 + 23 00 02 28 89 + 23 00 02 52 89 + 23 00 02 79 89 + 23 00 02 9F 89 + 23 00 02 C7 89 + 23 00 02 03 9E + 23 00 02 29 9E + 23 00 02 53 9E + 23 00 02 7A 9E + 23 00 02 A0 9E + 23 00 02 C8 9E + 23 00 02 09 00 + 23 00 02 05 B0 + 23 00 02 31 00 + 23 00 02 2B B0 + 23 00 02 5A 00 + 23 00 02 55 B0 + 23 00 02 80 00 + 23 00 02 7C B0 + 23 00 02 A7 00 + 23 00 02 A3 B0 + 23 00 02 CE 00 + 23 00 02 CA B0 + 23 00 02 06 C0 + 23 00 02 2D C0 + 23 00 02 56 C0 + 23 00 02 7D C0 + 23 00 02 A4 C0 + 23 00 02 CB C0 + 23 00 02 07 CF + 23 00 02 2F CF + 23 00 02 58 CF + 23 00 02 7E CF + 23 00 02 A5 CF + 23 00 02 CC CF + 23 00 02 08 DD + 23 00 02 30 DD + 23 00 02 59 DD + 23 00 02 7F DD + 23 00 02 A6 DD + 23 00 02 CD DD + 23 00 02 0E 15 + 23 00 02 0A E9 + 23 00 02 36 15 + 23 00 02 32 E9 + 23 00 02 5F 15 + 23 00 02 5B E9 + 23 00 02 85 15 + 23 00 02 81 E9 + 23 00 02 AD 15 + 23 00 02 A9 E9 + 23 00 02 D3 15 + 23 00 02 CF E9 + 23 00 02 0B 14 + 23 00 02 33 14 + 23 00 02 5C 14 + 23 00 02 82 14 + 23 00 02 AA 14 + 23 00 02 D0 14 + 23 00 02 0C 36 + 23 00 02 34 36 + 23 00 02 5D 36 + 23 00 02 83 36 + 23 00 02 AB 36 + 23 00 02 D1 36 + 23 00 02 0D 6B + 23 00 02 35 6B + 23 00 02 5E 6B + 23 00 02 84 6B + 23 00 02 AC 6B + 23 00 02 D2 6B + 23 00 02 13 5A + 23 00 02 0F 94 + 23 00 02 3B 5A + 23 00 02 37 94 + 23 00 02 64 5A + 23 00 02 60 94 + 23 00 02 8A 5A + 23 00 02 86 94 + 23 00 02 B2 5A + 23 00 02 AE 94 + 23 00 02 D8 5A + 23 00 02 D4 94 + 23 00 02 10 D1 + 23 00 02 38 D1 + 23 00 02 61 D1 + 23 00 02 87 D1 + 23 00 02 AF D1 + 23 00 02 D5 D1 + 23 00 02 11 04 + 23 00 02 39 04 + 23 00 02 62 04 + 23 00 02 88 04 + 23 00 02 B0 04 + 23 00 02 D6 04 + 23 00 02 12 05 + 23 00 02 3A 05 + 23 00 02 63 05 + 23 00 02 89 05 + 23 00 02 B1 05 + 23 00 02 D7 05 + 23 00 02 18 AA + 23 00 02 14 36 + 23 00 02 42 AA + 23 00 02 3D 36 + 23 00 02 69 AA + 23 00 02 65 36 + 23 00 02 8F AA + 23 00 02 8B 36 + 23 00 02 B7 AA + 23 00 02 B3 36 + 23 00 02 DD AA + 23 00 02 D9 36 + 23 00 02 15 74 + 23 00 02 3F 74 + 23 00 02 66 74 + 23 00 02 8C 74 + 23 00 02 B4 74 + 23 00 02 DA 74 + 23 00 02 16 9F + 23 00 02 40 9F + 23 00 02 67 9F + 23 00 02 8D 9F + 23 00 02 B5 9F + 23 00 02 DB 9F + 23 00 02 17 DC + 23 00 02 41 DC + 23 00 02 68 DC + 23 00 02 8E DC + 23 00 02 B6 DC + 23 00 02 DC DC + 23 00 02 1D FF + 23 00 02 19 03 + 23 00 02 47 FF + 23 00 02 43 03 + 23 00 02 6E FF + 23 00 02 6A 03 + 23 00 02 94 FF + 23 00 02 90 03 + 23 00 02 BC FF + 23 00 02 B8 03 + 23 00 02 E2 FF + 23 00 02 DE 03 + 23 00 02 1A 35 + 23 00 02 44 35 + 23 00 02 6B 35 + 23 00 02 91 35 + 23 00 02 B9 35 + 23 00 02 DF 35 + 23 00 02 1B 45 + 23 00 02 45 45 + 23 00 02 6C 45 + 23 00 02 92 45 + 23 00 02 BA 45 + 23 00 02 E0 45 + 23 00 02 1C 55 + 23 00 02 46 55 + 23 00 02 6D 55 + 23 00 02 93 55 + 23 00 02 BB 55 + 23 00 02 E1 55 + 23 00 02 22 FF + 23 00 02 1E 68 + 23 00 02 4C FF + 23 00 02 48 68 + 23 00 02 73 FF + 23 00 02 6F 68 + 23 00 02 99 FF + 23 00 02 95 68 + 23 00 02 C1 FF + 23 00 02 BD 68 + 23 00 02 E7 FF + 23 00 02 E3 68 + 23 00 02 1F 7E + 23 00 02 49 7E + 23 00 02 70 7E + 23 00 02 96 7E + 23 00 02 BE 7E + 23 00 02 E4 7E + 23 00 02 20 97 + 23 00 02 4A 97 + 23 00 02 71 97 + 23 00 02 97 97 + 23 00 02 BF 97 + 23 00 02 E5 97 + 23 00 02 21 B5 + 23 00 02 4B B5 + 23 00 02 72 B5 + 23 00 02 98 B5 + 23 00 02 C0 B5 + 23 00 02 E6 B5 + 23 00 02 25 F0 + 23 00 02 23 E8 + 23 00 02 4F F0 + 23 00 02 4D E8 + 23 00 02 76 F0 + 23 00 02 74 E8 + 23 00 02 9C F0 + 23 00 02 9A E8 + 23 00 02 C4 F0 + 23 00 02 C2 E8 + 23 00 02 EA F0 + 23 00 02 E8 E8 + 23 00 02 24 FF + 23 00 02 4E FF + 23 00 02 75 FF + 23 00 02 9B FF + 23 00 02 C3 FF + 23 00 02 E9 FF + 23 00 02 FE 3D + 23 00 02 00 04 + 23 00 02 FE 23 + 23 00 02 08 82 + 23 00 02 0A 00 + 23 00 02 0B 00 + 23 00 02 0C 01 + 23 00 02 16 00 + 23 00 02 18 02 + 23 00 02 1B 04 + 23 00 02 19 04 + 23 00 02 1C 81 + 23 00 02 1F 00 + 23 00 02 20 03 + 23 00 02 23 04 + 23 00 02 21 01 + 23 00 02 54 63 + 23 00 02 55 54 + 23 00 02 6E 45 + 23 00 02 6D 36 + 23 00 02 FE 3D + 23 00 02 55 78 + 23 00 02 FE 20 + 23 00 02 26 30 + 23 00 02 FE 3D + 23 00 02 20 71 + 23 00 02 50 8F + 23 00 02 51 8F + 23 00 02 FE 00 + 23 00 02 35 00 + 05 78 01 11 + 05 00 01 29 + ]; + + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + disp_timings0: display-timings { + native-mode = <&dsi_timing0>; + dsi_timing0: timing0 { + clock-frequency = <132000000>; + hactive = <1080>; + vactive = <1920>; + hfront-porch = <15>; + hsync-len = <4>; + hback-porch = <30>; + vfront-porch = <15>; + vsync-len = <2>; + vback-porch = <15>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; + +}; + +&dsi_in_vp0 { + status = "disabled"; +}; + +&dsi_in_vp1 { + status = "disabled"; +}; + +&dsi_in_vp2 { + status = "disabled"; +}; + +&dsi_in_vopl { + status = "disabled"; +}; + +&gmac0 { + /* Use rgmii-rxid mode to disable rx delay inside Soc */ + phy-mode = "rgmii-rxid"; + clock_in_out = "output"; + + snps,reset-gpio = <&gpio3 RK_PB7 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + /* Reset time is 20ms, 100ms for rtl8211f */ + snps,reset-delays-us = <0 20000 100000>; + + pinctrl-names = "default"; + pinctrl-0 = <ð0m0_miim + ð0m0_tx_bus2 + ð0m0_rx_bus2 + ð0m0_rgmii_clk + ð0m0_rgmii_bus + ðm0_clk0_25m_out>; + + tx_delay = <0x21>; + /* rx_delay = <0x3f>; */ + + phy-handle = <&rgmii_phy0>; + status = "okay"; +}; + +&gmac1 { + /* Use rgmii-rxid mode to disable rx delay inside Soc */ + phy-mode = "rgmii-rxid"; + clock_in_out = "output"; + + snps,reset-gpio = <&gpio3 RK_PB0 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + /* Reset time is 20ms, 100ms for rtl8211f */ + snps,reset-delays-us = <0 20000 100000>; + + pinctrl-names = "default"; + pinctrl-0 = <ð1m1_miim + ð1m1_tx_bus2 + ð1m1_rx_bus2 + ð1m1_rgmii_clk + ð1m1_rgmii_bus + ðm1_clk1_25m_out>; + + tx_delay = <0x22>; + /* rx_delay = <0x3f>; */ + + phy-handle = <&rgmii_phy1>; + status = "okay"; +}; + +&gpu { + mali-supply = <&vdd_gpu_s0>; + status = "okay"; +}; + +&hdmi { + status = "okay"; + enable-gpios = <&gpio4 RK_PC4 GPIO_ACTIVE_HIGH>; + rockchip,sda-falling-delay-ns = <360>; +}; + +&hdmi_in_vp0 { + status = "okay"; +}; + +&hdptxphy_hdmi { + status = "okay"; +}; + +&i2c0 { + pinctrl-0 = <&i2c0m1_xfer>; + status = "okay"; + + gt1x: gt1x@14 { + status = "okay"; + compatible = "goodix,gt1x"; + reg = <0x14>; + pinctrl-names = "default"; + pinctrl-0 = <&touch_gpio>; + goodix,rst-gpio = <&gpio0 RK_PC0 GPIO_ACTIVE_HIGH>; + goodix,irq-gpio = <&gpio0 RK_PC5 IRQ_TYPE_LEVEL_LOW>; + power-supply = <&vcc3v3_lcd_n>; + }; +}; + +&i2c1 { + status = "okay"; + + rk806: pmic@23 { + compatible = "rockchip,rk806"; + reg = <0x23>; + + interrupt-parent = <&gpio0>; + interrupts = <6 IRQ_TYPE_LEVEL_LOW>; + + pinctrl-names = "default", "pmic-power-off"; + pinctrl-0 = <&pmic_pins>, <&rk806_dvs1_null>, <&rk806_dvs2_null>, <&rk806_dvs3_null>; + pinctrl-1 = <&rk806_dvs1_pwrdn>; + + /* 2800mv-3500mv */ + low_voltage_threshold = <3000>; + /* 2700mv-3400mv */ + shutdown_voltage_threshold = <2700>; + /* 140 160 */ + shutdown_temperture_threshold = <160>; + hotdie_temperture_threshold = <115>; + + /* PWRON_ON_TIME: 0:500mS; 1:20mS */ + pwron-on-time-500ms; + + /* 0: restart PMU; + * 1: reset all the power off reset registers, + * forcing the state to switch to ACTIVE mode; + * 2: Reset all the power off reset registers, + * forcing the state to switch to ACTIVE mode, + * and simultaneously pull down the RESETB PIN for 5mS before releasing + */ + pmic-reset-func = <1>; + /* buck5 external feedback resister disable */ + buck5-feedback-disable; + + vcc1-supply = <&vcc_5v0_sys_s5>; + vcc2-supply = <&vcc_5v0_sys_s5>; + vcc3-supply = <&vcc_5v0_sys_s5>; + vcc4-supply = <&vcc_5v0_sys_s5>; + vcc5-supply = <&vcc_5v0_sys_s5>; + vcc6-supply = <&vcc_5v0_sys_s5>; + vcc7-supply = <&vcc_5v0_sys_s5>; + vcc8-supply = <&vcc_5v0_sys_s5>; + vcc9-supply = <&vcc_5v0_sys_s5>; + vcc10-supply = <&vcc_5v0_sys_s5>; + vcc11-supply = <&vcc_2v0_pldo_s3>; + vcc12-supply = <&vcc_5v0_sys_s5>; + vcc13-supply = <&vcc_1v1_nldo_s3>; + vcc14-supply = <&vcc_1v1_nldo_s3>; + vcca-supply = <&vcc_5v0_sys_s5>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk806: pinctrl_rk806 { + gpio-controller; + #gpio-cells = <2>; + + rk806_dvs1_null: rk806_dvs1_null { + pins = "gpio_pwrctrl2"; + function = "pin_fun0"; + }; + + rk806_dvs1_slp: rk806_dvs1_slp { + pins = "gpio_pwrctrl1"; + function = "pin_fun1"; + }; + + rk806_dvs1_pwrdn: rk806_dvs1_pwrdn { + pins = "gpio_pwrctrl1"; + function = "pin_fun2"; + }; + + rk806_dvs1_rst: rk806_dvs1_rst { + pins = "gpio_pwrctrl1"; + function = "pin_fun3"; + }; + + rk806_dvs2_null: rk806_dvs2_null { + pins = "gpio_pwrctrl2"; + function = "pin_fun0"; + }; + + rk806_dvs2_slp: rk806_dvs2_slp { + pins = "gpio_pwrctrl2"; + function = "pin_fun1"; + }; + + rk806_dvs2_pwrdn: rk806_dvs2_pwrdn { + pins = "gpio_pwrctrl2"; + function = "pin_fun2"; + }; + + rk806_dvs2_rst: rk806_dvs2_rst { + pins = "gpio_pwrctrl2"; + function = "pin_fun3"; + }; + + rk806_dvs2_dvs: rk806_dvs2_dvs { + pins = "gpio_pwrctrl2"; + function = "pin_fun4"; + }; + + rk806_dvs2_gpio: rk806_dvs2_gpio { + pins = "gpio_pwrctrl2"; + function = "pin_fun5"; + }; + + rk806_dvs3_null: rk806_dvs3_null { + pins = "gpio_pwrctrl3"; + function = "pin_fun0"; + }; + + rk806_dvs3_slp: rk806_dvs3_slp { + pins = "gpio_pwrctrl3"; + function = "pin_fun1"; + }; + + rk806_dvs3_pwrdn: rk806_dvs3_pwrdn { + pins = "gpio_pwrctrl3"; + function = "pin_fun2"; + }; + + rk806_dvs3_rst: rk806_dvs3_rst { + pins = "gpio_pwrctrl3"; + function = "pin_fun3"; + }; + + rk806_dvs3_dvs: rk806_dvs3_dvs { + pins = "gpio_pwrctrl3"; + function = "pin_fun4"; + }; + + rk806_dvs3_gpio: rk806_dvs3_gpio { + pins = "gpio_pwrctrl3"; + function = "pin_fun5"; + }; + }; + + regulators { + vdd_cpu_big_s0: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-init-microvolt = <850000>; + regulator-min-microvolt = <550000>; + regulator-max-microvolt = <950000>; + regulator-ramp-delay = <12500>; + regulator-name = "vdd_cpu_big_s0"; + regulator-enable-ramp-delay = <400>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_npu_s0: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <550000>; + regulator-max-microvolt = <950000>; + regulator-ramp-delay = <12500>; + regulator-name = "vdd_npu_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_cpu_lit_s0: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-init-microvolt = <850000>; + regulator-min-microvolt = <550000>; + regulator-max-microvolt = <950000>; + regulator-ramp-delay = <12500>; + regulator-name = "vdd_cpu_lit_s0"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <750000>; + }; + }; + + vcc_3v3_s3: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc_3v3_s3"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vdd_gpu_s0: DCDC_REG5 { + regulator-boot-on; + regulator-init-microvolt = <750000>; + regulator-min-microvolt = <550000>; + regulator-max-microvolt = <900000>; + regulator-ramp-delay = <12500>; + regulator-name = "vdd_gpu_s0"; + regulator-enable-ramp-delay = <400>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <850000>; + }; + }; + + vddq_ddr_s0: DCDC_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vddq_ddr_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_logic_s0: vdd_log_mem_s0: DCDC_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-init-microvolt = <750000>; + regulator-min-microvolt = <550000>; + regulator-max-microvolt = <800000>; + regulator-name = "vdd_logic_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_1v8_s3: DCDC_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_1v8_s3"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd2_ddr_s3: DCDC_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vdd2_ddr_s3"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vdd_ddr_s0: DCDC_REG10 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <550000>; + regulator-max-microvolt = <1200000>; + regulator-name = "vdd_ddr_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcca_1v8_s0: PLDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcca_1v8_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_1v8_cam: PLDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_1v8_cam"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdda_1v2_s0: PLDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-name = "vdda_1v2_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcca3v3_codec: PLDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcca3v3_codec"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vccio_sd_s0: PLDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_sd_s0"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcca1v8_pldo6_s3: PLDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcca1v8_pldo6_s3"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd_0v75_s3: NLDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <750000>; + regulator-name = "vdd_0v75_s3"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <750000>; + }; + }; + + vdda_ddr_pll_s0: NLDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <850000>; + regulator-max-microvolt = <850000>; + regulator-name = "vdda_ddr_pll_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdda0v75_hdmi_s0: NLDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <750000>; + regulator-name = "vdda0v75_hdmi_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdda_0v85_s0: NLDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <850000>; + regulator-max-microvolt = <850000>; + regulator-name = "vdda_0v85_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdda_0v75_s0: NLDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <750000>; + regulator-name = "vdda_0v75_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + }; +}; + +&i2c7 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c7m2_xfer>; + + hym8563: hym8563@51 { + compatible = "haoyu,hym8563"; + reg = <0x51>; + #clock-cells = <0>; + clock-frequency = <32768>; + clock-output-names = "hym8563"; + pinctrl-names = "default"; + pinctrl-0 = <&rtc_int>; + + interrupt-parent = <&gpio0>; + interrupts = ; + wakeup-source; + }; +}; + +&iep { + status = "okay"; +}; + +&iep_mmu { + status = "okay"; +}; + +&jpegd { + status = "okay"; +}; + +&jpege { + status = "okay"; +}; + +&jpeg_mmu { + status = "okay"; +}; + +&mdio0 { + rgmii_phy0: phy@1 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <0x1>; + clocks = <&cru REFCLKO25M_GMAC0_OUT>; + }; +}; + +&mdio1 { + rgmii_phy1: phy@1 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <0x1>; + clocks = <&cru REFCLKO25M_GMAC1_OUT>; + }; +}; + +&mipidcphy0 { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&pcie0 { + rockchip,skip-scan-in-resume; + reset-gpios = <&gpio4 RK_PC7 GPIO_ACTIVE_HIGH>; + vpcie3v3-supply = <&vcc_3v3_pcie>; + status = "okay"; +}; + +&pinctrl { + hym8563 { + rtc_int: rtc-int { + rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + gpio-leds { + leds_gpio: leds-gpio { + rockchip,pins = <3 16 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + pcie { + pcie_reset_gpio: pcie-reset-gpio { + rockchip,pins = <4 RK_PC7 RK_FUNC_GPIO &pcfg_pull_none>; + }; + pcie_wake_gpio: pcie-wake-gpio { + rockchip,pins = <0 RK_PD2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + pcie_pwren: pcie-pwren { + rockchip,pins = <2 RK_PC7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + sd { + sd_pwren: sd-pwren { + rockchip,pins = <0 RK_PB6 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + touch { + touch_gpio: touch-gpio { + rockchip,pins = + <0 RK_PC0 RK_FUNC_GPIO &pcfg_pull_up>, + <0 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + usb { + usb_host_pwren: usb-host-pwren { + rockchip,pins = <0 RK_PD3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + // usb_otg_pwren: usb-otg-pwren { + // rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>; + // }; + }; + + wireless-bluetooth { + uart7_gpios: uart7-gpios { + rockchip,pins = <2 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + bt_host_wake_irq: bt-host-wake-irq { + rockchip,pins = <0 RK_PB1 RK_FUNC_GPIO &pcfg_pull_down>; + }; + bt_reset_gpio: bt-reset-gpio { + rockchip,pins = <2 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; + }; + bt_wake_gpio: bt-wake-gpio { + rockchip,pins = <2 RK_PC6 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + wireless-wlan { + wifi_host_wake_irq: wifi-host-wake-irq { + rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_down>; + }; + + wifi_poweren_gpio: wifi-poweren-gpio { + rockchip,pins = <2 RK_PC0 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; +}; + +&pwm1_6ch_1 { + status = "okay"; + pinctrl-0 = <&pwm1m0_ch1>; +}; + +&pwm1_6ch_4 { + status = "okay"; + pinctrl-0 = <&pwm1m0_ch4>; +}; + +&rga2_core0 { + status = "okay"; +}; + +&rga2_core0_mmu { + status = "okay"; +}; + +&rga2_core1 { + status = "okay"; +}; + +&rga2_core1_mmu { + status = "okay"; +}; + +&rknpu { + rknpu-supply = <&vdd_npu_s0>; + status = "okay"; +}; + +&rknpu_mmu { + status = "okay"; +}; + +&rkvenc_ccu { + status = "okay"; +}; + +&rkvenc0 { + status = "okay"; +}; + +&rkvenc0_mmu { + status = "okay"; +}; + +&rkvenc1 { + status = "okay"; +}; + +&rkvenc1_mmu { + status = "okay"; + +}; + +&rkvdec { + status = "okay"; +}; + +&rkvdec_mmu { + status = "okay"; +}; + +&route_dsi { + status = "disabled"; + connect = <&vp1_out_dsi>; +}; + +&route_hdmi { + status = "okay"; + connect = <&vp0_out_hdmi>; +}; + +&sai2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&sai2m1_lrck + &sai2m1_sclk + &sai2m1_sdi + &sai2m1_sdo>; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcca_1v8_s0>; +}; + +&sdhci { + bus-width = <8>; + no-sdio; + no-sd; + non-removable; + max-frequency = <200000000>; + mmc-hs400-1_8v; + mmc-hs400-enhanced-strobe; + full-pwr-cycle-in-suspend; + status = "okay"; +}; + +&sdio { + max-frequency = <150000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc1m1_bus4 &sdmmc1m1_clk &sdmmc1m1_cmd>; + sd-uhs-sdr104; + status = "okay"; +}; + +&sdmmc { + supports-sd; + status = "okay"; +}; + +&rockchip_suspend { + status = "okay"; +}; + +&uart7 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&uart7m0_xfer &uart7m0_ctsn>; +}; + +&u2phy0 { + status = "okay"; +}; + +&u2phy1 { + status = "okay"; +}; + +&u2phy0_otg { + status = "okay"; +}; + +&u2phy1_otg { + phy-supply = <&vcc5v0_usb30_host1>; + status = "okay"; +}; + +&ufs { + reset-gpios = <&gpio4 RK_PD0 GPIO_ACTIVE_HIGH>; + status = "okay"; +}; + +&usbdp_phy { + status = "okay"; +}; + +&usbdp_phy_dp { + status = "disabled"; +}; + +&usbdp_phy_u3 { + status = "okay"; +}; + +&usb_drd0_dwc3 { + dr_mode = "peripheral"; + extcon = <&u2phy0>; + status = "okay"; +}; + +&usb_drd1_dwc3 { + dr_mode = "host"; + // extcon = <&u2phy1>; + status = "okay"; +}; + +&vdpp { + status = "okay"; +}; + +&vop { + status = "okay"; + vop-supply = <&vdd_logic_s0>; +}; + +&vop_mmu { + status = "okay"; +}; + +&vp2 { + assigned-clocks = <&cru DCLK_VP2_SRC>; + assigned-clock-parents = <&cru PLL_VPLL>; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3576-vehicle-evb-v21-maxim-max96712-dcphy0-ox03j10.dtsi b/arch/arm64/boot/dts/rockchip/rk3576-vehicle-evb-v21-maxim-max96712-dcphy0-ox03j10.dtsi new file mode 100644 index 000000000000..cffb04ea4958 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3576-vehicle-evb-v21-maxim-max96712-dcphy0-ox03j10.dtsi @@ -0,0 +1,729 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2025 Rockchip Electronics Co., Ltd. + * + */ +#include + +/ { + max96712_dcphy0_osc: max96712-dcphy0-oscillator { + compatible = "fixed-clock"; + #clock-cells = <1>; + clock-frequency = <25000000>; + clock-output-names = "max96712-dcphy0-osc"; + }; + + max96712_dcphy0_vcc1v2: max96712-dcphy0-vcc1v2 { + compatible = "regulator-fixed"; + regulator-name = "max96712_dcphy0_vcc1v2"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + startup-delay-us = <850>; + vin-supply = <&vcc_2v0_pldo_s3>; + }; + + max96712_dcphy0_vcc1v8: max96712-dcphy0-vcc1v8 { + compatible = "regulator-fixed"; + regulator-name = "max96712_dcphy0_vcc1v8"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + startup-delay-us = <200>; + vin-supply = <&vcc_2v0_pldo_s3>; + }; + + max96712_dcphy0_pwdn_regulator: max96712-dcphy0-pwdn-regulator { + compatible = "regulator-fixed"; + regulator-name = "max96712_dcphy0_pwdn"; + gpio = <&gpio2 RK_PA0 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&max96712_dcphy0_pwdn>; + enable-active-high; + startup-delay-us = <10000>; + off-on-delay-us = <5000>; + }; + + max96712_dcphy0_poc_regulator: max96712-dcphy0-poc-regulator { + compatible = "regulator-fixed"; + regulator-name = "max96712_dcphy0_poc"; + gpio = <&i2c7_nca9539_gpio 2 GPIO_ACTIVE_HIGH>; + enable-active-high; + startup-delay-us = <10000>; + off-on-delay-us = <5000>; + vin-supply = <&dcphy0_vcc12v_buck1>; + }; +}; + +&csi2_dcphy0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_dcphy0_in_max96712: endpoint@1 { + reg = <1>; + remote-endpoint = <&max96712_dcphy0_out>; + data-lanes = <1 2 3 4>; + }; + }; + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + csidcphy0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&mipi0_csi2_input>; + }; + }; + }; +}; + +&i2c2 { + max96712_dcphy0: max96712@29 { + compatible = "maxim4c,max96712"; + status = "okay"; + reg = <0x29>; + clock-names = "xvclk"; + clocks = <&max96712_dcphy0_osc 0>; + pinctrl-names = "default"; + pinctrl-0 = <&max96712_dcphy0_errb>, <&max96712_dcphy0_lock>; + power-domains = <&power RK3576_PD_VI>; + rockchip,grf = <&sys_grf>; + vcc1v2-supply = <&max96712_dcphy0_vcc1v2>; + vcc1v8-supply = <&max96712_dcphy0_vcc1v8>; + pwdn-supply = <&max96712_dcphy0_pwdn_regulator>; + lock-gpios = <&gpio2 RK_PA2 GPIO_ACTIVE_HIGH>; + + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "default"; + rockchip,camera-module-lens-name = "default"; + + port { + max96712_dcphy0_out: endpoint { + remote-endpoint = <&mipi_dcphy0_in_max96712>; + data-lanes = <1 2 3 4>; + }; + }; + + /* support mode config start */ + support-mode-config { + status = "okay"; + + bus-format = ; + sensor-width = <1920>; + sensor-height = <1280>; + max-fps-numerator = <10000>; + max-fps-denominator = <300000>; + bpp = <16>; + link-freq-idx = <20>; + }; + /* support mode config end */ + + /* serdes local device start */ + serdes-local-device { + status = "okay"; + + /* GMSL LINK config start */ + gmsl-links { + status = "okay"; + + link-vdd-ldo1-en = <1>; + link-vdd-ldo2-en = <1>; + + // Link A: link-id = 0 + gmsl-link-config-0 { + status = "okay"; + link-id = <0>; // Link ID: 0/1/2/3 + + link-type = <1>; // 0: GMSL1, 1: GMSL2 + link-rx-rate = <1>; // 0: 3GBPS, 1: 6GBPS + link-tx-rate = <0>; // 0: default for 187.5MBPS + + link-remote-cam = <&max96712_dcphy0_cam0>; // remote camera + + link-init-sequence { + seq-item-size = <5>; // reg-addr-len + reg-val-len * 2 + 1 + reg-addr-len = <2>; // 1: 8bits, 2: 16bits + reg-val-len = <1>; // 1: 8bits, 2: 16bits, 3: 24bits + + // reg_addr reg_val val_mask delay + init-sequence = [ + 14 D1 03 00 00 // VGAHiGain + 14 45 00 00 00 // Disable SSC + ]; + }; + }; + + // Link B: link-id = 1 + gmsl-link-config-1 { + status = "okay"; + link-id = <1>; // Link ID: 0/1/2/3 + + link-type = <1>; // 0: GMSL1, 1: GMSL2 + link-rx-rate = <1>; // 0: 3GBPS, 1: 6GBPS + link-tx-rate = <0>; // 0: default for 187.5MBPS + + link-remote-cam = <&max96712_dcphy0_cam1>; // remote camera + + link-init-sequence { + seq-item-size = <5>; // reg-addr-len + reg-val-len * 2 + 1 + reg-addr-len = <2>; // 1: 8bits, 2: 16bits + reg-val-len = <1>; // 1: 8bits, 2: 16bits, 3: 24bits + + // reg_addr reg_val val_mask delay + init-sequence = [ + 15 D1 03 00 00 // VGAHiGain + 15 45 00 00 00 // Disable SSC + ]; + }; + }; + + // Link C: link-id = 2 + gmsl-link-config-2 { + status = "okay"; + link-id = <2>; // Link ID: 0/1/2/3 + + link-type = <1>; // 0: GMSL1, 1: GMSL2 + link-rx-rate = <1>; // 0: 3GBPS, 1: 6GBPS + link-tx-rate = <0>; // 0: default for 187.5MBPS + + link-remote-cam = <&max96712_dcphy0_cam2>; // remote camera + + link-init-sequence { + seq-item-size = <5>; // reg-addr-len + reg-val-len * 2 + 1 + reg-addr-len = <2>; // 1: 8bits, 2: 16bits + reg-val-len = <1>; // 1: 8bits, 2: 16bits, 3: 24bits + + // reg_addr reg_val val_mask delay + init-sequence = [ + 16 D1 03 00 00 // VGAHiGain + 16 45 00 00 00 // Disable SSC + ]; + }; + }; + + // Link D: link-id = 3 + gmsl-link-config-3 { + status = "okay"; + link-id = <3>; // Link ID: 0/1/2/3 + + link-type = <1>; // 0: GMSL1, 1: GMSL2 + link-rx-rate = <1>; // 0: 3GBPS, 1: 6GBPS + link-tx-rate = <0>; // 0: default for 187.5MBPS + + link-remote-cam = <&max96712_dcphy0_cam3>; // remote camera + + link-init-sequence { + seq-item-size = <5>; // reg-addr-len + reg-val-len * 2 + 1 + reg-addr-len = <2>; // 1: 8bits, 2: 16bits + reg-val-len = <1>; // 1: 8bits, 2: 16bits, 3: 24bits + + // reg_addr reg_val val_mask delay + init-sequence = [ + 17 D1 03 00 00 // VGAHiGain + 17 45 00 00 00 // Disable SSC + ]; + }; + }; + }; + /* GMSL LINK config end */ + + /* VIDEO PIPE config start */ + video-pipes { + status = "okay"; + + // Video Pipe 0 + video-pipe-config-0 { + status = "okay"; + pipe-id = <0>; // Video Pipe ID: 0/1/2/3/4/5/6/7 + + pipe-idx = <2>; // Video Pipe X/Y/Z/U: 0/1/2/3 + link-idx = <0>; // Link A/B/C/D: 0/1/2/3 + + pipe-init-sequence { + seq-item-size = <5>; // reg-addr-len + reg-val-len * 2 + 1 + reg-addr-len = <2>; // 1: 8bits, 2: 16bits + reg-val-len = <1>; // 1: 8bits, 2: 16bits, 3: 24bits + + // reg_addr reg_val val_mask delay + init-sequence = [ + // Send YUV422, FS, and FE from Video Pipe 0 to Controller 1 + 09 0B 07 00 00 // Enable 0/1/2 SRC/DST Mappings + 09 2D 15 00 00 // SRC/DST 0/1/2 -> CSI2 Controller 1; + // For the following MSB 2 bits = VC, LSB 6 bits = DT + 09 0D 1e 00 00 // SRC0 VC = 0, DT = YUV422 8bit + 09 0E 1e 00 00 // DST0 VC = 0, DT = YUV422 8bit + 09 0F 00 00 00 // SRC1 VC = 0, DT = Frame Start + 09 10 00 00 00 // DST1 VC = 0, DT = Frame Start + 09 11 01 00 00 // SRC2 VC = 0, DT = Frame End + 09 12 01 00 00 // DST2 VC = 0, DT = Frame End + ]; + }; + }; + + // Video Pipe 1 + video-pipe-config-1 { + status = "okay"; + pipe-id = <1>; // Video Pipe 1: pipe-id = 1 + + pipe-idx = <2>; // Video Pipe X/Y/Z/U: 0/1/2/3 + link-idx = <1>; // Link A/B/C/D: 0/1/2/3 + + pipe-init-sequence { + seq-item-size = <5>; // reg-addr-len + reg-val-len * 2 + 1 + reg-addr-len = <2>; // 1: 8bits, 2: 16bits + reg-val-len = <1>; // 1: 8bits, 2: 16bits, 3: 24bits + + // reg_addr reg_val val_mask delay + init-sequence = [ + // Send YUV422, FS, and FE from Video Pipe 1 to Controller 1 + 09 4B 07 00 00 // Enable 0/1/2 SRC/DST Mappings + 09 6D 15 00 00 // SRC/DST 0/1/2 -> CSI2 Controller 1; + // For the following MSB 2 bits = VC, LSB 6 bits = DT + 09 4D 1e 00 00 // SRC0 VC = 0, DT = YUV422 8bit + 09 4E 5e 00 00 // DST0 VC = 1, DT = YUV422 8bit + 09 4F 00 00 00 // SRC1 VC = 0, DT = Frame Start + 09 50 40 00 00 // DST1 VC = 1, DT = Frame Start + 09 51 01 00 00 // SRC2 VC = 0, DT = Frame End + 09 52 41 00 00 // DST2 VC = 1, DT = Frame End + ]; + }; + }; + + // Video Pipe 2 + video-pipe-config-2 { + status = "okay"; + pipe-id = <2>; // Video Pipe ID: 0/1/2/3/4/5/6/7 + + pipe-idx = <2>; // Video Pipe X/Y/Z/U: 0/1/2/3 + link-idx = <2>; // Link A/B/C/D: 0/1/2/3 + + pipe-init-sequence { + seq-item-size = <5>; // reg-addr-len + reg-val-len * 2 + 1 + reg-addr-len = <2>; // 1: 8bits, 2: 16bits + reg-val-len = <1>; // 1: 8bits, 2: 16bits, 3: 24bits + + // reg_addr reg_val val_mask delay + init-sequence = [ + // Send YUV422, FS, and FE from Video Pipe 2 to Controller 1 + 09 8B 07 00 00 // Enable 0/1/2 SRC/DST Mappings + 09 AD 15 00 00 // SRC/DST 0/1/2 -> CSI2 Controller 1; + // For the following MSB 2 bits = VC, LSB 6 bits = DT + 09 8D 1e 00 00 // SRC0 VC = 0, DT = YUV422 8bit + 09 8E 9e 00 00 // DST0 VC = 2, DT = YUV422 8bit + 09 8F 00 00 00 // SRC1 VC = 0, DT = Frame Start + 09 90 80 00 00 // DST1 VC = 2, DT = Frame Start + 09 91 01 00 00 // SRC2 VC = 0, DT = Frame End + 09 92 81 00 00 // DST2 VC = 2, DT = Frame End + ]; + }; + }; + + // Video Pipe 3 + video-pipe-config-3 { + status = "okay"; + pipe-id = <3>; // Video Pipe ID: 0/1/2/3/4/5/6/7 + + pipe-idx = <2>; // Video Pipe X/Y/Z/U: 0/1/2/3 + link-idx = <3>; // Link A/B/C/D: 0/1/2/3 + + pipe-init-sequence { + seq-item-size = <5>; // reg-addr-len + reg-val-len * 2 + 1 + reg-addr-len = <2>; // 1: 8bits, 2: 16bits + reg-val-len = <1>; // 1: 8bits, 2: 16bits, 3: 24bits + + // reg_addr reg_val val_mask delay + init-sequence = [ + // Send YUV422, FS, and FE from Video Pipe 3 to Controller 1 + 09 CB 07 00 00 // Enable 0/1/2 SRC/DST Mappings + 09 ED 15 00 00 // SRC/DST 0/1/2 -> CSI2 Controller 1; + // For the following MSB 2 bits = VC, LSB 6 bits = DT + 09 CD 1e 00 00 // SRC0 VC = 0, DT = YUV422 8bit + 09 CE de 00 00 // DST0 VC = 3, DT = YUV422 8bit + 09 CF 00 00 00 // SRC1 VC = 0, DT = Frame Start + 09 D0 c0 00 00 // DST1 VC = 3, DT = Frame Start + 09 D1 01 00 00 // SRC2 VC = 0, DT = Frame End + 09 D2 c1 00 00 // DST2 VC = 3, DT = Frame End + ]; + }; + }; + }; + /* VIDEO PIPE config end */ + + /* MIPI TXPHY config start */ + mipi-txphys { + status = "okay"; + + phy-mode = <0>; // 0: 4Lanes, 1: 2Lanes + phy-force-clock-out = <1>; // 1: default for force clock out + phy-force-clk0-en = <1>; // provide MIPI clock: 0 = PHY1, 1 = PHY0 + phy-force-clk3-en = <0>; // provide MIPI clock: 0 = PHY2, 1 = PHY3 + + // MIPI TXPHY A: phy-id = 0 + mipi-txphy-config-0 { + status = "okay"; + phy-id = <0>; // MIPI TXPHY ID: 0/1/2/3 + + phy-type = <0>; // 0: DPHY, 1: CPHY + auto-deskew = <0x80>; + data-lane-num = <4>; + data-lane-map = <0x4>; + vc-ext-en = <0>; + }; + + // MIPI TXPHY B: phy-id = 1 + mipi-txphy-config-1 { + status = "okay"; + phy-id = <1>; // MIPI TXPHY ID: 0/1/2/3 + + phy-type = <0>; // 0: DPHY, 1: CPHY + auto-deskew = <0x80>; + data-lane-num = <4>; + data-lane-map = <0xe>; + vc-ext-en = <0>; + }; + }; + /* MIPI TXPHY config end */ + + /* local device extra init sequence */ + extra-init-sequence { + status = "disabled"; + + seq-item-size = <5>; // reg-addr-len + reg-val-len * 2 + 1 + reg-addr-len = <2>; // 1: 8bits, 2: 16bits + reg-val-len = <1>; // 1: 8bits, 2: 16bits, 3: 24bits + + // reg_addr reg_val val_mask delay + init-sequence = [ + // common init sequence such as fsync / gpio and so on + ]; + }; + }; + /* serdes local device end */ + + /* i2c-mux start */ + i2c-mux { + #address-cells = <1>; + #size-cells = <0>; + + i2c@0 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0>; + + // Note: Serializer node defined before camera node + max96712_dcphy0_ser0: max96717@51 { + compatible = "maxim,ser,max96717"; + reg = <0x51>; + + ser-i2c-addr-def = <0x40>; + + ser-init-sequence { + seq-item-size = <5>; // reg-addr-len + reg-val-len * 2 + 1 + reg-addr-len = <2>; // 1: 8bits, 2: 16bits + reg-val-len = <1>; // 1: 8bits, 2: 16bits, 3: 24bits + + // reg_addr reg_val val_mask delay + init-sequence = [ + 02 BE 00 00 01 // MFP0 GPIO_OUT = 0: MFP0 output is driven to 0 + 03 02 10 00 00 // Improve CMU voltage performance to improve link robustness + 14 17 00 00 00 // RLMS17 = 0x00: disable AGC/DFE adaptation + 14 32 7f 00 00 // RLMS32 = 0x7F: change OSN loop mode + 03 F0 59 00 00 // REFGEN_PREDEF_FREQ_ALT = 1: Alternative table, REFGEN_PREDEF_FREQ = 0x1: 24MHz + 00 03 03 00 00 // RCLKSEL = 0x3: Reference PLL output + 00 06 B1 00 01 // RCLKEN = 1: RCLK output is enabled + 02 BF 40 00 00 // MFP0 PULL_UPDN_SEL = 1: Pullup + 02 BE 10 00 01 // MFP0 GPIO_OUT = 1: MFP0 output is driven to 1 + ]; + }; + }; + + max96712_dcphy0_cam0: ox03j10@31 { + compatible = "maxim,ovti,ox03j10"; + reg = <0x31>; + + cam-i2c-addr-def = <0x36>; + + cam-remote-ser = <&max96712_dcphy0_ser0>; // remote serializer + + poc-supply = <&max96712_dcphy0_poc_regulator>; + + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "default"; + rockchip,camera-module-lens-name = "default"; + + /* port config start */ + port { + max96712_dcphy0_cam0_out: endpoint { + data-lanes = <1 2 3 4>; + }; + }; + /* port config end */ + }; + }; + + i2c@1 { + #address-cells = <1>; + #size-cells = <0>; + reg = <1>; + + // Note: Serializer node defined before camera node + max96712_dcphy0_ser1: max96717@52 { + compatible = "maxim,ser,max96717"; + reg = <0x52>; + + ser-i2c-addr-def = <0x40>; + + ser-init-sequence { + seq-item-size = <5>; // reg-addr-len + reg-val-len * 2 + 1 + reg-addr-len = <2>; // 1: 8bits, 2: 16bits + reg-val-len = <1>; // 1: 8bits, 2: 16bits, 3: 24bits + + // reg_addr reg_val val_mask delay + init-sequence = [ + 02 BE 00 00 01 // MFP0 GPIO_OUT = 0: MFP0 output is driven to 0 + 03 02 10 00 00 // Improve CMU voltage performance to improve link robustness + 14 17 00 00 00 // RLMS17 = 0x00: disable AGC/DFE adaptation + 14 32 7f 00 00 // RLMS32 = 0x7F: change OSN loop mode + 03 F0 59 00 00 // REFGEN_PREDEF_FREQ_ALT = 1: Alternative table, REFGEN_PREDEF_FREQ = 0x1: 24MHz + 00 03 03 00 00 // RCLKSEL = 0x3: Reference PLL output + 00 06 B1 00 01 // RCLKEN = 1: RCLK output is enabled + 02 BF 40 00 00 // MFP0 PULL_UPDN_SEL = 1: Pullup + 02 BE 10 00 01 // MFP0 GPIO_OUT = 1: MFP0 output is driven to 1 + ]; + }; + }; + + max96712_dcphy0_cam1: ox03j10@32 { + compatible = "maxim,ovti,ox03j10"; + reg = <0x32>; + + cam-i2c-addr-def = <0x36>; + + cam-remote-ser = <&max96712_dcphy0_ser1>; // remote serializer + + poc-supply = <&max96712_dcphy0_poc_regulator>; + + rockchip,camera-module-index = <1>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "default"; + rockchip,camera-module-lens-name = "default"; + + /* port config start */ + port { + max96712_dcphy0_cam1_out: endpoint { + data-lanes = <1 2 3 4>; + }; + }; + /* port config end */ + }; + }; + + i2c@2 { + #address-cells = <1>; + #size-cells = <0>; + reg = <2>; + + // Note: Serializer node defined before camera node + max96712_dcphy0_ser2: max96717@53 { + compatible = "maxim,ser,max96717"; + reg = <0x53>; + + ser-i2c-addr-def = <0x40>; + + ser-init-sequence { + seq-item-size = <5>; // reg-addr-len + reg-val-len * 2 + 1 + reg-addr-len = <2>; // 1: 8bits, 2: 16bits + reg-val-len = <1>; // 1: 8bits, 2: 16bits, 3: 24bits + + // reg_addr reg_val val_mask delay + init-sequence = [ + 02 BE 00 00 01 // MFP0 GPIO_OUT = 0: MFP0 output is driven to 0 + 03 02 10 00 00 // Improve CMU voltage performance to improve link robustness + 14 17 00 00 00 // RLMS17 = 0x00: disable AGC/DFE adaptation + 14 32 7f 00 00 // RLMS32 = 0x7F: change OSN loop mode + 03 F0 59 00 00 // REFGEN_PREDEF_FREQ_ALT = 1: Alternative table, REFGEN_PREDEF_FREQ = 0x1: 24MHz + 00 03 03 00 00 // RCLKSEL = 0x3: Reference PLL output + 00 06 B1 00 01 // RCLKEN = 1: RCLK output is enabled + 02 BF 40 00 00 // MFP0 PULL_UPDN_SEL = 1: Pullup + 02 BE 10 00 01 // MFP0 GPIO_OUT = 1: MFP0 output is driven to 1 + ]; + }; + }; + + max96712_dcphy0_cam2: ox03j10@33 { + compatible = "maxim,ovti,ox03j10"; + reg = <0x33>; + + cam-i2c-addr-def = <0x36>; + + cam-remote-ser = <&max96712_dcphy0_ser2>; // remote serializer + + poc-supply = <&max96712_dcphy0_poc_regulator>; + + rockchip,camera-module-index = <2>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "default"; + rockchip,camera-module-lens-name = "default"; + + /* port config start */ + port { + max96712_dcphy0_cam2_out: endpoint { + data-lanes = <1 2 3 4>; + }; + }; + /* port config end */ + }; + }; + + i2c@3 { + #address-cells = <1>; + #size-cells = <0>; + reg = <3>; + + // Note: Serializer node defined before camera node + max96712_dcphy0_ser3: max96717@54 { + compatible = "maxim,ser,max96717"; + reg = <0x54>; + + ser-i2c-addr-def = <0x40>; + + ser-init-sequence { + seq-item-size = <5>; // reg-addr-len + reg-val-len * 2 + 1 + reg-addr-len = <2>; // 1: 8bits, 2: 16bits + reg-val-len = <1>; // 1: 8bits, 2: 16bits, 3: 24bits + + // reg_addr reg_val val_mask delay + init-sequence = [ + 02 BE 00 00 01 // MFP0 GPIO_OUT = 0: MFP0 output is driven to 0 + 03 02 10 00 00 // Improve CMU voltage performance to improve link robustness + 14 17 00 00 00 // RLMS17 = 0x00: disable AGC/DFE adaptation + 14 32 7f 00 00 // RLMS32 = 0x7F: change OSN loop mode + 03 F0 59 00 00 // REFGEN_PREDEF_FREQ_ALT = 1: Alternative table, REFGEN_PREDEF_FREQ = 0x1: 24MHz + 00 03 03 00 00 // RCLKSEL = 0x3: Reference PLL output + 00 06 B1 00 01 // RCLKEN = 1: RCLK output is enabled + 02 BF 40 00 00 // MFP0 PULL_UPDN_SEL = 1: Pullup + 02 BE 10 00 01 // MFP0 GPIO_OUT = 1: MFP0 output is driven to 1 + ]; + }; + }; + + max96712_dcphy0_cam3: ox03j10@34 { + compatible = "maxim,ovti,ox03j10"; + reg = <0x34>; + + cam-i2c-addr-def = <0x36>; + + cam-remote-ser = <&max96712_dcphy0_ser3>; // remote serializer + + poc-supply = <&max96712_dcphy0_poc_regulator>; + + rockchip,camera-module-index = <3>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "default"; + rockchip,camera-module-lens-name = "default"; + + /* port config start */ + port { + max96712_dcphy0_cam3_out: endpoint { + data-lanes = <1 2 3 4>; + }; + }; + /* port config end */ + }; + }; + }; + /* i2c-mux end */ + }; +}; + +&mipi0_csi2 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi0_csi2_input: endpoint@1 { + reg = <1>; + remote-endpoint = <&csidcphy0_out>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + mipi0_csi2_output: endpoint@0 { + reg = <0>; + remote-endpoint = <&cif_mipi0_in>; + }; + }; + }; +}; + +&rkcif_mipi_lvds { + status = "okay"; + /* parameters for do cif reset detecting: + * index0: monitor mode, + 0 for idle, + 1 for continue, + 2 for trigger, + 3 for hotplug (for nextchip) + * index1: the frame id to start timer, + min is 2 + * index2: frame num of monitoring cycle + * index3: err time for keep monitoring + after finding out err (ms) + * index4: csi2 err reference val for resetting + */ + rockchip,cif-monitor = <3 2 1 1000 5>; + + port { + cif_mipi0_in: endpoint { + remote-endpoint = <&mipi0_csi2_output>; + }; + }; +}; + +&rkcif { + status = "okay"; + rockchip,android-usb-camerahal-enable; +}; + +&rkcif_mmu { + status = "okay"; +}; + +&pinctrl { + max96712-dcphy0 { + max96712_dcphy0_pwdn: max96712-dcphy0-pwdn { + rockchip,pins = <2 RK_PA0 RK_FUNC_GPIO &pcfg_output_low>; + }; + + max96712_dcphy0_errb: max96712-dcphy0-errb { + rockchip,pins = <2 RK_PA1 RK_FUNC_GPIO &pcfg_pull_none_smt>; + }; + + max96712_dcphy0_lock: max96712-dcphy0-lock { + rockchip,pins = <2 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none_smt>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3576-vehicle-evb-v21.dts b/arch/arm64/boot/dts/rockchip/rk3576-vehicle-evb-v21.dts new file mode 100644 index 000000000000..9e119c0a1b16 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3576-vehicle-evb-v21.dts @@ -0,0 +1,576 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2025 Rockchip Electronics Co., Ltd. + * + */ + +/dts-v1/; + +#include "rk3576.dtsi" +#include "rk3576-vehicle-evb-v20.dtsi" +#include "rk3576-vehicle-evb-v20-nca9539-io-expander.dtsi" +#include "rk3576-vehicle-evb-v20-serdes-mfd-display-maxim.dtsi" +#include "rk3576-vehicle-evb-v20-maxim-max96712-dphy0-ox03j10.dtsi" +#include "rk3576-vehicle-evb-v20-maxim-max96712-dphy3-sc233at.dtsi" +#include "rk3576-android.dtsi" + +/delete-node/ &vcc5v0_host_usb30; + +/ { + model = "Rockchip RK3576 VEHICLE EVB V21 Board"; + compatible = "rockchip,rk3576-vehicle-evb-v21", "rockchip,rk3576"; + + chosen: chosen { + bootargs = "earlycon=uart8250,mmio32,0x2ad40000 console=ttyFIQ0 rcupdate.rcu_expedited=1 rcu_nocbs=all spidev.bufsiz=131072"; + }; + + vehicle_dummy: vehicle-dummy { + status = "okay"; + compatible = "rockchip,vehicle-dummy-adc"; + io-channels = <&saradc 4>, <&saradc 5>, <&saradc 6>; + io-channel-names = "gear", "turn_left", "turn_right"; + }; + + vcc5v0_buck: vcc5v0-buck { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_buck"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + enable-active-high; + gpio = <&gpio0 RK_PD3 GPIO_ACTIVE_HIGH>; + vin-supply = <&vcc12v_dcin>; + pinctrl-names = "default"; + pinctrl-0 = <&vcc5v0_buck_en>; + startup-delay-us = <2500>; + off-on-delay-us = <1500>; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <5000000>; + }; + }; + + cluster_power_buck: cluster_power-buck { + compatible = "regulator-fixed"; + regulator-name = "cluster_power_buck"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + //enable-active-high; + gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_LOW>; + vin-supply = <&vcc_1v8_s0>; + pinctrl-names = "default"; + pinctrl-0 = <&cluster_buck_en>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + usb_otg_vcc5v_buck: usb_otg_vcc5v-buck { + compatible = "regulator-fixed"; + regulator-name = "usb_otg_vcc5v_buck"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + //enable-active-high; + gpio = <&i2c0_nca9539_gpio 0 GPIO_ACTIVE_HIGH>; + vin-supply = <&vcc5v0_buck>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <5000000>; + }; + }; + + usb_host_vcc5v_buck: usb_host_vcc5v-buck { + compatible = "regulator-fixed"; + regulator-name = "usb_host_vcc5v_buck"; + //regulator-boot-on; + //regulator-always-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + enable-active-high; + gpio = <&i2c0_nca9539_gpio 1 GPIO_ACTIVE_HIGH>; + vin-supply = <&vcc5v0_buck>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <5000000>; + }; + }; + + lcd1_vcc12v_buck: lcd1_vcc12v-buck { + compatible = "regulator-fixed"; + regulator-name = "lcd1_vcc12v_buck"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c0_nca9539_gpio 2 GPIO_ACTIVE_HIGH>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + lcd2_vcc12v_buck: lcd2_vcc12v-buck { + compatible = "regulator-fixed"; + regulator-name = "lcd2_vcc12v_buck"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c0_nca9539_gpio 3 GPIO_ACTIVE_HIGH>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + lcd1_ser_vcc5v_buck: lcd1_ser_vcc5v-buck { + compatible = "regulator-fixed"; + regulator-name = "lcd1_ser_vcc5v_buck"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + enable-active-high; + gpio = <&i2c0_nca9539_gpio 4 GPIO_ACTIVE_HIGH>; + vin-supply = <&vcc5v0_buck>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <5000000>; + }; + }; + + lcd2_ser_vcc5v_buck: lcd2_ser_vcc5v-buck { + compatible = "regulator-fixed"; + regulator-name = "lcd2_ser_vcc5v_buck"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + enable-active-high; + gpio = <&i2c0_nca9539_gpio 5 GPIO_ACTIVE_HIGH>; + vin-supply = <&vcc5v0_buck>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <5000000>; + }; + }; + + adsp_vcc12v_buck: adsp_vcc12v-buck { + compatible = "regulator-fixed"; + regulator-name = "adsp_vcc12v_buck"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c0_nca9539_gpio 6 GPIO_ACTIVE_HIGH>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + lcd3_vcc12v_buck: lcd3_vcc12v-buck { + compatible = "regulator-fixed"; + regulator-name = "lcd3_vcc12v_buck"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c7_nca9539_gpio 0 GPIO_ACTIVE_HIGH>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + lcd3_vcc5v_buck: lcd3_vcc5v-buck { + compatible = "regulator-fixed"; + regulator-name = "lcd3_vcc5v_buck"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + enable-active-high; + gpio = <&i2c7_nca9539_gpio 1 GPIO_ACTIVE_HIGH>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + dcphy0_vcc12v_buck1: dcphy0_vcc12v-buck1 { + compatible = "regulator-fixed"; + regulator-name = "dcphy0_vcc12v_buck1"; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c7_nca9539_gpio 3 GPIO_ACTIVE_HIGH>; + startup-delay-us = <2000>; + off-on-delay-us = <16000>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + dcphy0_vcc12v_buck2: dcphy0_vcc12v-buck2 { + compatible = "regulator-fixed"; + regulator-name = "dcphy0_vcc12v_buck2"; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c7_nca9539_gpio 4 GPIO_ACTIVE_HIGH>; + startup-delay-us = <2000>; + off-on-delay-us = <16000>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + dcphy0_vcc12v_buck3: dcphy0_vcc12v-buck3 { + compatible = "regulator-fixed"; + regulator-name = "dcphy0_vcc12v_buck3"; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c7_nca9539_gpio 5 GPIO_ACTIVE_HIGH>; + startup-delay-us = <2000>; + off-on-delay-us = <16000>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + dcphy0_vcc12v_buck4: dcphy0_vcc12v-buck4 { + compatible = "regulator-fixed"; + regulator-name = "dcphy0_vcc12v_buck4"; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c7_nca9539_gpio 6 GPIO_ACTIVE_HIGH>; + startup-delay-us = <2000>; + off-on-delay-us = <16000>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + dphy0_vcc12v_buck1: dphy0_vcc12v-buck1 { + compatible = "regulator-fixed"; + regulator-name = "dphy0_vcc12v_buck1"; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c7_nca9539_gpio 7 GPIO_ACTIVE_HIGH>; + startup-delay-us = <2000>; + off-on-delay-us = <16000>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + dphy0_vcc12v_buck2: dphy0_vcc12v-buck2 { + compatible = "regulator-fixed"; + regulator-name = "dphy0_vcc12v_buck2"; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c7_nca9539_gpio 8 GPIO_ACTIVE_HIGH>; + startup-delay-us = <2000>; + off-on-delay-us = <16000>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + dphy0_vcc12v_buck3: dphy0_vcc12v-buck3 { + compatible = "regulator-fixed"; + regulator-name = "dphy0_vcc12v_buck3"; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c7_nca9539_gpio 9 GPIO_ACTIVE_HIGH>; + startup-delay-us = <2000>; + off-on-delay-us = <16000>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + dphy0_vcc12v_buck4: dphy0_vcc12v-buck4 { + compatible = "regulator-fixed"; + regulator-name = "dphy0_vcc12v_buck4"; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c7_nca9539_gpio 10 GPIO_ACTIVE_HIGH>; + startup-delay-us = <2000>; + off-on-delay-us = <16000>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + dphy3_vcc12v_buck1: dphy3_vcc12v-buck1 { + compatible = "regulator-fixed"; + regulator-name = "dphy3_vcc12v_buck1"; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c7_nca9539_gpio 11 GPIO_ACTIVE_HIGH>; + startup-delay-us = <2000>; + off-on-delay-us = <16000>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + dphy3_vcc12v_buck2: dphy3_vcc12v-buck2 { + compatible = "regulator-fixed"; + regulator-name = "dphy3_vcc12v_buck2"; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c7_nca9539_gpio 12 GPIO_ACTIVE_HIGH>; + startup-delay-us = <2000>; + off-on-delay-us = <16000>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + dphy3_vcc12v_buck3: dphy3_vcc12v-buck3 { + compatible = "regulator-fixed"; + regulator-name = "dphy3_vcc12v_buck3"; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c7_nca9539_gpio 13 GPIO_ACTIVE_HIGH>; + startup-delay-us = <2000>; + off-on-delay-us = <16000>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + dphy3_vcc12v_buck4: dphy3_vcc12v-buck4 { + compatible = "regulator-fixed"; + regulator-name = "dphy3_vcc12v_buck4"; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c7_nca9539_gpio 14 GPIO_ACTIVE_HIGH>; + startup-delay-us = <2000>; + off-on-delay-us = <16000>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; +}; + +&dfi { + status = "disabled"; +}; + +&dmc { + status = "disabled"; +}; + +&gmac1 { + status = "disabled"; +}; + +&hym8563 { + status = "disabled"; +}; + +&i2c2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c2m0_xfer>; +}; + +/*edp*/ +&i2c3_max96752 { + use-reg-check-work; + vpower-supply = <&lcd1_vcc12v_buck>; +}; + +/*edp touch*/ +&i2c3_himax { + himax,irq-gpio = <&gpio0 RK_PB6 IRQ_TYPE_EDGE_FALLING>; +}; + +/*dp*/ +&i2c5_max96745 { + lock-gpios = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; +}; + +&i2c5_ilitek { + interrupt-parent = <&gpio4>; + interrupts = ; +}; + +/*dp*/ +&i2c5_max96752 { + use-reg-check-work; + vpower-supply = <&lcd2_vcc12v_buck>; +}; + +&i2c8_max96789 { + route-enable; +}; + +/*dsi*/ +&i2c8_max96752 { + use-reg-check-work; + vpower-supply = <&lcd3_vcc12v_buck>; +}; + +&dp2lvds_backlight0 { + pwms = <&pwm2_8ch_7 0 25000 0>; +}; + +&edp2lvds_backlight0 { + pwms = <&pwm0_2ch_0 0 25000 0>; +}; + +/* edp->serdes->lvds_panel */ +&pwm0_2ch_0 { + pinctrl-0 = <&pwm0m3_ch0>; +}; + +/* dp->serdes->lvds_panel */ +&pwm2_8ch_7 { + pinctrl-0 = <&pwm2m3_ch7>; +}; + +&pinctrl { + touch { + //dsi-i2c8 + touch_gpio_dsi: touch-gpio-dsi { + rockchip,pins = + <3 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + //dp-i2c5 + touch_gpio_dp: touch-gpio-dp { + rockchip,pins = <4 RK_PA0 RK_FUNC_GPIO &pcfg_pull_up>; + }; + //edp0-i2c3 + touch_gpio_edp: touch-gpio-edp { + rockchip,pins = + <0 RK_PB6 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + vcc5v0-buck { + vcc5v0_buck_en: vcc5v0-buck-en { + rockchip,pins = <0 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + cluster-buck { + cluster_buck_en: cluster-buck-en { + rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; +}; + +&rkvpss { + status = "okay"; +}; + +&rkvpss_mmu { + status = "okay"; +}; + +&rkvpss_vir0 { + status = "okay"; +}; + +&route_dsi { + status = "okay"; +}; + +&sdmmc { + status = "disabled"; +}; + +&ufs { + status = "disabled"; +}; + +&u2phy0 { + status = "okay"; +}; + +&u2phy0_otg { + //vbus-supply = <&usb_otg_vcc5v_buck>; + status = "okay"; +}; + +&u2phy1_otg { + phy-supply = <&usb_host_vcc5v_buck>; + status = "okay"; +}; + +&usbdp_phy { + maximum-speed = "high-speed"; + status = "okay"; +}; + +&usb_drd1_dwc3 { + snps,dis_u2_susphy_quirk; + snps,usb2-lpm-disable; + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3588-evb.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-evb.dtsi index 925445f42981..a2d471821611 100644 --- a/arch/arm64/boot/dts/rockchip/rk3588-evb.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3588-evb.dtsi @@ -296,6 +296,7 @@ reset-delay-ms = <10>; enable-delay-ms = <10>; prepare-delay-ms = <10>; + init-delay-ms = <10>; unprepare-delay-ms = <10>; disable-delay-ms = <60>; width-mm = <68>; @@ -630,6 +631,7 @@ reset-delay-ms = <10>; enable-delay-ms = <10>; prepare-delay-ms = <10>; + init-delay-ms = <10>; unprepare-delay-ms = <10>; disable-delay-ms = <10>; width-mm = <68>; diff --git a/arch/arm64/boot/dts/rockchip/rv1126b-evb1-v10-fastboot-emmc.dts b/arch/arm64/boot/dts/rockchip/rv1126b-evb1-v10-fastboot-emmc.dts new file mode 100644 index 000000000000..9ee73d5f6811 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rv1126b-evb1-v10-fastboot-emmc.dts @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2025 Rockchip Electronics Co., Ltd. + */ + +#include "rv1126b-evb1-v10.dts" +#include "rv1126b-fastboot-emmc.dtsi" + +/ { + model = "Rockchip RV1126B EVB1 V10 Arm64 Fastboot Board"; + compatible = "rockchip,rv1126b-evb1-v10-fastboot", "rockchip,rv1126b"; + + chosen { + bootargs = "loglevel=0 initcall_debug=0 earlycon=uart8250,mmio32,0x20810000 console=ttyFIQ0 root=/dev/rd0 rootfstype=erofs rootflags=dax snd_soc_core.prealloc_buffer_size_kbytes=16 coherent_pool=32K"; + }; +}; + +&ramdisk_r { + reg = <0x48c40000 (40 * 0x00100000)>; +}; + +&ramdisk_c { + reg = <0x4b440000 (20 * 0x00100000)>; +}; diff --git a/arch/arm64/boot/dts/rockchip/rv1126b-evb1-v10-fastboot-spi-nand.dts b/arch/arm64/boot/dts/rockchip/rv1126b-evb1-v10-fastboot-spi-nand.dts new file mode 100644 index 000000000000..a9d1046d7ff1 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rv1126b-evb1-v10-fastboot-spi-nand.dts @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2025 Rockchip Electronics Co., Ltd. + */ + +/dts-v1/; +#include "rv1126b-evb1-v10.dts" +#include "rv1126b-fastboot-spi-nand.dtsi" + +/ { + model = "Rockchip RV1126B EVB1 V10 Arm64 Board"; + compatible = "rockchip,rv1126b-evb1-v10-fastboot-spi-nand", "rockchip,rv1126b"; + + chosen { + bootargs = "loglevel=0 initcall_debug=0 earlycon=uart8250,mmio32,0x20810000 console=ttyFIQ0 root=/dev/rd0 rootfstype=erofs rootflags=dax snd_soc_core.prealloc_buffer_size_kbytes=16 coherent_pool=32K"; + }; +}; + +&fspi0 { + status = "okay"; + + flash@0 { + compatible = "spi-nand"; + reg = <0>; + spi-max-frequency = <75000000>; + spi-rx-bus-width = <4>; + spi-tx-bus-width = <1>; + }; +}; + +&ramdisk_r { + reg = <0x48c40000 (40 * 0x00100000)>; +}; + +&ramdisk_c { + reg = <0x4b440000 (20 * 0x00100000)>; +}; diff --git a/arch/arm64/boot/dts/rockchip/rv1126b-evb1-v10-fastboot-spi-nor.dts b/arch/arm64/boot/dts/rockchip/rv1126b-evb1-v10-fastboot-spi-nor.dts new file mode 100644 index 000000000000..3f14ea6c77be --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rv1126b-evb1-v10-fastboot-spi-nor.dts @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2025 Rockchip Electronics Co., Ltd. + */ + +/dts-v1/; +#include "rv1126b-evb1-v10.dts" +#include "rv1126b-fastboot-spi-nor.dtsi" + +/ { + model = "Rockchip RV1126B EVB1 V10 Arm64 Board"; + compatible = "rockchip,rv1126b-evb1-v10-fastboot-spi-nor", "rockchip,rv1126b"; + + chosen { + bootargs = "loglevel=0 initcall_debug=0 earlycon=uart8250,mmio32,0x20810000 console=ttyFIQ0 root=/dev/rd0 rootfstype=erofs rootflags=dax snd_soc_core.prealloc_buffer_size_kbytes=16 coherent_pool=32K"; + }; +}; + +&fspi0 { + status = "okay"; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <75000000>; + spi-rx-bus-width = <4>; + spi-tx-bus-width = <1>; + }; +}; + +&ramdisk_r { + reg = <0x48c40000 (20 * 0x00100000)>; +}; + +&ramdisk_c { + reg = <0x4a040000 (10 * 0x00100000)>; +}; diff --git a/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10-mcu-k350c4516t.dts b/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10-mcu-k350c4516t.dts index baa7b7a71285..9414553a06f3 100644 --- a/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10-mcu-k350c4516t.dts +++ b/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10-mcu-k350c4516t.dts @@ -6,6 +6,7 @@ /dts-v1/; #include +#include #include "rv1126b-evb2-v10.dts" / { @@ -15,6 +16,7 @@ &rgb { status = "okay"; + rockchip,data-map-mode = ; rockchip,data-sync-bypass; pinctrl-names = "default"; /* diff --git a/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10-sii9022-bt1120-to-hdmi.dts b/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10-sii9022-bt1120-to-hdmi.dts index 7ffacffa6ab3..d05397b88aab 100644 --- a/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10-sii9022-bt1120-to-hdmi.dts +++ b/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10-sii9022-bt1120-to-hdmi.dts @@ -6,6 +6,7 @@ /dts-v1/; #include +#include #include "rv1126b-evb2-v10.dts" / { @@ -60,6 +61,11 @@ &rgb { status = "okay"; + /* + * BT1120: ROCKCHIP_BT1120_DATA_MAP_DATA_WITH_DUMMY_0 + * BT656: ROCKCHIP_BT656_DATA_MAP_DATA_WITH_DUMMY_0 + */ + rockchip,data-map-mode = ; pinctrl-names = "default"; /* * bt1120_pins for bt1120 diff --git a/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10-tb-400w.dts b/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10-tb-400w.dts index de9f0fecf79f..08b6a95d551d 100644 --- a/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10-tb-400w.dts +++ b/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10-tb-400w.dts @@ -3,7 +3,11 @@ * Copyright (c) 2025 Rockchip Electronics Co., Ltd. */ -#include "rv1126b-evb2-v10.dts" +/dts-v1/; +#include "rv1126b.dtsi" +#include "rv1126b-evb.dtsi" +#include "rv1126b-evb2-v10.dtsi" +#include "rv1126b-thunder-boot-cam.dtsi" #include "rv1126b-thunder-boot-spi-nor.dtsi" / { @@ -15,6 +19,36 @@ }; }; +&i2c3 { + clock-frequency = <400000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c3m1_pins>; + rockchip,amp-shared; + status = "okay"; + + sc450ai: sc450ai@30 { + compatible = "smartsens,sc450ai"; + status = "okay"; + reg = <0x30>; + clocks = <&cru CLK_MIPI0_OUT2IO>; + clock-names = "xvclk"; + reset-gpios = <&gpio4 RK_PA7 GPIO_ACTIVE_LOW>; + pwdn-gpios = <&gpio4 RK_PA2 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&cam_clk0_pins>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "default"; + rockchip,camera-module-lens-name = "default"; + port { + cam0_out: endpoint { + remote-endpoint = <&csi_dphy_input0>; + data-lanes = <1 2 3 4>; + }; + }; + }; +}; + &ramdisk_r { reg = <0x48c40000 (40 * 0x00100000)>; }; @@ -22,3 +56,12 @@ &ramdisk_c { reg = <0x4b440000 (20 * 0x00100000)>; }; + +&rkisp_thunderboot { + /* reg's offset MUST match with RTOS */ + /* + * vicap, capture raw10, ceil(w*10/8/256)*256*h *4(buf num) + * e.g. 2688x1520: 0x14c8000 + */ + reg = <0x41300000 0x14c8000>; +}; diff --git a/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10.dts b/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10.dts index d1ceaa6832c6..96dd2a1089d5 100644 --- a/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10.dts +++ b/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10.dts @@ -6,405 +6,18 @@ /dts-v1/; #include "rv1126b.dtsi" #include "rv1126b-evb.dtsi" +#include "rv1126b-evb2-v10.dtsi" #include "rv1126b-evb-cam-csi0.dtsi" / { model = "Rockchip RV1126B EVB2 V10 Board"; compatible = "rockchip,rv1126b-evb2-v10", "rockchip,rv1126b"; - - gpio-keys { - compatible = "gpio-keys"; - autorepeat; - pinctrl-names = "default"; - pinctrl-0 = <&pwr_key>; - power-key { - gpios = <&gpio0 RK_PA4 GPIO_ACTIVE_LOW>; - linux,code = ; - label ="GPIO Key Power"; - debounce-interval = <100>; - wakeup-source; - /* gpio-key,wakeup; */ - }; - }; - - vcc5v0_dcin: vcc5v0-dcin { - compatible = "regulator-fixed"; - regulator-name = "vcc5v0_dcin"; - regulator-always-on; - regulator-boot-on; - regulator-min-microvolt = <5000000>; - regulator-max-microvolt = <5000000>; - vin-supply = <&vcc12v_dcin>; - }; - - vcc5v0_host: vcc5v0-host { - compatible = "regulator-fixed"; - regulator-name = "vcc5v0_host"; - regulator-boot-on; - regulator-always-on; - regulator-min-microvolt = <5000000>; - regulator-max-microvolt = <5000000>; - enable-active-high; - gpio = <&gpio0 RK_PD1 GPIO_ACTIVE_HIGH>; - vin-supply = <&vcc5v0_dcin>; - pinctrl-names = "default"; - pinctrl-0 = <&vcc5v0_host_en>; - }; - - vccsys_stb: vccsys-stb { - compatible = "regulator-fixed"; - regulator-name = "vccsys_stb"; - regulator-always-on; - regulator-boot-on; - regulator-min-microvolt = <5000000>; - regulator-max-microvolt = <5000000>; - vin-supply = <&vcc5v0_dcin>; - }; - - vcc3v3_stb: vcc3v3-stb { - compatible = "regulator-fixed"; - regulator-name = "vcc3v3_stb"; - regulator-always-on; - regulator-boot-on; - regulator-min-microvolt = <3300000>; - regulator-max-microvolt = <3300000>; - vin-supply = <&vccsys_stb>; - }; - - vcc1v8_pmu: vdd1_1v8_ddr: vcc1v8_pmu { - compatible = "regulator-fixed"; - regulator-name = "vcc3v3_stb"; - regulator-always-on; - regulator-boot-on; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; - vin-supply = <&vccsys_stb>; - }; - - vdd2_1v1_ddr: vdd2_1v1_ddr { - compatible = "regulator-fixed"; - regulator-name = "vdd2_1v1_ddr"; - regulator-always-on; - regulator-boot-on; - regulator-min-microvolt = <1100000>; - regulator-max-microvolt = <1100000>; - vin-supply = <&vccsys_stb>; - }; - - vddq_0v6_lp4x: vddq-0v6-lp4x { - compatible = "regulator-fixed"; - regulator-name = "vddq_0v6_lp4x"; - regulator-always-on; - regulator-boot-on; - regulator-min-microvolt = <600000>; - regulator-max-microvolt = <600000>; - vin-supply = <&vccsys_stb>; - }; - - vcc3v3_pmu: vcc3v3-pmu { - compatible = "regulator-fixed"; - regulator-name = "vcc3v3_pmu"; - regulator-always-on; - regulator-boot-on; - regulator-min-microvolt = <3300000>; - regulator-max-microvolt = <3300000>; - vin-supply = <&vccsys_stb>; - }; - - vcc0v9_pmu: vcc0v9-pmu { - compatible = "regulator-fixed"; - regulator-name = "vcc0v9_pmu"; - regulator-always-on; - regulator-boot-on; - regulator-min-microvolt = <900000>; - regulator-max-microvolt = <900000>; - vin-supply = <&vcc1v8_pmu>; - }; - - vcc_mipi: vcc-mipi{ - compatible = "regulator-fixed"; - regulator-name = "vcc_mipi"; - regulator-always-on; - regulator-boot-on; - vin-supply = <&vcc1v8_pmu>; - }; - - vcc_1v8: vcc-1v8 { - compatible = "regulator-fixed"; - regulator-name = "vcc_1v8"; - regulator-always-on; - regulator-boot-on; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; - vin-supply = <&vccsys_stb>; - }; - - vcc_3v3: vcc3v3_dev: vcc-3v3 { - compatible = "regulator-fixed"; - regulator-name = "vcc_3v3"; - regulator-always-on; - regulator-boot-on; - regulator-min-microvolt = <3300000>; - regulator-max-microvolt = <3300000>; - vin-supply = <&vccsys_stb>; - }; - - vcc_lcd: vcc-lcd { - compatible = "regulator-fixed"; - gpio = <&gpio7 RK_PA7 GPIO_ACTIVE_HIGH>; - regulator-name = "vcc_lcd"; - enable-active-high; - }; - - vcc_sd: vcc-sd { - compatible = "regulator-fixed"; - gpio = <&gpio0 RK_PB0 GPIO_ACTIVE_LOW>; - regulator-name = "vcc_sd"; - regulator-min-microvolt = <3300000>; - regulator-max-microvolt = <3300000>; - enable-active-low; - vin-supply = <&vcc_3v3>; - }; - - vccio_sd: vccio-sd { - compatible = "regulator-gpio"; - regulator-boot-on; - regulator-name = "vccio_sd"; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <3300000>; - pinctrl-names = "default"; - pinctrl-0 = <&sdmmc_volt>; - gpios = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>; - vin-supply = <&vccsys_stb>; - states = <1800000 0x0 - 3300000 0x1>; - }; - - vdd_log: vdd-log { - compatible = "pwm-regulator"; - pwms = <&pwm0_8ch_2 0 25000 1>; - regulator-name = "vdd_log"; - regulator-init-microvolt = <905000>; - regulator-min-microvolt = <810000>; - regulator-max-microvolt = <1006000>; - regulator-always-on; - regulator-boot-on; - pwm-supply = <&vccsys_stb>; - status = "okay"; - }; - - vdd_cpu: vdd-cpu { - compatible = "pwm-regulator"; - pwms = <&pwm0_8ch_0 0 25000 1>; - regulator-name = "vdd_cpu"; - regulator-init-microvolt = <950000>; - regulator-min-microvolt = <750000>; - regulator-max-microvolt = <1100000>; - regulator-always-on; - regulator-boot-on; - pwm-supply = <&vccsys_stb>; - status = "okay"; - }; - - vdd_npu: vdd-npu { - compatible = "pwm-regulator"; - pwms = <&pwm0_8ch_1 0 25000 1>; - regulator-name = "vdd_npu"; - regulator-init-microvolt = <950000>; - regulator-min-microvolt = <750000>; - regulator-max-microvolt = <1100000>; - regulator-always-on; - regulator-boot-on; - pwm-supply = <&vccsys_stb>; - status = "okay"; - }; - - wireless-bluetooth { - compatible = "bluetooth-platdata"; - uart_rts_gpios = <&gpio3 RK_PA6 GPIO_ACTIVE_LOW>; - pinctrl-names = "default", "rts_gpio"; - pinctrl-0 = <&uart2m0_rtsn_pins>; - pinctrl-1 = <&uart2_gpios>; - BT,power_gpio = <&gpio0 RK_PA0 GPIO_ACTIVE_HIGH>; - status = "okay"; - }; - - wireless-wlan { - compatible = "wlan-platdata"; - rockchip,grf = <&grf>; - pinctrl-names = "default"; - pinctrl-0 = <&wifi_wake_host>; - wifi_chip_type = "rk96x"; - WIFI,host_wake_irq = <&gpio0 RK_PC0 GPIO_ACTIVE_HIGH>; - status = "okay"; - }; -}; - -&acdcdig_dsm { - pa-ctl-gpios = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; - status = "okay"; -}; - -&acodec_sound { - status = "okay"; -}; - -&audio_codec { - status = "okay"; -}; - -&backlight { - pwms = <&pwm2_8ch_7 0 25000 0>; -}; - -&cpu0 { - cpu-supply = <&vdd_cpu>; -}; - -&display_subsystem { - status = "okay"; -}; - -&dsi { - status = "okay"; -}; - -&dsi_in_vop { - status = "okay"; -}; - -&dsi_panel { - power-supply = <&vcc_lcd>; -}; - -&fspi0 { - status = "okay"; - - flash@0 { - compatible = "jedec,spi-nor"; - reg = <0>; - spi-max-frequency = <100000000>; - spi-rx-bus-width = <4>; - spi-tx-bus-width = <1>; - }; -}; - -&gmac { - phy-mode = "rmii"; - clock_in_out = "input"; - phy-handle = <&rmii_phy>; - status = "okay"; }; &imx415 { reset-gpios = <&gpio4 RK_PA7 GPIO_ACTIVE_HIGH>; }; -&i2c2 { - pinctrl-0 = <&i2c2m1_pins>; - status = "okay"; - - gt1x: gt1x@14 { - compatible = "goodix,gt1x"; - reg = <0x14>; - pinctrl-names = "default"; - pinctrl-0 = <&touch_gpio>; - vdd_ana-supply = <&vcc_lcd>; - goodix,rst-gpio = <&gpio5 RK_PD6 GPIO_ACTIVE_HIGH>; - goodix,irq-gpio = <&gpio3 RK_PB7 IRQ_TYPE_LEVEL_LOW>; - }; -}; - -&mdio { - rmii_phy: ethernet-phy@2 { - compatible = "ethernet-phy-id0680.8101", "ethernet-phy-ieee802.3-c22"; - reg = <2>; - clocks = <&cru CLK_MACPHY>; - clock-frequency = <24000000>; - resets = <&cru SRST_RESETN_MACPHY>; - pinctrl-names = "default"; - pinctrl-0 = <&fephym2_pins>; - phy-is-integrated; - }; -}; - -&mipi_dphy { - status = "okay"; -}; - -&pinctrl { - buttons { - pwr_key: pwr-key { - rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_up>; - }; - }; - - sdmmc { - /omit-if-no-ref/ - sdmmc_volt: sdmmc-volt { - rockchip,pins = <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>; - }; - }; - - touch { - touch_gpio: touch-gpio { - rockchip,pins = - <3 RK_PB7 RK_FUNC_GPIO &pcfg_pull_up>, - <5 RK_PD6 RK_FUNC_GPIO &pcfg_pull_none>; - }; - }; - - usb { - vcc5v0_host_en: vcc5v0-host-en { - rockchip,pins = <0 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>; - }; - }; - - wireless-bluetooth { - uart2_gpios: uart2-gpios { - rockchip,pins = <3 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>; - }; - }; - - wireless-wlan { - wifi_wake_host: wifi-wake-host { - rockchip,pins = <0 RK_PB1 RK_FUNC_GPIO &pcfg_pull_up>; - }; - }; -}; - -&pwm0_8ch_0 { - status = "okay"; -}; - -&pwm0_8ch_1 { - status = "okay"; -}; - -&pwm0_8ch_2 { - status = "okay"; -}; - -&pwm2_8ch_7 { - status = "okay"; -}; - -&rkaiisp { - status = "okay"; -}; - -&rkaiisp_mmu { - status = "okay"; -}; - -&rkaiisp_vir0 { - status = "okay"; -}; - -&rknpu { - rknpu-supply = <&vdd_npu>; -}; - &rockchip_suspend { status = "okay"; @@ -562,26 +175,6 @@ >; }; -&route_dsi { - status = "okay"; -}; - -&rtc { - rockchip,rtc-suspend-bypass; - status = "okay"; -}; - -&sai2 { - rockchip,sai-rx-route = <1 0 2 3>; - status = "okay"; - /delete-property/ pinctrl-names; - /delete-property/ pinctrl-0; -}; - -&saradc0 { - vref-supply = <&vcc_1v8>; -}; - &sc450ai { reset-gpios = <&gpio4 RK_PA7 GPIO_ACTIVE_LOW>; }; @@ -590,56 +183,3 @@ reset-gpios = <&gpio4 RK_PA7 GPIO_ACTIVE_HIGH>; rockchip,camera-module-stb = <1>; }; - -&sdmmc0 { - max-frequency = <200000000>; - no-sdio; - no-mmc; - bus-width = <4>; - cap-mmc-highspeed; - cap-sd-highspeed; - disable-wp; - sd-uhs-sdr104; - vmmc-supply = <&vcc_sd>; - vqmmc-supply = <&vccio_sd>; - status = "okay"; -}; - -&sdmmc1 { - bus-width = <1>; - cap-sd-highspeed; - no-sd; - no-mmc; - max-frequency = <200000000>; - pinctrl-names = "default"; - pinctrl-0 = <&sdmmc1_clk_pins &sdmmc1_cmd_pins &sdmmc1_bus4_pins>; - keep-power-in-suspend; - non-removable; - //mmc-pwrseq = <&sdio_pwrseq>; - //sd-uhs-sdr104; - status = "okay"; -}; - -&uart2 { - status = "okay"; - pinctrl-names = "default"; - pinctrl-0 = <&uart2m0_xfer_pins &uart2m0_ctsn_pins>; -}; - -&usb2phy_host { - phy-supply = <&vcc5v0_host>; -}; - -&usb3phy { - status = "disabled"; -}; - -&usb_drd_dwc3 { - dr_mode = "otg"; - extcon = <&usb2phy>; - maximum-speed = "high-speed"; - phys = <&usb2phy_otg>; - phy-names = "usb2-phy"; - snps,dis_u2_susphy_quirk; - snps,usb2-lpm-disable; -}; diff --git a/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10.dtsi b/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10.dtsi new file mode 100644 index 000000000000..7f9cbcbc70b8 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10.dtsi @@ -0,0 +1,467 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2025 Rockchip Electronics Co., Ltd. + */ + +/ { + gpio-keys { + compatible = "gpio-keys"; + autorepeat; + pinctrl-names = "default"; + pinctrl-0 = <&pwr_key>; + power-key { + gpios = <&gpio0 RK_PA4 GPIO_ACTIVE_LOW>; + linux,code = ; + label ="GPIO Key Power"; + debounce-interval = <100>; + wakeup-source; + /* gpio-key,wakeup; */ + }; + }; + + vcc5v0_dcin: vcc5v0-dcin { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_dcin"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + vin-supply = <&vcc12v_dcin>; + }; + + vcc5v0_host: vcc5v0-host { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_host"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + enable-active-high; + gpio = <&gpio0 RK_PD1 GPIO_ACTIVE_HIGH>; + vin-supply = <&vcc5v0_dcin>; + pinctrl-names = "default"; + pinctrl-0 = <&vcc5v0_host_en>; + }; + + vccsys_stb: vccsys-stb { + compatible = "regulator-fixed"; + regulator-name = "vccsys_stb"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + vin-supply = <&vcc5v0_dcin>; + }; + + vcc3v3_stb: vcc3v3-stb { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_stb"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vccsys_stb>; + }; + + vcc1v8_pmu: vdd1_1v8_ddr: vcc1v8_pmu { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_stb"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + vin-supply = <&vccsys_stb>; + }; + + vdd2_1v1_ddr: vdd2_1v1_ddr { + compatible = "regulator-fixed"; + regulator-name = "vdd2_1v1_ddr"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1100000>; + regulator-max-microvolt = <1100000>; + vin-supply = <&vccsys_stb>; + }; + + vddq_0v6_lp4x: vddq-0v6-lp4x { + compatible = "regulator-fixed"; + regulator-name = "vddq_0v6_lp4x"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <600000>; + regulator-max-microvolt = <600000>; + vin-supply = <&vccsys_stb>; + }; + + vcc3v3_pmu: vcc3v3-pmu { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_pmu"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vccsys_stb>; + }; + + vcc0v9_pmu: vcc0v9-pmu { + compatible = "regulator-fixed"; + regulator-name = "vcc0v9_pmu"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + vin-supply = <&vcc1v8_pmu>; + }; + + vcc_mipi: vcc-mipi{ + compatible = "regulator-fixed"; + regulator-name = "vcc_mipi"; + regulator-always-on; + regulator-boot-on; + vin-supply = <&vcc1v8_pmu>; + }; + + vcc_1v8: vcc-1v8 { + compatible = "regulator-fixed"; + regulator-name = "vcc_1v8"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + vin-supply = <&vccsys_stb>; + }; + + vcc_3v3: vcc3v3_dev: vcc-3v3 { + compatible = "regulator-fixed"; + regulator-name = "vcc_3v3"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vccsys_stb>; + }; + + vcc_lcd: vcc-lcd { + compatible = "regulator-fixed"; + gpio = <&gpio7 RK_PA7 GPIO_ACTIVE_HIGH>; + regulator-name = "vcc_lcd"; + enable-active-high; + }; + + vcc_sd: vcc-sd { + compatible = "regulator-fixed"; + gpio = <&gpio0 RK_PB0 GPIO_ACTIVE_LOW>; + regulator-name = "vcc_sd"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + enable-active-low; + vin-supply = <&vcc_3v3>; + }; + + vccio_sd: vccio-sd { + compatible = "regulator-gpio"; + regulator-boot-on; + regulator-name = "vccio_sd"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc_volt>; + gpios = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>; + vin-supply = <&vccsys_stb>; + states = <1800000 0x0 + 3300000 0x1>; + }; + + vdd_log: vdd-log { + compatible = "pwm-regulator"; + pwms = <&pwm0_8ch_2 0 25000 1>; + regulator-name = "vdd_log"; + regulator-init-microvolt = <905000>; + regulator-min-microvolt = <810000>; + regulator-max-microvolt = <1006000>; + regulator-always-on; + regulator-boot-on; + pwm-supply = <&vccsys_stb>; + status = "okay"; + }; + + vdd_cpu: vdd-cpu { + compatible = "pwm-regulator"; + pwms = <&pwm0_8ch_0 0 25000 1>; + regulator-name = "vdd_cpu"; + regulator-init-microvolt = <950000>; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1100000>; + regulator-always-on; + regulator-boot-on; + pwm-supply = <&vccsys_stb>; + status = "okay"; + }; + + vdd_npu: vdd-npu { + compatible = "pwm-regulator"; + pwms = <&pwm0_8ch_1 0 25000 1>; + regulator-name = "vdd_npu"; + regulator-init-microvolt = <950000>; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1100000>; + regulator-always-on; + regulator-boot-on; + pwm-supply = <&vccsys_stb>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + uart_rts_gpios = <&gpio3 RK_PA6 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart2m0_rtsn_pins>; + pinctrl-1 = <&uart2_gpios>; + BT,power_gpio = <&gpio0 RK_PA0 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_wake_host>; + wifi_chip_type = "rk96x"; + WIFI,host_wake_irq = <&gpio0 RK_PC0 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&acdcdig_dsm { + pa-ctl-gpios = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; + status = "okay"; +}; + +&acodec_sound { + status = "okay"; +}; + +&audio_codec { + status = "okay"; +}; + +&backlight { + pwms = <&pwm2_8ch_7 0 25000 0>; +}; + +&cpu0 { + cpu-supply = <&vdd_cpu>; +}; + +&display_subsystem { + status = "okay"; +}; + +&dsi { + status = "okay"; +}; + +&dsi_in_vop { + status = "okay"; +}; + +&dsi_panel { + power-supply = <&vcc_lcd>; +}; + +&fspi0 { + status = "okay"; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <100000000>; + spi-rx-bus-width = <4>; + spi-tx-bus-width = <1>; + }; +}; + +&gmac { + phy-mode = "rmii"; + clock_in_out = "input"; + phy-handle = <&rmii_phy>; + status = "okay"; +}; + +&i2c2 { + pinctrl-0 = <&i2c2m1_pins>; + status = "okay"; + + gt1x: gt1x@14 { + compatible = "goodix,gt1x"; + reg = <0x14>; + pinctrl-names = "default"; + pinctrl-0 = <&touch_gpio>; + vdd_ana-supply = <&vcc_lcd>; + goodix,rst-gpio = <&gpio5 RK_PD6 GPIO_ACTIVE_HIGH>; + goodix,irq-gpio = <&gpio3 RK_PB7 IRQ_TYPE_LEVEL_LOW>; + }; +}; + +&mdio { + rmii_phy: ethernet-phy@2 { + compatible = "ethernet-phy-id0680.8101", "ethernet-phy-ieee802.3-c22"; + reg = <2>; + clocks = <&cru CLK_MACPHY>; + clock-frequency = <24000000>; + resets = <&cru SRST_RESETN_MACPHY>; + pinctrl-names = "default"; + pinctrl-0 = <&fephym2_pins>; + phy-is-integrated; + }; +}; + +&mipi_dphy { + status = "okay"; +}; + +&pinctrl { + buttons { + pwr_key: pwr-key { + rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + sdmmc { + /omit-if-no-ref/ + sdmmc_volt: sdmmc-volt { + rockchip,pins = <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + touch { + touch_gpio: touch-gpio { + rockchip,pins = + <3 RK_PB7 RK_FUNC_GPIO &pcfg_pull_up>, + <5 RK_PD6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + usb { + vcc5v0_host_en: vcc5v0-host-en { + rockchip,pins = <0 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-bluetooth { + uart2_gpios: uart2-gpios { + rockchip,pins = <3 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-wlan { + wifi_wake_host: wifi-wake-host { + rockchip,pins = <0 RK_PB1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; +}; + +&pwm0_8ch_0 { + status = "okay"; +}; + +&pwm0_8ch_1 { + status = "okay"; +}; + +&pwm0_8ch_2 { + status = "okay"; +}; + +&pwm2_8ch_7 { + status = "okay"; +}; + +&rkaiisp { + status = "okay"; +}; + +&rkaiisp_mmu { + status = "okay"; +}; + +&rkaiisp_vir0 { + status = "okay"; +}; + +&rknpu { + rknpu-supply = <&vdd_npu>; +}; + +&route_dsi { + status = "okay"; +}; + +&rtc { + rockchip,rtc-suspend-bypass; + status = "okay"; +}; + +&sai2 { + rockchip,sai-rx-route = <1 0 2 3>; + status = "okay"; + /delete-property/ pinctrl-names; + /delete-property/ pinctrl-0; +}; + +&saradc0 { + vref-supply = <&vcc_1v8>; +}; + +&sdmmc0 { + max-frequency = <200000000>; + no-sdio; + no-mmc; + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + disable-wp; + sd-uhs-sdr104; + vmmc-supply = <&vcc_sd>; + vqmmc-supply = <&vccio_sd>; + status = "okay"; +}; + +&sdmmc1 { + bus-width = <1>; + cap-sd-highspeed; + no-sd; + no-mmc; + max-frequency = <200000000>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc1_clk_pins &sdmmc1_cmd_pins &sdmmc1_bus4_pins>; + keep-power-in-suspend; + non-removable; + //mmc-pwrseq = <&sdio_pwrseq>; + //sd-uhs-sdr104; + status = "okay"; +}; + +&uart2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&uart2m0_xfer_pins &uart2m0_ctsn_pins>; +}; + +&usb2phy_host { + phy-supply = <&vcc5v0_host>; +}; + +&usb3phy { + status = "disabled"; +}; + +&usb_drd_dwc3 { + dr_mode = "otg"; + extcon = <&usb2phy>; + maximum-speed = "high-speed"; + phys = <&usb2phy_otg>; + phy-names = "usb2-phy"; + snps,dis_u2_susphy_quirk; + snps,usb2-lpm-disable; +}; diff --git a/arch/arm64/boot/dts/rockchip/rv1126b-fastboot-emmc.dtsi b/arch/arm64/boot/dts/rockchip/rv1126b-fastboot-emmc.dtsi new file mode 100644 index 000000000000..698e5b2148a4 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rv1126b-fastboot-emmc.dtsi @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2025 Rockchip Electronics Co., Ltd. + */ + +/ { + ramdisk: ramdisk { + compatible = "rockchip,ramdisk"; + memory-region = <&ramdisk_r>; + }; + + thunder_boot_mmc: thunder-boot-mmc { + compatible = "rockchip,thunder-boot-mmc"; + reg = <0x21470000 0x4000>; + clocks = <&cru HCLK_EMMC>, <&cru CCLK_EMMC>; + clock-names = "biu", "ciu"; + memory-region-src = <&ramdisk_c>; + memory-region-dst = <&ramdisk_r>; + memory-region-idmac = <&mmc_idmac>; + }; +}; + +&emmc { + memory-region-ecsd = <&mmc_ecsd>; + post-power-on-delay-ms = <0>; +}; + +&reserved_memory { + /* Should enable this node if the security feature is enabled, like TA. */ + tee: tee@48400000 { + reg = <0x48400000 0x00800000>; + status = "disabled"; + }; + + mmc_idmac: mmc@48000000 { + reg = <0x48000000 0x00400000>; + }; + + mmc_ecsd: mmc@47FFFE00{ + reg = <0x47FFFE00 0x00001000>; + }; + + ramdisk_r: ramdisk_r { + reg = <0x48c40000 (40 * 0x00100000)>; + }; + + ramdisk_c: ramdisk_c { + reg = <0x4b440000 (20 * 0x00100000)>; + }; +}; + +&hw_decompress { + memory-region = <&ramdisk_c>; + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rv1126b-fastboot-spi-nand.dtsi b/arch/arm64/boot/dts/rockchip/rv1126b-fastboot-spi-nand.dtsi new file mode 100644 index 000000000000..d31fb7748499 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rv1126b-fastboot-spi-nand.dtsi @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2025 Rockchip Electronics Co., Ltd. + */ + +/ { + ramdisk: ramdisk { + compatible = "rockchip,ramdisk"; + memory-region = <&ramdisk_r>; + }; +}; + +&reserved_memory { + /* Should enable this node if the security feature is enabled, like TA. */ + tee: tee@48400000 { + reg = <0x48400000 0x00800000>; + status = "disabled"; + }; + + ramdisk_r: ramdisk_r { + reg = <0x48c40000 (40 * 0x00100000)>; + }; + + ramdisk_c: ramdisk_c { + reg = <0x4b440000 (20 * 0x00100000)>; + }; +}; + +&hw_decompress { + memory-region = <&ramdisk_c>; + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rv1126b-fastboot-spi-nor.dtsi b/arch/arm64/boot/dts/rockchip/rv1126b-fastboot-spi-nor.dtsi new file mode 100644 index 000000000000..354484f5b66d --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rv1126b-fastboot-spi-nor.dtsi @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2025 Rockchip Electronics Co., Ltd. + */ + +/ { + ramdisk: ramdisk { + compatible = "rockchip,ramdisk"; + memory-region = <&ramdisk_r>; + }; + + thunder_boot_spi_nor: thunder-boot-spi-nor { + compatible = "rockchip,thunder-boot-sfc"; + reg = <0x21460000 0x4000>; + memory-region-src = <&ramdisk_c>; + memory-region-dst = <&ramdisk_r>; + }; +}; + +&reserved_memory { + /* Should enable this node if the security feature is enabled, like TA. */ + tee: tee@48400000 { + reg = <0x48400000 0x00800000>; + status = "disabled"; + }; + + ramdisk_r: ramdisk_r { + reg = <0x48c40000 (20 * 0x00100000)>; + }; + + ramdisk_c: ramdisk_c { + reg = <0x4a040000 (10 * 0x00100000)>; + }; +}; + +&hw_decompress { + memory-region = <&ramdisk_c>; + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rv1126b-thunder-boot-cam.dtsi b/arch/arm64/boot/dts/rockchip/rv1126b-thunder-boot-cam.dtsi new file mode 100644 index 000000000000..1a4eb65902e2 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rv1126b-thunder-boot-cam.dtsi @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2025 Rockchip Electronics Co., Ltd. + * + */ + +&csi2_dphy0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + csi_dphy_input0: endpoint@1 { + reg = <1>; + remote-endpoint = <&cam0_out>; + data-lanes = <1 2 3 4>; + }; + }; + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + csidphy0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&mipi0_csi2_input>; + }; + }; + }; +}; + +&mipi0_csi2 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi0_csi2_input: endpoint@1 { + reg = <1>; + remote-endpoint = <&csidphy0_out>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + mipi0_csi2_output: endpoint@0 { + reg = <0>; + remote-endpoint = <&cif_mipi_in0>; + }; + }; + }; +}; + +&rkcif { + status = "okay"; +}; + +&rkcif_mipi_lvds { + status = "okay"; + memory-region-thunderboot = <&rkisp_thunderboot>; + + port { + cif_mipi_in0: endpoint { + remote-endpoint = <&mipi0_csi2_output>; + }; + }; +}; + +&rkcif_mipi_lvds_sditf { + status = "okay"; + + port { + mipi_lvds_sditf: endpoint { + remote-endpoint = <&isp_vir0>; + }; + }; +}; + +&rkcif_mmu { + status = "okay"; +}; + +&rkisp { + status = "okay"; +}; + +&rkisp_mmu { + status = "okay"; +}; + +&rkisp_vir0 { + status = "okay"; + memory-region-thunderboot = <&rkisp_thunderboot>; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp_vir0: endpoint@0 { + reg = <0>; + remote-endpoint = <&mipi_lvds_sditf>; + }; + }; +}; + +&rkisp_vir0_sditf { + status = "okay"; +}; + +&rkvpss { + status = "okay"; + dvbm = <&rkdvbm>; +}; + +&rkvpss_mmu { + status = "okay"; +}; + +&rkvpss_vir0 { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rv1126b-thunder-boot.dtsi b/arch/arm64/boot/dts/rockchip/rv1126b-thunder-boot.dtsi index 890f84f832d1..dc1defc3ab39 100644 --- a/arch/arm64/boot/dts/rockchip/rv1126b-thunder-boot.dtsi +++ b/arch/arm64/boot/dts/rockchip/rv1126b-thunder-boot.dtsi @@ -8,6 +8,49 @@ compatible = "rockchip,ramdisk"; memory-region = <&ramdisk_r>; }; + + thunder_boot_rkisp: thunder-boot-rkisp { + compatible = "rockchip,thunder-boot-rkisp"; + clocks = <&cru CLK_HPMCU>, + <&cru PCLK_HPMCU_INTMUX>, <&cru PCLK_HPMCU_MAILBOX>, + <&cru PCLK_WDT_HPMCU>, <&cru HCLK_CACHE>, + <&cru PCLK_TIMER>, + <&cru ACLK_ISP>, <&cru HCLK_ISP>, + <&cru CLK_CORE_ISP>, <&cru ISP0CLK_VICAP>, + <&cru ACLK_VICAP>, <&cru HCLK_VICAP>, + <&cru DCLK_VICAP>, <&cru ISP0CLK_VICAP>, + <&cru PCLK_CSI2HOST0>, <&cru PCLK_CSI2HOST1>, + <&cru CLK_I2C4>, <&cru PCLK_I2C4>; + clock-names = "clk_hpmcu", + "pclk_hpmcu_intmux", "pclk_hpmcu_mailbox", + "pclk_wdt_hpmcu", "hclk_cache", + "pclk_timer", + "aclk_isp", "hclk_isp", + "clk_isp_core", "clk_isp_core_vicap", + "aclk_cif","hclk_cif", + "dclk_cif", "isp0clk_cif", + "pclk_csi2host0", "pclk_csi2host1", + "clk_i2c4", "pclk_i2c4"; + }; + + thunder_boot_service: thunder-boot-service { + compatible = "rockchip,thunder-boot-service"; + mbox-names = "amp-rx"; + mboxes = <&hpmcu_mbox0 0>; + resets = <&cru SRST_RESETN_HPMCU_FULL_CLUSTER>; + reset-names = "hpmcu_full_cluster"; + memory-region = <&rtos>; + status = "okay"; + }; +}; + +&hpmcu_mbox0 { + status = "okay"; +}; + +&hw_decompress { + memory-region = <&ramdisk_c>; + status = "okay"; }; &reserved_memory { @@ -88,8 +131,3 @@ reg = <0x49640000 (5 * 0x00100000)>; }; }; - -&hw_decompress { - memory-region = <&ramdisk_c>; - status = "okay"; -}; diff --git a/arch/arm64/boot/dts/rockchip/rv1126b.dtsi b/arch/arm64/boot/dts/rockchip/rv1126b.dtsi index aa31538be2e7..866f4238d7ea 100644 --- a/arch/arm64/boot/dts/rockchip/rv1126b.dtsi +++ b/arch/arm64/boot/dts/rockchip/rv1126b.dtsi @@ -940,7 +940,7 @@ <&cru CLK_UART_FRAC0>, <&cru CLK_UART_FRAC1>, <&cru CLK_CM_FRAC0>, <&cru CLK_CM_FRAC1>, <&cru CLK_CM_FRAC2>, <&cru CLK_AUDIO_FRAC0>, - <&cru CLK_AUDIO_FRAC1>, <&cru CLK_AISP_PLL>, + <&cru CLK_AUDIO_FRAC1>, <&cru CLK_AISP_PLL_SRC>, <&cru CLK_CPLL_DIV10>; assigned-clock-rates = <1188000000>, <1000000000>, @@ -951,7 +951,7 @@ <96000000>, <128000000>, <18432000>, <500000000>, <32768000>, <45158400>, - <49152000>, <393216000>, + <49152000>, <786432000>, <98304000>; }; @@ -1904,9 +1904,6 @@ clocks = <&cru CLK_TSADC>, <&cru PCLK_TSADC>, <&cru CLK_TSADC_PHYCTRL>; clock-names = "tsadc", "apb_pclk", "tsadc_phyctrl"; - resets = <&cru SRST_RESETN_TSADC>, <&cru SRST_PRESETN_TSADC>, - <&cru SRST_RESETN_TSADC_PHYCTRL>; - reset-names = "tsadc", "tsadc-apb", "tsadc-phy"; #thermal-sensor-cells = <1>; rockchip,grf = <&grf>; rockchip,hw-tshut-temp = <120000>; diff --git a/arch/arm64/boot/dts/rockchip/rv1126bp-evb-v14-dual-cam.dts b/arch/arm64/boot/dts/rockchip/rv1126bp-evb-v14-dual-cam.dts index 0f06f0ea6252..4a69dd9b6c7f 100644 --- a/arch/arm64/boot/dts/rockchip/rv1126bp-evb-v14-dual-cam.dts +++ b/arch/arm64/boot/dts/rockchip/rv1126bp-evb-v14-dual-cam.dts @@ -4,7 +4,7 @@ */ /dts-v1/; -#include "rv1126b.dtsi" +#include "rv1126bp.dtsi" #include "rv1126bp-evb.dtsi" #include "rv1126bp-evb-v14.dtsi" #include "rv1126bp-evb-v14-dual-cam.dtsi" diff --git a/arch/arm64/boot/dts/rockchip/rv1126bp-evb-v14.dts b/arch/arm64/boot/dts/rockchip/rv1126bp-evb-v14.dts index 89c248f9eb05..1c97ab42c895 100644 --- a/arch/arm64/boot/dts/rockchip/rv1126bp-evb-v14.dts +++ b/arch/arm64/boot/dts/rockchip/rv1126bp-evb-v14.dts @@ -4,7 +4,7 @@ */ /dts-v1/; -#include "rv1126b.dtsi" +#include "rv1126bp.dtsi" #include "rv1126bp-evb.dtsi" #include "rv1126bp-evb-v14.dtsi" #include "rv1126bp-evb-cam-csi0.dtsi" diff --git a/arch/arm64/boot/dts/rockchip/rv1126bp-evb.dtsi b/arch/arm64/boot/dts/rockchip/rv1126bp-evb.dtsi index 81acd65cf9cf..ec31468906b5 100644 --- a/arch/arm64/boot/dts/rockchip/rv1126bp-evb.dtsi +++ b/arch/arm64/boot/dts/rockchip/rv1126bp-evb.dtsi @@ -383,6 +383,17 @@ status = "okay"; }; +&rgb { + /* + * RV1126 compatible pin output mode 0. + * + * MCU: ROCKCHIP_MCU_DATA_MAP_DATA_1x24 + * BT1120: ROCKCHIP_BT1120_DATA_MAP_DATA_PADLO + * BT656: ROCKCHIP_BT656_DATA_MAP_PADLO + */ + rockchip,data-map-mode = <0>; +}; + &rkdvbm { status = "okay"; }; diff --git a/arch/arm64/boot/dts/rockchip/rv1126bp.dtsi b/arch/arm64/boot/dts/rockchip/rv1126bp.dtsi new file mode 100644 index 000000000000..f4e57e04e3d8 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rv1126bp.dtsi @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2025 Rockchip Electronics Co., Ltd. + */ + +#include "rv1126b.dtsi" + +/ { +}; diff --git a/arch/arm64/configs/rk3576_vehicle.config b/arch/arm64/configs/rk3576_vehicle.config index 0df4caf1b138..e58d71d0f0f2 100644 --- a/arch/arm64/configs/rk3576_vehicle.config +++ b/arch/arm64/configs/rk3576_vehicle.config @@ -39,6 +39,7 @@ CONFIG_HZ_1000=y # CONFIG_LIGHT_DEVICE is not set CONFIG_LOG_BUF_SHIFT=20 # CONFIG_MALI400 is not set +CONFIG_MALI_BIFROST=y # CONFIG_MALI_MIDGARD is not set # CONFIG_MFD_MAX96745 is not set # CONFIG_MFD_MAX96755F is not set @@ -57,8 +58,6 @@ CONFIG_REALTEK_PHY=y # CONFIG_REGULATOR_WL2868C is not set # CONFIG_REGULATOR_XZ3216 is not set # CONFIG_ROCKCHIP_CHARGER_MANAGER is not set -# CONFIG_ROCKCHIP_CLK_INV is not set -# CONFIG_ROCKCHIP_CLK_PVTM is not set CONFIG_ROCKCHIP_DRM_DIRECT_SHOW=y CONFIG_RTC_DRV_S35390A=y # CONFIG_SLUB_SYSFS is not set @@ -73,6 +72,7 @@ CONFIG_RTC_DRV_S35390A=y # CONFIG_SND_SOC_RK817 is not set # CONFIG_SND_SOC_RK_CODEC_DIGITAL is not set # CONFIG_SND_SOC_RT5640 is not set +CONFIG_SPI_SLAVE=y # CONFIG_TOUCHSCREEN_ELAN5515 is not set # CONFIG_TOUCHSCREEN_GSL3673 is not set # CONFIG_TOUCHSCREEN_GSLX680_PAD is not set @@ -93,6 +93,7 @@ CONFIG_TOUCHSCREEN_ILI210X=m # CONFIG_USB_NET_SMSC95XX is not set CONFIG_USB_OHCI_HCD=m CONFIG_USB_OHCI_HCD_PLATFORM=m +CONFIG_VEHICLE_CORE=y # CONFIG_VIDEO_AW36518 is not set # CONFIG_VIDEO_AW8601 is not set # CONFIG_VIDEO_CN3927V is not set @@ -117,6 +118,32 @@ CONFIG_VIDEO_MAXIM_SERDES=y # CONFIG_VIDEO_S5KJN1 is not set # CONFIG_VIDEO_SGM3784 is not set # CONFIG_VL6180 is not set +CONFIG_LARGE_PAGE_SUPPORT=y +# CONFIG_MALI_ARBITER_SUPPORT is not set +# CONFIG_MALI_ARBITRATION is not set +CONFIG_MALI_BIFROST_DEBUG=y +CONFIG_MALI_BIFROST_DEVFREQ=y +CONFIG_MALI_BIFROST_ENABLE_TRACE=y +CONFIG_MALI_BIFROST_EXPERT=y +CONFIG_MALI_BIFROST_FENCE_DEBUG=y +CONFIG_MALI_BIFROST_GATOR_SUPPORT=y +# CONFIG_MALI_BIFROST_NO_MALI is not set +# CONFIG_MALI_BIFROST_PRFCNT_SET_SECONDARY is not set +CONFIG_MALI_BIFROST_SYSTEM_TRACE=y +# CONFIG_MALI_CSF_SUPPORT is not set +# CONFIG_MALI_DMA_BUF_LEGACY_COMPAT is not set +# CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND is not set +# CONFIG_MALI_HW_ERRATA_1485982_NOT_AFFECTED is not set +# CONFIG_MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE is not set +# CONFIG_MALI_IS_FPGA is not set +# CONFIG_MALI_JOB_DUMP is not set +CONFIG_MALI_PLATFORM_NAME="rk" +CONFIG_MALI_PRFCNT_SET_PRIMARY=y +# CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS is not set +# CONFIG_MALI_PRFCNT_SET_TERTIARY is not set +CONFIG_MALI_REAL_HW=y +CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD=y +CONFIG_PAGE_MIGRATION_SUPPORT=y # CONFIG_ROCKCHIP_DRM_SELF_TEST is not set CONFIG_SERDES_DISPLAY_CHIP_MAXIM=y CONFIG_SERDES_DISPLAY_CHIP_MAXIM_MAX96745=y @@ -132,6 +159,15 @@ CONFIG_SERDES_DISPLAY_CHIP_ROCKCHIP_RKX121=y CONFIG_SERDES_DISPLAY_CHIP_ROHM=y CONFIG_SERDES_DISPLAY_CHIP_ROHM_BU18RL82=y CONFIG_SERDES_DISPLAY_CHIP_ROHM_BU18TL82=y +CONFIG_SPI_DYNAMIC=y +# CONFIG_SPI_SLAVE_ROCKCHIP_OBJ is not set +# CONFIG_SPI_SLAVE_SYSTEM_CONTROL is not set +# CONFIG_SPI_SLAVE_TIME is not set +CONFIG_VEHICLE_ADC=y +CONFIG_VEHICLE_DUMMY=y +CONFIG_VEHICLE_GPIO=y +CONFIG_VEHICLE_SPI=y +# CONFIG_VEHICLE_SPI_PROTOCOL is not set CONFIG_VIDEO_MAXIM_CAM_DUMMY=y # CONFIG_VIDEO_MAXIM_CAM_OS04A10 is not set CONFIG_VIDEO_MAXIM_CAM_OV231X=y @@ -144,7 +180,3 @@ CONFIG_VIDEO_MAXIM_SER_MAX9295=y CONFIG_VIDEO_MAXIM_SER_MAX96715=y CONFIG_VIDEO_MAXIM_SER_MAX96717=y # CONFIG_VIDEO_REVERSE_IMAGE is not set -CONFIG_MALI_BIFROST=y -CONFIG_MALI_PLATFORM_NAME="rk" -CONFIG_MALI_BIFROST_EXPERT=y -CONFIG_MALI_BIFROST_DEBUG=y diff --git a/arch/arm64/configs/rk3588_vehicle.config b/arch/arm64/configs/rk3588_vehicle.config index 722de0be5abf..1bf6dca4d8d8 100644 --- a/arch/arm64/configs/rk3588_vehicle.config +++ b/arch/arm64/configs/rk3588_vehicle.config @@ -40,6 +40,7 @@ CONFIG_HZ_1000=y CONFIG_LOG_BUF_SHIFT=20 # CONFIG_MALI400 is not set # CONFIG_MALI_MIDGARD is not set +CONFIG_MALI_VALHALL=y # CONFIG_MFD_MAX96745 is not set # CONFIG_MFD_MAX96755F is not set # CONFIG_MFD_RK618 is not set @@ -57,8 +58,6 @@ CONFIG_REALTEK_PHY=y # CONFIG_REGULATOR_WL2868C is not set # CONFIG_REGULATOR_XZ3216 is not set # CONFIG_ROCKCHIP_CHARGER_MANAGER is not set -# CONFIG_ROCKCHIP_CLK_INV is not set -# CONFIG_ROCKCHIP_CLK_PVTM is not set CONFIG_ROCKCHIP_DRM_DIRECT_SHOW=y CONFIG_RTC_DRV_S35390A=y # CONFIG_SLUB_SYSFS is not set @@ -73,6 +72,7 @@ CONFIG_RTC_DRV_S35390A=y # CONFIG_SND_SOC_RK817 is not set # CONFIG_SND_SOC_RK_CODEC_DIGITAL is not set # CONFIG_SND_SOC_RT5640 is not set +CONFIG_SPI_SLAVE=y # CONFIG_TOUCHSCREEN_ELAN5515 is not set # CONFIG_TOUCHSCREEN_GSL3673 is not set # CONFIG_TOUCHSCREEN_GSLX680_PAD is not set @@ -93,6 +93,7 @@ CONFIG_TOUCHSCREEN_ILI210X=m # CONFIG_USB_NET_SMSC95XX is not set CONFIG_USB_OHCI_HCD=m CONFIG_USB_OHCI_HCD_PLATFORM=m +CONFIG_VEHICLE_CORE=y # CONFIG_VIDEO_AW36518 is not set # CONFIG_VIDEO_AW8601 is not set # CONFIG_VIDEO_CN3927V is not set @@ -117,6 +118,30 @@ CONFIG_VIDEO_MAXIM_SERDES=y # CONFIG_VIDEO_S5KJN1 is not set # CONFIG_VIDEO_SGM3784 is not set # CONFIG_VL6180 is not set +CONFIG_MALI_CSF_INCLUDE_FW=y +# CONFIG_MALI_VALHALL_ARBITRATION is not set +# CONFIG_MALI_VALHALL_CORESIGHT is not set +# CONFIG_MALI_VALHALL_CORESTACK is not set +CONFIG_MALI_VALHALL_CSF_SUPPORT=y +CONFIG_MALI_VALHALL_DEBUG=y +CONFIG_MALI_VALHALL_DEVFREQ=y +# CONFIG_MALI_VALHALL_DMA_BUF_LEGACY_COMPAT is not set +# CONFIG_MALI_VALHALL_DMA_BUF_MAP_ON_DEMAND is not set +CONFIG_MALI_VALHALL_ENABLE_TRACE=y +CONFIG_MALI_VALHALL_EXPERT=y +CONFIG_MALI_VALHALL_FENCE_DEBUG=y +CONFIG_MALI_VALHALL_GATOR_SUPPORT=y +# CONFIG_MALI_VALHALL_HW_ERRATA_1485982_NOT_AFFECTED is not set +# CONFIG_MALI_VALHALL_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE is not set +# CONFIG_MALI_VALHALL_JOB_DUMP is not set +# CONFIG_MALI_VALHALL_NO_MALI is not set +CONFIG_MALI_VALHALL_PLATFORM_NAME="rk" +CONFIG_MALI_VALHALL_PRFCNT_SET_PRIMARY=y +# CONFIG_MALI_VALHALL_PRFCNT_SET_SECONDARY is not set +# CONFIG_MALI_VALHALL_PRFCNT_SET_TERTIARY is not set +CONFIG_MALI_VALHALL_REAL_HW=y +CONFIG_MALI_VALHALL_SYSTEM_TRACE=y +CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD=y # CONFIG_ROCKCHIP_DRM_SELF_TEST is not set CONFIG_SERDES_DISPLAY_CHIP_MAXIM=y CONFIG_SERDES_DISPLAY_CHIP_MAXIM_MAX96745=y @@ -132,6 +157,16 @@ CONFIG_SERDES_DISPLAY_CHIP_ROCKCHIP_RKX121=y CONFIG_SERDES_DISPLAY_CHIP_ROHM=y CONFIG_SERDES_DISPLAY_CHIP_ROHM_BU18RL82=y CONFIG_SERDES_DISPLAY_CHIP_ROHM_BU18TL82=y +CONFIG_SPI_DYNAMIC=y +# CONFIG_SPI_SLAVE_ROCKCHIP_OBJ is not set +# CONFIG_SPI_SLAVE_SYSTEM_CONTROL is not set +# CONFIG_SPI_SLAVE_TIME is not set +CONFIG_VALHALL_LARGE_PAGE_SUPPORT=y +CONFIG_VEHICLE_ADC=y +CONFIG_VEHICLE_DUMMY=y +CONFIG_VEHICLE_GPIO=y +CONFIG_VEHICLE_SPI=y +# CONFIG_VEHICLE_SPI_PROTOCOL is not set CONFIG_VIDEO_MAXIM_CAM_DUMMY=y CONFIG_VIDEO_MAXIM_CAM_OS04A10=y CONFIG_VIDEO_MAXIM_CAM_OV231X=y @@ -144,7 +179,3 @@ CONFIG_VIDEO_MAXIM_SER_MAX9295=y CONFIG_VIDEO_MAXIM_SER_MAX96715=y CONFIG_VIDEO_MAXIM_SER_MAX96717=y # CONFIG_VIDEO_REVERSE_IMAGE is not set -CONFIG_MALI_VALHALL=y -CONFIG_MALI_VALHALL_PLATFORM_NAME="rk" -CONFIG_MALI_VALHALL_EXPERT=y -CONFIG_MALI_VALHALL_DEBUG=y diff --git a/drivers/base/arm/memory_group_manager/memory_group_manager.c b/drivers/base/arm/memory_group_manager/memory_group_manager.c index 6789fe87075a..2c5780d42a62 100644 --- a/drivers/base/arm/memory_group_manager/memory_group_manager.c +++ b/drivers/base/arm/memory_group_manager/memory_group_manager.c @@ -310,6 +310,18 @@ static u64 example_mgm_update_gpu_pte(struct memory_group_manager_device *const if (WARN_ON(group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS)) return pte; + /* If a page is mapped uncached on the CPU but cached on the GPU, it will be considered to + * have Mismatched Memory Attributes (MMA), and the MMA_VIOLATION bit will be set in the + * pte_flags argument. + * + * If the system requires consistent memory attributes external to the GPU, system + * integrators must allocate one of the PBHA values (range 1-15) for this feature, and + * specify the value either via the mma-wa-id devicetree property or via the mma_wa_id + * module parameter, which is then passed into this function via the pbha_id parameter. The + * GPU will continue to use cached transactions internally, but use non-cacheable + * transactions externally. Note that system integrators may choose not to set the PBHA + * value here if their system does not require it. + */ if (pte_flags & BIT(MMA_VIOLATION)) { pr_warn_once("MMA violation! Applying PBHA override workaround to PTE\n"); pte |= ((u64)pbha_id << PTE_PBHA_SHIFT) & PTE_PBHA_MASK; diff --git a/drivers/clk/rockchip/clk-rv1126b.c b/drivers/clk/rockchip/clk-rv1126b.c index 1bff71eedba1..ed7cac4c2953 100644 --- a/drivers/clk/rockchip/clk-rv1126b.c +++ b/drivers/clk/rockchip/clk-rv1126b.c @@ -144,7 +144,7 @@ PNAME(clk_timer3_parents_p) = { "clk_timer_root", "mclk_asrc0", "mclk_asrc1" }; PNAME(clk_timer4_parents_p) = { "clk_timer_root", "mclk_asrc2", "mclk_asrc3" }; PNAME(clk_macphy_p) = { "xin24m", "clk_cpll_div20" }; PNAME(mux_ddrphy_p) = { "dpll", "aclk_sysmem" }; -PNAME(clk_cpll_div10_p) = { "gpll", "clk_aisp_pll" }; +PNAME(clk_cpll_div10_p) = { "gpll", "clk_aisp_pll_src" }; static struct rockchip_pll_clock rv1126b_pll_clks[] __initdata = { [gpll] = PLL(pll_rk3328, PLL_GPLL, "gpll", mux_pll_p, @@ -866,7 +866,7 @@ static struct rockchip_clk_branch rv1126b_clk_branches[] __initdata = { RV1126B_BUSCLKGATE_CON(2), 14, GFLAGS), GATE(HCLK_RKRNG_NS, "hclk_rkrng_ns", "hclk_rkrng_s_ns", 0, RV1126B_BUSCLKGATE_CON(2), 15, GFLAGS), - GATE(CLK_TIMER5, "clk_timer5", "clk_timer_root", 0, + GATE(CLK_TIMER5, "clk_timer5", "clk_timer_root", CLK_IS_CRITICAL, RV1126B_BUSCLKGATE_CON(2), 11, GFLAGS), GATE(PCLK_I2C0, "pclk_i2c0", "pclk_bus_root", 0, RV1126B_BUSCLKGATE_CON(3), 0, GFLAGS), @@ -924,9 +924,9 @@ static struct rockchip_clk_branch rv1126b_clk_branches[] __initdata = { RV1126B_BUSCLKGATE_CON(4), 14, GFLAGS), GATE(PCLK_UART7, "pclk_uart7", "pclk_bus_root", 0, RV1126B_BUSCLKGATE_CON(4), 15, GFLAGS), - GATE(PCLK_TSADC, "pclk_tsadc", "pclk_bus_root", 0, + GATE(PCLK_TSADC, "pclk_tsadc", "pclk_bus_root", CLK_IS_CRITICAL, RV1126B_BUSCLKGATE_CON(5), 0, GFLAGS), - GATE(CLK_TSADC, "clk_tsadc", "xin24m", 0, + GATE(CLK_TSADC, "clk_tsadc", "xin24m", CLK_IS_CRITICAL, RV1126B_BUSCLKGATE_CON(5), 1, GFLAGS), GATE(HCLK_SAI0, "hclk_sai0", "hclk_bus_root", 0, RV1126B_BUSCLKGATE_CON(5), 2, GFLAGS), @@ -962,7 +962,7 @@ static struct rockchip_clk_branch rv1126b_clk_branches[] __initdata = { RV1126B_BUSCLKGATE_CON(6), 4, GFLAGS), GATE(PCLK_OTP_MASK, "pclk_otp_mask", "pclk_bus_root", 0, RV1126B_BUSCLKGATE_CON(6), 6, GFLAGS), - GATE(CLK_TSADC_PHYCTRL, "clk_tsadc_phyctrl", "xin24m", 0, + GATE(CLK_TSADC_PHYCTRL, "clk_tsadc_phyctrl", "xin24m", CLK_IS_CRITICAL, RV1126B_BUSCLKGATE_CON(6), 8, GFLAGS), MUX(LRCK_SRC_ASRC0, "lrck_src_asrc0", lrck_src_asrc_p, 0, RV1126B_BUSCLKSEL_CON(3), 0, 3, MFLAGS), @@ -1016,9 +1016,11 @@ static struct rockchip_clk_branch rv1126b_clk_branches[] __initdata = { }; static struct rockchip_clk_branch rv1126b_clk_cpll_div10_v0[] __initdata = { - COMPOSITE(CLK_AISP_PLL, "clk_aisp_pll", mux_gpll_aupll_cpll_p, 0, - RV1126B_CLKSEL_CON(62), 4, 2, MFLAGS, 0, 3, DFLAGS, + COMPOSITE_NODIV(CLK_AISP_PLL_SRC, "clk_aisp_pll_src", mux_gpll_aupll_cpll_p, 0, + RV1126B_CLKSEL_CON(62), 4, 2, MFLAGS, RV1126B_CLKGATE_CON(5), 4, GFLAGS), + DIV(CLK_AISP_PLL, "clk_aisp_pll", "clk_aisp_pll_src", 0, + RV1126B_CLKSEL_CON(62), 0, 3, DFLAGS), COMPOSITE(CLK_CPLL_DIV10, "clk_cpll_div10", mux_gpll_cpll_p, 0, RV1126B_CLKSEL_CON(1), 15, 1, MFLAGS, 5, 5, DFLAGS, @@ -1026,9 +1028,11 @@ static struct rockchip_clk_branch rv1126b_clk_cpll_div10_v0[] __initdata = { }; static struct rockchip_clk_branch rv1126b_clk_cpll_div10[] __initdata = { - COMPOSITE(CLK_AISP_PLL, "clk_aisp_pll", mux_gpll_aupll_cpll_p, 0, - RV1126B_CLKSEL_CON(62), 4, 2, MFLAGS, 0, 3, DFLAGS, + COMPOSITE_NODIV(CLK_AISP_PLL_SRC, "clk_aisp_pll_src", mux_gpll_aupll_cpll_p, 0, + RV1126B_CLKSEL_CON(62), 4, 2, MFLAGS, RV1126B_CLKGATE_CON(5), 4, GFLAGS), + DIV(CLK_AISP_PLL, "clk_aisp_pll", "clk_aisp_pll_src", 0, + RV1126B_CLKSEL_CON(62), 0, 3, DFLAGS), COMPOSITE(CLK_CPLL_DIV10, "clk_cpll_div10", clk_cpll_div10_p, 0, RV1126B_CLKSEL_CON(1), 15, 1, MFLAGS, 5, 5, DFLAGS, diff --git a/drivers/gpu/arm/bifrost/context/mali_kbase_context.c b/drivers/gpu/arm/bifrost/context/mali_kbase_context.c index 2c7417bd6506..b9a6c2485de8 100644 --- a/drivers/gpu/arm/bifrost/context/mali_kbase_context.c +++ b/drivers/gpu/arm/bifrost/context/mali_kbase_context.c @@ -72,6 +72,75 @@ static struct kbase_process *find_process_node(struct rb_node *node, pid_t tgid) return kprcs; } +static ssize_t kbase_kctx_attr_show(struct kobject *kobj, struct attribute *attr, char *buf) +{ + struct kobj_attribute *kattr = container_of(attr, struct kobj_attribute, attr); + + if (kattr->show) + return kattr->show(kobj, kattr, buf); + + return -EIO; +} + +static ssize_t kbase_total_gpu_mem_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct kbase_context *kctx = container_of(kobj, struct kbase_context, kobj); + + return scnprintf(buf, PAGE_SIZE, "%zu\n", kctx->kprcs->total_gpu_pages << PAGE_SHIFT); +} + +static ssize_t kbase_private_gpu_mem_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct kbase_context *kctx = container_of(kobj, struct kbase_context, kobj); + struct kbase_process *kprcs = kctx->kprcs; + struct kbase_context *tmp_kctx; + size_t total_pages = 0; + + /* Sum up used_pages from all contexts in the process */ + list_for_each_entry(tmp_kctx, &kprcs->kctx_list, kprcs_link) { + total_pages += atomic_read(&tmp_kctx->used_pages); + } + + return scnprintf(buf, PAGE_SIZE, "%zu\n", total_pages << PAGE_SHIFT); +} + +static struct kobj_attribute kbase_total_gpu_mem_attr = { + .attr = { + .name = "total_gpu_mem", + .mode = 0444, + }, + .show = kbase_total_gpu_mem_show, + .store = NULL, +}; + +static struct kobj_attribute kbase_private_gpu_mem_attr = { + .attr = { + .name = "private_gpu_mem", + .mode = 0444, + }, + .show = kbase_private_gpu_mem_show, + .store = NULL, +}; + +static struct attribute *kbase_kctx_attrs[] = { + &kbase_total_gpu_mem_attr.attr, + &kbase_private_gpu_mem_attr.attr, + NULL, +}; + +static const struct attribute_group kbase_kctx_attr_group = { + .attrs = kbase_kctx_attrs, +}; + +static const struct sysfs_ops kbase_kctx_sysfs_ops = { + .show = kbase_kctx_attr_show, +}; + +static const struct kobj_type kbase_kctx_ktype = { + .sysfs_ops = &kbase_kctx_sysfs_ops, + .default_groups = (const struct attribute_group *[]) { &kbase_kctx_attr_group, NULL }, +}; + /** * kbase_insert_kctx_to_process - Initialise kbase process context. * @@ -100,6 +169,8 @@ static int kbase_insert_kctx_to_process(struct kbase_context *kctx) */ if (!kprcs) { struct rb_node **new = &prcs_root->rb_node, *parent = NULL; + char kctx_name[64]; + int ret = 0; kprcs = kzalloc(sizeof(*kprcs), GFP_KERNEL); if (kprcs == NULL) @@ -109,6 +180,15 @@ static int kbase_insert_kctx_to_process(struct kbase_context *kctx) kprcs->dma_buf_root = RB_ROOT; kprcs->total_gpu_pages = 0; + if (unlikely(!scnprintf(kctx_name, 64, "%d", tgid))) + return -ENOMEM; + + ret = kobject_init_and_add(&kctx->kobj, &kbase_kctx_ktype, kctx->kbdev->kprcs_kobj, kctx_name); + if (ret) { + dev_err(kctx->kbdev->dev, "Failed to create kctx kobject"); + kobject_put(&kctx->kobj); + } + while (*new) { struct kbase_process *prcs_node; @@ -260,6 +340,7 @@ static void kbase_remove_kctx_from_process(struct kbase_context *kctx) * we can remove it from the process rb_tree. */ if (list_empty(&kprcs->kctx_list)) { + kobject_put(&kctx->kobj); rb_erase(&kprcs->kprcs_node, &kctx->kbdev->process_root); /* Add checks, so that the terminating process Should not * hold any gpu_memory. diff --git a/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c b/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c index 9cd02fec4a10..46c91c6fcb6c 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c @@ -3182,6 +3182,56 @@ static ssize_t gpuinfo_show(struct device *dev, struct device_attribute *attr, c } static DEVICE_ATTR_RO(gpuinfo); +/** + * gpumem_private_show - Show callback for the gpumem_private sysfs entry. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer to receive the GPU memory information. + * + * This function is called to get the current number of pages used by the GPU. + * The returned value is in bytes. + * + * Return: The number of bytes output to @buf. + */ +static ssize_t private_gpu_mem_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct kbase_device *kbdev; + + CSTD_UNUSED(attr); + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + return scnprintf(buf, PAGE_SIZE, "%llu\n", (u64)atomic_read(&(kbdev->memdev.used_pages)) << PAGE_SHIFT); +} +static DEVICE_ATTR_RO(private_gpu_mem); + +/** + * total_gpu_mem_show - Show callback for the total_gpu_mem sysfs entry. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer to receive the GPU memory information. + * + * This function is called to get the total GPU memory including dmabuf memory. + * The returned value is in bytes. + * + * Return: The number of bytes output to @buf. + */ +static ssize_t total_gpu_mem_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct kbase_device *kbdev; + + CSTD_UNUSED(attr); + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + return scnprintf(buf, PAGE_SIZE, "%zu\n", kbdev->total_gpu_pages << PAGE_SHIFT); +} +static DEVICE_ATTR_RO(total_gpu_mem); + /** * dvfs_period_store - Store callback for the dvfs_period sysfs file. * @dev: The device with sysfs file is for @@ -5556,6 +5606,8 @@ static struct attribute *kbase_attrs[] = { &dev_attr_soft_job_timeout.attr, #endif /* !MALI_USE_CSF */ &dev_attr_gpuinfo.attr, + &dev_attr_total_gpu_mem.attr, + &dev_attr_private_gpu_mem.attr, &dev_attr_dvfs_period.attr, &dev_attr_pm_poweroff.attr, &dev_attr_reset_timeout.attr, @@ -5631,6 +5683,14 @@ int kbase_sysfs_init(struct kbase_device *kbdev) sysfs_remove_group(&kbdev->dev->kobj, &kbase_attr_group); } + kbdev->kprcs_kobj = kobject_create_and_add("kprcs", &kbdev->dev->kobj); + if (!kbdev->kprcs_kobj) { + dev_err(kbdev->dev, "Creation of kprcs sysfs group failed"); + sysfs_remove_group(&kbdev->dev->kobj, &kbase_mempool_attr_group); + sysfs_remove_group(&kbdev->dev->kobj, &kbase_scheduling_attr_group); + sysfs_remove_group(&kbdev->dev->kobj, &kbase_attr_group); + } + return err; } @@ -5639,6 +5699,7 @@ void kbase_sysfs_term(struct kbase_device *kbdev) sysfs_remove_group(&kbdev->dev->kobj, &kbase_mempool_attr_group); sysfs_remove_group(&kbdev->dev->kobj, &kbase_scheduling_attr_group); sysfs_remove_group(&kbdev->dev->kobj, &kbase_attr_group); + kobject_put(kbdev->kprcs_kobj); put_device(kbdev->dev); } diff --git a/drivers/gpu/arm/bifrost/mali_kbase_defs.h b/drivers/gpu/arm/bifrost/mali_kbase_defs.h index b97df15f7a17..b7b19b802eae 100755 --- a/drivers/gpu/arm/bifrost/mali_kbase_defs.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_defs.h @@ -1109,6 +1109,7 @@ struct kbase_device { unsigned int nr_regulators; #endif /* CONFIG_REGULATOR */ char devname[DEVNAME_SIZE]; + struct kobject *kprcs_kobj; u32 id; #if !IS_ENABLED(CONFIG_MALI_REAL_HW) @@ -1901,6 +1902,7 @@ struct kbase_sub_alloc { * is made on the device file. */ struct kbase_context { + struct kobject kobj; struct file *filp; struct kbase_device *kbdev; struct list_head kctx_list_link; diff --git a/drivers/gpu/arm/valhall/Kbuild b/drivers/gpu/arm/valhall/Kbuild index e2a58ea59d20..41afecc01294 100644 --- a/drivers/gpu/arm/valhall/Kbuild +++ b/drivers/gpu/arm/valhall/Kbuild @@ -63,7 +63,7 @@ endif # # Driver version string which is returned to userspace via an ioctl -MALI_RELEASE_NAME ?= '"g28p0-00eac0"' +MALI_RELEASE_NAME ?= '"g29p0-00eac0"' # Set up defaults if not defined by build system ifeq ($(CONFIG_MALI_VALHALL_DEBUG), y) MALI_UNIT_TEST = 1 @@ -79,13 +79,10 @@ MALI_PLATFORM_DIR := $(shell echo $(CONFIG_MALI_VALHALL_PLATFORM_NAME)) ifeq ($(CONFIG_MALI_VALHALL_CSF_SUPPORT),y) MALI_JIT_PRESSURE_LIMIT_BASE = 0 - MALI_USE_CSF = 1 else MALI_JIT_PRESSURE_LIMIT_BASE ?= 1 - MALI_USE_CSF ?= 0 endif - ifneq ($(CONFIG_MALI_VALHALL_KUTF), n) MALI_KERNEL_TEST_API ?= 1 else @@ -104,7 +101,6 @@ endif # ccflags-y = \ -DMALI_CUSTOMER_RELEASE=$(MALI_CUSTOMER_RELEASE) \ - -DMALI_USE_CSF=$(MALI_USE_CSF) \ -DMALI_KERNEL_TEST_API=$(MALI_KERNEL_TEST_API) \ -DMALI_UNIT_TEST=$(MALI_UNIT_TEST) \ -DMALI_COVERAGE=$(MALI_COVERAGE) \ @@ -112,7 +108,6 @@ ccflags-y = \ -DMALI_JIT_PRESSURE_LIMIT_BASE=$(MALI_JIT_PRESSURE_LIMIT_BASE) \ -DMALI_PLATFORM_DIR=$(MALI_PLATFORM_DIR) - ifeq ($(KBUILD_EXTMOD),) # in-tree ccflags-y +=-DMALI_KBASE_PLATFORM_PATH=../../$(src)/platform/$(CONFIG_MALI_VALHALL_PLATFORM_NAME) @@ -183,23 +178,6 @@ valhall_kbase-$(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) += \ mali_power_gpu_work_period_trace.o \ mali_kbase_gpu_metrics.o -ifneq ($(CONFIG_MALI_VALHALL_CSF_SUPPORT),y) - valhall_kbase-y += \ - mali_kbase_jm.o \ - mali_kbase_dummy_job_wa.o \ - mali_kbase_debug_job_fault.o \ - mali_kbase_event.o \ - mali_kbase_jd.o \ - mali_kbase_jd_debugfs.o \ - mali_kbase_js.o \ - mali_kbase_js_ctx_attr.o \ - mali_kbase_kinstr_jm.o - - valhall_kbase-$(CONFIG_SYNC_FILE) += \ - mali_kbase_fence_ops.o \ - mali_kbase_fence.o -endif - INCLUDE_SUBDIR = \ $(src)/arbiter/Kbuild \ diff --git a/drivers/gpu/arm/valhall/Kconfig b/drivers/gpu/arm/valhall/Kconfig index d821acbdafe0..b54098a24b3d 100644 --- a/drivers/gpu/arm/valhall/Kconfig +++ b/drivers/gpu/arm/valhall/Kconfig @@ -324,15 +324,6 @@ config MALI_VALHALL_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE endif -config MALI_VALHALL_ARBITRATION - tristate "Enable Virtualization reference code" - depends on MALI_VALHALL - default n - help - Enables the build of several reference modules used in the reference - virtualization setup for Mali - If unsure, say N. - config MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD bool "Enable per-application GPU metrics tracepoints" depends on MALI_VALHALL diff --git a/drivers/gpu/arm/valhall/Makefile b/drivers/gpu/arm/valhall/Makefile index f5871b01a3be..1bc641fb1119 100644 --- a/drivers/gpu/arm/valhall/Makefile +++ b/drivers/gpu/arm/valhall/Makefile @@ -39,7 +39,6 @@ ifeq ($(MALI_KCONFIG_EXT_PREFIX),) CONFIG_MALI_VALHALL_PLATFORM_NAME ?= "devicetree" CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD ?= y CONFIG_MALI_BIFROST_GATOR_SUPPORT ?= y - CONFIG_MALI_VALHALL_ARBITRATION ?= n CONFIG_MALI_KUTF_PTM_TESTS ?= n @@ -143,7 +142,6 @@ ifeq ($(MALI_KCONFIG_EXT_PREFIX),) endif else # Prevent misuse when CONFIG_MALI_BIFROST=n - CONFIG_MALI_VALHALL_ARBITRATION = n CONFIG_MALI_VALHALL_KUTF = n CONFIG_MALI_VALHALL_KUTF_IRQ_TEST = n CONFIG_MALI_VALHALL_KUTF_CLK_RATE_TRACE = n @@ -155,7 +153,6 @@ ifeq ($(MALI_KCONFIG_EXT_PREFIX),) CONFIG_MALI_BIFROST \ CONFIG_MALI_VALHALL_CSF_SUPPORT \ CONFIG_MALI_BIFROST_GATOR_SUPPORT \ - CONFIG_MALI_VALHALL_ARBITRATION \ CONFIG_MALI_KUTF_PTM_TESTS \ CONFIG_MALI_VALHALL_REAL_HW \ CONFIG_MALI_BIFROST_DEVFREQ \ diff --git a/drivers/gpu/arm/valhall/arbiter/mali_kbase_arbif.c b/drivers/gpu/arm/valhall/arbiter/mali_kbase_arbif.c index 49b42a6ec2c0..b69b315ccd3d 100644 --- a/drivers/gpu/arm/valhall/arbiter/mali_kbase_arbif.c +++ b/drivers/gpu/arm/valhall/arbiter/mali_kbase_arbif.c @@ -36,264 +36,6 @@ #error "Unsupported Mali Arbiter interface version." #endif -static void on_max_config(struct device *dev, uint32_t max_l2_slices, uint32_t max_core_mask) -{ - struct kbase_device *kbdev; - - if (!dev) { - pr_err("%s(): dev is NULL", __func__); - return; - } - - kbdev = dev_get_drvdata(dev); - if (!kbdev) { - dev_err(dev, "%s(): kbdev is NULL", __func__); - return; - } - - if (!max_l2_slices || !max_core_mask) { - dev_dbg(dev, "%s(): max_config ignored as one of the fields is zero", __func__); - return; - } - - /* set the max config info in the kbase device */ - kbase_arbiter_set_max_config(kbdev, max_l2_slices, max_core_mask); -} - -/** - * on_update_freq() - Updates GPU clock frequency - * @dev: arbiter interface device handle - * @freq: GPU clock frequency value reported from arbiter - * - * call back function to update GPU clock frequency with - * new value from arbiter - */ -static void on_update_freq(struct device *dev, uint32_t freq) -{ - struct kbase_device *kbdev; - - if (!dev) { - pr_err("%s(): dev is NULL", __func__); - return; - } - - kbdev = dev_get_drvdata(dev); - if (!kbdev) { - dev_err(dev, "%s(): kbdev is NULL", __func__); - return; - } - - kbase_arbiter_pm_update_gpu_freq(&kbdev->arb.arb_freq, freq); -} - -/** - * on_gpu_stop() - sends KBASE_VM_GPU_STOP_EVT event on VM stop - * @dev: arbiter interface device handle - * - * call back function to signal a GPU STOP event from arbiter interface - */ -static void on_gpu_stop(struct device *dev) -{ - struct kbase_device *kbdev; - - if (!dev) { - pr_err("%s(): dev is NULL", __func__); - return; - } - - kbdev = dev_get_drvdata(dev); - if (!kbdev) { - dev_err(dev, "%s(): kbdev is NULL", __func__); - return; - } - - KBASE_TLSTREAM_TL_ARBITER_STOP_REQUESTED(kbdev, kbdev); - KBASE_KTRACE_ADD(kbdev, ARB_GPU_STOP_REQUESTED, NULL, 0); - kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_STOP_EVT); -} - -/** - * on_gpu_granted() - sends KBASE_VM_GPU_GRANTED_EVT event on GPU granted - * @dev: arbiter interface device handle - * - * call back function to signal a GPU GRANT event from arbiter interface - */ -static void on_gpu_granted(struct device *dev) -{ - struct kbase_device *kbdev; - - if (!dev) { - pr_err("%s(): dev is NULL", __func__); - return; - } - - kbdev = dev_get_drvdata(dev); - if (!kbdev) { - dev_err(dev, "%s(): kbdev is NULL", __func__); - return; - } - - KBASE_TLSTREAM_TL_ARBITER_GRANTED(kbdev, kbdev); - KBASE_KTRACE_ADD(kbdev, ARB_GPU_GRANTED, NULL, 0); - kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_GRANTED_EVT); -} - -/** - * on_gpu_lost() - sends KBASE_VM_GPU_LOST_EVT event on GPU granted - * @dev: arbiter interface device handle - * - * call back function to signal a GPU LOST event from arbiter interface - */ -static void on_gpu_lost(struct device *dev) -{ - struct kbase_device *kbdev; - - if (!dev) { - pr_err("%s(): dev is NULL", __func__); - return; - } - - kbdev = dev_get_drvdata(dev); - if (!kbdev) { - dev_err(dev, "%s(): kbdev is NULL", __func__); - return; - } - KBASE_TLSTREAM_TL_ARBITER_LOST(kbdev, kbdev); - KBASE_KTRACE_ADD(kbdev, ARB_GPU_LOST, NULL, 0); - kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_LOST_EVT); -} - -static int kbase_arbif_of_init(struct kbase_device *kbdev) -{ - struct arbiter_if_dev *arb_if; - struct device_node *arbiter_if_node; - struct platform_device *pdev; - - if (!IS_ENABLED(CONFIG_OF)) { - /* - * Return -ENODEV in the event CONFIG_OF is not available and let the - * internal AW check for suitability for arbitration. - */ - return -ENODEV; - } - - arbiter_if_node = of_parse_phandle(kbdev->dev->of_node, "arbiter-if", 0); - if (!arbiter_if_node) - arbiter_if_node = of_parse_phandle(kbdev->dev->of_node, "arbiter_if", 0); - if (!arbiter_if_node) { - dev_dbg(kbdev->dev, "No arbiter_if in Device Tree"); - /* no arbiter interface defined in device tree */ - kbdev->arb.arb_dev = NULL; - kbdev->arb.arb_if = NULL; - return -ENODEV; - } - - pdev = of_find_device_by_node(arbiter_if_node); - if (!pdev) { - dev_err(kbdev->dev, "Failed to find arbiter_if device"); - return -EPROBE_DEFER; - } - - if (!pdev->dev.driver || !try_module_get(pdev->dev.driver->owner)) { - dev_err(kbdev->dev, "arbiter_if driver not available"); - put_device(&pdev->dev); - return -EPROBE_DEFER; - } - kbdev->arb.arb_dev = &pdev->dev; - arb_if = platform_get_drvdata(pdev); - if (!arb_if) { - dev_err(kbdev->dev, "arbiter_if driver not ready"); - module_put(pdev->dev.driver->owner); - put_device(&pdev->dev); - return -EPROBE_DEFER; - } - - kbdev->arb.arb_if = arb_if; - return 0; -} - -static void kbase_arbif_of_term(struct kbase_device *kbdev) -{ - if (!IS_ENABLED(CONFIG_OF)) - return; - - if (kbdev->arb.arb_dev) { - module_put(kbdev->arb.arb_dev->driver->owner); - put_device(kbdev->arb.arb_dev); - } - kbdev->arb.arb_dev = NULL; -} - - -/** - * kbase_arbif_init() - Kbase Arbiter interface initialisation. - * @kbdev: The kbase device structure for the device (must be a valid pointer) - * - * Initialise Kbase Arbiter interface and assign callback functions. - * - * Return: - * * 0 - the interface was initialized or was not specified - * * in the device tree. - * * -EFAULT - the interface was specified but failed to initialize. - * * -EPROBE_DEFER - module dependencies are not yet available. - */ -int kbase_arbif_init(struct kbase_device *kbdev) -{ - struct arbiter_if_arb_vm_ops ops; - struct arbiter_if_dev *arb_if; - int err = 0; - - /* Tries to init with 'arbiter-if' if present in devicetree */ - err = kbase_arbif_of_init(kbdev); - - if (err == -ENODEV) { - /* devicetree does not support arbitration */ - return -EPERM; - } - - if (err) - return err; - - ops.arb_vm_gpu_stop = on_gpu_stop; - ops.arb_vm_gpu_granted = on_gpu_granted; - ops.arb_vm_gpu_lost = on_gpu_lost; - ops.arb_vm_max_config = on_max_config; - ops.arb_vm_update_freq = on_update_freq; - - kbdev->arb.arb_freq.arb_freq = 0; - kbdev->arb.arb_freq.freq_updated = false; - mutex_init(&kbdev->arb.arb_freq.arb_freq_lock); - - arb_if = kbdev->arb.arb_if; - - if (arb_if == NULL) { - dev_err(kbdev->dev, "No arbiter interface present"); - goto failure_term; - } - - if (!arb_if->vm_ops.vm_arb_register_dev) { - dev_err(kbdev->dev, "arbiter_if registration callback not present"); - goto failure_term; - } - - /* register kbase arbiter_if callbacks */ - err = arb_if->vm_ops.vm_arb_register_dev(arb_if, kbdev->dev, &ops); - if (err) { - dev_err(kbdev->dev, "Failed to register with arbiter. (err = %d)", err); - goto failure_term; - } - - return 0; - -failure_term: - { - kbase_arbif_of_term(kbdev); - } - - if (err != -EPROBE_DEFER) - err = -EFAULT; - return err; -} /** * kbase_arbif_destroy() - De-init Kbase arbiter interface @@ -308,9 +50,6 @@ void kbase_arbif_destroy(struct kbase_device *kbdev) if (arb_if && arb_if->vm_ops.vm_arb_unregister_dev) arb_if->vm_ops.vm_arb_unregister_dev(kbdev->arb.arb_if); - { - kbase_arbif_of_term(kbdev); - } kbdev->arb.arb_if = NULL; } diff --git a/drivers/gpu/arm/valhall/arbiter/mali_kbase_arbif.h b/drivers/gpu/arm/valhall/arbiter/mali_kbase_arbif.h index c77792115e4d..8da8e3c1d0d9 100644 --- a/drivers/gpu/arm/valhall/arbiter/mali_kbase_arbif.h +++ b/drivers/gpu/arm/valhall/arbiter/mali_kbase_arbif.h @@ -51,21 +51,6 @@ enum kbase_arbif_evt { }; -/** - * kbase_arbif_init() - Initialize the arbiter interface functionality. - * @kbdev: The kbase device structure for the device (must be a valid pointer) - * - * Initialize the arbiter interface and also determines - * if Arbiter functionality is required. - * - * Return: - * * 0 - the interface was initialized or was not specified - * * in the device tree. - * * -EFAULT - the interface was specified but failed to initialize. - * * -EPROBE_DEFER - module dependencies are not yet available. - */ -int kbase_arbif_init(struct kbase_device *kbdev); - /** * kbase_arbif_destroy() - Cleanups the arbiter interface functionality. * @kbdev: The kbase device structure for the device (must be a valid pointer) diff --git a/drivers/gpu/arm/valhall/arbiter/mali_kbase_arbiter_pm.c b/drivers/gpu/arm/valhall/arbiter/mali_kbase_arbiter_pm.c index 1378e6486c37..57952106429a 100644 --- a/drivers/gpu/arm/valhall/arbiter/mali_kbase_arbiter_pm.c +++ b/drivers/gpu/arm/valhall/arbiter/mali_kbase_arbiter_pm.c @@ -31,9 +31,7 @@ #include #include -#if MALI_USE_CSF #include -#endif /* A dmesg warning will occur if the GPU is not granted * after the following time (in milliseconds) has ellapsed. @@ -299,7 +297,9 @@ int kbase_arbiter_pm_early_init(struct kbase_device *kbdev) arb_vm_state->vm_request_timer.function = request_timer_callback; kbdev->pm.arb_vm_state = arb_vm_state; - err = kbase_arbif_init(kbdev); + /* platform does not support arbitration */ + err = -EPERM; + if (err) { if (err != -EPERM) dev_err(kbdev->dev, "Failed to initialise arbif module. (err = %d)", err); @@ -360,10 +360,8 @@ void kbase_arbiter_pm_early_term(struct kbase_device *kbdev) cancel_request_timer(kbdev); mutex_lock(&arb_vm_state->vm_state_lock); - if (arb_vm_state->vm_state > KBASE_VM_STATE_STOPPED_GPU_REQUESTED) { - kbase_io_clear_status(kbdev->io, KBASE_IO_STATUS_AW_REMOVED); + if (arb_vm_state->vm_state > KBASE_VM_STATE_STOPPED_GPU_REQUESTED) kbase_arbif_gpu_stopped(kbdev, false); - } mutex_unlock(&arb_vm_state->vm_state_lock); destroy_workqueue(arb_vm_state->vm_arb_wq); kbase_arbif_destroy(kbdev); @@ -410,6 +408,7 @@ void kbase_arbiter_pm_vm_stopped(struct kbase_device *kbdev) { bool request_gpu = false; struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + unsigned long flags; lockdep_assert_held(&arb_vm_state->vm_state_lock); @@ -420,17 +419,6 @@ void kbase_arbiter_pm_vm_stopped(struct kbase_device *kbdev) dev_dbg(kbdev->dev, "%s %s\n", __func__, kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state)); - /* - * Release the interrupts on external arb_if to address Xen requirements. - * Interrupts are not released with internal arb_if as the IRQs are required - * to handle messaging to/from Arbiter/Resource Group. - */ - if (arb_vm_state->interrupts_installed - ) { - arb_vm_state->interrupts_installed = false; - kbase_release_interrupts(kbdev); - } - switch (arb_vm_state->vm_state) { case KBASE_VM_STATE_STOPPING_ACTIVE: request_gpu = true; @@ -447,7 +435,17 @@ void kbase_arbiter_pm_vm_stopped(struct kbase_device *kbdev) break; } - kbase_io_clear_status(kbdev->io, KBASE_IO_STATUS_AW_REMOVED); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + if (kbdev->pm.backend.gpu_lost_pending) { + { + kbdev->csf.firmware_reload_needed = true; + dev_warn(kbdev->dev, "%s(): GPU LOST detected, marking MCU for cold boot", + __func__); + } + } + kbase_io_set_status(kbdev->io, KBASE_IO_STATUS_AW_REMOVED); + kbdev->pm.backend.gpu_lost_pending = false; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); { kbase_arbif_gpu_stopped(kbdev, request_gpu); @@ -550,16 +548,6 @@ static void kbase_arbiter_pm_vm_gpu_start(struct kbase_device *kbdev) break; case KBASE_VM_STATE_STOPPED_GPU_REQUESTED: kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_STARTING); - arb_vm_state->interrupts_installed = true; - /* - * Re-install interrupts that were released for external arb_if to - * address Xen requirements. Interrupts are not released with internal - * arb_if as the IRQs are required to handle messaging to/from - * Arbiter/Resource Group. - */ - { - kbase_install_interrupts(kbdev); - } /* * GPU GRANTED received while in stop can be a result of a * repartitioning. @@ -569,7 +557,6 @@ static void kbase_arbiter_pm_vm_gpu_start(struct kbase_device *kbdev) queue_work(arb_vm_state->vm_arb_wq, &arb_vm_state->vm_resume_work); break; case KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT: - kbase_io_clear_status(kbdev->io, KBASE_IO_STATUS_AW_REMOVED); kbase_arbif_gpu_stopped(kbdev, false); kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_SUSPENDED); break; @@ -637,7 +624,6 @@ static void kbase_arbiter_pm_vm_gpu_stop(struct kbase_device *kbdev) static void kbase_gpu_lost(struct kbase_device *kbdev) { struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; - bool handle_gpu_lost = false; lockdep_assert_held(&arb_vm_state->vm_state_lock); @@ -648,13 +634,11 @@ static void kbase_gpu_lost(struct kbase_device *kbdev) dev_warn(kbdev->dev, "GPU lost in state %s", kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state)); kbase_arbiter_pm_vm_gpu_stop(kbdev); - handle_gpu_lost = true; break; case KBASE_VM_STATE_STOPPING_IDLE: case KBASE_VM_STATE_STOPPING_ACTIVE: case KBASE_VM_STATE_SUSPEND_PENDING: - dev_dbg(kbdev->dev, "GPU lost while stopping"); - handle_gpu_lost = true; + dev_warn(kbdev->dev, "GPU lost while stopping"); break; case KBASE_VM_STATE_SUSPENDED: case KBASE_VM_STATE_STOPPED: @@ -668,8 +652,6 @@ static void kbase_gpu_lost(struct kbase_device *kbdev) default: break; } - if (handle_gpu_lost) - kbase_pm_handle_gpu_lost(kbdev); } /** @@ -798,6 +780,7 @@ static void kbase_arbiter_pm_vm_os_resume(struct kbase_device *kbdev) void kbase_arbiter_pm_vm_event(struct kbase_device *kbdev, enum kbase_arbif_evt evt) { struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + unsigned long flags; if (!kbase_has_arbiter(kbdev)) return; @@ -816,6 +799,13 @@ void kbase_arbiter_pm_vm_event(struct kbase_device *kbdev, enum kbase_arbif_evt break; case KBASE_VM_GPU_LOST_EVT: dev_dbg(kbdev->dev, "KBASE_ARBIF_GPU_LOST_EVT!"); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + if (!kbdev->pm.backend.gpu_lost_pending) { + dev_dbg(kbdev->dev, "skipping GPU_LOST handling"); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + break; + } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); kbase_gpu_lost(kbdev); break; case KBASE_VM_OS_SUSPEND_EVENT: @@ -933,10 +923,6 @@ int kbase_arbiter_pm_ctx_active_handle_suspend(struct kbase_device *kbdev, struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; int res = 0; -#if !MALI_USE_CSF - CSTD_UNUSED(sched_lock_held); -#endif - if (!kbase_has_arbiter(kbdev)) return res; @@ -949,7 +935,8 @@ int kbase_arbiter_pm_ctx_active_handle_suspend(struct kbase_device *kbdev, kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_STOPPED_GPU_REQUESTED); kbase_arbif_gpu_request(kbdev); start_request_timer(kbdev); - } else if (arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_ACTIVE) { + } else if (suspend_handler != KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE && + arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_ACTIVE) { res = 1; break; } else if (arb_vm_state->vm_state == KBASE_VM_STATE_INITIALIZING_WITH_GPU) @@ -985,15 +972,11 @@ int kbase_arbiter_pm_ctx_active_handle_suspend(struct kbase_device *kbdev, atomic_inc(&kbdev->pm.gpu_users_waiting); mutex_unlock(&arb_vm_state->vm_state_lock); kbase_pm_unlock(kbdev); -#if MALI_USE_CSF if (sched_lock_held) kbase_csf_scheduler_unlock(kbdev); -#endif kbase_arbiter_pm_vm_wait_gpu_assignment(kbdev); -#if MALI_USE_CSF if (sched_lock_held) kbase_csf_scheduler_lock(kbdev); -#endif kbase_pm_lock(kbdev); mutex_lock(&arb_vm_state->vm_state_lock); atomic_dec(&kbdev->pm.gpu_users_waiting); diff --git a/drivers/gpu/arm/valhall/backend/gpu/Kbuild b/drivers/gpu/arm/valhall/backend/gpu/Kbuild index 78f362d9046e..dd6b3bb038c8 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/Kbuild +++ b/drivers/gpu/arm/valhall/backend/gpu/Kbuild @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -33,16 +33,6 @@ valhall_kbase-y += \ backend/gpu/mali_kbase_l2_mmu_config.o \ backend/gpu/mali_kbase_clk_rate_trace_mgr.o -ifeq ($(MALI_USE_CSF),0) - valhall_kbase-y += \ - backend/gpu/mali_kbase_instr_backend.o \ - backend/gpu/mali_kbase_jm_as.o \ - backend/gpu/mali_kbase_debug_job_fault_backend.o \ - backend/gpu/mali_kbase_jm_hw.o \ - backend/gpu/mali_kbase_jm_rb.o \ - backend/gpu/mali_kbase_js_backend.o -endif - valhall_kbase-$(CONFIG_MALI_VALHALL_DEVFREQ) += \ backend/gpu/mali_kbase_devfreq.o diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_cache_policy_backend.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_cache_policy_backend.c index aa8436420234..ed42d5349978 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_cache_policy_backend.c +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_cache_policy_backend.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -27,7 +27,6 @@ void kbase_cache_set_coherency_mode(struct kbase_device *kbdev, u32 mode) kbdev->current_gpu_coherency_mode = mode; -#if MALI_USE_CSF if (kbdev->gpu_props.gpu_id.arch_id >= GPU_ID_ARCH_MAKE(12, 0, 1)) { /* AMBA_ENABLE present from 12.0.1 */ u32 val = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(AMBA_ENABLE)); @@ -38,14 +37,10 @@ void kbase_cache_set_coherency_mode(struct kbase_device *kbdev, u32 mode) /* Fallback to COHERENCY_ENABLE for older versions */ kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(COHERENCY_ENABLE), mode); } -#else /* MALI_USE_CSF */ - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(COHERENCY_ENABLE), mode); -#endif /* MALI_USE_CSF */ } void kbase_amba_set_shareable_cache_support(struct kbase_device *kbdev) { -#if MALI_USE_CSF /* AMBA registers only present from 12.0.1 */ if (kbdev->gpu_props.gpu_id.arch_id < GPU_ID_ARCH_MAKE(12, 0, 1)) @@ -60,5 +55,4 @@ void kbase_amba_set_shareable_cache_support(struct kbase_device *kbdev) kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(AMBA_ENABLE), val); } } -#endif /* MALI_USE_CSF */ } diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_debug_job_fault_backend.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_debug_job_fault_backend.c deleted file mode 100644 index af8d1e3af87c..000000000000 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_debug_job_fault_backend.c +++ /dev/null @@ -1,167 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2012-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include -#include -#include -#include "mali_kbase_debug_job_fault.h" - -#if IS_ENABLED(CONFIG_DEBUG_FS) - -/*GPU_CONTROL_REG(r)*/ -static unsigned int gpu_control_reg_snapshot[] = { GPU_CONTROL_ENUM(GPU_ID), - GPU_CONTROL_ENUM(SHADER_READY), - GPU_CONTROL_ENUM(TILER_READY), - GPU_CONTROL_ENUM(L2_READY) }; - -/* JOB_CONTROL_REG(r) */ -static unsigned int job_control_reg_snapshot[] = { JOB_CONTROL_ENUM(JOB_IRQ_MASK), - JOB_CONTROL_ENUM(JOB_IRQ_STATUS) }; - -/* JOB_SLOT_REG(n,r) */ -static unsigned int job_slot_reg_snapshot[] = { - JOB_SLOT_ENUM(0, HEAD) - JOB_SLOT_BASE_ENUM(0), - JOB_SLOT_ENUM(0, TAIL) - JOB_SLOT_BASE_ENUM(0), - JOB_SLOT_ENUM(0, AFFINITY) - JOB_SLOT_BASE_ENUM(0), - JOB_SLOT_ENUM(0, CONFIG) - JOB_SLOT_BASE_ENUM(0), - JOB_SLOT_ENUM(0, STATUS) - JOB_SLOT_BASE_ENUM(0), - JOB_SLOT_ENUM(0, HEAD_NEXT) - JOB_SLOT_BASE_ENUM(0), - JOB_SLOT_ENUM(0, AFFINITY_NEXT) - JOB_SLOT_BASE_ENUM(0), - JOB_SLOT_ENUM(0, CONFIG_NEXT) - JOB_SLOT_BASE_ENUM(0) -}; - -/*MMU_CONTROL_REG(r)*/ -static unsigned int mmu_reg_snapshot[] = { MMU_CONTROL_ENUM(IRQ_MASK), - MMU_CONTROL_ENUM(IRQ_STATUS) }; - -/* MMU_AS_REG(n,r) */ -static unsigned int as_reg_snapshot[] = { MMU_AS_ENUM(0, TRANSTAB) - MMU_AS_BASE_ENUM(0), - MMU_AS_ENUM(0, TRANSCFG) - MMU_AS_BASE_ENUM(0), - MMU_AS_ENUM(0, MEMATTR) - MMU_AS_BASE_ENUM(0), - MMU_AS_ENUM(0, FAULTSTATUS) - MMU_AS_BASE_ENUM(0), - MMU_AS_ENUM(0, FAULTADDRESS) - MMU_AS_BASE_ENUM(0), - MMU_AS_ENUM(0, STATUS) - MMU_AS_BASE_ENUM(0) }; - -bool kbase_debug_job_fault_reg_snapshot_init(struct kbase_context *kctx, int reg_range) -{ - uint i, j; - int offset = 0; - uint slot_number; - uint as_number; - - if (kctx->reg_dump == NULL) - return false; - - slot_number = kctx->kbdev->gpu_props.num_job_slots; - as_number = kctx->kbdev->gpu_props.num_address_spaces; - - /* get the GPU control registers*/ - for (i = 0; i < ARRAY_SIZE(gpu_control_reg_snapshot); i++) { - kctx->reg_dump[offset] = gpu_control_reg_snapshot[i]; - if (kbase_reg_is_size64(kctx->kbdev, kctx->reg_dump[offset])) - offset += 4; - else - offset += 2; - } - - /* get the Job control registers*/ - for (i = 0; i < ARRAY_SIZE(job_control_reg_snapshot); i++) { - kctx->reg_dump[offset] = job_control_reg_snapshot[i]; - if (kbase_reg_is_size64(kctx->kbdev, kctx->reg_dump[offset])) - offset += 4; - else - offset += 2; - } - - /* get the Job Slot registers*/ - for (j = 0; j < slot_number; j++) { - for (i = 0; i < ARRAY_SIZE(job_slot_reg_snapshot); i++) { - kctx->reg_dump[offset] = JOB_SLOT_BASE_OFFSET(j) + job_slot_reg_snapshot[i]; - if (kbase_reg_is_size64(kctx->kbdev, kctx->reg_dump[offset])) - offset += 4; - else - offset += 2; - } - } - - /* get the MMU registers*/ - for (i = 0; i < ARRAY_SIZE(mmu_reg_snapshot); i++) { - kctx->reg_dump[offset] = mmu_reg_snapshot[i]; - if (kbase_reg_is_size64(kctx->kbdev, kctx->reg_dump[offset])) - offset += 4; - else - offset += 2; - } - - /* get the Address space registers*/ - for (j = 0; j < as_number; j++) { - for (i = 0; i < ARRAY_SIZE(as_reg_snapshot); i++) { - kctx->reg_dump[offset] = MMU_AS_BASE_OFFSET(j) + as_reg_snapshot[i]; - if (kbase_reg_is_size64(kctx->kbdev, kctx->reg_dump[offset])) - offset += 4; - else - offset += 2; - } - } - - WARN_ON(offset >= (reg_range * 2 / 4)); - - /* set the termination flag*/ - kctx->reg_dump[offset] = REGISTER_DUMP_TERMINATION_FLAG; - kctx->reg_dump[offset + 1] = REGISTER_DUMP_TERMINATION_FLAG; - - dev_dbg(kctx->kbdev->dev, "kbase_job_fault_reg_snapshot_init:%d\n", offset); - - return true; -} - -bool kbase_job_fault_get_reg_snapshot(struct kbase_context *kctx) -{ - int offset = 0; - u32 reg_enum; - u64 val64; - - if (kctx->reg_dump == NULL) - return false; - - while (kctx->reg_dump[offset] != REGISTER_DUMP_TERMINATION_FLAG) { - reg_enum = kctx->reg_dump[offset]; - /* Get register offset from enum */ - kbase_reg_get_offset(kctx->kbdev, reg_enum, &kctx->reg_dump[offset]); - - if (kbase_reg_is_size64(kctx->kbdev, reg_enum)) { - val64 = kbase_reg_read64(kctx->kbdev, reg_enum); - - /* offset computed offset to get _HI offset */ - kctx->reg_dump[offset + 2] = kctx->reg_dump[offset] + 4; - - kctx->reg_dump[offset + 1] = (u32)(val64 & 0xFFFFFFFF); - kctx->reg_dump[offset + 3] = (u32)(val64 >> 32); - offset += 4; - } else { - kctx->reg_dump[offset + 1] = kbase_reg_read32(kctx->kbdev, reg_enum); - offset += 2; - } - } - return true; -} - -#endif diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_defs.h b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_defs.h index bff0b7235736..fa745bf6b471 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_defs.h +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_defs.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -98,12 +98,6 @@ struct slot_rb { * The hwaccess_lock (a spinlock) must be held when accessing this structure */ struct kbase_backend_data { -#if !MALI_USE_CSF - struct slot_rb slot_rb[BASE_JM_MAX_NR_SLOTS]; - struct hrtimer scheduling_timer; - - bool timer_running; -#endif bool suspend_timer; atomic_t reset_gpu; diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_devfreq.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_devfreq.c index 22be72239fe3..db6a475a5301 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_devfreq.c +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_devfreq.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2025 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -195,7 +195,7 @@ static int kbase_devfreq_status(struct device *dev, struct devfreq_dev_status *s stat->current_frequency = kbdev->current_nominal_freq; stat->private_data = NULL; -#if MALI_USE_CSF && defined CONFIG_DEVFREQ_THERMAL +#if defined CONFIG_DEVFREQ_THERMAL if (!kbdev->devfreq_profile.is_cooling_device) kbase_ipa_reset_data(kbdev); #endif @@ -399,7 +399,6 @@ static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev) opp_freq); continue; } -#if MALI_USE_CSF if (kbase_csf_dev_has_ne(kbdev)) { u64 neural_present = kbdev->gpu_props.neural_present; u64 sc_with_ne = shader_present & neural_present; @@ -423,7 +422,6 @@ static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev) continue; } } -#endif /* MALI_USE_CSF */ core_count_p = of_get_property(node, "opp-core-count", NULL); if (core_count_p) { diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_gpuprops_backend.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_gpuprops_backend.c index 1cd3dafcb452..cdca4d3ed6b8 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_gpuprops_backend.c +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_gpuprops_backend.c @@ -44,22 +44,10 @@ int kbase_backend_gpuprops_get(struct kbase_device *kbdev, struct kbasep_gpuprop if (kbase_reg_is_valid(kbdev, GPU_CONTROL_ENUM(STACK_PRESENT))) regdump->stack_present = kbase_reg_read64(kbdev, GPU_CONTROL_ENUM(STACK_PRESENT)); -#if !MALI_USE_CSF - regdump->js_present = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(JS_PRESENT)); - /* Not a valid register on TMIX */ - - /* TGOx specific register */ - if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_THREAD_TLS_ALLOC)) - regdump->thread_tls_alloc = - kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(THREAD_TLS_ALLOC)); -#endif /* !MALI_USE_CSF */ - regdump->thread_max_threads = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(THREAD_MAX_THREADS)); if (kbase_reg_is_valid(kbdev, GPU_CONTROL_ENUM(THREAD_MAX_WORKGROUP_SIZE))) regdump->thread_max_workgroup_size = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(THREAD_MAX_WORKGROUP_SIZE)); -#if MALI_USE_CSF -#endif /* MALI_USE_CSF */ if (kbase_reg_is_valid(kbdev, GPU_CONTROL_ENUM(THREAD_MAX_BARRIER_SIZE))) regdump->thread_max_barrier_size = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(THREAD_MAX_BARRIER_SIZE)); @@ -73,7 +61,6 @@ int kbase_backend_gpuprops_get(struct kbase_device *kbdev, struct kbasep_gpuprop if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_CORE_FEATURES)) regdump->core_features = KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(CORE_FEATURES)); -#if MALI_USE_CSF if (kbase_reg_is_valid(kbdev, GPU_CONTROL_ENUM(GPU_FEATURES))) regdump->gpu_features = KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(GPU_FEATURES)); /* Only applicable to GPUs with power control domain registers */ @@ -82,19 +69,11 @@ int kbase_backend_gpuprops_get(struct kbase_device *kbdev, struct kbasep_gpuprop regdump->neural_present = kbase_reg_read64(kbdev, HOST_POWER_ENUM(NEURAL_PRESENT)); } -#endif /* MALI_USE_CSF */ regdump->tiler_features = KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(TILER_FEATURES)); regdump->l2_features = KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(L2_FEATURES)); regdump->mem_features = KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(MEM_FEATURES)); regdump->mmu_features = KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(MMU_FEATURES)); -#if !MALI_USE_CSF - for (i = 0; i < GPU_MAX_JOB_SLOTS; i++) - regdump->js_features[i] = kbase_reg_read32(kbdev, GPU_JS_FEATURES_OFFSET(i)); -#endif /* !MALI_USE_CSF */ - -#if MALI_USE_CSF -#endif /* MALI_USE_CSF */ { for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++) regdump->texture_features[i] = @@ -131,14 +110,12 @@ int kbase_backend_gpuprops_get_l2_features(struct kbase_device *kbdev, regdump->l2_features = KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(L2_FEATURES)); regdump->l2_config = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(L2_CONFIG)); -#if MALI_USE_CSF if (kbase_hw_has_l2_slice_hash_feature(kbdev)) { uint i; for (i = 0; i < GPU_L2_SLICE_HASH_COUNT; i++) regdump->l2_slice_hash[i] = kbase_reg_read32(kbdev, GPU_L2_SLICE_HASH_OFFSET(i)); } -#endif /* MALI_USE_CSF */ if (!kbase_io_has_gpu(kbdev)) return -EIO; diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_instr_backend.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_instr_backend.c deleted file mode 100644 index 62bb507c64d1..000000000000 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_instr_backend.c +++ /dev/null @@ -1,479 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * GPU backend instrumentation APIs. - */ - -#include -#include -#include -#include -#include -#include - -#define WAIT_FOR_DUMP_TIMEOUT_MS 5000 - -static int wait_prfcnt_ready(struct kbase_device *kbdev) -{ - u32 val; - const u32 timeout_us = - kbase_get_timeout_ms(kbdev, KBASE_PRFCNT_ACTIVE_TIMEOUT) * USEC_PER_MSEC; - const int err = kbase_reg_poll32_timeout(kbdev, GPU_CONTROL_ENUM(GPU_STATUS), val, - !(val & GPU_STATUS_PRFCNT_ACTIVE), 0, timeout_us, - false); - if (err) { - dev_err(kbdev->dev, "PRFCNT_ACTIVE bit stuck\n"); - return -EBUSY; - } - return 0; -} - -int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbase_instr_hwcnt_enable *enable) -{ - unsigned long flags; - int err = -EINVAL; - u32 irq_mask; - u32 prfcnt_config; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - /* alignment failure */ - if ((enable->dump_buffer == 0ULL) || (enable->dump_buffer & (2048 - 1))) - return err; - - spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - - if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_DISABLED) { - /* Instrumentation is already enabled */ - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - return err; - } - - if (!kbase_io_has_gpu(kbdev)) { - /* GPU has been removed by Arbiter */ - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - return err; - } - - /* Enable interrupt */ - irq_mask = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_MASK)); - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_MASK), - irq_mask | PRFCNT_SAMPLE_COMPLETED); - - /* In use, this context is the owner */ - kbdev->hwcnt.kctx = kctx; - /* Remember the dump address so we can reprogram it later */ - kbdev->hwcnt.addr = enable->dump_buffer; - kbdev->hwcnt.addr_bytes = enable->dump_buffer_bytes; - - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - - /* Configure */ - prfcnt_config = (u32)kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT; -#ifdef CONFIG_MALI_VALHALL_PRFCNT_SET_SELECT_VIA_DEBUG_FS - prfcnt_config |= (u32)kbdev->hwcnt.backend.override_counter_set - << PRFCNT_CONFIG_SETSELECT_SHIFT; -#else - prfcnt_config |= (u32)enable->counter_set << PRFCNT_CONFIG_SETSELECT_SHIFT; -#endif - - /* Wait until prfcnt config register can be written */ - err = wait_prfcnt_ready(kbdev); - if (err) - return err; - - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(PRFCNT_CONFIG), - prfcnt_config | PRFCNT_CONFIG_MODE_OFF); - - /* Wait until prfcnt is disabled before writing configuration registers */ - err = wait_prfcnt_ready(kbdev); - if (err) - return err; - - kbase_reg_write64(kbdev, GPU_CONTROL_ENUM(PRFCNT_BASE), enable->dump_buffer); - - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(PRFCNT_JM_EN), enable->fe_bm); - - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(PRFCNT_SHADER_EN), enable->shader_bm); - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(PRFCNT_MMU_L2_EN), enable->mmu_l2_bm); - - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(PRFCNT_TILER_EN), enable->tiler_bm); - - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(PRFCNT_CONFIG), - prfcnt_config | PRFCNT_CONFIG_MODE_MANUAL); - - spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; - kbdev->hwcnt.backend.triggered = 1; - wake_up(&kbdev->hwcnt.backend.wait); - - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - - dev_dbg(kbdev->dev, "HW counters dumping set-up for context %pK", kctx); - return 0; -} - -static void kbasep_instr_hwc_disable_hw_prfcnt(struct kbase_device *kbdev) -{ - u32 irq_mask; - - lockdep_assert_held(&kbdev->hwaccess_lock); - lockdep_assert_held(&kbdev->hwcnt.lock); - - if (!kbase_io_has_gpu(kbdev)) - /* GPU has been removed by Arbiter */ - return; - - /* Disable interrupt */ - irq_mask = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_MASK)); - - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_MASK), - irq_mask & ~PRFCNT_SAMPLE_COMPLETED); - - /* Wait until prfcnt config register can be written, then disable the counters. - * Return value is ignored as we are disabling anyway. - */ - wait_prfcnt_ready(kbdev); - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(PRFCNT_CONFIG), 0); - - kbdev->hwcnt.kctx = NULL; - kbdev->hwcnt.addr = 0ULL; - kbdev->hwcnt.addr_bytes = 0ULL; -} - -int kbase_instr_hwcnt_disable_internal(struct kbase_context *kctx) -{ - unsigned long flags, pm_flags; - struct kbase_device *kbdev = kctx->kbdev; - const unsigned long timeout = msecs_to_jiffies(WAIT_FOR_DUMP_TIMEOUT_MS); - unsigned int remaining; - - while (1) { - spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); - spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - - if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_UNRECOVERABLE_ERROR) { - /* Instrumentation is in unrecoverable error state, - * there is nothing for us to do. - */ - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); - /* Already disabled, return no error. */ - return 0; - } - - if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DISABLED) { - /* Instrumentation is not enabled */ - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); - return -EINVAL; - } - - if (kbdev->hwcnt.kctx != kctx) { - /* Instrumentation has been setup for another context */ - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); - return -EINVAL; - } - - if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_IDLE) - break; - - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); - - /* Ongoing dump/setup - wait for its completion */ - remaining = wait_event_timeout(kbdev->hwcnt.backend.wait, - kbdev->hwcnt.backend.triggered != 0, timeout); - - if (remaining == 0) - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_UNRECOVERABLE_ERROR; - } - - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED; - kbdev->hwcnt.backend.triggered = 0; - - kbasep_instr_hwc_disable_hw_prfcnt(kbdev); - - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); - - dev_dbg(kbdev->dev, "HW counters dumping disabled for context %pK", kctx); - - return 0; -} - -int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx) -{ - unsigned long flags; - int err = -EINVAL; - struct kbase_device *kbdev = kctx->kbdev; - - spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - - if (kbdev->hwcnt.kctx != kctx) { - /* The instrumentation has been setup for another context */ - goto unlock; - } - - if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_IDLE) { - /* HW counters are disabled or another dump is ongoing, or we're - * resetting, or we are in unrecoverable error state. - */ - goto unlock; - } - - if (!kbase_io_has_gpu(kbdev)) { - /* GPU has been removed by Arbiter */ - goto unlock; - } - - kbdev->hwcnt.backend.triggered = 0; - - /* Mark that we're dumping - the PF handler can signal that we faulted - */ - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DUMPING; - - /* Wait until prfcnt is ready to request dump */ - err = wait_prfcnt_ready(kbdev); - if (err) - goto unlock; - - /* Reconfigure the dump address */ - kbase_reg_write64(kbdev, GPU_CONTROL_ENUM(PRFCNT_BASE), kbdev->hwcnt.addr); - - /* Start dumping */ - KBASE_KTRACE_ADD(kbdev, CORE_GPU_PRFCNT_SAMPLE, NULL, kbdev->hwcnt.addr); - - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_COMMAND), GPU_COMMAND_PRFCNT_SAMPLE); - - dev_dbg(kbdev->dev, "HW counters dumping done for context %pK", kctx); - -unlock: - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - return err; -} -KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_request_dump); - -bool kbase_instr_hwcnt_dump_complete(struct kbase_context *kctx, bool *const success) -{ - unsigned long flags; - bool complete = false; - struct kbase_device *kbdev = kctx->kbdev; - - spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - - if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_IDLE) { - *success = true; - complete = true; - } else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) { - *success = false; - complete = true; - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; - } - - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - - return complete; -} -KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_dump_complete); - -void kbase_instr_hwcnt_sample_done(struct kbase_device *kbdev) -{ - unsigned long flags; - - spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - - /* If the state is in unrecoverable error, we already wake_up the waiter - * and don't need to do any action when sample is done. - */ - - if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) { - kbdev->hwcnt.backend.triggered = 1; - wake_up(&kbdev->hwcnt.backend.wait); - } else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DUMPING) { - /* All finished and idle */ - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; - kbdev->hwcnt.backend.triggered = 1; - wake_up(&kbdev->hwcnt.backend.wait); - } - - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -} - -int kbase_instr_hwcnt_wait_for_dump(struct kbase_context *kctx) -{ - struct kbase_device *kbdev = kctx->kbdev; - unsigned long flags; - int err; - - unsigned long remaining; - const unsigned long timeout = msecs_to_jiffies(WAIT_FOR_DUMP_TIMEOUT_MS); - - /* Wait for dump & cache clean to complete */ - remaining = wait_event_timeout(kbdev->hwcnt.backend.wait, - kbdev->hwcnt.backend.triggered != 0, timeout); - if (remaining == 0) { - err = -ETIME; - /* Set the backend state so it's clear things have gone bad (could be a HW issue) - */ - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_UNRECOVERABLE_ERROR; - goto timed_out; - } - - spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - - if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) { - err = -EINVAL; - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; - } else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_UNRECOVERABLE_ERROR) { - err = -EIO; - } else { - /* Dump done */ - KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_IDLE); - err = 0; - } - - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -timed_out: - return err; -} - -int kbase_instr_hwcnt_clear(struct kbase_context *kctx) -{ - unsigned long flags; - int err = -EINVAL; - struct kbase_device *kbdev = kctx->kbdev; - - spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - - /* Check it's the context previously set up and we're not in IDLE - * state. - */ - if (kbdev->hwcnt.kctx != kctx || kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_IDLE) - goto unlock; - - if (!kbase_io_has_gpu(kbdev)) { - /* GPU has been removed by Arbiter */ - goto unlock; - } - - /* Wait until prfcnt is ready to clear */ - err = wait_prfcnt_ready(kbdev); - if (err) - goto unlock; - - /* Clear the counters */ - KBASE_KTRACE_ADD(kbdev, CORE_GPU_PRFCNT_CLEAR, NULL, 0); - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_COMMAND), GPU_COMMAND_PRFCNT_CLEAR); - -unlock: - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - return err; -} -KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_clear); - -void kbase_instr_hwcnt_on_unrecoverable_error(struct kbase_device *kbdev) -{ - unsigned long flags; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - - /* If we already in unrecoverable error state, early return. */ - if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_UNRECOVERABLE_ERROR) { - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - return; - } - - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_UNRECOVERABLE_ERROR; - - /* Need to disable HW if it's not disabled yet. */ - if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_DISABLED) - kbasep_instr_hwc_disable_hw_prfcnt(kbdev); - - /* Wake up any waiters. */ - kbdev->hwcnt.backend.triggered = 1; - wake_up(&kbdev->hwcnt.backend.wait); - - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -} -KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_on_unrecoverable_error); - -void kbase_instr_hwcnt_on_before_reset(struct kbase_device *kbdev) -{ - unsigned long flags; - - spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - - /* A reset is the only way to exit the unrecoverable error state */ - if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_UNRECOVERABLE_ERROR) - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED; - - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -} -KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_on_before_reset); - -int kbase_instr_backend_init(struct kbase_device *kbdev) -{ - spin_lock_init(&kbdev->hwcnt.lock); - - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED; - - init_waitqueue_head(&kbdev->hwcnt.backend.wait); - -#ifdef CONFIG_MALI_VALHALL_PRFCNT_SET_SELECT_VIA_DEBUG_FS -/* Use the build time option for the override default. */ -#if defined(CONFIG_MALI_VALHALL_PRFCNT_SET_SECONDARY) - kbdev->hwcnt.backend.override_counter_set = KBASE_HWCNT_PHYSICAL_SET_SECONDARY; -#elif defined(CONFIG_MALI_VALHALL_PRFCNT_SET_TERTIARY) - kbdev->hwcnt.backend.override_counter_set = KBASE_HWCNT_PHYSICAL_SET_TERTIARY; -#else - /* Default to primary */ - kbdev->hwcnt.backend.override_counter_set = KBASE_HWCNT_PHYSICAL_SET_PRIMARY; -#endif -#endif - return 0; -} - -void kbase_instr_backend_term(struct kbase_device *kbdev) -{ - CSTD_UNUSED(kbdev); -} - -#ifdef CONFIG_MALI_VALHALL_PRFCNT_SET_SELECT_VIA_DEBUG_FS -void kbase_instr_backend_debugfs_init(struct kbase_device *kbdev) -{ - /* No validation is done on the debugfs input. Invalid input could cause - * performance counter errors. This is acceptable since this is a debug - * only feature and users should know what they are doing. - * - * Valid inputs are the values accepted bythe SET_SELECT bits of the - * PRFCNT_CONFIG register as defined in the architecture specification. - */ - debugfs_create_u8("hwcnt_set_select", 0644, kbdev->mali_debugfs_directory, - (u8 *)&kbdev->hwcnt.backend.override_counter_set); -} -#endif diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_irq_linux.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_irq_linux.c index 11bb1c57ca96..05290d403ad5 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_irq_linux.c +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_irq_linux.c @@ -62,12 +62,8 @@ static irqreturn_t kbase_job_irq_handler(int irq, void *data) dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); -#if MALI_USE_CSF /* call the csf interrupt handler */ kbase_csf_interrupt(kbdev, val); -#else - kbase_job_done(kbdev, val); -#endif spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); @@ -108,7 +104,6 @@ static irqreturn_t kbase_mmu_irq_handler(int irq, void *data) return IRQ_HANDLED; } -#if MALI_USE_CSF static irqreturn_t kbase_pwr_irq_handler(int irq, void *data) { unsigned long flags; @@ -138,7 +133,6 @@ static irqreturn_t kbase_pwr_irq_handler(int irq, void *data) return irq_state; } -#endif /* MALI_USE_CSF */ static irqreturn_t kbase_gpuonly_irq_handler(int irq, void *data) @@ -181,7 +175,6 @@ static irqreturn_t kbase_gpuonly_irq_handler(int irq, void *data) static irqreturn_t kbase_gpu_irq_handler(int irq, void *data) { irqreturn_t irq_state = kbase_gpuonly_irq_handler(irq, data); -#if MALI_USE_CSF struct kbase_device *kbdev = kbase_untag(data); /* Skip if HOST_POWER page is not available */ @@ -189,7 +182,6 @@ static irqreturn_t kbase_gpu_irq_handler(int irq, void *data) if (kbase_pwr_irq_handler(irq, data) == IRQ_HANDLED) irq_state = IRQ_HANDLED; } -#endif /* MALI_USE_CSF */ return irq_state; } @@ -350,6 +342,40 @@ static enum hrtimer_restart kbasep_test_interrupt_timeout(struct hrtimer *timer) return HRTIMER_NORESTART; } +struct interrupt_masks { + u32 job; + u32 mmu; + u32 gpu; + u32 pwr; +}; + +static void store_interrupt_masks(struct interrupt_masks *masks, struct kbase_device *const kbdev) +{ + /* Store all interrupt masks */ + masks->job = kbase_reg_read32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_MASK)); + masks->mmu = kbase_reg_read32(kbdev, MMU_CONTROL_ENUM(IRQ_MASK)); + masks->gpu = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_MASK)); + if (kbdev->pm.backend.has_host_pwr_iface) + masks->pwr = kbase_reg_read32(kbdev, HOST_POWER_ENUM(PWR_IRQ_MASK)); + + /* Set all masks to 0 to disable all interrupt sources */ + kbase_reg_write32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_MASK), 0x0); + kbase_reg_write32(kbdev, MMU_CONTROL_ENUM(IRQ_MASK), 0x0); + kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_MASK), 0x0); + if (kbdev->pm.backend.has_host_pwr_iface) + kbase_reg_write32(kbdev, HOST_POWER_ENUM(PWR_IRQ_MASK), 0x0); +} + +static void restore_interrupt_masks(struct interrupt_masks *masks, struct kbase_device *const kbdev) +{ + /* Restore all interrupt masks to their previous values */ + kbase_reg_write32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_MASK), masks->job); + kbase_reg_write32(kbdev, MMU_CONTROL_ENUM(IRQ_MASK), masks->mmu); + kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_MASK), masks->gpu); + if (kbdev->pm.backend.has_host_pwr_iface) + kbase_reg_write32(kbdev, HOST_POWER_ENUM(PWR_IRQ_MASK), masks->pwr); +} + /** * validate_interrupt - Validate an interrupt * @kbdev: Kbase device @@ -364,9 +390,9 @@ static enum hrtimer_restart kbasep_test_interrupt_timeout(struct hrtimer *timer) static int validate_interrupt(struct kbase_device *const kbdev, u32 tag) { int err = 0; + struct interrupt_masks masks = { 0 }; irq_handler_t handler; const int irq = (kbdev->nr_irqs == 1) ? 0 : tag; - u32 old_mask_val; u16 mask_offset; u16 rawstat_offset; @@ -389,10 +415,8 @@ static int validate_interrupt(struct kbase_device *const kbdev, u32 tag) return -EINVAL; } - /* store old mask */ - old_mask_val = kbase_reg_read32(kbdev, mask_offset); - /* mask interrupts */ - kbase_reg_write32(kbdev, mask_offset, 0x0); + /* store masks and disable all possible interrupt sources */ + store_interrupt_masks(&masks, kbdev); if (kbdev->irqs[irq].irq) { /* release original handler and install test handler */ @@ -441,8 +465,8 @@ static int validate_interrupt(struct kbase_device *const kbdev, u32 tag) err = -EINVAL; } } - /* restore old mask */ - kbase_reg_write32(kbdev, mask_offset, old_mask_val); + /* restore old masks */ + restore_interrupt_masks(&masks, kbdev); return err; } @@ -487,7 +511,6 @@ int kbase_install_interrupts(struct kbase_device *kbdev) { u32 irq_index; -#if MALI_USE_CSF if (kbdev->gpu_props.gpu_id.arch_id >= GPU_ID_ARCH_MAKE(14, 8, 0)) { if (kbdev->nr_irqs != 1) { dev_err(kbdev->dev, "Incorrect number of irq entries (%u)", kbdev->nr_irqs); @@ -499,7 +522,6 @@ int kbase_install_interrupts(struct kbase_device *kbdev) return -EINVAL; } } -#endif /* MALI_USE_CSF */ for (irq_index = 0; irq_index < kbdev->nr_irqs; irq_index++) { const int result = request_irq(kbdev->irqs[irq_index].irq, kbase_get_interrupt_handler(kbdev, irq_index), diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_as.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_as.c deleted file mode 100644 index 960ac9905343..000000000000 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_as.c +++ /dev/null @@ -1,237 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Register backend context / address space management - */ - -#include -#include -#include - -/** - * assign_and_activate_kctx_addr_space - Assign an AS to a context - * @kbdev: Kbase device - * @kctx: Kbase context - * @current_as: Address Space to assign - * - * Assign an Address Space (AS) to a context, and add the context to the Policy. - * - * This includes - * setting up the global runpool_irq structure and the context on the AS, - * Activating the MMU on the AS, - * Allowing jobs to be submitted on the AS. - * - * Context: - * kbasep_js_kctx_info.jsctx_mutex held, - * kbasep_js_device_data.runpool_mutex held, - * AS transaction mutex held, - * Runpool IRQ lock held - */ -static void assign_and_activate_kctx_addr_space(struct kbase_device *kbdev, - struct kbase_context *kctx, - struct kbase_as *current_as) -{ - struct kbasep_js_device_data *js_devdata = &kbdev->js_data; - - CSTD_UNUSED(current_as); - - lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex); - lockdep_assert_held(&js_devdata->runpool_mutex); - lockdep_assert_held(&kbdev->hwaccess_lock); - -#if !MALI_USE_CSF - /* Attribute handling */ - kbasep_js_ctx_attr_runpool_retain_ctx(kbdev, kctx); -#endif - - /* Allow it to run jobs */ - kbasep_js_set_submit_allowed(js_devdata, kctx); - - kbase_js_runpool_inc_context_count(kbdev, kctx); -} - -bool kbase_backend_use_ctx_sched(struct kbase_device *kbdev, struct kbase_context *kctx, - unsigned int js) -{ - int i; - - if (kbdev->hwaccess.active_kctx[js] == kctx) { - /* Context is already active */ - return true; - } - - for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { - if (kbdev->as_to_kctx[i] == kctx) { - /* Context already has ASID - mark as active */ - return true; - } - } - - /* Context does not have address space assigned */ - return false; -} - -void kbase_backend_release_ctx_irq(struct kbase_device *kbdev, struct kbase_context *kctx) -{ - int as_nr = kctx->as_nr; - - if (as_nr == KBASEP_AS_NR_INVALID) { - WARN(1, "Attempting to release context without ASID\n"); - return; - } - - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (atomic_read(&kctx->refcount) != 1) { - WARN(1, "Attempting to release active ASID\n"); - return; - } - - kbasep_js_clear_submit_allowed(&kbdev->js_data, kctx); - - kbase_ctx_sched_release_ctx(kctx); - kbase_js_runpool_dec_context_count(kbdev, kctx); -} - -void kbase_backend_release_ctx_noirq(struct kbase_device *kbdev, struct kbase_context *kctx) -{ - CSTD_UNUSED(kbdev); - CSTD_UNUSED(kctx); -} - -int kbase_backend_find_and_release_free_address_space(struct kbase_device *kbdev, - struct kbase_context *kctx) -{ - struct kbasep_js_device_data *js_devdata; - struct kbasep_js_kctx_info *js_kctx_info; - unsigned long flags; - int i; - - js_devdata = &kbdev->js_data; - js_kctx_info = &kctx->jctx.sched_info; - - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - mutex_lock(&js_devdata->runpool_mutex); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { - struct kbasep_js_kctx_info *as_js_kctx_info; - struct kbase_context *as_kctx; - - as_kctx = kbdev->as_to_kctx[i]; - as_js_kctx_info = &as_kctx->jctx.sched_info; - - /* Don't release privileged or active contexts, or contexts with - * jobs running. - * Note that a context will have at least 1 reference (which - * was previously taken by kbasep_js_schedule_ctx()) until - * descheduled. - */ - if (as_kctx && !kbase_ctx_flag(as_kctx, KCTX_PRIVILEGED) && - atomic_read(&as_kctx->refcount) == 1) { - if (!kbase_ctx_sched_inc_refcount_nolock(as_kctx)) { - WARN(1, "Failed to retain active context\n"); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&js_devdata->runpool_mutex); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - - return KBASEP_AS_NR_INVALID; - } - - kbasep_js_clear_submit_allowed(js_devdata, as_kctx); - - /* Drop and retake locks to take the jsctx_mutex on the - * context we're about to release without violating lock - * ordering - */ - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&js_devdata->runpool_mutex); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - - /* Release context from address space */ - mutex_lock(&as_js_kctx_info->ctx.jsctx_mutex); - mutex_lock(&js_devdata->runpool_mutex); - - kbasep_js_runpool_release_ctx_nolock(kbdev, as_kctx); - - if (!kbase_ctx_flag(as_kctx, KCTX_SCHEDULED)) { - kbasep_js_runpool_requeue_or_kill_ctx(kbdev, as_kctx, true); - - mutex_unlock(&js_devdata->runpool_mutex); - mutex_unlock(&as_js_kctx_info->ctx.jsctx_mutex); - - return i; - } - - /* Context was retained while locks were dropped, - * continue looking for free AS - */ - mutex_unlock(&js_devdata->runpool_mutex); - mutex_unlock(&as_js_kctx_info->ctx.jsctx_mutex); - - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - mutex_lock(&js_devdata->runpool_mutex); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - } - } - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - mutex_unlock(&js_devdata->runpool_mutex); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - - return KBASEP_AS_NR_INVALID; -} - -bool kbase_backend_use_ctx(struct kbase_device *kbdev, struct kbase_context *kctx, int as_nr) -{ - struct kbasep_js_device_data *js_devdata; - struct kbase_as *new_address_space = NULL; - int js; - - js_devdata = &kbdev->js_data; - - for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { - if (kbdev->hwaccess.active_kctx[js] == kctx) { - WARN(1, "Context is already scheduled in\n"); - return false; - } - } - - new_address_space = &kbdev->as[as_nr]; - - lockdep_assert_held(&js_devdata->runpool_mutex); - lockdep_assert_held(&kbdev->mmu_hw_mutex); - lockdep_assert_held(&kbdev->hwaccess_lock); - - assign_and_activate_kctx_addr_space(kbdev, kctx, new_address_space); - - if (kbase_ctx_flag(kctx, KCTX_PRIVILEGED)) { - /* We need to retain it to keep the corresponding address space - */ - kbase_ctx_sched_retain_ctx_refcount(kctx); - } - - return true; -} diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_hw.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_hw.c deleted file mode 100644 index d815fd80ebb9..000000000000 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_hw.c +++ /dev/null @@ -1,1280 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Base kernel job manager APIs - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -static void kbasep_try_reset_gpu_early_locked(struct kbase_device *kbdev); -static u64 kbasep_apply_limited_core_mask(const struct kbase_device *kbdev, const u64 affinity, - const u64 limited_core_mask); - -static u64 kbase_job_write_affinity(struct kbase_device *kbdev, base_jd_core_req core_req, - unsigned int js, const u64 limited_core_mask) -{ - u64 affinity; - bool skip_affinity_check = false; - - if ((core_req & (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T)) == BASE_JD_REQ_T) { - /* Tiler-only atom, affinity value can be programed as 0 */ - affinity = 0; - skip_affinity_check = true; - } else if ((core_req & - (BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP))) { - affinity = kbdev->pm.backend.shaders_avail & kbdev->pm.debug_core_mask[js]; - - /* Bifrost onwards GPUs only have 1 coherent group which is equal to - * shader_present - */ - affinity &= kbdev->gpu_props.curr_config.shader_present; - } else { - /* Use all cores */ - affinity = kbdev->pm.backend.shaders_avail & kbdev->pm.debug_core_mask[js]; - } - - if (core_req & BASE_JD_REQ_LIMITED_CORE_MASK) { - /* Limiting affinity due to BASE_JD_REQ_LIMITED_CORE_MASK by applying the limited core mask. */ - affinity = kbasep_apply_limited_core_mask(kbdev, affinity, limited_core_mask); - } - - if (unlikely(!affinity && !skip_affinity_check)) { -#ifdef CONFIG_MALI_VALHALL_DEBUG - u64 shaders_ready = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER); - - WARN_ON(!(shaders_ready & kbdev->pm.backend.shaders_avail)); -#endif - - affinity = kbdev->pm.backend.shaders_avail; - - if (core_req & BASE_JD_REQ_LIMITED_CORE_MASK) { - /* Limiting affinity again to make sure it only enables shader cores with backed TLS memory. */ - affinity = - kbasep_apply_limited_core_mask(kbdev, affinity, limited_core_mask); - -#ifdef CONFIG_MALI_VALHALL_DEBUG - /* affinity should never be 0 */ - WARN_ON(!affinity); -#endif - } - } - - kbase_reg_write64(kbdev, JOB_SLOT_OFFSET(js, AFFINITY_NEXT), affinity); - - return affinity; -} - -static inline bool kbasep_jm_wait_js_free(struct kbase_device *kbdev, unsigned int js, - struct kbase_context *kctx) -{ - u32 val; - const u32 timeout_us = kbdev->js_data.js_free_wait_time_ms * USEC_PER_MSEC; - /* wait for the JS_COMMAND_NEXT register to reach the given status value */ - const int err = kbase_reg_poll32_timeout(kbdev, JOB_SLOT_OFFSET(js, COMMAND_NEXT), val, - !val, 0, timeout_us, false); - - if (!err) - return true; - - dev_err(kbdev->dev, "Timeout in waiting for job slot %u to become free for ctx %d_%u", js, - kctx->tgid, kctx->id); - - return false; -} - -int kbase_job_hw_submit(struct kbase_device *kbdev, struct kbase_jd_atom *katom, unsigned int js) -{ - struct kbase_context *kctx; - u32 cfg; - u64 jc_head = katom->jc; - u64 affinity; - struct slot_rb *ptr_slot_rb = &kbdev->hwaccess.backend.slot_rb[js]; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - kctx = katom->kctx; - - /* Command register must be available */ - if (!kbasep_jm_wait_js_free(kbdev, js, kctx)) - return -EPERM; - - dev_dbg(kctx->kbdev->dev, "Write JS_HEAD_NEXT 0x%llx for atom %pK\n", jc_head, - (void *)katom); - - kbase_reg_write64(kbdev, JOB_SLOT_OFFSET(js, HEAD_NEXT), jc_head); - - affinity = kbase_job_write_affinity(kbdev, katom->core_req, js, kctx->limited_core_mask); - - /* start MMU, medium priority, cache clean/flush on end, clean/flush on - * start - */ - cfg = (u32)kctx->as_nr; - - if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_FLUSH_REDUCTION) && - !(kbdev->serialize_jobs & KBASE_SERIALIZE_RESET)) - cfg |= JS_CONFIG_ENABLE_FLUSH_REDUCTION; - - if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_START)) { - /* Force a cache maintenance operation if the newly submitted - * katom to the slot is from a different kctx. For a JM GPU - * that has the feature KBASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER, - * applies a FLUSH_INV_SHADER_OTHER. Otherwise, do a - * FLUSH_CLEAN_INVALIDATE. - */ - u64 tagged_kctx = ptr_slot_rb->last_kctx_tagged; - - if (tagged_kctx != SLOT_RB_NULL_TAG_VAL && tagged_kctx != SLOT_RB_TAG_KCTX(kctx)) { - if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER)) - cfg |= JS_CONFIG_START_FLUSH_INV_SHADER_OTHER; - else - cfg |= JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE; - } else - cfg |= JS_CONFIG_START_FLUSH_NO_ACTION; - } else - cfg |= JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE; - - if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_END) && - !(kbdev->serialize_jobs & KBASE_SERIALIZE_RESET)) - cfg |= JS_CONFIG_END_FLUSH_NO_ACTION; - else if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_CLEAN_ONLY_SAFE)) - cfg |= JS_CONFIG_END_FLUSH_CLEAN; - else - cfg |= JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE; - - cfg |= JS_CONFIG_THREAD_PRI(8); - - if (katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED) - cfg |= JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK; - - if (!ptr_slot_rb->job_chain_flag) { - cfg |= JS_CONFIG_JOB_CHAIN_FLAG; - katom->atom_flags |= KBASE_KATOM_FLAGS_JOBCHAIN; - ptr_slot_rb->job_chain_flag = true; - } else { - katom->atom_flags &= ~KBASE_KATOM_FLAGS_JOBCHAIN; - ptr_slot_rb->job_chain_flag = false; - } - - kbase_reg_write32(kbdev, JOB_SLOT_OFFSET(js, CONFIG_NEXT), cfg); - - if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_FLUSH_REDUCTION)) - kbase_reg_write32(kbdev, JOB_SLOT_OFFSET(js, FLUSH_ID_NEXT), katom->flush_id); - - /* Write an approximate start timestamp. - * It's approximate because there might be a job in the HEAD register. - */ - katom->start_timestamp = ktime_get_raw(); - - /* GO ! */ - dev_dbg(kbdev->dev, "JS: Submitting atom %pK from ctx %pK to js[%d] with head=0x%llx", - katom, kctx, js, jc_head); - - KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, JM_SUBMIT, kctx, katom, jc_head, js, (u32)affinity); - - KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT(kbdev, kctx, js, kbase_jd_atom_id(kctx, katom), - TL_JS_EVENT_START); - - KBASE_TLSTREAM_TL_ATTRIB_ATOM_CONFIG(kbdev, katom, jc_head, affinity, cfg); - KBASE_TLSTREAM_TL_RET_CTX_LPU(kbdev, kctx, &kbdev->gpu_props.js_features[katom->slot_nr]); - KBASE_TLSTREAM_TL_RET_ATOM_AS(kbdev, katom, &kbdev->as[kctx->as_nr]); - KBASE_TLSTREAM_TL_RET_ATOM_LPU(kbdev, katom, &kbdev->gpu_props.js_features[js], - "ctx_nr,atom_nr"); - kbase_kinstr_jm_atom_hw_submit(katom); - - /* Update the slot's last katom submission kctx */ - ptr_slot_rb->last_kctx_tagged = SLOT_RB_TAG_KCTX(kctx); - - trace_sysgraph_gpu(SGR_SUBMIT, kctx->id, kbase_jd_atom_id(kctx, katom), js); - - kbase_reg_write32(kbdev, JOB_SLOT_OFFSET(js, COMMAND_NEXT), JS_COMMAND_START); - - return 0; -} - -/** - * kbasep_job_slot_update_head_start_timestamp - Update timestamp - * @kbdev: kbase device - * @js: job slot - * @end_timestamp: timestamp - * - * Update the start_timestamp of the job currently in the HEAD, based on the - * fact that we got an IRQ for the previous set of completed jobs. - * - * The estimate also takes into account the time the job was submitted, to - * work out the best estimate (which might still result in an over-estimate to - * the calculated time spent) - */ -static void kbasep_job_slot_update_head_start_timestamp(struct kbase_device *kbdev, unsigned int js, - ktime_t end_timestamp) -{ - ktime_t timestamp_diff; - struct kbase_jd_atom *katom; - - /* Checking the HEAD position for the job slot */ - katom = kbase_gpu_inspect(kbdev, js, 0); - if (katom != NULL) { - timestamp_diff = ktime_sub(end_timestamp, katom->start_timestamp); - if (ktime_to_ns(timestamp_diff) >= 0) { - /* Only update the timestamp if it's a better estimate - * than what's currently stored. This is because our - * estimate that accounts for the throttle time may be - * too much of an overestimate - */ - katom->start_timestamp = end_timestamp; - } - } -} - -/** - * kbasep_trace_tl_event_lpu_softstop - Call event_lpu_softstop timeline - * tracepoint - * @kbdev: kbase device - * @js: job slot - * - * Make a tracepoint call to the instrumentation module informing that - * softstop happened on given lpu (job slot). - */ -static void kbasep_trace_tl_event_lpu_softstop(struct kbase_device *kbdev, unsigned int js) -{ - KBASE_TLSTREAM_TL_EVENT_LPU_SOFTSTOP(kbdev, &kbdev->gpu_props.js_features[js]); -} - -void kbase_job_done(struct kbase_device *kbdev, u32 done) -{ - u32 count = 0; - ktime_t end_timestamp; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - KBASE_KTRACE_ADD_JM(kbdev, JM_IRQ, NULL, NULL, 0, done); - - end_timestamp = ktime_get_raw(); - - while (done) { - unsigned int i; - u32 failed = done >> 16; - - /* treat failed slots as finished slots */ - u32 finished = (done & 0xFFFF) | failed; - - /* Note: This is inherently unfair, as we always check for lower - * numbered interrupts before the higher numbered ones. - */ - i = (unsigned int)ffs((int)finished) - 1u; - - do { - u32 nr_done; - u32 active; - u32 completion_code = BASE_JD_EVENT_DONE; /* assume OK */ - u64 job_tail = 0; - - if (failed & (1u << i)) { - /* read out the job slot status code if the job - * slot reported failure - */ - completion_code = - kbase_reg_read32(kbdev, JOB_SLOT_OFFSET(i, STATUS)); - - if (completion_code == BASE_JD_EVENT_STOPPED) { - u64 job_head; - - KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT(kbdev, NULL, i, 0, - TL_JS_EVENT_SOFT_STOP); - - kbasep_trace_tl_event_lpu_softstop(kbdev, i); - - /* Soft-stopped job - read the value of - * JS_TAIL so that the job chain can - * be resumed - */ - job_tail = - kbase_reg_read64(kbdev, JOB_SLOT_OFFSET(i, TAIL)); - job_head = - kbase_reg_read64(kbdev, JOB_SLOT_OFFSET(i, HEAD)); - /* For a soft-stopped job chain js_tail should - * same as the js_head, but if not then the - * job chain was incorrectly marked as - * soft-stopped. In such case we should not - * be resuming the job chain from js_tail and - * report the completion_code as UNKNOWN. - */ - if (job_tail != job_head) - completion_code = BASE_JD_EVENT_UNKNOWN; - - } else if (completion_code == BASE_JD_EVENT_NOT_STARTED) { - /* PRLAM-10673 can cause a TERMINATED - * job to come back as NOT_STARTED, - * but the error interrupt helps us - * detect it - */ - completion_code = BASE_JD_EVENT_TERMINATED; - } - - kbase_gpu_irq_evict(kbdev, i, completion_code); - - /* Some jobs that encounter a BUS FAULT may - * result in corrupted state causing future - * jobs to hang. Reset GPU before allowing - * any other jobs on the slot to continue. - */ - if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TTRX_3076)) { - if (completion_code == BASE_JD_EVENT_JOB_BUS_FAULT) { - if (kbase_prepare_to_reset_gpu_locked( - kbdev, RESET_FLAGS_NONE)) - kbase_reset_gpu_locked(kbdev); - } - } - } - - kbase_reg_write32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_CLEAR), - done & ((1u << i) | (1u << (i + 16)))); - active = kbase_reg_read32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_JS_STATE)); - - if (((active >> i) & 1) == 0 && (((done >> (i + 16)) & 1) == 0)) { - /* There is a potential race we must work - * around: - * - * 1. A job slot has a job in both current and - * next registers - * 2. The job in current completes - * successfully, the IRQ handler reads - * RAWSTAT and calls this function with the - * relevant bit set in "done" - * 3. The job in the next registers becomes the - * current job on the GPU - * 4. Sometime before the JOB_IRQ_CLEAR line - * above the job on the GPU _fails_ - * 5. The IRQ_CLEAR clears the done bit but not - * the failed bit. This atomically sets - * JOB_IRQ_JS_STATE. However since both jobs - * have now completed the relevant bits for - * the slot are set to 0. - * - * If we now did nothing then we'd incorrectly - * assume that _both_ jobs had completed - * successfully (since we haven't yet observed - * the fail bit being set in RAWSTAT). - * - * So at this point if there are no active jobs - * left we check to see if RAWSTAT has a failure - * bit set for the job slot. If it does we know - * that there has been a new failure that we - * didn't previously know about, so we make sure - * that we record this in active (but we wait - * for the next loop to deal with it). - * - * If we were handling a job failure (i.e. done - * has the relevant high bit set) then we know - * that the value read back from - * JOB_IRQ_JS_STATE is the correct number of - * remaining jobs because the failed job will - * have prevented any futher jobs from starting - * execution. - */ - u32 rawstat = - kbase_reg_read32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_RAWSTAT)); - - if ((rawstat >> (i + 16)) & 1) { - /* There is a failed job that we've - * missed - add it back to active - */ - active |= (1u << i); - } - } - - dev_dbg(kbdev->dev, "Job ended with status 0x%08X\n", completion_code); - - nr_done = kbase_backend_nr_atoms_submitted(kbdev, i); - nr_done -= (active >> i) & 1; - nr_done -= (active >> (i + 16)) & 1; - - if (nr_done == 0 || nr_done > SLOT_RB_SIZE) { - dev_warn(kbdev->dev, "Spurious interrupt on slot %u", i); - - goto spurious; - } - - count += nr_done; - - while (nr_done) { - if (likely(nr_done == 1)) { - kbase_gpu_complete_hw(kbdev, i, completion_code, job_tail, - &end_timestamp); - kbase_jm_try_kick_all(kbdev); - } else { - /* More than one job has completed. - * Since this is not the last job being - * reported this time it must have - * passed. This is because the hardware - * will not allow further jobs in a job - * slot to complete until the failed job - * is cleared from the IRQ status. - */ - kbase_gpu_complete_hw(kbdev, i, BASE_JD_EVENT_DONE, 0, - &end_timestamp); -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) - /* Increment the end timestamp value by 1 ns to - * avoid having the same value for 'start_time_ns' - * and 'end_time_ns' for the 2nd atom whose job - * completion IRQ got merged with the 1st atom. - */ - end_timestamp = ktime_add(end_timestamp, ns_to_ktime(1)); -#endif - } - nr_done--; - } -spurious: - done = kbase_reg_read32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_RAWSTAT)); - - failed = done >> 16; - finished = (done & 0xFFFF) | failed; - if (done) - end_timestamp = ktime_get_raw(); - } while (finished & (1u << i)); - - kbasep_job_slot_update_head_start_timestamp(kbdev, i, end_timestamp); - } - - if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) == KBASE_RESET_GPU_COMMITTED) { - /* If we're trying to reset the GPU then we might be able to do - * it early (without waiting for a timeout) because some jobs - * have completed - */ - kbasep_try_reset_gpu_early_locked(kbdev); - } - KBASE_KTRACE_ADD_JM(kbdev, JM_IRQ_END, NULL, NULL, 0, count); -} - -void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, unsigned int js, - u32 action, base_jd_core_req core_reqs, - struct kbase_jd_atom *target_katom) -{ -#if KBASE_KTRACE_ENABLE - u32 status_reg_before; - u64 job_in_head_before; - u32 status_reg_after; - - WARN_ON(action & (~(u32)JS_COMMAND_MASK)); - - /* Check the head pointer */ - job_in_head_before = kbase_reg_read64(kbdev, JOB_SLOT_OFFSET(js, HEAD)); - status_reg_before = kbase_reg_read32(kbdev, JOB_SLOT_OFFSET(js, STATUS)); -#endif - - if (action == JS_COMMAND_SOFT_STOP) { - if (kbase_jd_katom_is_protected(target_katom)) { -#ifdef CONFIG_MALI_VALHALL_DEBUG - dev_dbg(kbdev->dev, - "Attempt made to soft-stop a job that cannot be soft-stopped. core_reqs = 0x%x", - (unsigned int)core_reqs); -#else - CSTD_UNUSED(core_reqs); -#endif /* CONFIG_MALI_VALHALL_DEBUG */ - return; - } - - /* We are about to issue a soft stop, so mark the atom as having - * been soft stopped - */ - target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_SOFT_STOPPED; - - /* Mark the point where we issue the soft-stop command */ - KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ISSUE(kbdev, target_katom); - - action = (target_katom->atom_flags & KBASE_KATOM_FLAGS_JOBCHAIN) ? - JS_COMMAND_SOFT_STOP_1 : - JS_COMMAND_SOFT_STOP_0; - } else if (action == JS_COMMAND_HARD_STOP) { - target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_HARD_STOPPED; - - action = (target_katom->atom_flags & KBASE_KATOM_FLAGS_JOBCHAIN) ? - JS_COMMAND_HARD_STOP_1 : - JS_COMMAND_HARD_STOP_0; - } - - kbase_reg_write32(kbdev, JOB_SLOT_OFFSET(js, COMMAND), action); - -#if KBASE_KTRACE_ENABLE - status_reg_after = kbase_reg_read32(kbdev, JOB_SLOT_OFFSET(js, STATUS)); - if (status_reg_after == BASE_JD_EVENT_ACTIVE) { - struct kbase_jd_atom *head; - struct kbase_context *head_kctx; - - head = kbase_gpu_inspect(kbdev, js, 0); - if (unlikely(!head)) { - dev_err(kbdev->dev, "Can't get a katom from js(%d)\n", js); - return; - } - head_kctx = head->kctx; - - if (status_reg_before == BASE_JD_EVENT_ACTIVE) - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_CHECK_HEAD, head_kctx, head, - job_in_head_before, js); - else - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, 0, js); - - switch (action) { - case JS_COMMAND_SOFT_STOP: - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP, head_kctx, head, head->jc, js); - break; - case JS_COMMAND_SOFT_STOP_0: - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP_0, head_kctx, head, head->jc, - js); - break; - case JS_COMMAND_SOFT_STOP_1: - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP_1, head_kctx, head, head->jc, - js); - break; - case JS_COMMAND_HARD_STOP: - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP, head_kctx, head, head->jc, js); - break; - case JS_COMMAND_HARD_STOP_0: - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_0, head_kctx, head, head->jc, - js); - break; - case JS_COMMAND_HARD_STOP_1: - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_1, head_kctx, head, head->jc, - js); - break; - default: - WARN(1, "Unknown action %d on atom %pK in kctx %pK\n", action, - (void *)target_katom, (void *)target_katom->kctx); - break; - } - } else { - if (status_reg_before == BASE_JD_EVENT_ACTIVE) - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, - job_in_head_before, js); - else - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, 0, js); - - switch (action) { - case JS_COMMAND_SOFT_STOP: - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP, NULL, NULL, 0, js); - break; - case JS_COMMAND_SOFT_STOP_0: - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP_0, NULL, NULL, 0, js); - break; - case JS_COMMAND_SOFT_STOP_1: - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP_1, NULL, NULL, 0, js); - break; - case JS_COMMAND_HARD_STOP: - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP, NULL, NULL, 0, js); - break; - case JS_COMMAND_HARD_STOP_0: - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_0, NULL, NULL, 0, js); - break; - case JS_COMMAND_HARD_STOP_1: - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_1, NULL, NULL, 0, js); - break; - default: - WARN(1, "Unknown action %d on atom %pK in kctx %pK\n", action, - (void *)target_katom, (void *)target_katom->kctx); - break; - } - } -#endif -} - -void kbase_backend_jm_kill_running_jobs_from_kctx(struct kbase_context *kctx) -{ - struct kbase_device *kbdev = kctx->kbdev; - unsigned int i; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) - kbase_job_slot_hardstop(kctx, i, NULL); -} - -void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx, - struct kbase_jd_atom *target_katom) -{ - struct kbase_device *kbdev; - unsigned int target_js = target_katom->slot_nr; - int i; - bool stop_sent = false; - - kbdev = kctx->kbdev; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - for (i = 0; i < kbase_backend_nr_atoms_on_slot(kbdev, target_js); i++) { - struct kbase_jd_atom *slot_katom; - - slot_katom = kbase_gpu_inspect(kbdev, target_js, i); - if (!slot_katom) - continue; - - if (kbase_js_atom_runs_before(kbdev, target_katom, slot_katom, - KBASE_ATOM_ORDERING_FLAG_SEQNR)) { - if (!stop_sent) - KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITIZED(kbdev, target_katom); - - kbase_job_slot_softstop(kbdev, target_js, slot_katom); - stop_sent = true; - } - } -} - -void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx) -{ - struct kbase_device *kbdev = kctx->kbdev; - unsigned long timeout = msecs_to_jiffies(ZAP_TIMEOUT); - - timeout = wait_event_timeout(kctx->jctx.zero_jobs_wait, kctx->jctx.job_nr == 0, - (long)timeout); - - if (timeout != 0) - timeout = wait_event_timeout(kctx->jctx.sched_info.ctx.is_scheduled_wait, - !kbase_ctx_flag(kctx, KCTX_SCHEDULED), (long)timeout); - - /* Neither wait timed out; all done! */ - if (timeout != 0) - goto exit; - - if (kbase_prepare_to_reset_gpu(kbdev, RESET_FLAGS_HWC_UNRECOVERABLE_ERROR)) { - dev_err(kbdev->dev, - "Issuing GPU soft-reset because jobs failed to be killed (within %d ms) as part of context termination (e.g. process exit)\n", - ZAP_TIMEOUT); - kbase_reset_gpu(kbdev); - } - - /* Wait for the reset to complete */ - kbase_reset_gpu_wait(kbdev); -exit: - dev_dbg(kbdev->dev, "Zap: Finished Context %pK", kctx); - - /* Ensure that the signallers of the waitqs have finished */ - mutex_lock(&kctx->jctx.lock); - mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); - mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); - mutex_unlock(&kctx->jctx.lock); -} - -u32 kbase_backend_get_current_flush_id(struct kbase_device *kbdev) -{ - u32 flush_id = 0; - - if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_FLUSH_REDUCTION)) { - mutex_lock(&kbdev->pm.lock); - if (kbase_io_is_gpu_powered(kbdev)) - flush_id = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(LATEST_FLUSH)); - mutex_unlock(&kbdev->pm.lock); - } - - return flush_id; -} - -int kbase_job_slot_init(struct kbase_device *kbdev) -{ - CSTD_UNUSED(kbdev); - return 0; -} -KBASE_EXPORT_TEST_API(kbase_job_slot_init); - -void kbase_job_slot_halt(struct kbase_device *kbdev) -{ - CSTD_UNUSED(kbdev); -} - -void kbase_job_slot_term(struct kbase_device *kbdev) -{ - CSTD_UNUSED(kbdev); -} -KBASE_EXPORT_TEST_API(kbase_job_slot_term); - -/** - * kbase_job_slot_softstop_swflags - Soft-stop a job with flags - * @kbdev: The kbase device - * @js: The job slot to soft-stop - * @target_katom: The job that should be soft-stopped (or NULL for any job) - * @sw_flags: Flags to pass in about the soft-stop - * - * Context: - * The job slot lock must be held when calling this function. - * The job slot must not already be in the process of being soft-stopped. - * - * Soft-stop the specified job slot, with extra information about the stop - * - * Where possible any job in the next register is evicted before the soft-stop. - */ -void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, unsigned int js, - struct kbase_jd_atom *target_katom, u32 sw_flags) -{ - dev_dbg(kbdev->dev, "Soft-stop atom %pK with flags 0x%x (s:%d)\n", target_katom, sw_flags, - js); - - if (sw_flags & JS_COMMAND_MASK) { - WARN(true, "Atom %pK in kctx %pK received non-NOP flags %d\n", (void *)target_katom, - target_katom ? (void *)target_katom->kctx : NULL, sw_flags); - sw_flags &= ~((u32)JS_COMMAND_MASK); - } - kbase_backend_soft_hard_stop_slot(kbdev, NULL, js, target_katom, - JS_COMMAND_SOFT_STOP | sw_flags); -} - -void kbase_job_slot_softstop(struct kbase_device *kbdev, unsigned int js, - struct kbase_jd_atom *target_katom) -{ - kbase_job_slot_softstop_swflags(kbdev, js, target_katom, 0u); -} - -void kbase_job_slot_hardstop(struct kbase_context *kctx, unsigned int js, - struct kbase_jd_atom *target_katom) -{ - struct kbase_device *kbdev = kctx->kbdev; - bool stopped; - - stopped = kbase_backend_soft_hard_stop_slot(kbdev, kctx, js, target_katom, - JS_COMMAND_HARD_STOP); - CSTD_UNUSED(stopped); -} - -void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action, - base_jd_core_req core_reqs, struct kbase_jd_atom *target_katom) -{ - u32 hw_action = action & JS_COMMAND_MASK; - - CSTD_UNUSED(core_reqs); - - /* For soft-stop, don't enter if soft-stop not allowed, or isn't - * causing disjoint. - */ - if (hw_action == JS_COMMAND_SOFT_STOP && (kbase_jd_katom_is_protected(target_katom) || - (0 == (action & JS_COMMAND_SW_CAUSES_DISJOINT)))) - return; - - /* Nothing to do if already logged disjoint state on this atom */ - if (target_katom->atom_flags & KBASE_KATOM_FLAG_IN_DISJOINT) - return; - - target_katom->atom_flags |= KBASE_KATOM_FLAG_IN_DISJOINT; - kbase_disjoint_state_up(kbdev); -} - -void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, struct kbase_jd_atom *target_katom) -{ - if (target_katom->atom_flags & KBASE_KATOM_FLAG_IN_DISJOINT) { - target_katom->atom_flags &= ~KBASE_KATOM_FLAG_IN_DISJOINT; - kbase_disjoint_state_down(kbdev); - } -} - -int kbase_reset_gpu_prevent_and_wait(struct kbase_device *kbdev) -{ - CSTD_UNUSED(kbdev); - WARN(true, "%s Not implemented for JM GPUs", __func__); - return -EINVAL; -} - -int kbase_reset_gpu_try_prevent(struct kbase_device *kbdev) -{ - CSTD_UNUSED(kbdev); - WARN(true, "%s Not implemented for JM GPUs", __func__); - return -EINVAL; -} - -void kbase_reset_gpu_allow(struct kbase_device *kbdev) -{ - CSTD_UNUSED(kbdev); - WARN(true, "%s Not implemented for JM GPUs", __func__); -} - -void kbase_reset_gpu_assert_prevented(struct kbase_device *kbdev) -{ - CSTD_UNUSED(kbdev); - WARN(true, "%s Not implemented for JM GPUs", __func__); -} - -void kbase_reset_gpu_assert_failed_or_prevented(struct kbase_device *kbdev) -{ - CSTD_UNUSED(kbdev); - WARN(true, "%s Not implemented for JM GPUs", __func__); -} - -static void kbase_debug_dump_registers(struct kbase_device *kbdev) -{ - unsigned int i; - - kbase_io_history_dump(kbdev); - - dev_err(kbdev->dev, "Register state:"); - dev_err(kbdev->dev, " GPU_IRQ_RAWSTAT=0x%08x GPU_STATUS=0x%08x", - kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_RAWSTAT)), - kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_STATUS))); - dev_err(kbdev->dev, " JOB_IRQ_RAWSTAT=0x%08x JOB_IRQ_JS_STATE=0x%08x", - kbase_reg_read32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_RAWSTAT)), - kbase_reg_read32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_JS_STATE))); - for (i = 0; i < 3; i++) { - dev_err(kbdev->dev, " JS%u_STATUS=0x%08x JS%u_HEAD=0x%016llx", i, - kbase_reg_read32(kbdev, JOB_SLOT_OFFSET(i, STATUS)), i, - kbase_reg_read64(kbdev, JOB_SLOT_OFFSET(i, HEAD))); - } - dev_err(kbdev->dev, " MMU_IRQ_RAWSTAT=0x%08x GPU_FAULTSTATUS=0x%08x", - kbase_reg_read32(kbdev, MMU_CONTROL_ENUM(IRQ_RAWSTAT)), - kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_FAULTSTATUS))); - dev_err(kbdev->dev, " GPU_IRQ_MASK=0x%08x JOB_IRQ_MASK=0x%08x MMU_IRQ_MASK=0x%08x", - kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_MASK)), - kbase_reg_read32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_MASK)), - kbase_reg_read32(kbdev, MMU_CONTROL_ENUM(IRQ_MASK))); - dev_err(kbdev->dev, " PWR_OVERRIDE0=0x%08x PWR_OVERRIDE1=0x%08x", - kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(PWR_OVERRIDE0)), - kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(PWR_OVERRIDE1))); - dev_err(kbdev->dev, " SHADER_CONFIG=0x%08x L2_MMU_CONFIG=0x%08x", - kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(SHADER_CONFIG)), - kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(L2_MMU_CONFIG))); - dev_err(kbdev->dev, " TILER_CONFIG=0x%08x JM_CONFIG=0x%08x", - kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(TILER_CONFIG)), - kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(JM_CONFIG))); -} - -static void kbasep_reset_timeout_worker(struct work_struct *data) -{ - unsigned long flags; - struct kbase_device *kbdev; - ktime_t end_timestamp = ktime_get_raw(); - struct kbasep_js_device_data *js_devdata; - bool silent = false; - - kbdev = container_of(data, struct kbase_device, hwaccess.backend.reset_work); - - js_devdata = &kbdev->js_data; - - if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) == KBASE_RESET_GPU_SILENT) - silent = true; - - if (kbase_is_quick_reset_enabled(kbdev)) - silent = true; - - KBASE_KTRACE_ADD_JM(kbdev, JM_BEGIN_RESET_WORKER, NULL, NULL, 0u, 0); - - /* Disable GPU hardware counters. - * This call will block until counters are disabled. - */ - kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); - - /* Make sure the timer has completed - this cannot be done from - * interrupt context, so this cannot be done within - * kbasep_try_reset_gpu_early. - */ - hrtimer_cancel(&kbdev->hwaccess.backend.reset_timer); - - if (kbase_pm_context_active_handle_suspend(kbdev, - KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { - /* This would re-activate the GPU. Since it's already idle, - * there's no need to reset it - */ - atomic_set(&kbdev->hwaccess.backend.reset_gpu, KBASE_RESET_GPU_NOT_PENDING); - kbase_disjoint_state_down(kbdev); - wake_up(&kbdev->hwaccess.backend.reset_wait); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - return; - } - - WARN(kbdev->irq_reset_flush, "%s: GPU reset already in flight\n", __func__); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - spin_lock(&kbdev->mmu_mask_change); - kbase_pm_reset_start_locked(kbdev); - - /* We're about to flush out the IRQs and their bottom half's */ - kbdev->irq_reset_flush = true; - - /* Disable IRQ to avoid IRQ handlers to kick in after releasing the - * spinlock; this also clears any outstanding interrupts - */ - kbase_pm_disable_interrupts_nolock(kbdev); - - spin_unlock(&kbdev->mmu_mask_change); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - /* Ensure that any IRQ handlers have finished - * Must be done without any locks IRQ handlers will take - */ - kbase_synchronize_irqs(kbdev); - - /* Flush out any in-flight work items */ - kbase_flush_mmu_wqs(kbdev); - - /* The flush has completed so reset the active indicator */ - kbdev->irq_reset_flush = false; - - if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TMIX_8463)) { - u64 val; - const u32 timeout_us = - kbase_get_timeout_ms(kbdev, KBASE_CLEAN_CACHE_TIMEOUT) * USEC_PER_MSEC; - /* Ensure that L2 is not transitioning when we send the reset command */ - const int err = kbase_reg_poll64_timeout(kbdev, GPU_CONTROL_ENUM(L2_PWRTRANS), val, - !val, 0, timeout_us, false); - - WARN(err, "L2 power transition timed out while trying to reset\n"); - } - - mutex_lock(&kbdev->pm.lock); - /* We hold the pm lock, so there ought to be a current policy */ - if (unlikely(!kbdev->pm.backend.pm_current_policy)) - dev_warn(kbdev->dev, "No power policy set!"); - - /* All slot have been soft-stopped and we've waited - * SOFT_STOP_RESET_TIMEOUT for the slots to clear, at this point we - * assume that anything that is still left on the GPU is stuck there and - * we'll kill it when we reset the GPU - */ - - if (!silent) - dev_err(kbdev->dev, "Resetting GPU (allowing up to %d ms)", RESET_TIMEOUT); - - /* Output the state of some interesting registers to help in the - * debugging of GPU resets - */ - if (!silent) - kbase_debug_dump_registers(kbdev); - - /* Complete any jobs that were still on the GPU */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbdev->protected_mode = false; - if (!kbdev->pm.backend.protected_entry_transition_override) - kbase_backend_reset(kbdev, &end_timestamp); - kbase_pm_metrics_update(kbdev, NULL); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - /* Tell hardware counters a reset is about to occur. - * If the instr backend is in an unrecoverable error state (e.g. due to - * HW being unresponsive), this will transition the backend out of - * it, on the assumption a reset will fix whatever problem there was. - */ - kbase_instr_hwcnt_on_before_reset(kbdev); - - /* Reset the GPU */ - kbase_pm_init_hw(kbdev, 0); - - mutex_unlock(&kbdev->pm.lock); - - mutex_lock(&js_devdata->runpool_mutex); - - mutex_lock(&kbdev->mmu_hw_mutex); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_ctx_sched_restore_all_as(kbdev); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kbdev->mmu_hw_mutex); - - kbase_pm_enable_interrupts(kbdev); - - kbase_disjoint_state_down(kbdev); - - mutex_unlock(&js_devdata->runpool_mutex); - - mutex_lock(&kbdev->pm.lock); - - kbase_pm_reset_complete(kbdev); - - /* Find out what cores are required now */ - kbase_pm_update_cores_state(kbdev); - - /* Synchronously request and wait for those cores, because if - * instrumentation is enabled it would need them immediately. - */ - kbase_pm_wait_for_desired_state(kbdev); - - mutex_unlock(&kbdev->pm.lock); - - atomic_set(&kbdev->hwaccess.backend.reset_gpu, KBASE_RESET_GPU_NOT_PENDING); - - wake_up(&kbdev->hwaccess.backend.reset_wait); - if (!silent) - dev_err(kbdev->dev, "Reset complete"); - - /* Try submitting some jobs to restart processing */ - KBASE_KTRACE_ADD_JM(kbdev, JM_SUBMIT_AFTER_RESET, NULL, NULL, 0u, 0); - kbase_js_sched_all(kbdev); - - /* Process any pending slot updates */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_backend_slot_update(kbdev); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - kbase_pm_context_idle(kbdev); - - /* Re-enable GPU hardware counters */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - KBASE_KTRACE_ADD_JM(kbdev, JM_END_RESET_WORKER, NULL, NULL, 0u, 0); -} - -static enum hrtimer_restart kbasep_reset_timer_callback(struct hrtimer *timer) -{ - struct kbase_device *kbdev = - container_of(timer, struct kbase_device, hwaccess.backend.reset_timer); - - /* Reset still pending? */ - if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, KBASE_RESET_GPU_COMMITTED, - KBASE_RESET_GPU_HAPPENING) == KBASE_RESET_GPU_COMMITTED) - queue_work(kbdev->hwaccess.backend.reset_workq, - &kbdev->hwaccess.backend.reset_work); - - return HRTIMER_NORESTART; -} - -/* - * If all jobs are evicted from the GPU then we can reset the GPU - * immediately instead of waiting for the timeout to elapse - */ - -static void kbasep_try_reset_gpu_early_locked(struct kbase_device *kbdev) -{ - unsigned int i; - u32 pending_jobs = 0; - - /* Count the number of jobs */ - for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) - pending_jobs += kbase_backend_nr_atoms_submitted(kbdev, i); - - if (pending_jobs > 0) { - /* There are still jobs on the GPU - wait */ - return; - } - - /* To prevent getting incorrect registers when dumping failed job, - * skip early reset. - */ - if (atomic_read(&kbdev->job_fault_debug) > 0) - return; - - /* Check that the reset has been committed to (i.e. kbase_reset_gpu has - * been called), and that no other thread beat this thread to starting - * the reset - */ - if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, KBASE_RESET_GPU_COMMITTED, - KBASE_RESET_GPU_HAPPENING) != KBASE_RESET_GPU_COMMITTED) { - /* Reset has already occurred */ - return; - } - - queue_work(kbdev->hwaccess.backend.reset_workq, &kbdev->hwaccess.backend.reset_work); -} - -static void kbasep_try_reset_gpu_early(struct kbase_device *kbdev) -{ - unsigned long flags; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbasep_try_reset_gpu_early_locked(kbdev); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -} - -/** - * kbase_prepare_to_reset_gpu_locked - Prepare for resetting the GPU - * @kbdev: kbase device - * @flags: Bitfield indicating impact of reset (see flag defines) - * - * This function soft-stops all the slots to ensure that as many jobs as - * possible are saved. - * - * Return: boolean which should be interpreted as follows: - * true - Prepared for reset, kbase_reset_gpu_locked should be called. - * false - Another thread is performing a reset, kbase_reset_gpu should - * not be called. - */ -bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev, unsigned int flags) -{ - unsigned int i; - - if (kbase_io_is_aw_removed(kbdev)) { - /* GPU access has been removed, reset will be done by - * Arbiter instead - */ - return false; - } - - if (flags & RESET_FLAGS_HWC_UNRECOVERABLE_ERROR) - kbase_instr_hwcnt_on_unrecoverable_error(kbdev); - - if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, KBASE_RESET_GPU_NOT_PENDING, - KBASE_RESET_GPU_PREPARED) != KBASE_RESET_GPU_NOT_PENDING) { - /* Some other thread is already resetting the GPU */ - return false; - } - - kbase_disjoint_state_up(kbdev); - - for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) - kbase_job_slot_softstop(kbdev, i, NULL); - - return true; -} - -bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev, unsigned int flags) -{ - unsigned long lock_flags; - bool ret; - - spin_lock_irqsave(&kbdev->hwaccess_lock, lock_flags); - ret = kbase_prepare_to_reset_gpu_locked(kbdev, flags); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, lock_flags); - - return ret; -} -KBASE_EXPORT_TEST_API(kbase_prepare_to_reset_gpu); - -/* - * This function should be called after kbase_prepare_to_reset_gpu if it - * returns true. It should never be called without a corresponding call to - * kbase_prepare_to_reset_gpu. - * - * After this function is called (or not called if kbase_prepare_to_reset_gpu - * returned false), the caller should wait for - * kbdev->hwaccess.backend.reset_waitq to be signalled to know when the reset - * has completed. - */ -void kbase_reset_gpu(struct kbase_device *kbdev) -{ - /* Note this is an assert/atomic_set because it is a software issue for - * a race to be occurring here - */ - if (WARN_ON(atomic_read(&kbdev->hwaccess.backend.reset_gpu) != KBASE_RESET_GPU_PREPARED)) - return; - atomic_set(&kbdev->hwaccess.backend.reset_gpu, KBASE_RESET_GPU_COMMITTED); - - if (!kbase_is_quick_reset_enabled(kbdev)) - dev_err(kbdev->dev, - "Preparing to soft-reset GPU: Waiting (up to %d ms) for all jobs to complete soft-stop\n", - kbdev->reset_timeout_ms); - - hrtimer_start(&kbdev->hwaccess.backend.reset_timer, - HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms), HRTIMER_MODE_REL); - - /* Try resetting early */ - kbasep_try_reset_gpu_early(kbdev); -} -KBASE_EXPORT_TEST_API(kbase_reset_gpu); - -void kbase_reset_gpu_locked(struct kbase_device *kbdev) -{ - /* Note this is an assert/atomic_set because it is a software issue for - * a race to be occurring here - */ - if (WARN_ON(atomic_read(&kbdev->hwaccess.backend.reset_gpu) != KBASE_RESET_GPU_PREPARED)) - return; - atomic_set(&kbdev->hwaccess.backend.reset_gpu, KBASE_RESET_GPU_COMMITTED); - - if (!kbase_is_quick_reset_enabled(kbdev)) - dev_err(kbdev->dev, - "Preparing to soft-reset GPU: Waiting (up to %d ms) for all jobs to complete soft-stop\n", - kbdev->reset_timeout_ms); - hrtimer_start(&kbdev->hwaccess.backend.reset_timer, - HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms), HRTIMER_MODE_REL); - - /* Try resetting early */ - kbasep_try_reset_gpu_early_locked(kbdev); -} - -int kbase_reset_gpu_silent(struct kbase_device *kbdev) -{ - if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, KBASE_RESET_GPU_NOT_PENDING, - KBASE_RESET_GPU_SILENT) != KBASE_RESET_GPU_NOT_PENDING) { - /* Some other thread is already resetting the GPU */ - return -EAGAIN; - } - - kbase_disjoint_state_up(kbdev); - - queue_work(kbdev->hwaccess.backend.reset_workq, &kbdev->hwaccess.backend.reset_work); - - return 0; -} - -bool kbase_reset_gpu_is_active(struct kbase_device *kbdev) -{ - if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) == KBASE_RESET_GPU_NOT_PENDING) - return false; - - return true; -} - -bool kbase_reset_gpu_is_not_pending(struct kbase_device *kbdev) -{ - return atomic_read(&kbdev->hwaccess.backend.reset_gpu) == KBASE_RESET_GPU_NOT_PENDING; -} - -int kbase_reset_gpu_wait(struct kbase_device *kbdev) -{ - wait_event(kbdev->hwaccess.backend.reset_wait, - atomic_read(&kbdev->hwaccess.backend.reset_gpu) == KBASE_RESET_GPU_NOT_PENDING); - - return 0; -} -KBASE_EXPORT_TEST_API(kbase_reset_gpu_wait); - -int kbase_reset_gpu_init(struct kbase_device *kbdev) -{ - kbdev->hwaccess.backend.reset_workq = alloc_workqueue("Mali reset workqueue", 0, 1); - if (kbdev->hwaccess.backend.reset_workq == NULL) - return -ENOMEM; - - INIT_WORK(&kbdev->hwaccess.backend.reset_work, kbasep_reset_timeout_worker); - - hrtimer_init(&kbdev->hwaccess.backend.reset_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - kbdev->hwaccess.backend.reset_timer.function = kbasep_reset_timer_callback; - - return 0; -} - -void kbase_reset_gpu_term(struct kbase_device *kbdev) -{ - destroy_workqueue(kbdev->hwaccess.backend.reset_workq); -} - -static u64 kbasep_apply_limited_core_mask(const struct kbase_device *kbdev, const u64 affinity, - const u64 limited_core_mask) -{ - const u64 result = affinity & limited_core_mask; - -#ifdef CONFIG_MALI_VALHALL_DEBUG - dev_dbg(kbdev->dev, - "Limiting affinity due to BASE_JD_REQ_LIMITED_CORE_MASK from 0x%lx to 0x%lx (mask is 0x%lx)\n", - (unsigned long)affinity, (unsigned long)result, (unsigned long)limited_core_mask); -#else - CSTD_UNUSED(kbdev); -#endif - - return result; -} diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_internal.h b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_internal.h deleted file mode 100644 index 2dd651a096fd..000000000000 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_internal.h +++ /dev/null @@ -1,140 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2011-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Job Manager backend-specific low-level APIs. - */ - -#ifndef _KBASE_JM_HWACCESS_H_ -#define _KBASE_JM_HWACCESS_H_ - -#include -#include -#include - -#include -#include - -/** - * kbase_job_done_slot() - Complete the head job on a particular job-slot - * @kbdev: Device pointer - * @s: Job slot - * @completion_code: Completion code of job reported by GPU - * @job_tail: Job tail address reported by GPU - * @end_timestamp: Timestamp of job completion - */ -void kbase_job_done_slot(struct kbase_device *kbdev, int s, u32 completion_code, u64 job_tail, - ktime_t *end_timestamp); - -/** - * kbase_job_hw_submit() - Submit a job to the GPU - * @kbdev: Device pointer - * @katom: Atom to submit - * @js: Job slot to submit on - * - * The caller must check kbasep_jm_is_submit_slots_free() != false before - * calling this. - * - * The following locking conditions are made on the caller: - * - it must hold the hwaccess_lock - * - * Return: 0 if the job was successfully submitted to hardware, an error otherwise. - */ -int kbase_job_hw_submit(struct kbase_device *kbdev, struct kbase_jd_atom *katom, unsigned int js); - -#if !MALI_USE_CSF -/** - * kbasep_job_slot_soft_or_hard_stop_do_action() - Perform a soft or hard stop - * on the specified atom - * @kbdev: Device pointer - * @js: Job slot to stop on - * @action: The action to perform, either JS_COMMAND_HARD_STOP or - * JS_COMMAND_SOFT_STOP - * @core_reqs: Core requirements of atom to stop - * @target_katom: Atom to stop - * - * The following locking conditions are made on the caller: - * - it must hold the hwaccess_lock - */ -void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, unsigned int js, - u32 action, base_jd_core_req core_reqs, - struct kbase_jd_atom *target_katom); -#endif /* !MALI_USE_CSF */ - -/** - * kbase_backend_soft_hard_stop_slot() - Soft or hard stop jobs on a given job - * slot belonging to a given context. - * @kbdev: Device pointer - * @kctx: Context pointer. May be NULL - * @katom: Specific atom to stop. May be NULL - * @js: Job slot to hard stop - * @action: The action to perform, either JS_COMMAND_HARD_STOP or - * JS_COMMAND_SOFT_STOP - * - * If no context is provided then all jobs on the slot will be soft or hard - * stopped. - * - * If a katom is provided then only that specific atom will be stopped. In this - * case the kctx parameter is ignored. - * - * Jobs that are on the slot but are not yet on the GPU will be unpulled and - * returned to the job scheduler. - * - * Return: true if an atom was stopped, false otherwise - */ -bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, struct kbase_context *kctx, - unsigned int js, struct kbase_jd_atom *katom, u32 action); - -/** - * kbase_job_slot_init - Initialise job slot framework - * @kbdev: Device pointer - * - * Called on driver initialisation - * - * Return: 0 on success - */ -int kbase_job_slot_init(struct kbase_device *kbdev); - -/** - * kbase_job_slot_halt - Halt the job slot framework - * @kbdev: Device pointer - * - * Should prevent any further job slot processing - */ -void kbase_job_slot_halt(struct kbase_device *kbdev); - -/** - * kbase_job_slot_term - Terminate job slot framework - * @kbdev: Device pointer - * - * Called on driver termination - */ -void kbase_job_slot_term(struct kbase_device *kbdev); - -/** - * kbase_gpu_cache_clean - Cause a GPU cache clean & flush - * @kbdev: Device pointer - * - * Caller must not be in IRQ context - */ -void kbase_gpu_cache_clean(struct kbase_device *kbdev); - -#endif /* _KBASE_JM_HWACCESS_H_ */ diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_rb.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_rb.c deleted file mode 100644 index b211dd9748ca..000000000000 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_rb.c +++ /dev/null @@ -1,1793 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Register-based HW access backend specific APIs - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) -#include -#endif -#include -#include -#include -#include -#include - -/** - * SLOT_RB_EMPTY - Return whether the specified ringbuffer is empty. - * - * @rb: ring buffer - * - * Note: HW access lock must be held - */ -#define SLOT_RB_EMPTY(rb) (rb->write_idx == rb->read_idx) - -/** - * SLOT_RB_ENTRIES - Return number of atoms currently in the specified ringbuffer. - * - * @rb: ring buffer - * - * Note: HW access lock must be held - */ -#define SLOT_RB_ENTRIES(rb) ((int)(s8)(rb->write_idx - rb->read_idx)) - -static void kbase_gpu_release_atom(struct kbase_device *kbdev, struct kbase_jd_atom *katom, - ktime_t *end_timestamp); - -/** - * kbase_gpu_enqueue_atom - Enqueue an atom in the HW access ringbuffer - * @kbdev: Device pointer - * @katom: Atom to enqueue - * - * Context: Caller must hold the HW access lock - */ -static void kbase_gpu_enqueue_atom(struct kbase_device *kbdev, struct kbase_jd_atom *katom) -{ - struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[katom->slot_nr]; - - WARN_ON(SLOT_RB_ENTRIES(rb) >= SLOT_RB_SIZE); - - lockdep_assert_held(&kbdev->hwaccess_lock); - - rb->entries[rb->write_idx & SLOT_RB_MASK].katom = katom; - rb->write_idx++; - - katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED; -} - -/** - * kbase_gpu_dequeue_atom - Remove an atom from the HW access ringbuffer, once - * it has been completed - * @kbdev: Device pointer - * @js: Job slot to remove atom from - * @end_timestamp: Pointer to timestamp of atom completion. May be NULL, in - * which case current time will be used. - * - * Context: Caller must hold the HW access lock - * - * Return: Atom removed from ringbuffer - */ -static struct kbase_jd_atom *kbase_gpu_dequeue_atom(struct kbase_device *kbdev, unsigned int js, - ktime_t *end_timestamp) -{ - struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; - struct kbase_jd_atom *katom; - - if (SLOT_RB_EMPTY(rb)) { - WARN(1, "GPU ringbuffer unexpectedly empty\n"); - return NULL; - } - - lockdep_assert_held(&kbdev->hwaccess_lock); - - katom = rb->entries[rb->read_idx & SLOT_RB_MASK].katom; - - kbase_gpu_release_atom(kbdev, katom, end_timestamp); - - rb->read_idx++; - - katom->gpu_rb_state = KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB; - - return katom; -} - -struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, unsigned int js, int idx) -{ - struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - if ((SLOT_RB_ENTRIES(rb) - 1) < idx) - return NULL; /* idx out of range */ - - return rb->entries[(rb->read_idx + idx) & SLOT_RB_MASK].katom; -} - -struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev, unsigned int js) -{ - struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; - - if (SLOT_RB_EMPTY(rb)) - return NULL; - - return rb->entries[(rb->write_idx - 1) & SLOT_RB_MASK].katom; -} - -bool kbase_gpu_atoms_submitted_any(struct kbase_device *kbdev) -{ - unsigned int js; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { - int i; - - for (i = 0; i < SLOT_RB_SIZE; i++) { - struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); - - if (katom && katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) - return true; - } - } - return false; -} - -u32 kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, unsigned int js) -{ - u32 nr = 0; - int i; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - for (i = 0; i < SLOT_RB_SIZE; i++) { - struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); - - if (katom && (katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED)) - nr++; - } - - return nr; -} - -int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, unsigned int js) -{ - int nr = 0; - int i; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - for (i = 0; i < SLOT_RB_SIZE; i++) { - if (kbase_gpu_inspect(kbdev, js, i)) - nr++; - } - - return nr; -} - -static int kbase_gpu_nr_atoms_on_slot_min(struct kbase_device *kbdev, unsigned int js, - enum kbase_atom_gpu_rb_state min_rb_state) -{ - int nr = 0; - int i; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - for (i = 0; i < SLOT_RB_SIZE; i++) { - struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); - - if (katom && (katom->gpu_rb_state >= min_rb_state)) - nr++; - } - - return nr; -} - -/** - * check_secure_atom - Check if the given atom is in the given secure state and - * has a ringbuffer state of at least - * KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION - * @katom: Atom pointer - * @secure: Desired secure state - * - * Return: true if atom is in the given state, false otherwise - */ -static bool check_secure_atom(struct kbase_jd_atom *katom, bool secure) -{ - if (katom->gpu_rb_state >= KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION && - ((kbase_jd_katom_is_protected(katom) && secure) || - (!kbase_jd_katom_is_protected(katom) && !secure))) - return true; - - return false; -} - -/** - * kbase_gpu_check_secure_atoms - Check if there are any atoms in the given - * secure state in the ringbuffers of at least - * state - * KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE - * @kbdev: Device pointer - * @secure: Desired secure state - * - * Return: true if any atoms are in the given state, false otherwise - */ -static bool kbase_gpu_check_secure_atoms(struct kbase_device *kbdev, bool secure) -{ - unsigned int js; - - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { - int i; - - for (i = 0; i < SLOT_RB_SIZE; i++) { - struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); - - if (katom) { - if (check_secure_atom(katom, secure)) - return true; - } - } - } - - return false; -} - -int kbase_backend_slot_free(struct kbase_device *kbdev, unsigned int js) -{ - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) != KBASE_RESET_GPU_NOT_PENDING) { - /* The GPU is being reset - so prevent submission */ - return 0; - } - - return SLOT_RB_SIZE - kbase_backend_nr_atoms_on_slot(kbdev, js); -} - -/** - * trace_atom_completion_for_gpu_metrics - Report the completion of atom for the - * purpose of emitting power/gpu_work_period - * tracepoint. - * - * @katom: Pointer to the atom that completed execution on GPU. - * @end_timestamp: Pointer to the timestamp of atom completion. May be NULL, in - * which case current time will be used. - * - * The function would also report the start for an atom that was in the HEAD_NEXT - * register. - * - * Note: Caller must hold the HW access lock. - */ -static inline void trace_atom_completion_for_gpu_metrics(struct kbase_jd_atom *const katom, - ktime_t *end_timestamp) -{ -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) - u64 complete_ns; - struct kbase_context *kctx = katom->kctx; - struct kbase_jd_atom *queued = kbase_gpu_inspect(kctx->kbdev, katom->slot_nr, 1); - -#ifdef CONFIG_MALI_VALHALL_DEBUG - WARN_ON(!kbase_gpu_inspect(kctx->kbdev, katom->slot_nr, 0)); -#endif - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - if (unlikely(queued == katom)) - return; - - /* A protected atom and a non-protected atom cannot be in the RB_SUBMITTED - * state at the same time in the job slot ringbuffer. Atom submission state - * machine prevents the submission of a non-protected atom until all - * protected atoms have completed and GPU has exited the protected mode. - * This implies that if the queued atom is in RB_SUBMITTED state, it shall - * be a protected atom and so we can return early. - */ - if (unlikely(kbase_jd_katom_is_protected(katom))) - return; - - if (likely(end_timestamp)) - complete_ns = ktime_to_ns(*end_timestamp); - else - complete_ns = ktime_get_raw_ns(); - - kbase_gpu_metrics_ctx_end_activity(kctx, complete_ns); - if (queued && queued->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) - kbase_gpu_metrics_ctx_start_activity(queued->kctx, complete_ns); -#else - CSTD_UNUSED(katom); - CSTD_UNUSED(end_timestamp); -#endif -} - -static void kbase_gpu_release_atom(struct kbase_device *kbdev, struct kbase_jd_atom *katom, - ktime_t *end_timestamp) -{ - struct kbase_context *kctx = katom->kctx; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - switch (katom->gpu_rb_state) { - case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: - /* Should be impossible */ - WARN(1, "Attempting to release atom not in ringbuffer\n"); - break; - - case KBASE_ATOM_GPU_RB_SUBMITTED: - trace_atom_completion_for_gpu_metrics(katom, end_timestamp); - kbase_kinstr_jm_atom_hw_release(katom); - /* Inform power management at start/finish of atom so it can - * update its GPU utilisation metrics. Mark atom as not - * submitted beforehand. - */ - katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; - kbase_pm_metrics_update(kbdev, end_timestamp); - - /* Inform platform at start/finish of atom */ - kbasep_platform_event_atom_complete(katom); - - if (katom->core_req & BASE_JD_REQ_PERMON) - kbase_pm_release_gpu_cycle_counter_nolock(kbdev); - - KBASE_TLSTREAM_TL_NRET_ATOM_LPU(kbdev, katom, - &kbdev->gpu_props.js_features[katom->slot_nr]); - KBASE_TLSTREAM_TL_NRET_ATOM_AS(kbdev, katom, &kbdev->as[kctx->as_nr]); - KBASE_TLSTREAM_TL_NRET_CTX_LPU(kbdev, kctx, - &kbdev->gpu_props.js_features[katom->slot_nr]); - - /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ - fallthrough; - case KBASE_ATOM_GPU_RB_READY: - /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ - fallthrough; - case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: - break; - - case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: - if (kbase_jd_katom_is_protected(katom) && - (katom->protected_state.enter != KBASE_ATOM_ENTER_PROTECTED_CHECK) && - (katom->protected_state.enter != KBASE_ATOM_ENTER_PROTECTED_HWCNT)) { - kbase_pm_protected_override_disable(kbdev); - kbase_pm_update_cores_state_nolock(kbdev); - } - if (kbase_jd_katom_is_protected(katom) && - (katom->protected_state.enter == KBASE_ATOM_ENTER_PROTECTED_IDLE_L2)) - kbase_pm_protected_entry_override_disable(kbdev); - if (!kbase_jd_katom_is_protected(katom) && - (katom->protected_state.exit != KBASE_ATOM_EXIT_PROTECTED_CHECK) && - (katom->protected_state.exit != KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT)) { - kbase_pm_protected_override_disable(kbdev); - kbase_pm_update_cores_state_nolock(kbdev); - } - - if (katom->protected_state.enter != KBASE_ATOM_ENTER_PROTECTED_CHECK || - katom->protected_state.exit != KBASE_ATOM_EXIT_PROTECTED_CHECK) - kbdev->protected_mode_transition = false; - - /* If the atom is at KBASE_ATOM_ENTER_PROTECTED_HWCNT state, it means - * one of two events prevented it from progressing to the next state and - * ultimately reach protected mode: - * - hwcnts were enabled, and the atom had to schedule a worker to - * disable them. - * - the hwcnts were already disabled, but some other error occurred. - * In the first case, if the worker has not yet completed - * (kbdev->protected_mode_hwcnt_disabled == false), we need to re-enable - * them and signal to the worker they have already been enabled - */ - if (kbase_jd_katom_is_protected(katom) && - (katom->protected_state.enter == KBASE_ATOM_ENTER_PROTECTED_HWCNT)) { - kbdev->protected_mode_hwcnt_desired = true; - if (kbdev->protected_mode_hwcnt_disabled) { - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); - kbdev->protected_mode_hwcnt_disabled = false; - } - } - - /* If the atom has suspended hwcnt but has not yet entered - * protected mode, then resume hwcnt now. If the GPU is now in - * protected mode then hwcnt will be resumed by GPU reset so - * don't resume it here. - */ - if (kbase_jd_katom_is_protected(katom) && - ((katom->protected_state.enter == KBASE_ATOM_ENTER_PROTECTED_IDLE_L2) || - (katom->protected_state.enter == KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY) || - (katom->protected_state.enter == KBASE_ATOM_ENTER_PROTECTED_FINISHED))) { - WARN_ON(!kbdev->protected_mode_hwcnt_disabled); - kbdev->protected_mode_hwcnt_desired = true; - if (kbdev->protected_mode_hwcnt_disabled) { - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); - kbdev->protected_mode_hwcnt_disabled = false; - } - } - - if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TGOX_R1_1234)) { - if (katom->atom_flags & KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT) { - kbase_pm_protected_l2_override(kbdev, false); - katom->atom_flags &= ~KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT; - } - } - - /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ - fallthrough; - case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: - /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ - fallthrough; - case KBASE_ATOM_GPU_RB_WAITING_BLOCKED: - /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ - fallthrough; - case KBASE_ATOM_GPU_RB_RETURN_TO_JS: - break; - } - - katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED; - katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; -} - -static void kbase_gpu_mark_atom_for_return(struct kbase_device *kbdev, struct kbase_jd_atom *katom) -{ - lockdep_assert_held(&kbdev->hwaccess_lock); - - KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, JM_MARK_FOR_RETURN_TO_JS, katom->kctx, katom, - katom->jc, katom->slot_nr, katom->event_code); - kbase_gpu_release_atom(kbdev, katom, NULL); - katom->gpu_rb_state = KBASE_ATOM_GPU_RB_RETURN_TO_JS; -} - -/** - * other_slots_busy - Determine if any job slots other than @js are currently - * running atoms - * @kbdev: Device pointer - * @js: Job slot - * - * Return: true if any slots other than @js are busy, false otherwise - */ -static inline bool other_slots_busy(struct kbase_device *kbdev, unsigned int js) -{ - unsigned int slot; - - for (slot = 0; slot < kbdev->gpu_props.num_job_slots; slot++) { - if (slot == js) - continue; - - if (kbase_gpu_nr_atoms_on_slot_min(kbdev, slot, KBASE_ATOM_GPU_RB_SUBMITTED)) - return true; - } - - return false; -} - -static inline bool kbase_gpu_in_protected_mode(struct kbase_device *kbdev) -{ - return kbdev->protected_mode; -} - -static void kbase_gpu_disable_coherent(struct kbase_device *kbdev) -{ - lockdep_assert_held(&kbdev->hwaccess_lock); - - /* - * When entering into protected mode, we must ensure that the - * GPU is not operating in coherent mode as well. This is to - * ensure that no protected memory can be leaked. - */ - if (kbdev->system_coherency == COHERENCY_ACE) - kbase_cache_set_coherency_mode(kbdev, COHERENCY_ACE_LITE); -} - -static int kbase_gpu_protected_mode_enter(struct kbase_device *kbdev) -{ - int err = -EINVAL; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - WARN_ONCE(!kbdev->protected_ops, - "Cannot enter protected mode: protected callbacks not specified.\n"); - - if (kbdev->protected_ops) { - /* Switch GPU to protected mode */ - err = kbdev->protected_ops->protected_mode_enable(kbdev->protected_dev); - - if (err) { - dev_warn(kbdev->dev, "Failed to enable protected mode: %d\n", err); - } else { - kbdev->protected_mode = true; - kbase_ipa_protection_mode_switch_event(kbdev); - } - } - - return err; -} - -static int kbase_gpu_protected_mode_reset(struct kbase_device *kbdev) -{ - lockdep_assert_held(&kbdev->hwaccess_lock); - - WARN_ONCE(!kbdev->protected_ops, - "Cannot exit protected mode: protected callbacks not specified.\n"); - - if (!kbdev->protected_ops) - return -EINVAL; - - /* The protected mode disable callback will be called as part of reset - */ - return kbase_reset_gpu_silent(kbdev); -} - -static int kbase_jm_protected_entry(struct kbase_device *kbdev, struct kbase_jd_atom **katom, - int idx, unsigned int js) -{ - int err = 0; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - err = kbase_gpu_protected_mode_enter(kbdev); - - /* - * Regardless of result before this call, we are no longer - * transitioning the GPU. - */ - - kbdev->protected_mode_transition = false; - kbase_pm_protected_override_disable(kbdev); - kbase_pm_update_cores_state_nolock(kbdev); - - KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END(kbdev, kbdev); - if (err) { - /* - * Failed to switch into protected mode. - * - * At this point we expect: - * katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION && - * katom->protected_state.enter = KBASE_ATOM_ENTER_PROTECTED_FINISHED - * ==> - * kbdev->protected_mode_hwcnt_disabled = false - */ - katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID; - kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); - /* - * Only return if head atom or previous atom - * already removed - as atoms must be returned - * in order. - */ - if (idx == 0 || katom[0]->gpu_rb_state == KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { - kbase_gpu_dequeue_atom(kbdev, js, NULL); - kbase_jm_return_atom_to_js(kbdev, katom[idx]); - } - - return -EINVAL; - } - - /* - * Protected mode sanity checks. - */ - WARN(kbase_jd_katom_is_protected(katom[idx]) != kbase_gpu_in_protected_mode(kbdev), - "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)", - kbase_jd_katom_is_protected(katom[idx]), kbase_gpu_in_protected_mode(kbdev)); - katom[idx]->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; - - return err; -} - -static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, struct kbase_jd_atom **katom, - int idx, unsigned int js) -{ - int err = 0; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - switch (katom[idx]->protected_state.enter) { - case KBASE_ATOM_ENTER_PROTECTED_CHECK: - KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START(kbdev, kbdev); - /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV - * should ensure that we are not already transitiong, and that - * there are no atoms currently on the GPU. - */ - WARN_ON(kbdev->protected_mode_transition); - WARN_ON(kbase_gpu_atoms_submitted_any(kbdev)); - /* If hwcnt is disabled, it means we didn't clean up correctly - * during last exit from protected mode. - */ - WARN_ON(kbdev->protected_mode_hwcnt_disabled); - - katom[idx]->protected_state.enter = KBASE_ATOM_ENTER_PROTECTED_HWCNT; - - kbdev->protected_mode_transition = true; - - /* ***TRANSITION TO HIGHER STATE*** */ - fallthrough; - case KBASE_ATOM_ENTER_PROTECTED_HWCNT: - /* See if we can get away with disabling hwcnt atomically */ - kbdev->protected_mode_hwcnt_desired = false; - if (!kbdev->protected_mode_hwcnt_disabled) { - if (kbase_hwcnt_context_disable_atomic(kbdev->hwcnt_gpu_ctx)) - kbdev->protected_mode_hwcnt_disabled = true; - } - - /* We couldn't disable atomically, so kick off a worker */ - if (!kbdev->protected_mode_hwcnt_disabled) { - kbase_hwcnt_context_queue_work(kbdev->hwcnt_gpu_ctx, - &kbdev->protected_mode_hwcnt_disable_work); - return -EAGAIN; - } - - /* Once reaching this point GPU must be switched to protected - * mode or hwcnt re-enabled. - */ - - if (kbase_pm_protected_entry_override_enable(kbdev)) - return -EAGAIN; - - /* - * Not in correct mode, begin protected mode switch. - * Entering protected mode requires us to power down the L2, - * and drop out of fully coherent mode. - */ - katom[idx]->protected_state.enter = KBASE_ATOM_ENTER_PROTECTED_IDLE_L2; - - kbase_pm_protected_override_enable(kbdev); - /* - * Only if the GPU reset hasn't been initiated, there is a need - * to invoke the state machine to explicitly power down the - * shader cores and L2. - */ - if (!kbdev->pm.backend.protected_entry_transition_override) - kbase_pm_update_cores_state_nolock(kbdev); - - /* ***TRANSITION TO HIGHER STATE*** */ - fallthrough; - case KBASE_ATOM_ENTER_PROTECTED_IDLE_L2: - /* Avoid unnecessary waiting on non-ACE platforms. */ - if (kbdev->system_coherency == COHERENCY_ACE) { - if (kbdev->pm.backend.l2_always_on) { - /* - * If the GPU reset hasn't completed, then L2 - * could still be powered up. - */ - if (kbase_reset_gpu_is_active(kbdev)) - return -EAGAIN; - } - - if (kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2) || - kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2) || - !kbase_io_has_gpu(kbdev)) { - /* - * The L2 is still powered, wait for all - * the users to finish with it before doing - * the actual reset. - */ - return -EAGAIN; - } - } - - katom[idx]->protected_state.enter = KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY; - - /* ***TRANSITION TO HIGHER STATE*** */ - fallthrough; - case KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY: - /* - * When entering into protected mode, we must ensure that the - * GPU is not operating in coherent mode as well. This is to - * ensure that no protected memory can be leaked. - */ - kbase_gpu_disable_coherent(kbdev); - - kbase_pm_protected_entry_override_disable(kbdev); - - if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TGOX_R1_1234)) { - /* - * Power on L2 caches; this will also result in the - * correct value written to coherency enable register. - */ - kbase_pm_protected_l2_override(kbdev, true); - - /* - * Set the flag on the atom that additional - * L2 references are taken. - */ - katom[idx]->atom_flags |= KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT; - } - - katom[idx]->protected_state.enter = KBASE_ATOM_ENTER_PROTECTED_FINISHED; - - if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TGOX_R1_1234)) - return -EAGAIN; - - /* ***TRANSITION TO HIGHER STATE*** */ - fallthrough; - case KBASE_ATOM_ENTER_PROTECTED_FINISHED: - if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TGOX_R1_1234)) { - /* - * Check that L2 caches are powered and, if so, - * enter protected mode. - */ - if (kbdev->pm.backend.l2_state == KBASE_L2_ON) { - /* - * Remove additional L2 reference and reset - * the atom flag which denotes it. - */ - if (katom[idx]->atom_flags & KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT) { - kbase_pm_protected_l2_override(kbdev, false); - katom[idx]->atom_flags &= - ~KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT; - } - - err = kbase_jm_protected_entry(kbdev, katom, idx, js); - - if (err) - return err; - } else { - /* - * still waiting for L2 caches to power up - */ - return -EAGAIN; - } - } else { - err = kbase_jm_protected_entry(kbdev, katom, idx, js); - - if (err) - return err; - } - } - - return 0; -} - -static int kbase_jm_exit_protected_mode(struct kbase_device *kbdev, struct kbase_jd_atom **katom, - int idx, unsigned int js) -{ - int err = 0; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - switch (katom[idx]->protected_state.exit) { - case KBASE_ATOM_EXIT_PROTECTED_CHECK: - KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START(kbdev, kbdev); - /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV - * should ensure that we are not already transitiong, and that - * there are no atoms currently on the GPU. - */ - WARN_ON(kbdev->protected_mode_transition); - WARN_ON(kbase_gpu_atoms_submitted_any(kbdev)); - - /* - * Exiting protected mode requires a reset, but first the L2 - * needs to be powered down to ensure it's not active when the - * reset is issued. - */ - katom[idx]->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_IDLE_L2; - - kbdev->protected_mode_transition = true; - kbase_pm_protected_override_enable(kbdev); - kbase_pm_update_cores_state_nolock(kbdev); - - /* ***TRANSITION TO HIGHER STATE*** */ - fallthrough; - case KBASE_ATOM_EXIT_PROTECTED_IDLE_L2: - if (kbdev->pm.backend.l2_state != KBASE_L2_OFF) { - /* - * The L2 is still powered, wait for all the users to - * finish with it before doing the actual reset. - */ - return -EAGAIN; - } - katom[idx]->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_RESET; - - /* ***TRANSITION TO HIGHER STATE*** */ - fallthrough; - case KBASE_ATOM_EXIT_PROTECTED_RESET: - /* L2 cache has been turned off (which is needed prior to the reset of GPU - * to exit the protected mode), so the override flag can be safely cleared. - * Even if L2 cache is powered up again before the actual reset, it should - * not be an issue (there are no jobs running on the GPU). - */ - kbase_pm_protected_override_disable(kbdev); - - /* Issue the reset to the GPU */ - err = kbase_gpu_protected_mode_reset(kbdev); - - if (err == -EAGAIN) - return -EAGAIN; - - if (err) { - kbdev->protected_mode_transition = false; - - /* Failed to exit protected mode, fail atom */ - katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID; - kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); - /* Only return if head atom or previous atom - * already removed - as atoms must be returned in order - */ - if (idx == 0 || - katom[0]->gpu_rb_state == KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { - kbase_gpu_dequeue_atom(kbdev, js, NULL); - kbase_jm_return_atom_to_js(kbdev, katom[idx]); - } - - /* If we're exiting from protected mode, hwcnt must have - * been disabled during entry. - */ - WARN_ON(!kbdev->protected_mode_hwcnt_disabled); - kbdev->protected_mode_hwcnt_desired = true; - if (kbdev->protected_mode_hwcnt_disabled) { - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); - kbdev->protected_mode_hwcnt_disabled = false; - } - - return -EINVAL; - } - - katom[idx]->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT; - - /* ***TRANSITION TO HIGHER STATE*** */ - fallthrough; - case KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT: - /* A GPU reset is issued when exiting protected mode. Once the - * reset is done all atoms' state will also be reset. For this - * reason, if the atom is still in this state we can safely - * say that the reset has not completed i.e., we have not - * finished exiting protected mode yet. - */ - return -EAGAIN; - } - - return 0; -} - -void kbase_backend_slot_update(struct kbase_device *kbdev) -{ - unsigned int js; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (kbase_reset_gpu_is_active(kbdev) || !kbase_io_has_gpu(kbdev)) - return; - - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { - struct kbase_jd_atom *katom[2]; - int idx; - - katom[0] = kbase_gpu_inspect(kbdev, js, 0); - katom[1] = kbase_gpu_inspect(kbdev, js, 1); - WARN_ON(katom[1] && !katom[0]); - - for (idx = 0; idx < SLOT_RB_SIZE; idx++) { - bool cores_ready; -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) - bool trace_atom_submit_for_gpu_metrics = true; -#endif - int ret; - - if (!katom[idx]) - continue; - - switch (katom[idx]->gpu_rb_state) { - case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: - /* Should be impossible */ - WARN(1, "Attempting to update atom not in ringbuffer\n"); - break; - - case KBASE_ATOM_GPU_RB_WAITING_BLOCKED: - if (katom[idx]->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) - break; - - katom[idx]->gpu_rb_state = - KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV; - - /* ***TRANSITION TO HIGHER STATE*** */ - fallthrough; - case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: - if (kbase_gpu_check_secure_atoms( - kbdev, !kbase_jd_katom_is_protected(katom[idx]))) - break; - - if ((idx == 1) && (kbase_jd_katom_is_protected(katom[0]) != - kbase_jd_katom_is_protected(katom[1]))) - break; - - if (kbdev->protected_mode_transition) - break; - - katom[idx]->gpu_rb_state = - KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION; - - /* ***TRANSITION TO HIGHER STATE*** */ - fallthrough; - case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: - - /* - * Exiting protected mode must be done before - * the references on the cores are taken as - * a power down the L2 is required which - * can't happen after the references for this - * atom are taken. - */ - - if (!kbase_gpu_in_protected_mode(kbdev) && - kbase_jd_katom_is_protected(katom[idx])) { - /* Atom needs to transition into protected mode. */ - ret = kbase_jm_enter_protected_mode(kbdev, katom, idx, js); - if (ret) - break; - } else if (kbase_gpu_in_protected_mode(kbdev) && - !kbase_jd_katom_is_protected(katom[idx])) { - /* Atom needs to transition out of protected mode. */ - ret = kbase_jm_exit_protected_mode(kbdev, katom, idx, js); - if (ret) - break; - } - katom[idx]->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; - - /* Atom needs no protected mode transition. */ - - katom[idx]->gpu_rb_state = - KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE; - - /* ***TRANSITION TO HIGHER STATE*** */ - fallthrough; - case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: - if (katom[idx]->will_fail_event_code) { - kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); - /* Set EVENT_DONE so this atom will be - * completed, not unpulled. - */ - katom[idx]->event_code = BASE_JD_EVENT_DONE; - /* Only return if head atom or previous - * atom already removed - as atoms must - * be returned in order. - */ - if (idx == 0 || katom[0]->gpu_rb_state == - KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { - kbase_gpu_dequeue_atom(kbdev, js, NULL); - kbase_jm_return_atom_to_js(kbdev, katom[idx]); - } - break; - } - - cores_ready = kbase_pm_cores_requested(kbdev, true); - - if (!cores_ready) - break; - - katom[idx]->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; - - /* ***TRANSITION TO HIGHER STATE*** */ - fallthrough; - case KBASE_ATOM_GPU_RB_READY: - - if (idx == 1) { - enum kbase_atom_gpu_rb_state atom_0_gpu_rb_state = - katom[0]->gpu_rb_state; - -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) - trace_atom_submit_for_gpu_metrics = - (atom_0_gpu_rb_state == - KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB); -#endif - - /* Only submit if head atom or previous - * atom already submitted - */ - if ((atom_0_gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED && - atom_0_gpu_rb_state != - KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB)) - break; - - /* If intra-slot serialization in use - * then don't submit atom to NEXT slot - */ - if (kbdev->serialize_jobs & KBASE_SERIALIZE_INTRA_SLOT) - break; - } - - /* If inter-slot serialization in use then don't - * submit atom if any other slots are in use - */ - if ((kbdev->serialize_jobs & KBASE_SERIALIZE_INTER_SLOT) && - other_slots_busy(kbdev, js)) - break; - - /* Check if this job needs the cycle counter - * enabled before submission - */ - if (katom[idx]->core_req & BASE_JD_REQ_PERMON) - kbase_pm_request_gpu_cycle_counter_l2_is_on(kbdev); - - if (!kbase_job_hw_submit(kbdev, katom[idx], js)) { - katom[idx]->gpu_rb_state = KBASE_ATOM_GPU_RB_SUBMITTED; - - /* Inform power management at start/finish of - * atom so it can update its GPU utilisation - * metrics. - */ - kbase_pm_metrics_update(kbdev, - &katom[idx]->start_timestamp); - - /* Inform platform at start/finish of atom */ - kbasep_platform_event_atom_submit(katom[idx]); -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) - if (likely(trace_atom_submit_for_gpu_metrics && - !kbase_jd_katom_is_protected(katom[idx]))) - kbase_gpu_metrics_ctx_start_activity( - katom[idx]->kctx, - ktime_to_ns(katom[idx]->start_timestamp)); -#endif - } else { - if (katom[idx]->core_req & BASE_JD_REQ_PERMON) - kbase_pm_release_gpu_cycle_counter_nolock(kbdev); - - break; - } - - /* ***TRANSITION TO HIGHER STATE*** */ - fallthrough; - case KBASE_ATOM_GPU_RB_SUBMITTED: - break; - - case KBASE_ATOM_GPU_RB_RETURN_TO_JS: - /* Only return if head atom or previous atom - * already removed - as atoms must be returned - * in order - */ - if (idx == 0 || - katom[0]->gpu_rb_state == KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { - kbase_gpu_dequeue_atom(kbdev, js, NULL); - kbase_jm_return_atom_to_js(kbdev, katom[idx]); - } - break; - } - } - } -} - -void kbase_backend_run_atom(struct kbase_device *kbdev, struct kbase_jd_atom *katom) -{ - lockdep_assert_held(&kbdev->hwaccess_lock); - dev_dbg(kbdev->dev, "Backend running atom %pK\n", (void *)katom); - - kbase_gpu_enqueue_atom(kbdev, katom); - kbase_backend_slot_update(kbdev); -} - -/** - * kbase_rb_atom_might_depend - determine if one atom in the slot ringbuffer - * might depend on another from the same kctx - * @katom_a: dependee atom - * @katom_b: atom to query - * - * This can be used on atoms that belong to different slot ringbuffers - * - * Return: true if @katom_b might depend on @katom_a, false if it cannot depend. - */ -static inline bool kbase_rb_atom_might_depend(const struct kbase_jd_atom *katom_a, - const struct kbase_jd_atom *katom_b) -{ - if (katom_a->kctx != katom_b->kctx) - return false; - return (katom_b->pre_dep || (katom_b->atom_flags & (KBASE_KATOM_FLAG_X_DEP_BLOCKED | - KBASE_KATOM_FLAG_FAIL_BLOCKER))); -} - -static inline void kbase_gpu_remove_atom(struct kbase_device *kbdev, struct kbase_jd_atom *katom, - u32 action, bool disjoint) -{ - struct kbase_context *kctx = katom->kctx; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT; - kbase_gpu_mark_atom_for_return(kbdev, katom); - kbase_jsctx_slot_prio_blocked_set(kctx, katom->slot_nr, katom->sched_priority); - - if (disjoint) - kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, katom); -} - -/** - * kbase_gpu_irq_evict - evict a slot's JS_HEAD_NEXT atom from the HW if it is - * related to a failed JS_HEAD atom - * @kbdev: kbase device - * @js: job slot to check - * @completion_code: completion code of the failed atom - * - * Note: 'STOPPED' atoms are considered 'failed', as they are in the HW, but - * unlike other failure codes we _can_ re-run them. - * - * This forms step 1 in a 2-step process of removing any related atoms from a - * slot's JS_HEAD_NEXT (ringbuffer index 1), should there have - * been a 'failure' on an atom in JS_HEAD (ringbuffer index 0). - * - * This step only removes the atoms from the HW, and marks them as - * (potentially) ready to run again. - * - * Step 2 is on marking the JS_HEAD atom as complete - * (kbase_gpu_complete_hw()), to dequeue said atoms and return them to the JS - * as appropriate, or re-submit them. - * - * Hence, this function must evict at a minimum the atoms related to the atom - * in JS_HEAD that kbase_gpu_complete_hw() will also dequeue. It is acceptable - * if this function evicts more atoms than kbase_gpu_complete_hw() dequeues, as - * the next kbase_backend_slot_update() will resubmit any remaining. - * - * Return: true if an atom was evicted, false otherwise. - */ -bool kbase_gpu_irq_evict(struct kbase_device *kbdev, unsigned int js, u32 completion_code) -{ - struct kbase_jd_atom *katom; - struct kbase_jd_atom *next_katom; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - katom = kbase_gpu_inspect(kbdev, js, 0); - if (!katom) { - dev_err(kbdev->dev, "Can't get a katom from js(%u)\n", js); - return false; - } - next_katom = kbase_gpu_inspect(kbdev, js, 1); - - if (next_katom && next_katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED && - (kbase_rb_atom_might_depend(katom, next_katom) || - kbase_js_atom_runs_before(kbdev, katom, next_katom, 0u)) && - kbase_reg_read64(kbdev, JOB_SLOT_OFFSET(js, HEAD_NEXT)) != 0) { - kbase_reg_write32(kbdev, JOB_SLOT_OFFSET(js, COMMAND_NEXT), JS_COMMAND_NOP); - - if (completion_code == BASE_JD_EVENT_STOPPED) { - kbase_gpu_remove_atom(kbdev, next_katom, JS_COMMAND_SOFT_STOP, false); - KBASE_TLSTREAM_TL_NRET_ATOM_LPU( - kbdev, next_katom, - &kbdev->gpu_props.js_features[next_katom->slot_nr]); - KBASE_TLSTREAM_TL_NRET_ATOM_AS(kbdev, next_katom, - &kbdev->as[next_katom->kctx->as_nr]); - KBASE_TLSTREAM_TL_NRET_CTX_LPU( - kbdev, next_katom->kctx, - &kbdev->gpu_props.js_features[next_katom->slot_nr]); - } else { - next_katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; - - if (next_katom->core_req & BASE_JD_REQ_PERMON) - kbase_pm_release_gpu_cycle_counter_nolock(kbdev); - } - - /* On evicting the next_katom, the last submission kctx on the - * given job slot then reverts back to the one that owns katom. - * The aim is to enable the next submission that can determine - * if the read only shader core L1 cache should be invalidated. - */ - kbdev->hwaccess.backend.slot_rb[js].last_kctx_tagged = - SLOT_RB_TAG_KCTX(katom->kctx); - - return true; - } - - return false; -} - -/** - * kbase_gpu_complete_hw - complete the atom in a slot's JS_HEAD - * @kbdev: kbase device - * @js: job slot to check - * @completion_code: completion code of the completed atom - * @job_tail: value read from JS_TAIL, for STOPPED atoms - * @end_timestamp: pointer to approximate ktime value when the katom completed - * - * Among other operations, this also executes step 2 of a 2-step process of - * removing any related atoms from a slot's JS_HEAD_NEXT (ringbuffer index 1), - * should there have been a 'failure' on an atom in JS_HEAD (ringbuffer index - * 0). The first step is done in kbase_gpu_irq_evict(). - * - * Note: 'STOPPED' atoms are considered 'failed', as they are in the HW, but - * unlike other failure codes we _can_ re-run them. - * - * When the JS_HEAD atom is considered to be 'failed', then this will dequeue - * and return to the JS some (usually all) of the atoms evicted from the HW - * during the kbase_gpu_irq_evict() for that JS_HEAD atom. If it dequeues an - * atom, that atom must not have been running or must already be evicted, as - * otherwise we would be in the incorrect state of having an atom both running - * on the HW and returned to the JS. - */ - -void kbase_gpu_complete_hw(struct kbase_device *kbdev, unsigned int js, u32 completion_code, - u64 job_tail, ktime_t *end_timestamp) -{ - struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0); - struct kbase_context *kctx = NULL; - - if (unlikely(!katom)) { - dev_err(kbdev->dev, "Can't get a katom from js(%d)\n", js); - return; - } - - kctx = katom->kctx; - - dev_dbg(kbdev->dev, "Atom %pK completed on hw with code 0x%x and job_tail 0x%llx (s:%d)\n", - (void *)katom, completion_code, job_tail, js); - - lockdep_assert_held(&kbdev->hwaccess_lock); - - /* - * When a hard-stop is followed close after a soft-stop, the completion - * code may be set to STOPPED, even though the job is terminated - */ - if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TMIX_8438)) { - if (completion_code == BASE_JD_EVENT_STOPPED && - (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_HARD_STOPPED)) { - completion_code = BASE_JD_EVENT_TERMINATED; - } - } - - if ((katom->core_req & BASE_JD_REQ_SKIP_CACHE_END) && - completion_code != BASE_JD_EVENT_DONE && !(completion_code & BASE_JD_SW_EVENT)) { - /* When a job chain fails, on a T60x or when - * BASE_JD_REQ_SKIP_CACHE_END is set, the GPU cache is not - * flushed. To prevent future evictions causing possible memory - * corruption we need to flush the cache manually before any - * affected memory gets reused. - */ - katom->need_cache_flush_cores_retained = true; - } - - katom = kbase_gpu_dequeue_atom(kbdev, js, end_timestamp); - - if (completion_code == BASE_JD_EVENT_STOPPED) { - struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js, 0); - - /* - * Dequeue next atom from ringbuffers on same slot if required. - * This atom will already have been removed from the NEXT - * registers by kbase_gpu_soft_hard_stop_slot(), to ensure that - * the atoms on this slot are returned in the correct order. - */ - if (next_katom && kbase_js_atom_runs_before(kbdev, katom, next_katom, 0u)) { - WARN_ON(next_katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED); - kbase_gpu_dequeue_atom(kbdev, js, end_timestamp); - kbase_jm_return_atom_to_js(kbdev, next_katom); - } - } else if (completion_code != BASE_JD_EVENT_DONE) { - struct kbasep_js_device_data *js_devdata = &kbdev->js_data; - unsigned int i; - - if (!kbase_ctx_flag(katom->kctx, KCTX_DYING) && - !kbase_ctx_flag(katom->kctx, KCTX_PAGE_FAULT_REPORT_SKIP)) { - dev_warn(kbdev->dev, "error detected from slot %d, job status 0x%08x (%s)", - js, completion_code, kbase_gpu_exception_name(completion_code)); - - } - -#if KBASE_KTRACE_DUMP_ON_JOB_SLOT_ERROR != 0 - KBASE_KTRACE_DUMP(kbdev); -#endif - kbasep_js_clear_submit_allowed(js_devdata, katom->kctx); - - /* - * Remove all atoms on the same context from ringbuffers. This - * will not remove atoms that are already on the GPU, as these - * are guaranteed not to have fail dependencies on the failed - * atom. - */ - for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) { - struct kbase_jd_atom *katom_idx0 = kbase_gpu_inspect(kbdev, i, 0); - struct kbase_jd_atom *katom_idx1 = kbase_gpu_inspect(kbdev, i, 1); - - if (katom_idx0 && kbase_rb_atom_might_depend(katom, katom_idx0) && - katom_idx0->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { - /* Dequeue katom_idx0 from ringbuffer */ - kbase_gpu_dequeue_atom(kbdev, i, end_timestamp); - - if (katom_idx1 && kbase_rb_atom_might_depend(katom, katom_idx1) && - katom_idx0->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { - /* Dequeue katom_idx1 from ringbuffer */ - kbase_gpu_dequeue_atom(kbdev, i, end_timestamp); - - katom_idx1->event_code = BASE_JD_EVENT_STOPPED; - kbase_jm_return_atom_to_js(kbdev, katom_idx1); - } - katom_idx0->event_code = BASE_JD_EVENT_STOPPED; - kbase_jm_return_atom_to_js(kbdev, katom_idx0); - - } else if (katom_idx1 && kbase_rb_atom_might_depend(katom, katom_idx1) && - katom_idx1->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { - /* Can not dequeue this atom yet - will be - * dequeued when atom at idx0 completes - */ - katom_idx1->event_code = BASE_JD_EVENT_STOPPED; - kbase_gpu_mark_atom_for_return(kbdev, katom_idx1); - } - } - } - - KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, JM_JOB_DONE, kctx, katom, katom->jc, js, - completion_code); - - if (job_tail != 0 && job_tail != katom->jc) { - /* Some of the job has been executed */ - dev_dbg(kbdev->dev, "Update job chain address of atom %pK to resume from 0x%llx\n", - (void *)katom, job_tail); - - /* Some of the job has been executed, so we update the job chain address to where - * we should resume from - */ - katom->jc = job_tail; - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_UPDATE_HEAD, katom->kctx, katom, job_tail, js); - } - - /* Only update the event code for jobs that weren't cancelled */ - if (katom->event_code != BASE_JD_EVENT_JOB_CANCELLED) - katom->event_code = (enum base_jd_event_code)completion_code; - - /* Complete the job, and start new ones - * - * Also defer remaining work onto the workqueue: - * - Re-queue Soft-stopped jobs - * - For any other jobs, queue the job back into the dependency system - * - Schedule out the parent context if necessary, and schedule a new - * one in. - */ - if (kbdev->serialize_jobs & KBASE_SERIALIZE_RESET) - kbase_reset_gpu_silent(kbdev); - - if (completion_code == BASE_JD_EVENT_STOPPED) - katom = kbase_jm_return_atom_to_js(kbdev, katom); - else - katom = kbase_jm_complete(kbdev, katom, end_timestamp); - - if (katom) { - dev_dbg(kbdev->dev, "Cross-slot dependency %pK has become runnable.\n", - (void *)katom); - - /* Cross-slot dependency has now become runnable. Try to submit it. */ - - /* Check if there are lower priority jobs to soft stop */ - kbase_job_slot_ctx_priority_check_locked(kctx, katom); - - kbase_jm_try_kick(kbdev, 1UL << katom->slot_nr); - } - - /* For partial shader core off L2 cache flush */ - kbase_pm_update_state(kbdev); - - /* Job completion may have unblocked other atoms. Try to update all job - * slots - */ - kbase_backend_slot_update(kbdev); -} - -void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp) -{ - unsigned int js; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - /* Reset should always take the GPU out of protected mode */ - WARN_ON(kbase_gpu_in_protected_mode(kbdev)); - - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { - int atom_idx = 0; - int idx; - - for (idx = 0; idx < SLOT_RB_SIZE; idx++) { - struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, atom_idx); - bool keep_in_jm_rb = false; - - if (!katom) - break; - if (katom->protected_state.exit == KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT) { - /* protected mode sanity checks */ - WARN(kbase_jd_katom_is_protected(katom) != - kbase_gpu_in_protected_mode(kbdev), - "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)", - kbase_jd_katom_is_protected(katom), - kbase_gpu_in_protected_mode(kbdev)); - WARN(!(kbase_jd_katom_is_protected(katom) && js == 0) && - kbase_jd_katom_is_protected(katom), - "Protected atom on JS%u not supported", js); - } - if ((katom->gpu_rb_state < KBASE_ATOM_GPU_RB_SUBMITTED) && - !kbase_ctx_flag(katom->kctx, KCTX_DYING)) - keep_in_jm_rb = true; - - kbase_gpu_release_atom(kbdev, katom, NULL); - - /* - * If the atom wasn't on HW when the reset was issued - * then leave it in the RB and next time we're kicked - * it will be processed again from the starting state. - */ - if (kbase_io_has_gpu(kbdev) && keep_in_jm_rb) { - katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; - /* As the atom was not removed, increment the - * index so that we read the correct atom in the - * next iteration. - */ - atom_idx++; - continue; - } - - /* - * The atom was on the HW when the reset was issued - * all we can do is fail the atom. - */ - kbase_gpu_dequeue_atom(kbdev, js, NULL); - katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; - kbase_jm_complete(kbdev, katom, end_timestamp); - } - - /* Clear the slot's last katom submission kctx on reset */ - kbdev->hwaccess.backend.slot_rb[js].last_kctx_tagged = SLOT_RB_NULL_TAG_VAL; - } - - /* Re-enable GPU hardware counters if we're resetting from protected - * mode. - */ - kbdev->protected_mode_hwcnt_desired = true; - if (kbdev->protected_mode_hwcnt_disabled) { - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); - kbdev->protected_mode_hwcnt_disabled = false; - - KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END(kbdev, kbdev); - } - - kbdev->protected_mode_transition = false; - kbase_pm_protected_override_disable(kbdev); -} - -/** - * should_stop_next_atom - given a soft/hard stop action, determine if the next - * atom on a slot should be stopped - * @kbdev: kbase devices - * @head_katom: atom currently in the JS_HEAD - * @next_katom: atom currently in the JS_HEAD_NEXT - * @action: COMMAND_<...> action for soft/hard-stop - * - * This is used in cases where @head_katom is the target of the soft/hard-stop. - * It only makes sense to call this when @head_katom and @next_katom are from - * the same slot. - * - * Return: true if @next_katom should also be stopped with the given action, - * false otherwise - */ -static bool should_stop_next_atom(struct kbase_device *kbdev, - const struct kbase_jd_atom *head_katom, - const struct kbase_jd_atom *next_katom, u32 action) -{ - bool ret = false; - u32 hw_action = action & JS_COMMAND_MASK; - - switch (hw_action) { - case JS_COMMAND_SOFT_STOP: - ret = kbase_js_atom_runs_before(kbdev, head_katom, next_katom, 0u); - break; - case JS_COMMAND_HARD_STOP: - /* Unlike soft-stop, a hard-stop targeting a particular atom - * should not cause atoms from unrelated contexts to be - * removed - */ - ret = (head_katom->kctx == next_katom->kctx); - break; - default: - /* Other stop actions are possible, but the driver should not - * be generating them at this point in the call chain - */ - WARN(1, "Unexpected stop action: 0x%.8x", hw_action); - break; - } - return ret; -} - -static inline void kbase_gpu_stop_atom(struct kbase_device *kbdev, unsigned int js, - struct kbase_jd_atom *katom, u32 action) -{ - struct kbase_context *kctx = katom->kctx; - u32 hw_action = action & JS_COMMAND_MASK; - - kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, katom); - kbasep_job_slot_soft_or_hard_stop_do_action(kbdev, js, hw_action, katom->core_req, katom); - kbase_jsctx_slot_prio_blocked_set(kctx, js, katom->sched_priority); -} - -static int should_stop_x_dep_slot(struct kbase_jd_atom *katom) -{ - if (katom->x_post_dep) { - struct kbase_jd_atom *dep_atom = katom->x_post_dep; - - if (dep_atom->gpu_rb_state != KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB && - dep_atom->gpu_rb_state != KBASE_ATOM_GPU_RB_RETURN_TO_JS) - return (int)dep_atom->slot_nr; - } - return -1; -} - -bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, struct kbase_context *kctx, - unsigned int js, struct kbase_jd_atom *katom, u32 action) -{ - struct kbase_jd_atom *katom_idx0; - struct kbase_context *kctx_idx0 = NULL; - struct kbase_jd_atom *katom_idx1; - struct kbase_context *kctx_idx1 = NULL; - - bool katom_idx0_valid, katom_idx1_valid; - - bool ret = false; - - int stop_x_dep_idx0 = -1, stop_x_dep_idx1 = -1; - int prio_idx0 = 0, prio_idx1 = 0; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - katom_idx0 = kbase_gpu_inspect(kbdev, js, 0); - katom_idx1 = kbase_gpu_inspect(kbdev, js, 1); - - if (katom_idx0) { - kctx_idx0 = katom_idx0->kctx; - prio_idx0 = katom_idx0->sched_priority; - } - if (katom_idx1) { - kctx_idx1 = katom_idx1->kctx; - prio_idx1 = katom_idx1->sched_priority; - } - - if (katom) { - katom_idx0_valid = (katom_idx0 == katom); - if (katom_idx1) - katom_idx1_valid = (katom_idx1 == katom); - else - katom_idx1_valid = false; - } else { - katom_idx0_valid = (katom_idx0 && (!kctx || kctx_idx0 == kctx)); - katom_idx1_valid = (katom_idx1 && (!kctx || kctx_idx1 == kctx)); - } - /* If there's an atom in JS_HEAD_NEXT that we haven't already decided - * to stop, but we're stopping the JS_HEAD atom, see if they are - * related/ordered in some way that would require the same stop action - */ - if (!katom_idx1_valid && katom_idx0_valid && katom_idx1) - katom_idx1_valid = should_stop_next_atom(kbdev, katom_idx0, katom_idx1, action); - - if (katom_idx0_valid) - stop_x_dep_idx0 = should_stop_x_dep_slot(katom_idx0); - if (katom_idx1_valid) - stop_x_dep_idx1 = should_stop_x_dep_slot(katom_idx1); - - if (katom_idx0_valid) { - if (katom_idx0->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { - /* Simple case - just dequeue and return */ - kbase_gpu_dequeue_atom(kbdev, js, NULL); - if (katom_idx1_valid) { - kbase_gpu_dequeue_atom(kbdev, js, NULL); - katom_idx1->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT; - kbase_jm_return_atom_to_js(kbdev, katom_idx1); - kbase_jsctx_slot_prio_blocked_set(kctx_idx1, js, prio_idx1); - } - - katom_idx0->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT; - kbase_jm_return_atom_to_js(kbdev, katom_idx0); - kbase_jsctx_slot_prio_blocked_set(kctx_idx0, js, prio_idx0); - } else { - /* katom_idx0 is on GPU */ - if (katom_idx1_valid && - katom_idx1->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) { - /* katom_idx0 and katom_idx1 are on GPU */ - - if (kbase_reg_read32(kbdev, JOB_SLOT_OFFSET(js, COMMAND_NEXT)) == - 0) { - /* idx0 has already completed - stop - * idx1 if needed - */ - if (katom_idx1_valid) { - kbase_gpu_stop_atom(kbdev, js, katom_idx1, action); - ret = true; - } - } else { - /* idx1 is in NEXT registers - attempt - * to remove - */ - kbase_reg_write32(kbdev, JOB_SLOT_OFFSET(js, COMMAND_NEXT), - JS_COMMAND_NOP); - - if (kbase_reg_read64(kbdev, - JOB_SLOT_OFFSET(js, HEAD_NEXT)) != 0) { - /* idx1 removed successfully, - * will be handled in IRQ - */ - kbase_gpu_remove_atom(kbdev, katom_idx1, action, - true); - /* Revert the last_context. */ - kbdev->hwaccess.backend.slot_rb[js] - .last_kctx_tagged = - SLOT_RB_TAG_KCTX(katom_idx0->kctx); - - stop_x_dep_idx1 = - should_stop_x_dep_slot(katom_idx1); - - /* stop idx0 if still on GPU */ - kbase_gpu_stop_atom(kbdev, js, katom_idx0, action); - ret = true; - } else if (katom_idx1_valid) { - /* idx0 has already completed, - * stop idx1 if needed - */ - kbase_gpu_stop_atom(kbdev, js, katom_idx1, action); - ret = true; - } - } - } else if (katom_idx1_valid) { - /* idx1 not on GPU but must be dequeued*/ - - /* idx1 will be handled in IRQ */ - kbase_gpu_remove_atom(kbdev, katom_idx1, action, false); - /* stop idx0 */ - /* This will be repeated for anything removed - * from the next registers, since their normal - * flow was also interrupted, and this function - * might not enter disjoint state e.g. if we - * don't actually do a hard stop on the head - * atom - */ - kbase_gpu_stop_atom(kbdev, js, katom_idx0, action); - ret = true; - } else { - /* no atom in idx1 */ - /* just stop idx0 */ - kbase_gpu_stop_atom(kbdev, js, katom_idx0, action); - ret = true; - } - } - } else if (katom_idx1_valid) { - if (katom_idx1->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { - /* Mark for return */ - /* idx1 will be returned once idx0 completes */ - kbase_gpu_remove_atom(kbdev, katom_idx1, action, false); - } else { - /* idx1 is on GPU */ - if (kbase_reg_read32(kbdev, JOB_SLOT_OFFSET(js, COMMAND_NEXT)) == 0) { - /* idx0 has already completed - stop idx1 */ - kbase_gpu_stop_atom(kbdev, js, katom_idx1, action); - ret = true; - } else { - /* idx1 is in NEXT registers - attempt to - * remove - */ - kbase_reg_write32(kbdev, JOB_SLOT_OFFSET(js, COMMAND_NEXT), - JS_COMMAND_NOP); - - if (kbase_reg_read64(kbdev, JOB_SLOT_OFFSET(js, HEAD_NEXT)) != 0) { - /* idx1 removed successfully, will be - * handled in IRQ once idx0 completes - */ - kbase_gpu_remove_atom(kbdev, katom_idx1, action, false); - /* Revert the last_context, or mark as purged */ - kbdev->hwaccess.backend.slot_rb[js].last_kctx_tagged = - kctx_idx0 ? SLOT_RB_TAG_KCTX(katom_idx0->kctx) : - SLOT_RB_TAG_PURGED; - } else { - /* idx0 has already completed - stop - * idx1 - */ - kbase_gpu_stop_atom(kbdev, js, katom_idx1, action); - ret = true; - } - } - } - } - - if (stop_x_dep_idx0 != -1) - kbase_backend_soft_hard_stop_slot(kbdev, kctx, (unsigned int)stop_x_dep_idx0, NULL, - action); - - if (stop_x_dep_idx1 != -1) - kbase_backend_soft_hard_stop_slot(kbdev, kctx, (unsigned int)stop_x_dep_idx1, NULL, - action); - - return ret; -} - -void kbase_backend_cache_clean(struct kbase_device *kbdev, struct kbase_jd_atom *katom) -{ - if (katom->need_cache_flush_cores_retained) { - kbase_gpu_start_cache_clean(kbdev, GPU_COMMAND_CACHE_CLN_INV_FULL); - kbase_gpu_wait_cache_clean(kbdev); - - katom->need_cache_flush_cores_retained = false; - } -} - -void kbase_backend_complete_wq(struct kbase_device *kbdev, struct kbase_jd_atom *katom) -{ - /* - * If cache flush required due to HW workaround then perform the flush - * now - */ - kbase_backend_cache_clean(kbdev, katom); -} - -void kbase_backend_complete_wq_post_sched(struct kbase_device *kbdev, base_jd_core_req core_req) -{ - CSTD_UNUSED(core_req); - - if (!kbdev->pm.active_count) { - kbase_pm_lock(kbdev); - kbase_pm_update_active(kbdev); - kbase_pm_unlock(kbdev); - } -} - -void kbase_gpu_dump_slots(struct kbase_device *kbdev) -{ - unsigned long flags; - unsigned int js; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - dev_info(kbdev->dev, "%s:\n", __func__); - - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { - int idx; - - for (idx = 0; idx < SLOT_RB_SIZE; idx++) { - struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, idx); - - if (katom) - dev_info(kbdev->dev, " js%u idx%d : katom=%pK gpu_rb_state=%d\n", - js, idx, katom, katom->gpu_rb_state); - else - dev_info(kbdev->dev, " js%u idx%d : empty\n", js, idx); - } - } - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -} - -void kbase_backend_slot_kctx_purge_locked(struct kbase_device *kbdev, struct kbase_context *kctx) -{ - unsigned int js; - bool tracked = false; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { - u64 tagged_kctx = kbdev->hwaccess.backend.slot_rb[js].last_kctx_tagged; - - if (tagged_kctx == SLOT_RB_TAG_KCTX(kctx)) { - /* Marking the slot kctx tracking field is purged */ - kbdev->hwaccess.backend.slot_rb[js].last_kctx_tagged = SLOT_RB_TAG_PURGED; - tracked = true; - } - } - - if (tracked) { - /* The context had run some jobs before the purge, other slots - * in SLOT_RB_NULL_TAG_VAL condition needs to be marked as - * purged as well. - */ - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { - if (kbdev->hwaccess.backend.slot_rb[js].last_kctx_tagged == - SLOT_RB_NULL_TAG_VAL) - kbdev->hwaccess.backend.slot_rb[js].last_kctx_tagged = - SLOT_RB_TAG_PURGED; - } - } -} diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_rb.h b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_rb.h deleted file mode 100644 index 32be0bf44655..000000000000 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_rb.h +++ /dev/null @@ -1,77 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2014-2018, 2020-2022 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Register-based HW access backend specific APIs - */ - -#ifndef _KBASE_HWACCESS_GPU_H_ -#define _KBASE_HWACCESS_GPU_H_ - -#include - -/** - * kbase_gpu_irq_evict - Evict an atom from a NEXT slot - * - * @kbdev: Device pointer - * @js: Job slot to evict from - * @completion_code: Event code from job that was run. - * - * Evict the atom in the NEXT slot for the specified job slot. This function is - * called from the job complete IRQ handler when the previous job has failed. - * - * Return: true if job evicted from NEXT registers, false otherwise - */ -bool kbase_gpu_irq_evict(struct kbase_device *kbdev, unsigned int js, u32 completion_code); - -/** - * kbase_gpu_complete_hw - Complete an atom on job slot js - * - * @kbdev: Device pointer - * @js: Job slot that has completed - * @completion_code: Event code from job that has completed - * @job_tail: The tail address from the hardware if the job has partially - * completed - * @end_timestamp: Time of completion - */ -void kbase_gpu_complete_hw(struct kbase_device *kbdev, unsigned int js, u32 completion_code, - u64 job_tail, ktime_t *end_timestamp); - -/** - * kbase_gpu_inspect - Inspect the contents of the HW access ringbuffer - * - * @kbdev: Device pointer - * @js: Job slot to inspect - * @idx: Index into ringbuffer. 0 is the job currently running on - * the slot, 1 is the job waiting, all other values are invalid. - * Return: The atom at that position in the ringbuffer - * or NULL if no atom present - */ -struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, unsigned int js, int idx); - -/** - * kbase_gpu_dump_slots - Print the contents of the slot ringbuffers - * - * @kbdev: Device pointer - */ -void kbase_gpu_dump_slots(struct kbase_device *kbdev); - -#endif /* _KBASE_HWACCESS_GPU_H_ */ diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_js_backend.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_js_backend.c deleted file mode 100644 index dd0d915be48c..000000000000 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_js_backend.c +++ /dev/null @@ -1,371 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Register-based HW access backend specific job scheduler APIs - */ - -#include -#include -#include -#include -#include -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) -#include -#endif - -/* - * Hold the runpool_mutex for this - */ -static inline bool timer_callback_should_run(struct kbase_device *kbdev, int nr_running_ctxs) -{ - lockdep_assert_held(&kbdev->js_data.runpool_mutex); - -#ifdef CONFIG_MALI_VALHALL_DEBUG - if (kbdev->js_data.softstop_always) { - /* Debug support for allowing soft-stop on a single context */ - return true; - } -#endif /* CONFIG_MALI_VALHALL_DEBUG */ - - if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_9435)) { - /* Timeouts would have to be 4x longer (due to micro- - * architectural design) to support OpenCL conformance tests, so - * only run the timer when there's: - * - 2 or more CL contexts - * - 1 or more GLES contexts - * - * NOTE: We will treat a context that has both Compute and Non- - * Compute jobs will be treated as an OpenCL context (hence, we - * don't check KBASEP_JS_CTX_ATTR_NON_COMPUTE). - */ - { - int nr_compute_ctxs = kbasep_js_ctx_attr_count_on_runpool( - kbdev, KBASEP_JS_CTX_ATTR_COMPUTE); - int nr_noncompute_ctxs = nr_running_ctxs - nr_compute_ctxs; - - return (bool)(nr_compute_ctxs >= 2 || nr_noncompute_ctxs > 0); - } - } else { - /* Run the timer callback whenever you have at least 1 context - */ - return (bool)(nr_running_ctxs > 0); - } -} - -static enum hrtimer_restart timer_callback(struct hrtimer *timer) -{ - unsigned long flags; - struct kbase_device *kbdev; - struct kbasep_js_device_data *js_devdata; - struct kbase_backend_data *backend; - unsigned int s; - bool reset_needed = false; - - KBASE_DEBUG_ASSERT(timer != NULL); - - backend = container_of(timer, struct kbase_backend_data, scheduling_timer); - kbdev = container_of(backend, struct kbase_device, hwaccess.backend); - js_devdata = &kbdev->js_data; - - /* Loop through the slots */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - for (s = 0; s < kbdev->gpu_props.num_job_slots; s++) { - struct kbase_jd_atom *atom = NULL; - - if (kbase_backend_nr_atoms_on_slot(kbdev, s) > 0) { - atom = kbase_gpu_inspect(kbdev, s, 0); - KBASE_DEBUG_ASSERT(atom != NULL); - } - - if (atom != NULL) { - /* The current version of the model doesn't support - * Soft-Stop - */ - if (!kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_5736)) { - u32 ticks = atom->ticks++; - -#if !defined(CONFIG_MALI_VALHALL_JOB_DUMP) && !defined(CONFIG_MALI_VECTOR_DUMP) - u32 soft_stop_ticks, hard_stop_ticks, gpu_reset_ticks; - if (atom->core_req & BASE_JD_REQ_ONLY_COMPUTE) { - soft_stop_ticks = js_devdata->soft_stop_ticks_cl; - hard_stop_ticks = js_devdata->hard_stop_ticks_cl; - gpu_reset_ticks = js_devdata->gpu_reset_ticks_cl; - } else { - soft_stop_ticks = js_devdata->soft_stop_ticks; - if (kbase_is_quick_reset_enabled(kbdev)) { - hard_stop_ticks = 2; - gpu_reset_ticks = 3; - } else { - hard_stop_ticks = js_devdata->hard_stop_ticks_ss; - gpu_reset_ticks = js_devdata->gpu_reset_ticks_ss; - } - } - - /* If timeouts have been changed then ensure - * that atom tick count is not greater than the - * new soft_stop timeout. This ensures that - * atoms do not miss any of the timeouts due to - * races between this worker and the thread - * changing the timeouts. - */ - if (backend->timeouts_updated && ticks > soft_stop_ticks) - ticks = atom->ticks = soft_stop_ticks; - - /* Job is Soft-Stoppable */ - if (ticks == soft_stop_ticks) { - /* Job has been scheduled for at least - * js_devdata->soft_stop_ticks ticks. - * Soft stop the slot so we can run - * other jobs. - */ -#if !KBASE_DISABLE_SCHEDULING_SOFT_STOPS - int disjoint_threshold = - KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD; - u32 softstop_flags = 0u; - - dev_dbg(kbdev->dev, "Soft-stop"); - /* nr_user_contexts_running is updated - * with the runpool_mutex, but we can't - * take that here. - * - * However, if it's about to be - * increased then the new context can't - * run any jobs until they take the - * hwaccess_lock, so it's OK to observe - * the older value. - * - * Similarly, if it's about to be - * decreased, the last job from another - * context has already finished, so - * it's not too bad that we observe the - * older value and register a disjoint - * event when we try soft-stopping - */ - if (js_devdata->nr_user_contexts_running >= - disjoint_threshold) - softstop_flags |= JS_COMMAND_SW_CAUSES_DISJOINT; - - kbase_job_slot_softstop_swflags(kbdev, s, atom, - softstop_flags); -#endif - } else if (ticks == hard_stop_ticks) { - /* Job has been scheduled for at least - * js_devdata->hard_stop_ticks_ss ticks. - * It should have been soft-stopped by - * now. Hard stop the slot. - */ -#if !KBASE_DISABLE_SCHEDULING_HARD_STOPS - u32 ms = js_devdata->scheduling_period_ns / 1000000u; - if (!kbase_is_quick_reset_enabled(kbdev)) - dev_warn( - kbdev->dev, - "JS: Job Hard-Stopped (took more than %u ticks at %u ms/tick)", - ticks, ms); - kbase_job_slot_hardstop(atom->kctx, s, atom); -#endif - } else if (ticks == gpu_reset_ticks) { - /* Job has been scheduled for at least - * js_devdata->gpu_reset_ticks_ss ticks. - * It should have left the GPU by now. - * Signal that the GPU needs to be - * reset. - */ - reset_needed = true; - } -#else /* !CONFIG_MALI_VALHALL_JOB_DUMP */ - /* NOTE: During CONFIG_MALI_VALHALL_JOB_DUMP, we use - * the alternate timeouts, which makes the hard- - * stop and GPU reset timeout much longer. We - * also ensure that we don't soft-stop at all. - */ - if (ticks == js_devdata->soft_stop_ticks) { - /* Job has been scheduled for at least - * js_devdata->soft_stop_ticks. We do - * not soft-stop during - * CONFIG_MALI_VALHALL_JOB_DUMP, however. - */ - dev_dbg(kbdev->dev, "Soft-stop"); - } else if (ticks == js_devdata->hard_stop_ticks_dumping) { - /* Job has been scheduled for at least - * js_devdata->hard_stop_ticks_dumping - * ticks. Hard stop the slot. - */ -#if !KBASE_DISABLE_SCHEDULING_HARD_STOPS - u32 ms = js_devdata->scheduling_period_ns / 1000000u; - dev_warn( - kbdev->dev, - "JS: Job Hard-Stopped (took more than %u ticks at %u ms/tick)", - ticks, ms); - kbase_job_slot_hardstop(atom->kctx, s, atom); -#endif - } else if (ticks == js_devdata->gpu_reset_ticks_dumping) { - /* Job has been scheduled for at least - * js_devdata->gpu_reset_ticks_dumping - * ticks. It should have left the GPU by - * now. Signal that the GPU needs to be - * reset. - */ - reset_needed = true; - } -#endif /* !CONFIG_MALI_VALHALL_JOB_DUMP */ - } - } - } - if (reset_needed) { - if (kbase_is_quick_reset_enabled(kbdev)) - dev_err(kbdev->dev, "quick reset"); - else - dev_err(kbdev->dev, - "JS: Job has been on the GPU for too long (JS_RESET_TICKS_SS/DUMPING timeout hit). Issuing GPU soft-reset to resolve."); - - if (kbase_prepare_to_reset_gpu_locked(kbdev, RESET_FLAGS_NONE)) - kbase_reset_gpu_locked(kbdev); - } - /* the timer is re-issued if there is contexts in the run-pool */ - - if (backend->timer_running) - hrtimer_start(&backend->scheduling_timer, - HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns), - HRTIMER_MODE_REL); - - backend->timeouts_updated = false; - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - return HRTIMER_NORESTART; -} - -void kbase_backend_ctx_count_changed(struct kbase_device *kbdev) -{ - struct kbasep_js_device_data *js_devdata = &kbdev->js_data; - struct kbase_backend_data *backend = &kbdev->hwaccess.backend; - unsigned long flags; - /* Timer must stop if we are suspending */ - const bool suspend_timer = backend->suspend_timer; - const int nr_running_ctxs = atomic_read(&kbdev->js_data.nr_contexts_runnable); - - lockdep_assert_held(&js_devdata->runpool_mutex); - - if (suspend_timer || !timer_callback_should_run(kbdev, nr_running_ctxs)) { - /* Take spinlock to force synchronisation with timer */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - backend->timer_running = false; - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - /* From now on, return value of timer_callback_should_run() - * will also cause the timer to not requeue itself. Its return - * value cannot change, because it depends on variables updated - * with the runpool_mutex held, which the caller of this must - * also hold - */ - hrtimer_cancel(&backend->scheduling_timer); - } - - if (!suspend_timer && timer_callback_should_run(kbdev, nr_running_ctxs) && - !backend->timer_running) { - /* Take spinlock to force synchronisation with timer */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - backend->timer_running = true; - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - hrtimer_start(&backend->scheduling_timer, - HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns), - HRTIMER_MODE_REL); - - KBASE_KTRACE_ADD_JM(kbdev, JS_POLICY_TIMER_START, NULL, NULL, 0u, 0u); - } - -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) - if (unlikely(suspend_timer)) { - js_devdata->gpu_metrics_timer_needed = false; - /* Cancel the timer as System suspend is happening */ - hrtimer_cancel(&js_devdata->gpu_metrics_timer); - js_devdata->gpu_metrics_timer_running = false; - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - /* Explicitly emit the tracepoint on System suspend */ - kbase_gpu_metrics_emit_tracepoint(kbdev, ktime_get_raw_ns()); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - return; - } - - if (!nr_running_ctxs) { - /* Just set the flag to not restart the timer on expiry */ - js_devdata->gpu_metrics_timer_needed = false; - return; - } - - /* There are runnable contexts so the timer is needed */ - if (!js_devdata->gpu_metrics_timer_needed) { - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - js_devdata->gpu_metrics_timer_needed = true; - /* No need to restart the timer if it is already running. */ - if (!js_devdata->gpu_metrics_timer_running) { - hrtimer_start(&js_devdata->gpu_metrics_timer, - HR_TIMER_DELAY_NSEC(kbase_gpu_metrics_get_tp_emit_interval()), - HRTIMER_MODE_REL); - js_devdata->gpu_metrics_timer_running = true; - } - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - } -#endif -} - -int kbase_backend_timer_init(struct kbase_device *kbdev) -{ - struct kbase_backend_data *backend = &kbdev->hwaccess.backend; - - hrtimer_init(&backend->scheduling_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - backend->scheduling_timer.function = timer_callback; - backend->timer_running = false; - - return 0; -} - -void kbase_backend_timer_term(struct kbase_device *kbdev) -{ - struct kbase_backend_data *backend = &kbdev->hwaccess.backend; - - hrtimer_cancel(&backend->scheduling_timer); -} - -void kbase_backend_timer_suspend(struct kbase_device *kbdev) -{ - struct kbase_backend_data *backend = &kbdev->hwaccess.backend; - - backend->suspend_timer = true; - - kbase_backend_ctx_count_changed(kbdev); -} - -void kbase_backend_timer_resume(struct kbase_device *kbdev) -{ - struct kbase_backend_data *backend = &kbdev->hwaccess.backend; - - backend->suspend_timer = false; - - kbase_backend_ctx_count_changed(kbdev); -} - -void kbase_backend_timeouts_changed(struct kbase_device *kbdev) -{ - struct kbase_backend_data *backend = &kbdev->hwaccess.backend; - - backend->timeouts_updated = true; -} diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_js_internal.h b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_js_internal.h deleted file mode 100644 index 4f7c371a1f1a..000000000000 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_js_internal.h +++ /dev/null @@ -1,72 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2014-2015, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Register-based HW access backend specific job scheduler APIs - */ - -#ifndef _KBASE_JS_BACKEND_H_ -#define _KBASE_JS_BACKEND_H_ - -/** - * kbase_backend_timer_init() - Initialise the JS scheduling timer - * @kbdev: Device pointer - * - * This function should be called at driver initialisation - * - * Return: 0 on success - */ -int kbase_backend_timer_init(struct kbase_device *kbdev); - -/** - * kbase_backend_timer_term() - Terminate the JS scheduling timer - * @kbdev: Device pointer - * - * This function should be called at driver termination - */ -void kbase_backend_timer_term(struct kbase_device *kbdev); - -/** - * kbase_backend_timer_suspend - Suspend is happening, stop the JS scheduling - * timer - * @kbdev: Device pointer - * - * This function should be called on suspend, after the active count has reached - * zero. This is required as the timer may have been started on job submission - * to the job scheduler, but before jobs are submitted to the GPU. - * - * Caller must hold runpool_mutex. - */ -void kbase_backend_timer_suspend(struct kbase_device *kbdev); - -/** - * kbase_backend_timer_resume - Resume is happening, re-evaluate the JS - * scheduling timer - * @kbdev: Device pointer - * - * This function should be called on resume. Note that is not guaranteed to - * re-start the timer, only evalute whether it should be re-started. - * - * Caller must hold runpool_mutex. - */ -void kbase_backend_timer_resume(struct kbase_device *kbdev); - -#endif /* _KBASE_JS_BACKEND_H_ */ diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_model_dummy.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_model_dummy.c index 527d0c5717e6..d752f2d9a6da 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_model_dummy.c +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_model_dummy.c @@ -36,7 +36,6 @@ #include -#if MALI_USE_CSF #include /* Index of the last value register for each type of core, with the 1st value @@ -55,11 +54,8 @@ static u32 gpu_control_base_addr; /* Array for storing the value of SELECT register for each type of core */ static u64 ipa_ctl_select_config[KBASE_IPA_CORE_TYPE_NUM]; static u32 ipa_control_timer_enabled; -#endif -#if MALI_USE_CSF static u32 sysc_alloc_regs[SYSC_ALLOC_COUNT]; -#endif #define LO_MASK(M) ((M)&0xFFFFFFFF) #define HI_MASK(M) ((M)&0xFFFFFFFF00000000) @@ -69,13 +65,8 @@ static u32 sysc_alloc_regs[SYSC_ALLOC_COUNT]; * significant bits, which are set to THREAD_FEATURES_IMPLEMENTATION_TECHNOLOGY_SOFTWARE in * midgard_model_read_reg(). */ -#if MALI_USE_CSF #define THREAD_FEATURES_PARTIAL(MAX_REGISTERS, MAX_TASK_QUEUE, MAX_TG_SPLIT) \ ((MAX_REGISTERS) | ((MAX_TASK_QUEUE) << 24)) -#else -#define THREAD_FEATURES_PARTIAL(MAX_REGISTERS, MAX_TASK_QUEUE, MAX_TG_SPLIT) \ - ((MAX_REGISTERS) | ((MAX_TASK_QUEUE) << 16) | ((MAX_TG_SPLIT) << 24)) -#endif struct error_status_t hw_error_status; @@ -133,24 +124,18 @@ enum pwr_on_index { INDEX_TILER, INDEX_SHADER, INDEX_STACK, -#if MALI_USE_CSF INDEX_BASE, INDEX_NEURAL, -#endif INDEX_DOMAIN_COUNT }; struct dummy_model_t { int reset_completed; int reset_completed_mask; -#if !MALI_USE_CSF - int prfcnt_sample_completed; -#endif /* !MALI_USE_CSF */ int power_changed_mask; /* 2 bits: _ALL,_SINGLE */ int power_changed; /* 1 bit */ bool clean_caches_completed; bool clean_caches_completed_irq_enabled; -#if MALI_USE_CSF bool flush_pa_range_completed; bool flush_pa_range_completed_irq_enabled; /* Representations of COMMAND_NOT_ALLOWED and COMMAND_INVALID bits in @@ -164,7 +149,6 @@ struct dummy_model_t { u64 command_arg; /* PWR_CMDARG register */ u64 gov_core_mask; -#endif uint32_t domain_power_on[INDEX_DOMAIN_COUNT]; u32 coherency_enable; unsigned int job_irq_js_state; @@ -462,9 +446,6 @@ static const struct control_reg_values_t all_control_reg_values[] = { static struct { spinlock_t access_lock; -#if !MALI_USE_CSF - unsigned long prfcnt_base; -#endif /* !MALI_USE_CSF */ u32 *prfcnt_base_cpu; u32 time; @@ -474,11 +455,7 @@ static struct { u64 l2_present; u64 shader_present; -#if !MALI_USE_CSF - u64 jm_counters[KBASE_DUMMY_MODEL_COUNTER_PER_CORE]; -#else u64 cshw_counters[KBASE_DUMMY_MODEL_COUNTER_PER_CORE]; -#endif /* !MALI_USE_CSF */ u64 tiler_counters[KBASE_DUMMY_MODEL_COUNTER_PER_CORE]; u64 l2_counters[KBASE_DUMMY_MODEL_MAX_MEMSYS_BLOCKS * KBASE_DUMMY_MODEL_COUNTER_PER_CORE]; u64 shader_counters[KBASE_DUMMY_MODEL_MAX_SHADER_CORES * KBASE_DUMMY_MODEL_COUNTER_PER_CORE]; @@ -510,7 +487,6 @@ static u32 get_implementation_register(u32 reg, return 0; } -#if MALI_USE_CSF static u32 hctrl_get_implementation_register(u32 reg, const struct control_reg_values_t *const control_reg_values) @@ -531,7 +507,6 @@ hctrl_get_implementation_register(u32 reg, return 0; } -#endif void gpu_device_set_data(void *model, void *data) { @@ -553,7 +528,6 @@ static char *no_mali_gpu = CONFIG_MALI_VALHALL_NO_MALI_DEFAULT_GPU; module_param(no_mali_gpu, charp, 0000); MODULE_PARM_DESC(no_mali_gpu, "GPU to identify as"); -#if MALI_USE_CSF static u32 gpu_model_get_prfcnt_value(enum kbase_ipa_core_type core_type, u32 cnt_idx, bool is_low_word) { @@ -628,7 +602,6 @@ static u32 gpu_model_get_prfcnt_value(enum kbase_ipa_core_type core_type, u32 cn else return (value >> 32); } -#endif /* MALI_USE_CSF */ /** * gpu_model_clear_prfcnt_values_nolock - Clear performance counter values @@ -639,18 +612,13 @@ static u32 gpu_model_get_prfcnt_value(enum kbase_ipa_core_type core_type, u32 cn static void gpu_model_clear_prfcnt_values_nolock(void) { lockdep_assert_held(&performance_counters.access_lock); -#if !MALI_USE_CSF - memset(performance_counters.jm_counters, 0, sizeof(performance_counters.jm_counters)); -#else memset(performance_counters.cshw_counters, 0, sizeof(performance_counters.cshw_counters)); -#endif /* !MALI_USE_CSF */ memset(performance_counters.tiler_counters, 0, sizeof(performance_counters.tiler_counters)); memset(performance_counters.l2_counters, 0, sizeof(performance_counters.l2_counters)); memset(performance_counters.shader_counters, 0, sizeof(performance_counters.shader_counters)); } -#if MALI_USE_CSF void gpu_model_clear_prfcnt_values(void) { unsigned long flags; @@ -660,7 +628,6 @@ void gpu_model_clear_prfcnt_values(void) spin_unlock_irqrestore(&performance_counters.access_lock, flags); } KBASE_EXPORT_TEST_API(gpu_model_clear_prfcnt_values); -#endif /* MALI_USE_CSF */ /** * gpu_model_dump_prfcnt_blocks() - Dump performance counter values to buffer @@ -689,11 +656,9 @@ static void gpu_model_dump_prfcnt_blocks(u64 *values, u32 *out_index, u32 block_ for (block_idx = 0; block_idx < block_count; block_idx++) { /* only dump values if core is present */ if (!(blocks_present & (1U << block_idx))) { -#if MALI_USE_CSF /* if CSF dump zeroed out block */ memset(&prfcnt_base[*out_index], 0, KBASE_DUMMY_MODEL_BLOCK_SIZE); *out_index += KBASE_DUMMY_MODEL_VALUES_PER_BLOCK; -#endif /* MALI_USE_CSF */ continue; } @@ -724,13 +689,8 @@ static void gpu_model_dump_nolock(void) lockdep_assert_held(&performance_counters.access_lock); -#if !MALI_USE_CSF - gpu_model_dump_prfcnt_blocks(performance_counters.jm_counters, &index, 1, - performance_counters.prfcnt_en.fe, 0x1); -#else gpu_model_dump_prfcnt_blocks(performance_counters.cshw_counters, &index, 1, performance_counters.prfcnt_en.fe, 0x1); -#endif /* !MALI_USE_CSF */ gpu_model_dump_prfcnt_blocks(performance_counters.tiler_counters, &index, 1, performance_counters.prfcnt_en.tiler, DUMMY_IMPLEMENTATION_TILER_PRESENT); @@ -763,16 +723,6 @@ static void gpu_model_raise_irq(void *model, u32 irq) gpu_device_raise_irq(model, irq); } -#if !MALI_USE_CSF -static void midgard_model_dump_prfcnt(void) -{ - unsigned long flags; - - spin_lock_irqsave(&performance_counters.access_lock, flags); - gpu_model_dump_nolock(); - spin_unlock_irqrestore(&performance_counters.access_lock, flags); -} -#else void gpu_model_prfcnt_dump_request(u32 *sample_buf, struct gpu_model_prfcnt_en enable_maps) { unsigned long flags; @@ -796,7 +746,6 @@ void gpu_model_glb_request_job_irq(void *model) spin_unlock_irqrestore(&hw_error_status.access_lock, flags); gpu_model_raise_irq(model, MODEL_LINUX_JOB_IRQ); } -#endif /* !MALI_USE_CSF */ static void init_register_statuses(struct dummy_model_t *dummy) { @@ -830,117 +779,6 @@ static void update_register_statuses(struct dummy_model_t *dummy, u32 job_slot) if (hw_error_status.errors_mask & IS_A_JOB_ERROR) { if (job_slot == hw_error_status.current_job_slot) { -#if !MALI_USE_CSF - if (hw_error_status.js_status[job_slot] == 0) { - /* status reg is clean; it can be written */ - - switch (hw_error_status.errors_mask & IS_A_JOB_ERROR) { - case KBASE_JOB_INTERRUPTED: - hw_error_status.js_status[job_slot] = JS_STATUS_INTERRUPTED; - break; - - case KBASE_JOB_STOPPED: - hw_error_status.js_status[job_slot] = JS_STATUS_STOPPED; - break; - - case KBASE_JOB_TERMINATED: - hw_error_status.js_status[job_slot] = JS_STATUS_TERMINATED; - break; - - case KBASE_JOB_CONFIG_FAULT: - hw_error_status.js_status[job_slot] = - JS_STATUS_CONFIG_FAULT; - break; - - case KBASE_JOB_POWER_FAULT: - hw_error_status.js_status[job_slot] = JS_STATUS_POWER_FAULT; - break; - - case KBASE_JOB_READ_FAULT: - hw_error_status.js_status[job_slot] = JS_STATUS_READ_FAULT; - break; - - case KBASE_JOB_WRITE_FAULT: - hw_error_status.js_status[job_slot] = JS_STATUS_WRITE_FAULT; - break; - - case KBASE_JOB_AFFINITY_FAULT: - hw_error_status.js_status[job_slot] = - JS_STATUS_AFFINITY_FAULT; - break; - - case KBASE_JOB_BUS_FAULT: - hw_error_status.js_status[job_slot] = JS_STATUS_BUS_FAULT; - break; - - case KBASE_INSTR_INVALID_PC: - hw_error_status.js_status[job_slot] = - JS_STATUS_INSTR_INVALID_PC; - break; - - case KBASE_INSTR_INVALID_ENC: - hw_error_status.js_status[job_slot] = - JS_STATUS_INSTR_INVALID_ENC; - break; - - case KBASE_INSTR_TYPE_MISMATCH: - hw_error_status.js_status[job_slot] = - JS_STATUS_INSTR_TYPE_MISMATCH; - break; - - case KBASE_INSTR_OPERAND_FAULT: - hw_error_status.js_status[job_slot] = - JS_STATUS_INSTR_OPERAND_FAULT; - break; - - case KBASE_INSTR_TLS_FAULT: - hw_error_status.js_status[job_slot] = - JS_STATUS_INSTR_TLS_FAULT; - break; - - case KBASE_INSTR_BARRIER_FAULT: - hw_error_status.js_status[job_slot] = - JS_STATUS_INSTR_BARRIER_FAULT; - break; - - case KBASE_INSTR_ALIGN_FAULT: - hw_error_status.js_status[job_slot] = - JS_STATUS_INSTR_ALIGN_FAULT; - break; - - case KBASE_DATA_INVALID_FAULT: - hw_error_status.js_status[job_slot] = - JS_STATUS_DATA_INVALID_FAULT; - break; - - case KBASE_TILE_RANGE_FAULT: - hw_error_status.js_status[job_slot] = - JS_STATUS_TILE_RANGE_FAULT; - break; - - case KBASE_ADDR_RANGE_FAULT: - hw_error_status.js_status[job_slot] = - JS_STATUS_ADDRESS_RANGE_FAULT; - break; - - case KBASE_OUT_OF_MEMORY: - hw_error_status.js_status[job_slot] = - JS_STATUS_OUT_OF_MEMORY; - break; - - case KBASE_UNKNOWN: - hw_error_status.js_status[job_slot] = JS_STATUS_UNKNOWN; - break; - - default: - model_error_log(KBASE_CORE, - "\nAtom Chain 0x%llx: Invalid Error Mask!", - hw_error_status.current_jc); - break; - } - } -#endif /* !MALI_USE_CSF */ - /* we set JOB_FAIL_ */ hw_error_status.job_irq_rawstat |= (dummy->slots[job_slot].job_complete_irq_asserted) @@ -1049,40 +887,6 @@ static void update_register_statuses(struct dummy_model_t *dummy, u32 job_slot) hw_error_status.errors_mask = 0; /*clear error mask */ } -#if !MALI_USE_CSF -static void update_job_irq_js_state(struct dummy_model_t *dummy, int mask) -{ - int i; - - lockdep_assert_held(&hw_error_status.access_lock); - pr_debug("%s", "Updating the JS_ACTIVE register"); - - for (i = 0; i < NUM_SLOTS; i++) { - int slot_active = dummy->slots[i].job_active; - int next_busy = dummy->slots[i].job_queued; - - if ((mask & (1 << i)) || (mask & (1 << (i + 16)))) { - /* clear the bits we're updating */ - dummy->job_irq_js_state &= ~((1 << (16 + i)) | (1 << i)); - if (hw_error_status.js_status[i]) { - dummy->job_irq_js_state |= next_busy << (i + 16); - if (mask & (1 << (i + 16))) { - /* clear job slot status */ - hw_error_status.js_status[i] = 0; - /* continue execution of jobchain */ - dummy->slots[i].job_active = dummy->slots[i].job_queued; - } - } else { - /* set bits if needed */ - dummy->job_irq_js_state |= - ((slot_active << i) | (next_busy << (i + 16))); - } - } - } - pr_debug("The new snapshot is 0x%08X\n", dummy->job_irq_js_state); -} -#endif /* !MALI_USE_CSF */ - /** * find_gpu_rev() - Append GPU HW revision to the GPU name, if needed. * @new_gpu_name: Caller-allocated string for the new GPU name. @@ -1121,6 +925,9 @@ static const struct control_reg_values_t *find_control_reg_values(const char *gp const struct control_reg_values_t *ret = NULL; char *gpu_with_rev = kmalloc(strlen(gpu) + GPU_REV_STR_LEN + 1, GFP_KERNEL); + if (gpu_with_rev == NULL) + return NULL; + /* Fix gpu name in case there are multiple entries for different HW versions. */ find_gpu_rev(gpu_with_rev, gpu); @@ -1164,10 +971,15 @@ void *midgard_model_create(struct kbase_device *kbdev) dummy->job_irq_js_state = 0; init_register_statuses(dummy); dummy->control_reg_values = find_control_reg_values(no_mali_gpu); + + if (dummy->control_reg_values == NULL) { + kfree(dummy); + return NULL; + } + dummy->arch_id = get_arch_id(dummy->control_reg_values->gpu_id); gpu_control_base_addr = GPU_CONTROL_BASE; -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface) { performance_counters.l2_present = hctrl_get_implementation_register( GET_HOST_POWER_REG(HOST_POWER_L2_PRESENT_LO), @@ -1175,9 +987,7 @@ void *midgard_model_create(struct kbase_device *kbdev) performance_counters.shader_present = hctrl_get_implementation_register( GET_HOST_POWER_REG(HOST_POWER_SHADER_PRESENT_LO), dummy->control_reg_values); - } else -#endif /* MALI_USE_CSF */ - { + } else { performance_counters.l2_present = get_implementation_register( GET_GPU_CONTROL_REG(L2_PRESENT_LO), dummy->control_reg_values); performance_counters.shader_present = get_implementation_register( @@ -1299,73 +1109,6 @@ void midgard_model_write_reg(void *h, u32 addr, u32 value) spin_lock_irqsave(&hw_error_status.access_lock, flags); -#if !MALI_USE_CSF - if ((addr >= JOB_CONTROL_REG(JOB_SLOT0)) && (addr < (JOB_CONTROL_REG(JOB_SLOT15) + 0x80))) { - unsigned int slot_idx = (addr >> 7) & 0xf; - - KBASE_DEBUG_ASSERT(slot_idx < NUM_SLOTS); - if (addr == JOB_SLOT_REG(slot_idx, JS_HEAD_NEXT_LO)) { - hw_error_status.current_jc &= ~((u64)(0xFFFFFFFF)); - hw_error_status.current_jc |= (u64)value; - } - if (addr == JOB_SLOT_REG(slot_idx, JS_HEAD_NEXT_HI)) { - hw_error_status.current_jc &= (u64)0xFFFFFFFF; - hw_error_status.current_jc |= ((u64)value) << 32; - } - if (addr == JOB_SLOT_REG(slot_idx, JS_COMMAND_NEXT) && value == 1) { - pr_debug("%s", "start detected"); - KBASE_DEBUG_ASSERT(!dummy->slots[slot_idx].job_active || - !dummy->slots[slot_idx].job_queued); - if ((dummy->slots[slot_idx].job_active) || - (hw_error_status.job_irq_rawstat & (1 << (slot_idx + 16)))) { - pr_debug( - "~~~~~~~~~~~ Start: job slot is already active or there are IRQ pending ~~~~~~~~~"); - dummy->slots[slot_idx].job_queued = 1; - } else { - dummy->slots[slot_idx].job_active = 1; - } - } - - if (addr == JOB_SLOT_REG(slot_idx, JS_COMMAND_NEXT) && value == 0) - dummy->slots[slot_idx].job_queued = 0; - - if ((addr == JOB_SLOT_REG(slot_idx, JS_COMMAND)) && - (value == JS_COMMAND_SOFT_STOP || value == JS_COMMAND_HARD_STOP)) { - /*dummy->slots[slot_idx].job_active = 0; */ - hw_error_status.current_job_slot = slot_idx; - if (value == JS_COMMAND_SOFT_STOP) { - hw_error_status.errors_mask = KBASE_JOB_STOPPED; - } else { /*value == 3 */ - - if (dummy->slots[slot_idx].job_disabled != 0) { - pr_debug("enabling slot after HARD_STOP"); - dummy->slots[slot_idx].job_disabled = 0; - } - hw_error_status.errors_mask = KBASE_JOB_TERMINATED; - } - } - } else if (addr == JOB_CONTROL_REG(JOB_IRQ_CLEAR)) { - int i; - - for (i = 0; i < NUM_SLOTS; i++) { - if (value & ((1u << i) | (1u << (i + 16)))) - dummy->slots[i].job_complete_irq_asserted = 0; - /* hw_error_status.js_status[i] is cleared in - * update_job_irq_js_state - */ - } - pr_debug("%s", "job irq cleared"); - update_job_irq_js_state(dummy, value); - /*remove error condition for JOB */ - hw_error_status.job_irq_rawstat &= ~(value); - hw_error_status.job_irq_status &= ~(value); - } else if (addr == JOB_CONTROL_REG(JOB_IRQ_MASK)) { - int i; - - for (i = 0; i < NUM_SLOTS; i++) - dummy->slots[i].job_irq_mask = (value >> i) & 0x01; - pr_debug("job irq mask to value %x", value); -#else /* MALI_USE_CSF */ if (addr == JOB_CONTROL_REG(JOB_IRQ_CLEAR)) { pr_debug("%s", "job irq cleared"); @@ -1376,25 +1119,16 @@ void midgard_model_write_reg(void *h, u32 addr, u32 value) hw_error_status.job_irq_status |= value; } else if (addr == JOB_CONTROL_REG(JOB_IRQ_MASK)) { /* ignore JOB_IRQ_MASK as it is handled by CSFFW */ -#endif /* !MALI_USE_CSF */ } else if (addr == GET_GPU_CONTROL_REG(GPU_IRQ_MASK)) { pr_debug("GPU_IRQ_MASK set to 0x%x", value); -#if MALI_USE_CSF if (!dummy->kbdev->pm.backend.has_host_pwr_iface) { dummy->reset_completed_mask = (value >> 8) & 0x01; dummy->power_changed_mask = (value >> 9) & 0x03; } -#else - dummy->reset_completed_mask = (value >> 8) & 0x01; - dummy->power_changed_mask = (value >> 9) & 0x03; -#endif dummy->clean_caches_completed_irq_enabled = (value & (1u << 17)) != 0u; -#if MALI_USE_CSF dummy->flush_pa_range_completed_irq_enabled = (value & (1u << 20)) != 0u; -#endif } else if (addr == GET_GPU_CONTROL_REG(COHERENCY_ENABLE)) { dummy->coherency_enable = value; -#if MALI_USE_CSF } else if (addr == GET_HOST_POWER_REG(PWR_IRQ_MASK)) { pr_debug("PWR_IRQ_MASK set to 0x%x", value); dummy->power_changed_mask = (value & PWR_IRQ_POWER_CHANGED_SINGLE) | @@ -1531,28 +1265,12 @@ void midgard_model_write_reg(void *h, u32 addr, u32 value) if (value & (POWER_CHANGED_SINGLE | POWER_CHANGED_ALL)) dummy->power_changed = 0; } -#else - } else if (addr == GET_GPU_CONTROL_REG(GPU_IRQ_CLEAR)) { - if (value & RESET_COMPLETED) { - pr_debug("%s", "gpu RESET_COMPLETED irq cleared"); - dummy->reset_completed = 0; - } - if (value & (POWER_CHANGED_SINGLE | POWER_CHANGED_ALL)) - dummy->power_changed = 0; -#endif /* MALI_USE_CSF */ if (value & CLEAN_CACHES_COMPLETED) dummy->clean_caches_completed = false; -#if MALI_USE_CSF if (value & (1u << 20)) dummy->flush_pa_range_completed = false; -#endif /* MALI_USE_CSF */ - -#if !MALI_USE_CSF - if (value & PRFCNT_SAMPLE_COMPLETED) /* (1 << 16) */ - dummy->prfcnt_sample_completed = 0; -#endif /* !MALI_USE_CSF */ /*update error status */ hw_error_status.gpu_error_irq &= ~(value); @@ -1566,41 +1284,26 @@ void midgard_model_write_reg(void *h, u32 addr, u32 value) /* completed reset instantly */ dummy->reset_completed = 1; break; -#if MALI_USE_CSF case GPU_COMMAND_CACHE_CLN_INV_L2: case GPU_COMMAND_CACHE_CLN_INV_L2_LSC: case GPU_COMMAND_CACHE_CLN_INV_FULL: -#else - case GPU_COMMAND_CLEAN_CACHES: - case GPU_COMMAND_CLEAN_INV_CACHES: -#endif pr_debug("clean caches requested"); dummy->clean_caches_completed = true; break; -#if MALI_USE_CSF case GPU_COMMAND_FLUSH_PA_RANGE_CLN_INV_L2: case GPU_COMMAND_FLUSH_PA_RANGE_CLN_INV_L2_LSC: case GPU_COMMAND_FLUSH_PA_RANGE_CLN_INV_FULL: pr_debug("pa range flush requested"); dummy->flush_pa_range_completed = true; break; -#endif /* MALI_USE_CSF */ -#if !MALI_USE_CSF - case GPU_COMMAND_PRFCNT_SAMPLE: - midgard_model_dump_prfcnt(); - dummy->prfcnt_sample_completed = 1; -#endif /* !MALI_USE_CSF */ default: break; } -#if MALI_USE_CSF } else if (addr >= GET_GPU_CONTROL_REG(GPU_COMMAND_ARG0_LO) && addr <= GET_GPU_CONTROL_REG(GPU_COMMAND_ARG1_HI)) { /* Writes ignored */ -#endif } else if (addr == GET_GPU_CONTROL_REG(L2_CONFIG)) { dummy->l2_config = value; -#if MALI_USE_CSF } else if (addr >= CSF_HW_DOORBELL_PAGE_OFFSET && addr < CSF_HW_DOORBELL_PAGE_OFFSET + (dummy->kbdev->csf.num_doorbells * CSF_HW_DOORBELL_PAGE_SIZE)) { @@ -1650,7 +1353,6 @@ void midgard_model_write_reg(void *h, u32 addr, u32 value) } } else if (addr == GPU_GOV_CORE_MASK_OFFSET) { dummy->gov_core_mask = value; -#endif /* MALI_USE_CSF */ } else if (addr == MMU_CONTROL_REG(MMU_IRQ_MASK)) { hw_error_status.mmu_irq_mask = value; } else if (addr == MMU_CONTROL_REG(MMU_IRQ_CLEAR)) { @@ -1659,6 +1361,9 @@ void midgard_model_write_reg(void *h, u32 addr, u32 value) (addr <= MMU_STAGE1_REG(MMU_AS_REG(15, AS_STATUS)))) { u32 mem_addr_space = (addr - MMU_STAGE1_REG(MMU_AS_REG(0, AS_TRANSTAB_LO))) >> MMU_STAGE1_AS_SHIFT; + if (WARN_ON(mem_addr_space >= NUM_MMU_AS)) { + return; + } switch (addr & 0x3F) { case AS_COMMAND: @@ -1737,32 +1442,6 @@ void midgard_model_write_reg(void *h, u32 addr, u32 value) } } else { switch (addr) { -#if !MALI_USE_CSF - case PRFCNT_BASE_LO: - performance_counters.prfcnt_base = - HI_MASK(performance_counters.prfcnt_base) | value; - performance_counters.prfcnt_base_cpu = - (u32 *)(uintptr_t)performance_counters.prfcnt_base; - break; - case PRFCNT_BASE_HI: - performance_counters.prfcnt_base = - LO_MASK(performance_counters.prfcnt_base) | (((u64)value) << 32); - performance_counters.prfcnt_base_cpu = - (u32 *)(uintptr_t)performance_counters.prfcnt_base; - break; - case PRFCNT_JM_EN: - performance_counters.prfcnt_en.fe = value; - break; - case PRFCNT_SHADER_EN: - performance_counters.prfcnt_en.shader = value; - break; - case PRFCNT_TILER_EN: - performance_counters.prfcnt_en.tiler = value; - break; - case PRFCNT_MMU_L2_EN: - performance_counters.prfcnt_en.l2 = value; - break; -#endif /* !MALI_USE_CSF */ case TILER_PWRON_LO: dummy->domain_power_on[INDEX_TILER] |= value & DUMMY_IMPLEMENTATION_TILER_PRESENT; @@ -1813,13 +1492,8 @@ void midgard_model_write_reg(void *h, u32 addr, u32 value) case PWR_KEY: case PWR_OVERRIDE0: case PWR_OVERRIDE1: -#if MALI_USE_CSF case SHADER_PWRFEATURES: case CSF_CONFIG: -#else /* !MALI_USE_CSF */ - case JM_CONFIG: - case PRFCNT_CONFIG: -#endif /* MALI_USE_CSF */ case SHADER_CONFIG: case TILER_CONFIG: case L2_MMU_CONFIG: @@ -1847,15 +1521,7 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value) spin_lock_irqsave(&hw_error_status.access_lock, flags); *value = 0; /* 0 by default */ -#if !MALI_USE_CSF - if (addr == JOB_CONTROL_REG(JOB_IRQ_JS_STATE)) { - pr_debug("%s", "JS_ACTIVE being read"); - - *value = dummy->job_irq_js_state; - } else if (addr == GET_GPU_CONTROL_REG(GPU_ID)) { -#else /* !MALI_USE_CSF */ if (addr == GET_GPU_CONTROL_REG(GPU_ID)) { -#endif /* !MALI_USE_CSF */ *value = dummy->control_reg_values->gpu_id & U32_MAX; } else if (addr == JOB_CONTROL_REG(JOB_IRQ_RAWSTAT)) { *value = hw_error_status.job_irq_rawstat; @@ -1863,42 +1529,21 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value) } else if (addr == JOB_CONTROL_REG(JOB_IRQ_STATUS)) { *value = hw_error_status.job_irq_status; pr_debug("JS_IRQ_STATUS being read %x", *value); -#if !MALI_USE_CSF - } else if (addr == JOB_CONTROL_REG(JOB_IRQ_MASK)) { - int i; - - *value = 0; - for (i = 0; i < NUM_SLOTS; i++) - *value |= dummy->slots[i].job_irq_mask << i; - pr_debug("JS_IRQ_MASK being read %x", *value); -#else /* !MALI_USE_CSF */ } else if (addr == JOB_CONTROL_REG(JOB_IRQ_MASK)) { /* ignore JOB_IRQ_MASK as it is handled by CSFFW */ -#endif /* !MALI_USE_CSF */ } else if (addr == GET_GPU_CONTROL_REG(GPU_IRQ_MASK)) { *value = (dummy->reset_completed_mask << 8) | ((dummy->clean_caches_completed_irq_enabled ? 1u : 0u) << 17) | -#if MALI_USE_CSF ((dummy->flush_pa_range_completed_irq_enabled ? 1u : 0u) << 20) | -#endif (dummy->power_changed_mask << 9) | (1u << 7) | 1u; pr_debug("GPU_IRQ_MASK read %x", *value); } else if (addr == GET_GPU_CONTROL_REG(GPU_IRQ_RAWSTAT)) { *value = ((dummy->clean_caches_completed ? 1u : 0u) << 17) | -#if MALI_USE_CSF ((dummy->flush_pa_range_completed ? 1u : 0u) << 20) | -#else - (dummy->prfcnt_sample_completed ? PRFCNT_SAMPLE_COMPLETED : 0) | -#endif hw_error_status.gpu_error_irq; -#if MALI_USE_CSF if (!dummy->kbdev->pm.backend.has_host_pwr_iface) *value |= (dummy->power_changed << 9) | (dummy->power_changed << 10) | (dummy->reset_completed << 8); -#else - *value |= (dummy->power_changed << 9) | (dummy->power_changed << 10) | - (dummy->reset_completed << 8); -#endif pr_debug("GPU_IRQ_RAWSTAT read %x", *value); } else if (addr == GET_GPU_CONTROL_REG(GPU_IRQ_STATUS)) { @@ -1907,40 +1552,25 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value) 1u : 0u) << 17) | -#if MALI_USE_CSF (((dummy->flush_pa_range_completed && dummy->flush_pa_range_completed_irq_enabled) ? 1u : 0u) << 20) | -#else - (dummy->prfcnt_sample_completed ? PRFCNT_SAMPLE_COMPLETED : 0) | -#endif hw_error_status.gpu_error_irq; -#if MALI_USE_CSF if (!dummy->kbdev->pm.backend.has_host_pwr_iface) *value |= ((dummy->power_changed && (dummy->power_changed_mask & 0x1)) << 9) | ((dummy->power_changed && (dummy->power_changed_mask & 0x2)) << 10) | ((dummy->reset_completed & dummy->reset_completed_mask) << 8); -#else - *value |= ((dummy->power_changed && (dummy->power_changed_mask & 0x1)) << 9) | - ((dummy->power_changed && (dummy->power_changed_mask & 0x2)) << 10) | - ((dummy->reset_completed & dummy->reset_completed_mask) << 8); -#endif pr_debug("GPU_IRQ_STAT read %x", *value); } else if (addr == GET_GPU_CONTROL_REG(GPU_STATUS)) { *value = 0; -#if !MALI_USE_CSF - } else if (addr == GET_GPU_CONTROL_REG(LATEST_FLUSH)) { - *value = 0; -#endif } else if (addr == GET_GPU_CONTROL_REG(GPU_FAULTSTATUS)) { *value = hw_error_status.gpu_fault_status; } else if (addr == GET_GPU_CONTROL_REG(L2_CONFIG)) { *value = dummy->l2_config; -#if MALI_USE_CSF } else if ((addr >= GET_GPU_CONTROL_REG(SYSC_ALLOC0)) && (addr < GET_GPU_CONTROL_REG(SYSC_ALLOC(SYSC_ALLOC_COUNT)))) { u32 alloc_reg = (addr - GET_GPU_CONTROL_REG(SYSC_ALLOC0)) >> 2; @@ -2042,8 +1672,6 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value) addr == GET_HOST_POWER_REG(HOST_POWER_NEURAL_PWRTRANS_LO) || addr == GET_HOST_POWER_REG(HOST_POWER_NEURAL_PWRTRANS_HI) || - addr == GET_HOST_POWER_REG(HOST_POWER_L2_PWRACTIVE_LO) || - addr == GET_HOST_POWER_REG(HOST_POWER_L2_PWRACTIVE_HI) || addr == GET_HOST_POWER_REG(HOST_POWER_TILER_PWRACTIVE_LO) || addr == GET_HOST_POWER_REG(HOST_POWER_TILER_PWRACTIVE_HI) || addr == GET_HOST_POWER_REG(HOST_POWER_SHADER_PWRACTIVE_LO) || @@ -2066,7 +1694,6 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value) "Dummy model register access: Reading unknown control reg 0x%x\n", addr); } -#endif } else if ((addr >= GET_GPU_CONTROL_REG(SHADER_PRESENT_LO)) && (addr <= GET_GPU_CONTROL_REG(L2_MMU_CONFIG))) { if (addr == GET_GPU_CONTROL_REG(SHADER_PRESENT_LO) || @@ -2106,8 +1733,6 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value) addr == GET_GPU_CONTROL_REG(SHADER_PWRTRANS_HI) || addr == GET_GPU_CONTROL_REG(STACK_PWRTRANS_LO) || addr == GET_GPU_CONTROL_REG(STACK_PWRTRANS_HI) || - addr == GET_GPU_CONTROL_REG(L2_PWRACTIVE_LO) || - addr == GET_GPU_CONTROL_REG(L2_PWRACTIVE_HI) || addr == GET_GPU_CONTROL_REG(TILER_PWRACTIVE_LO) || addr == GET_GPU_CONTROL_REG(TILER_PWRACTIVE_HI) || addr == GET_GPU_CONTROL_REG(SHADER_PWRACTIVE_LO) || @@ -2131,51 +1756,6 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value) addr); } -#if !MALI_USE_CSF - } else if ((addr >= JOB_CONTROL_REG(JOB_SLOT0)) && - (addr < (JOB_CONTROL_REG(JOB_SLOT15) + 0x80))) { - int slot_idx = (addr >> 7) & 0xf; - int sub_reg = addr & 0x7F; - - KBASE_DEBUG_ASSERT(slot_idx < NUM_SLOTS); - switch (sub_reg) { - case JS_HEAD_NEXT_LO: - *value = (u32)((hw_error_status.current_jc) & 0xFFFFFFFF); - break; - case JS_HEAD_NEXT_HI: - *value = (u32)(hw_error_status.current_jc >> 32); - break; - case JS_STATUS: - if (hw_error_status.js_status[slot_idx]) - *value = hw_error_status.js_status[slot_idx]; - else /* 0x08 means active, 0x00 idle */ - *value = (dummy->slots[slot_idx].job_active) << 3; - break; - case JS_COMMAND_NEXT: - *value = dummy->slots[slot_idx].job_queued; - break; - - /** - * The dummy model does not implement these registers - * avoid printing error messages - */ - case JS_HEAD_HI: - case JS_HEAD_LO: - case JS_TAIL_HI: - case JS_TAIL_LO: - case JS_FLUSH_ID_NEXT: - break; - - default: - model_error_log( - KBASE_CORE, - "Dummy model register access: unknown job slot reg 0x%02X being read\n", - sub_reg); - break; - } - } else if (addr == GET_GPU_CONTROL_REG(JS_PRESENT)) { - *value = 0x7; -#endif /* !MALI_USE_CSF */ } else if (addr == GET_GPU_CONTROL_REG(AS_PRESENT)) { *value = dummy->control_reg_values->as_present; } else if (addr >= GET_GPU_CONTROL_REG(TEXTURE_FEATURES_0) && @@ -2188,27 +1768,6 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value) *value = 0x9f81ffff; else if (addr == GET_GPU_CONTROL_REG(TEXTURE_FEATURES_3)) *value = 0; -#if !MALI_USE_CSF - } else if (addr >= GPU_CONTROL_REG(JS0_FEATURES) && - addr <= GPU_CONTROL_REG(JS15_FEATURES)) { - switch (addr) { - case GPU_CONTROL_REG(JS0_FEATURES): - *value = 0x20e; - break; - - case GPU_CONTROL_REG(JS1_FEATURES): - *value = 0x1fe; - break; - - case GPU_CONTROL_REG(JS2_FEATURES): - *value = 0x7e; - break; - - default: - *value = 0; - break; - } -#endif /* !MALI_USE_CSF */ } else if (addr >= GET_GPU_CONTROL_REG(L2_FEATURES) && addr <= GET_GPU_CONTROL_REG(MMU_FEATURES)) { if (addr == GET_GPU_CONTROL_REG(L2_FEATURES)) @@ -2242,8 +1801,11 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value) } else if ((addr >= MMU_STAGE1_REG(MMU_AS_REG(0, AS_TRANSTAB_LO))) && (addr <= MMU_STAGE1_REG(MMU_AS_REG(15, AS_STATUS)))) { - u32 mem_addr_space = (addr - MMU_STAGE1_REG(MMU_AS_REG(0, AS_TRANSTAB_LO))) >> - MMU_STAGE1_AS_SHIFT; + u32 mem_addr_space; + + mem_addr_space = (addr - MMU_STAGE1_REG(MMU_AS_REG(0, AS_TRANSTAB_LO))) >> + MMU_STAGE1_AS_SHIFT; + switch (addr & 0x3F) { case AS_TRANSTAB_LO: *value = (u32)(hw_error_status.as_transtab[mem_addr_space] & 0xffffffff); @@ -2289,7 +1851,6 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value) *value = hw_error_status.mmu_irq_rawstat; } else if (addr == MMU_CONTROL_REG(MMU_IRQ_STATUS)) { *value = hw_error_status.mmu_irq_mask & hw_error_status.mmu_irq_rawstat; -#if MALI_USE_CSF } else if (addr == IPA_CONTROL_REG(STATUS) || addr == IPA_CONTROL_REG(STATUS) + GPU_GOV_IPA_CONTROL_OFFSET) { *value = (ipa_control_timer_enabled << 31); @@ -2382,7 +1943,6 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value) *value = gpu_model_get_prfcnt_value(KBASE_IPA_CORE_TYPE_NEURAL, counter_index, is_low_word); -#endif /* MALI_USE_CSF */ } else if (addr == GET_GPU_CONTROL_REG(GPU_FEATURES_LO)) { *value = dummy->control_reg_values->gpu_features_lo; } else if (addr == GET_GPU_CONTROL_REG(GPU_FEATURES_HI)) { @@ -2473,13 +2033,8 @@ int gpu_model_set_dummy_prfcnt_user_sample(u32 __user *data, u32 size) } spin_lock_irqsave(&performance_counters.access_lock, flags); -#if !MALI_USE_CSF - offset = set_user_sample_core_type(performance_counters.jm_counters, user_data, offset, - size, 1); -#else offset = set_user_sample_core_type(performance_counters.cshw_counters, user_data, offset, size, 1); -#endif /* !MALI_USE_CSF */ offset = set_user_sample_core_type(performance_counters.tiler_counters, user_data, offset, size, hweight64(DUMMY_IMPLEMENTATION_TILER_PRESENT)); offset = set_user_sample_core_type(performance_counters.l2_counters, user_data, offset, @@ -2499,13 +2054,8 @@ void gpu_model_set_dummy_prfcnt_kernel_sample(u64 *data, u32 size) u32 offset = 0; spin_lock_irqsave(&performance_counters.access_lock, flags); -#if !MALI_USE_CSF - offset = set_kernel_sample_core_type(performance_counters.jm_counters, data, offset, size, - 1); -#else offset = set_kernel_sample_core_type(performance_counters.cshw_counters, data, offset, size, 1); -#endif /* !MALI_USE_CSF */ offset = set_kernel_sample_core_type(performance_counters.tiler_counters, data, offset, size, hweight64(DUMMY_IMPLEMENTATION_TILER_PRESENT)); offset = set_kernel_sample_core_type(performance_counters.l2_counters, data, offset, size, diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_model_dummy.h b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_model_dummy.h index 115d31a6f560..c57c3020c51a 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_model_dummy.h +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_model_dummy.h @@ -202,7 +202,6 @@ void gpu_model_set_dummy_prfcnt_cores(struct kbase_device *kbdev, u64 l2_present /* Clear the counter values array maintained by the dummy model */ void gpu_model_clear_prfcnt_values(void); -#if MALI_USE_CSF /** * gpu_model_prfcnt_dump_request() - Request performance counter sample dump. * @sample_buf: Pointer to KBASE_DUMMY_MODEL_MAX_VALUES_PER_SAMPLE sized array @@ -217,7 +216,6 @@ void gpu_model_prfcnt_dump_request(uint32_t *sample_buf, struct gpu_model_prfcnt * @model: Model pointer returned by midgard_model_create(). */ void gpu_model_glb_request_job_irq(void *model); -#endif /* MALI_USE_CSF */ extern struct error_status_t hw_error_status; diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_always_on.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_always_on.c index 37c35ee9bd32..12bb7448f619 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_always_on.c +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_always_on.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -67,9 +67,7 @@ const struct kbase_pm_policy kbase_pm_always_on_policy_ops = { always_on_get_core_active, /* get_core_active */ NULL, /* handle_event */ KBASE_PM_POLICY_ID_ALWAYS_ON, /* id */ -#if MALI_USE_CSF ALWAYS_ON_PM_SCHED_FLAGS, /* pm_sched_flags */ -#endif }; KBASE_EXPORT_TEST_API(kbase_pm_always_on_policy_ops); diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_backend.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_backend.c index a61809dd2fda..28665a8f1ee1 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_backend.c +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_backend.c @@ -29,20 +29,13 @@ #include #include -#if !MALI_USE_CSF -#include -#include -#include -#else #include #include #include #include -#endif /* !MALI_USE_CSF */ #include #include #include -#include #include @@ -99,9 +92,6 @@ void kbase_pm_register_access_enable(struct kbase_device *kbdev) if (callbacks) callbacks->power_on_callback(kbdev); - if (WARN_ON(kbase_io_is_aw_removed(kbdev))) - dev_err(kbdev->dev, "Attempting to power on while GPU lost\n"); - kbase_io_clear_status(kbdev->io, KBASE_IO_STATUS_GPU_OFF); } @@ -124,6 +114,7 @@ int kbase_hwaccess_pm_init(struct kbase_device *kbdev) KBASE_DEBUG_ASSERT(kbdev != NULL); mutex_init(&kbdev->pm.lock); + kbdev->pm.runtime_suspend_result = 0; kbdev->pm.backend.gpu_poweroff_wait_wq = alloc_workqueue("kbase_pm_poweroff_wait", WQ_HIGHPRI | WQ_UNBOUND, 1); @@ -133,22 +124,17 @@ int kbase_hwaccess_pm_init(struct kbase_device *kbdev) INIT_WORK(&kbdev->pm.backend.gpu_poweroff_wait_work, kbase_pm_gpu_poweroff_wait_wq); kbdev->pm.backend.ca_cores_enabled = ~0ull; - kbase_io_clear_status(kbdev->io, KBASE_IO_STATUS_AW_REMOVED); + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) + kbdev->pm.backend.ca_gov_cores_enabled = ~0ull; + init_waitqueue_head(&kbdev->pm.backend.gpu_in_desired_state_wait); -#if !MALI_USE_CSF - /* Initialise the metrics subsystem */ - ret = kbasep_pm_metrics_init(kbdev); - if (ret) - return ret; -#else mutex_init(&kbdev->pm.backend.policy_change_lock); kbdev->pm.backend.policy_change_clamp_state_to_off = false; /* Due to dependency on kbase_ipa_control, the metrics subsystem can't * be initialized here. */ CSTD_UNUSED(ret); -#endif init_waitqueue_head(&kbdev->pm.backend.reset_done_wait); kbdev->pm.backend.reset_done = false; @@ -162,12 +148,10 @@ int kbase_hwaccess_pm_init(struct kbase_device *kbdev) init_waitqueue_head(&kbdev->pm.backend.poweroff_wait); -#if MALI_USE_CSF /* Select the power interface that the GPU is using. */ kbdev->pm.backend.has_host_pwr_iface = kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_POWER_CONTROL); kbdev->pm.backend.pwr_cntl_delegated = false; -#endif if (kbase_pm_ca_init(kbdev) != 0) goto workq_fail; @@ -181,24 +165,27 @@ int kbase_hwaccess_pm_init(struct kbase_device *kbdev) INIT_WORK(&kbdev->pm.backend.hwcnt_disable_work, kbase_pm_hwcnt_disable_worker); kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) - kbdev->pm.backend.gpu_sleep_allowed = 0; - if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GPU_SLEEP) && - !kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TURSEHW_1997) && - kbdev->pm.backend.callback_power_runtime_gpu_active && - kbdev->pm.backend.callback_power_runtime_gpu_idle) - set_bit(KBASE_GPU_SUPPORTS_GPU_SLEEP, &kbdev->pm.backend.gpu_sleep_allowed); + if (IS_ENABLED(CONFIG_PM)) { + kbdev->pm.backend.gpu_sleep_allowed = 0; + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GPU_SLEEP) && + !kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TURSEHW_1997) && + kbdev->pm.backend.callback_power_runtime_gpu_active && + kbdev->pm.backend.callback_power_runtime_gpu_idle) + set_bit(KBASE_GPU_SUPPORTS_GPU_SLEEP, &kbdev->pm.backend.gpu_sleep_allowed); - kbdev->pm.backend.apply_hw_issue_TITANHW_2938_wa = - kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TITANHW_2938) && - test_bit(KBASE_GPU_SUPPORTS_GPU_SLEEP, &kbdev->pm.backend.gpu_sleep_allowed); + kbdev->pm.backend.apply_hw_issue_TITANHW_2938_wa = + kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TITANHW_2938) && + test_bit(KBASE_GPU_SUPPORTS_GPU_SLEEP, + &kbdev->pm.backend.gpu_sleep_allowed); - /* FW Sleep-on-Idle is only available in certain architecture revisions */ - if ((kbdev->gpu_props.gpu_id.arch_major > 11) || - ((kbdev->gpu_props.gpu_id.arch_major == 11) && - (kbdev->gpu_props.gpu_id.arch_minor >= 8) && (kbdev->gpu_props.gpu_id.arch_rev >= 10))) - set_bit(KBASE_GPU_SUPPORTS_FW_SLEEP_ON_IDLE, &kbdev->pm.backend.gpu_sleep_allowed); -#endif + /* FW Sleep-on-Idle is only available in certain architecture revisions */ + if ((kbdev->gpu_props.gpu_id.arch_major > 11) || + ((kbdev->gpu_props.gpu_id.arch_major == 11) && + (kbdev->gpu_props.gpu_id.arch_minor >= 8) && + (kbdev->gpu_props.gpu_id.arch_rev >= 10))) + set_bit(KBASE_GPU_SUPPORTS_FW_SLEEP_ON_IDLE, + &kbdev->pm.backend.gpu_sleep_allowed); + } if (IS_ENABLED(CONFIG_MALI_VALHALL_HW_ERRATA_1485982_NOT_AFFECTED)) return 0; @@ -225,9 +212,6 @@ pm_state_machine_fail: kbase_pm_policy_term(kbdev); kbase_pm_ca_term(kbdev); workq_fail: -#if !MALI_USE_CSF - kbasep_pm_metrics_term(kbdev); -#endif return -EINVAL; } @@ -259,7 +243,6 @@ void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume) */ } -#if MALI_USE_CSF static bool wait_cond_mmu_fault_handling_in_gpu_poweroff_wait_wq(struct kbase_device *kbdev, int faults_pending) { @@ -272,7 +255,6 @@ static bool wait_cond_mmu_fault_handling_in_gpu_poweroff_wait_wq(struct kbase_de return cond; } -#endif /** * wait_for_mmu_fault_handling_in_gpu_poweroff_wait() - Wait for pending MMU @@ -288,7 +270,6 @@ static bool wait_cond_mmu_fault_handling_in_gpu_poweroff_wait_wq(struct kbase_de */ static void wait_for_mmu_fault_handling_in_gpu_poweroff_wait(struct kbase_device *kbdev) { -#if MALI_USE_CSF bool reset_triggered = false; int ret = 0; @@ -318,19 +299,12 @@ static void wait_for_mmu_fault_handling_in_gpu_poweroff_wait(struct kbase_device } } while (ret); kbdev->pm.backend.waiting_for_mmu_fault_handling = false; -#else - kbase_pm_unlock(kbdev); - kbase_flush_mmu_wqs(kbdev); - kbase_pm_lock(kbdev); -#endif } static void pm_handle_power_off(struct kbase_device *kbdev) { struct kbase_pm_backend_data *backend = &kbdev->pm.backend; -#if MALI_USE_CSF enum kbase_mcu_state mcu_state; -#endif unsigned long flags; lockdep_assert_held(&kbdev->pm.lock); @@ -339,28 +313,24 @@ static void pm_handle_power_off(struct kbase_device *kbdev) return; spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) - if (kbdev->pm.backend.gpu_wakeup_override) { + + if (IS_ENABLED(CONFIG_PM) && kbdev->pm.backend.gpu_wakeup_override) { spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); return; } -#endif + WARN_ON(backend->shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF || backend->l2_state != KBASE_L2_OFF); -#if MALI_USE_CSF mcu_state = backend->mcu_state; WARN_ON(!kbase_pm_is_mcu_inactive(kbdev, mcu_state)); -#endif spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) - if (backend->callback_power_runtime_gpu_idle) { + if (IS_ENABLED(CONFIG_PM) && backend->callback_power_runtime_gpu_idle) { WARN_ON(backend->gpu_idled); backend->callback_power_runtime_gpu_idle(kbdev); backend->gpu_idled = true; return; } -#endif /* Disable interrupts and turn the clock off */ if (unlikely(!kbase_pm_clock_off(kbdev))) { @@ -398,21 +368,12 @@ void kbase_pm_handle_gpu_poweroff_wait_work(struct kbase_device *kbdev) KBASE_KTRACE_ADD(kbdev, PM_POWEROFF_WAIT_WQ, NULL, 0); -#if !MALI_USE_CSF - /* Wait for power transitions to complete. We do this with no locks held - * so that we don't deadlock with any pending workqueues. - */ - kbase_pm_wait_for_desired_state(kbdev); -#endif - kbase_pm_lock(kbdev); -#if MALI_USE_CSF spin_lock_irqsave(&kbdev->hwaccess_lock, flags); if (pm->backend.invoke_poweroff_wait_wq_when_l2_off || !pm->backend.poweroff_wait_in_progress) goto wakeup_exit; spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#endif pm_handle_power_off(kbdev); @@ -421,18 +382,11 @@ void kbase_pm_handle_gpu_poweroff_wait_work(struct kbase_device *kbdev) if (backend->poweron_required) { backend->poweron_required = false; kbdev->pm.backend.l2_desired = true; -#if MALI_USE_CSF kbdev->pm.backend.mcu_desired = true; -#endif kbase_pm_update_state(kbdev); kbase_pm_update_cores_state_nolock(kbdev); -#if !MALI_USE_CSF - kbase_backend_slot_update(kbdev); -#endif /* !MALI_USE_CSF */ } -#if MALI_USE_CSF wakeup_exit: -#endif spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); kbase_pm_unlock(kbdev); wake_up(&kbdev->pm.backend.poweroff_wait); @@ -589,28 +543,20 @@ static void kbase_pm_hwcnt_disable_worker(struct work_struct *data) */ backend->hwcnt_disabled = true; kbase_pm_update_state(kbdev); -#if !MALI_USE_CSF - kbase_backend_slot_update(kbdev); -#endif /* !MALI_USE_CSF */ } else { /* PM state was updated while we were doing the disable, * so we need to undo the disable we just performed. */ -#if MALI_USE_CSF unsigned long lock_flags; kbase_csf_scheduler_spin_lock(kbdev, &lock_flags); -#endif kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); -#if MALI_USE_CSF kbase_csf_scheduler_spin_unlock(kbdev, lock_flags); -#endif } spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) /** * kbase_pm_do_poweroff_sync - Do the synchronous power down of GPU * @@ -679,7 +625,6 @@ out: kbase_pm_unlock(kbdev); return ret; } -#endif void kbase_pm_do_poweroff(struct kbase_device *kbdev) { @@ -695,12 +640,7 @@ void kbase_pm_do_poweroff(struct kbase_device *kbdev) if (kbdev->pm.backend.poweroff_wait_in_progress) goto unlock_hwaccess; -#if MALI_USE_CSF kbdev->pm.backend.mcu_desired = false; -#else - /* Force all cores off */ - kbdev->pm.backend.shaders_desired = false; -#endif kbdev->pm.backend.l2_desired = false; kbdev->pm.backend.poweroff_wait_in_progress = true; @@ -744,7 +684,10 @@ int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, unsigned int flags) kbase_pm_unlock(kbdev); return ret; } -#if MALI_USE_CSF + + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) + kbdev->pm.sysfs_gov_core_mask = kbdev->gpu_props.shader_present; + kbdev->pm.debug_core_mask = kbdev->gpu_props.shader_present; spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); /* Set the initial value for 'shaders_avail'. It would be later @@ -755,25 +698,19 @@ int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, unsigned int flags) */ kbdev->pm.backend.shaders_avail = kbase_pm_ca_get_core_mask(kbdev); spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); -#else - kbdev->pm.debug_core_mask_all = kbdev->pm.debug_core_mask[0] = - kbdev->pm.debug_core_mask[1] = kbdev->pm.debug_core_mask[2] = - kbdev->gpu_props.shader_present; -#endif /* Pretend the GPU is active to prevent a power policy turning the GPU * cores off */ kbdev->pm.active_count = 1; -#if MALI_USE_CSF && KBASE_PM_RUNTIME - if (kbdev->pm.backend.callback_power_runtime_gpu_active) { + + if (IS_ENABLED(CONFIG_PM) && kbdev->pm.backend.callback_power_runtime_gpu_active) { /* Take the RPM reference count to match with the internal * PM reference count */ kbdev->pm.backend.callback_power_runtime_gpu_active(kbdev); WARN_ON(kbdev->pm.backend.gpu_idled); } -#endif spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock, irq_flags); /* Ensure cycle counter is off */ @@ -790,12 +727,10 @@ int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, unsigned int flags) kbdev->pm.backend.gpu_ready = true; /* Turn on the GPU and any cores needed by the policy */ -#if MALI_USE_CSF /* Turn on the L2 caches, needed for firmware boot */ spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); kbdev->pm.backend.l2_desired = true; spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); -#endif kbase_pm_do_poweron(kbdev, false); kbase_pm_unlock(kbdev); @@ -806,15 +741,14 @@ void kbase_hwaccess_pm_halt(struct kbase_device *kbdev) { KBASE_DEBUG_ASSERT(kbdev != NULL); -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) - WARN_ON(kbase_pm_do_poweroff_sync(kbdev)); -#else - mutex_lock(&kbdev->pm.lock); - kbase_pm_do_poweroff(kbdev); - mutex_unlock(&kbdev->pm.lock); - - kbase_pm_wait_for_poweroff_work_complete(kbdev); -#endif + if (IS_ENABLED(CONFIG_PM)) + WARN_ON(kbase_pm_do_poweroff_sync(kbdev)); + else { + mutex_lock(&kbdev->pm.lock); + kbase_pm_do_poweroff(kbdev); + mutex_unlock(&kbdev->pm.lock); + kbase_pm_wait_for_poweroff_work_complete(kbdev); + } } KBASE_EXPORT_TEST_API(kbase_hwaccess_pm_halt); @@ -827,17 +761,12 @@ void kbase_hwaccess_pm_term(struct kbase_device *kbdev) cancel_work_sync(&kbdev->pm.backend.hwcnt_disable_work); - if (kbdev->pm.backend.hwcnt_disabled) { + if (kbdev->pm.backend.hwcnt_disabled && kbdev->csf.firmware_hctl_core_pwr) { unsigned long flags; -#if MALI_USE_CSF + kbase_csf_scheduler_spin_lock(kbdev, &flags); kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); kbase_csf_scheduler_spin_unlock(kbdev, flags); -#else - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#endif } /* Free any resources the policy allocated */ @@ -845,16 +774,11 @@ void kbase_hwaccess_pm_term(struct kbase_device *kbdev) kbase_pm_policy_term(kbdev); kbase_pm_ca_term(kbdev); -#if !MALI_USE_CSF - /* Shut down the metrics subsystem */ - kbasep_pm_metrics_term(kbdev); -#else if (WARN_ON(mutex_is_locked(&kbdev->pm.backend.policy_change_lock))) { mutex_lock(&kbdev->pm.backend.policy_change_lock); mutex_unlock(&kbdev->pm.backend.policy_change_lock); } mutex_destroy(&kbdev->pm.backend.policy_change_lock); -#endif destroy_workqueue(kbdev->pm.backend.gpu_poweroff_wait_wq); } @@ -866,48 +790,23 @@ void kbase_pm_power_changed(struct kbase_device *kbdev) spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbase_pm_update_state(kbdev); -#if !MALI_USE_CSF - kbase_backend_slot_update(kbdev); -#endif /* !MALI_USE_CSF */ - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } -#if MALI_USE_CSF void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, u64 new_core_mask) { lockdep_assert_held(&kbdev->hwaccess_lock); lockdep_assert_held(&kbdev->pm.lock); - kbdev->pm.debug_core_mask = new_core_mask; - kbase_pm_update_dynamic_cores_onoff(kbdev); + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) { + kbdev->pm.sysfs_gov_core_mask = new_core_mask; + kbase_pm_ca_set_gov_core_mask_nolock(kbdev, SYSFS_COREMASK, new_core_mask); + } else { + kbdev->pm.debug_core_mask = new_core_mask; + kbase_pm_update_dynamic_cores_onoff(kbdev); + } } KBASE_EXPORT_TEST_API(kbase_pm_set_debug_core_mask); -#else -void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, u64 *new_core_mask, - size_t new_core_mask_size) -{ - size_t i; - - lockdep_assert_held(&kbdev->hwaccess_lock); - lockdep_assert_held(&kbdev->pm.lock); - - if (kbase_dummy_job_wa_enabled(kbdev)) { - dev_warn_once( - kbdev->dev, - "Change of core mask not supported for slot 0 as dummy job WA is enabled"); - new_core_mask[0] = kbdev->pm.debug_core_mask[0]; - } - - kbdev->pm.debug_core_mask_all = 0; - for (i = 0; i < new_core_mask_size; i++) { - kbdev->pm.debug_core_mask[i] = new_core_mask[i]; - kbdev->pm.debug_core_mask_all |= new_core_mask[i]; - } - - kbase_pm_update_dynamic_cores_onoff(kbdev); -} -#endif /* MALI_USE_CSF */ void kbase_hwaccess_pm_gpu_active(struct kbase_device *kbdev) { @@ -923,35 +822,23 @@ int kbase_hwaccess_pm_suspend(struct kbase_device *kbdev) { int ret = 0; -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) - ret = kbase_pm_do_poweroff_sync(kbdev); - if (ret) - return ret; -#else - /* Force power off the GPU and all cores (regardless of policy), only - * after the PM active count reaches zero (otherwise, we risk turning it - * off prematurely) - */ - kbase_pm_lock(kbdev); + if (IS_ENABLED(CONFIG_PM)) { + ret = kbase_pm_do_poweroff_sync(kbdev); + if (ret) + return ret; + } else { + /* Force power off the GPU and all cores (regardless of policy), only + * after the PM active count reaches zero (otherwise, we risk turning it + * off prematurely) + */ + kbase_pm_lock(kbdev); + kbase_pm_do_poweroff(kbdev); + kbase_pm_unlock(kbdev); - kbase_pm_do_poweroff(kbdev); - -#if !MALI_USE_CSF - kbase_backend_timer_suspend(kbdev); -#endif /* !MALI_USE_CSF */ - - kbase_pm_unlock(kbdev); - - ret = kbase_pm_wait_for_poweroff_work_complete(kbdev); - if (ret) { -#if !MALI_USE_CSF - mutex_lock(&kbdev->js_data.runpool_mutex); - kbase_backend_timer_resume(kbdev); - mutex_unlock(&kbdev->js_data.runpool_mutex); -#endif /* !MALI_USE_CSF */ - return ret; + ret = kbase_pm_wait_for_poweroff_work_complete(kbdev); + if (ret) + return ret; } -#endif WARN_ON(kbase_io_is_gpu_powered(kbdev)); WARN_ON(atomic_read(&kbdev->faults_pending)); @@ -976,10 +863,6 @@ void kbase_hwaccess_pm_resume(struct kbase_device *kbdev) } kbase_pm_do_poweron(kbdev, true); -#if !MALI_USE_CSF - kbase_backend_timer_resume(kbdev); -#endif /* !MALI_USE_CSF */ - kbase_pm_unlock(kbdev); } @@ -995,24 +878,23 @@ void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev) return; } -#if MALI_USE_CSF - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - if (kbase_io_is_gpu_powered(kbdev) && !kbase_io_is_aw_removed(kbdev)) { + if (kbase_io_is_aw_removed(kbdev)) { unsigned long flags_sched; - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); /* GPU is no longer mapped to VM. So no interrupts will * be received and Mali registers have been replaced by * dummy RAM */ - WARN(kbase_io_has_gpu(kbdev), "GPU is still available after GPU lost event\n"); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + kbase_csf_scheduler_spin_lock(kbdev, &flags_sched); if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) != KBASE_RESET_GPU_NOT_PENDING) dev_warn(kbdev->dev, "GPU reset pending at the time of GPU lost event"); atomic_set(&kbdev->hwaccess.backend.reset_gpu, KBASE_RESET_GPU_NOT_PENDING); kbase_csf_scheduler_spin_unlock(kbdev, flags_sched); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); kbase_synchronize_irqs(kbdev); @@ -1030,60 +912,13 @@ void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev) spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbdev->protected_mode = false; kbase_pm_update_state(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); /* Cancel any pending HWC dumps */ kbase_hwcnt_backend_csf_on_unrecoverable_error(&kbdev->hwcnt_gpu_iface); } - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#else - /* Releasing the VM state lock here is safe because - * we are guaranteed to be in either STOPPING_IDLE, - * STOPPING_ACTIVE or SUSPEND_PENDING at this point. - * The only transitions that are valid from here are to - * STOPPED, STOPPED_GPU_REQUESTED or SUSPENDED which can - * only happen at the completion of the GPU lost handling. - */ - mutex_unlock(&arb_vm_state->vm_state_lock); - mutex_lock(&kbdev->pm.lock); - mutex_lock(&arb_vm_state->vm_state_lock); - if (kbase_io_is_gpu_powered(kbdev) && !kbase_io_is_aw_removed(kbdev)) { - ktime_t end_timestamp = ktime_get_raw(); - - /* GPU is no longer mapped to VM. So no interrupts will - * be received and Mali registers have been replaced by - * dummy RAM - */ - WARN(kbase_io_has_gpu(kbdev), "GPU is still available after GPU lost event\n"); - - /* Full GPU reset will have been done by hypervisor, so cancel */ - atomic_set(&kbdev->hwaccess.backend.reset_gpu, KBASE_RESET_GPU_NOT_PENDING); - hrtimer_cancel(&kbdev->hwaccess.backend.reset_timer); - - kbase_synchronize_irqs(kbdev); - - /* Clear all jobs running on the GPU */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbdev->protected_mode = false; - kbase_backend_reset(kbdev, &end_timestamp); - kbase_pm_metrics_update(kbdev, NULL); - kbase_pm_update_state(kbdev); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - /* Cancel any pending HWC dumps */ - spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DUMPING || - kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) { - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_FAULT; - kbdev->hwcnt.backend.triggered = 1; - wake_up(&kbdev->hwcnt.backend.wait); - } - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - } - mutex_unlock(&kbdev->pm.lock); -#endif /* MALI_USE_CSF */ } -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) int kbase_pm_force_mcu_wakeup_after_sleep(struct kbase_device *kbdev) { unsigned long flags; @@ -1108,17 +943,15 @@ static int pm_handle_mcu_sleep_on_runtime_suspend(struct kbase_device *kbdev) lockdep_assert_held(&kbdev->csf.scheduler.lock); lockdep_assert_held(&kbdev->pm.lock); -#ifdef CONFIG_MALI_VALHALL_DEBUG /* In case of no active CSG on slot, powering up L2 could be skipped and * proceed directly to suspend GPU. * ToDo: firmware has to be reloaded after wake-up as no halt command * has been sent when GPU was put to sleep mode. */ - if (!kbase_csf_scheduler_get_nr_active_csgs(kbdev)) + if (IS_ENABLED(CONFIG_MALI_VALHALL_DEBUG) && !kbase_csf_scheduler_get_nr_active_csgs(kbdev)) dev_info( kbdev->dev, "No active CSGs. Can skip the power up of L2 and go for suspension directly"); -#endif ret = kbase_pm_force_mcu_wakeup_after_sleep(kbdev); if (ret) { @@ -1300,5 +1133,3 @@ out: return ret; } - -#endif diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_ca.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_ca.c index 32c51152cbbd..27ab4162c325 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_ca.c +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_ca.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2013-2024 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2013-2025 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -27,19 +27,37 @@ #include #include #include -#include -int kbase_pm_ca_init(struct kbase_device *kbdev) -{ #ifdef CONFIG_MALI_VALHALL_DEVFREQ +static void pm_init_cores_enabled_mask(struct kbase_device *kbdev) +{ struct kbase_pm_backend_data *pm_backend = &kbdev->pm.backend; if (kbdev->current_core_mask) pm_backend->ca_cores_enabled = kbdev->current_core_mask; else pm_backend->ca_cores_enabled = kbdev->gpu_props.shader_present; +} + +static void pm_init_gov_cores_enabled_mask(struct kbase_device *kbdev) +{ + struct kbase_pm_backend_data *pm_backend = &kbdev->pm.backend; + + if (kbdev->current_core_mask) + pm_backend->ca_gov_cores_enabled = kbdev->current_core_mask; + else + pm_backend->ca_gov_cores_enabled = kbdev->gpu_props.shader_present; +} #endif +int kbase_pm_ca_init(struct kbase_device *kbdev) +{ +#ifdef CONFIG_MALI_VALHALL_DEVFREQ + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) + pm_init_gov_cores_enabled_mask(kbdev); + + pm_init_cores_enabled_mask(kbdev); +#endif return 0; } @@ -48,61 +66,119 @@ void kbase_pm_ca_term(struct kbase_device *kbdev) CSTD_UNUSED(kbdev); } -#ifdef CONFIG_MALI_VALHALL_DEVFREQ -void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask) +void kbase_pm_ca_set_gov_core_mask_nolock(struct kbase_device *kbdev, enum mask_type core_mask_type, + u64 core_mask) { struct kbase_pm_backend_data *pm_backend = &kbdev->pm.backend; - unsigned long flags; -#if MALI_USE_CSF - u64 old_core_mask = 0; - bool mmu_sync_needed = false; - if (!IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI) && - kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_GPU2019_3901)) { - mmu_sync_needed = true; - down_write(&kbdev->csf.mmu_sync_sem); + if (!kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) { + dev_warn( + kbdev->dev, + "This function requires Kbase to have access to GOV_CORE_MASK register, cannot proceed\n"); + return; + } + + lockdep_assert_held(&kbdev->hwaccess_lock); + + /** A value of ZERO means disabling. + * When disabling, store the last used mask for re-enabling + */ + if (core_mask_type == SYSFS_COREMASK) { + if (core_mask != 0) + pm_backend->ca_gov_cores_enabled = core_mask; + else +#ifdef CONFIG_MALI_VALHALL_DEVFREQ + pm_backend->ca_gov_cores_enabled = kbdev->current_core_mask; +#else + pm_backend->ca_gov_cores_enabled = + kbdev->gpu_props.curr_config.shader_present; +#endif + } +#ifdef CONFIG_MALI_VALHALL_DEVFREQ + /* sysfs core mask takes priority over OPP mask when sysfs core mask is set */ + else if (core_mask_type == DEVFREQ_COREMASK) { + if (core_mask == 0) { + dev_warn(kbdev->dev, + "Required core_mask cannot be zero when sysfs usage disabled\n"); + return; + } + /* if sysfs non-zero then no need to re-write value */ + if (kbdev->pm.sysfs_gov_core_mask) + return; + + pm_backend->ca_gov_cores_enabled = core_mask; } #endif - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + /** after all checks, write the to GOV_CORE_MASK register if GPU powered, + * otherwise value will be applied on next reboot. + */ + if (kbase_io_is_gpu_powered(kbdev)) + kbase_reg_write64(kbdev, GPU_GOVERNOR_ENUM(GOV_CORE_MASK), + pm_backend->ca_gov_cores_enabled); +} -#if MALI_USE_CSF - if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) { - if (kbase_io_is_gpu_powered(kbdev)) { - kbase_reg_write64(kbdev, GPU_GOVERNOR_ENUM(GOV_CORE_MASK), - core_mask & kbdev->pm.debug_core_mask); - } +void kbase_pm_ca_set_gov_core_mask(struct kbase_device *kbdev, enum mask_type core_mask_type, + u64 core_mask) +{ + unsigned long flags; - goto unlock; + if (!kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) { + dev_warn( + kbdev->dev, + "This function requires Kbase to have access to GOV_CORE_MASK register, cannot proceed\n"); + return; } + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_pm_ca_set_gov_core_mask_nolock(kbdev, core_mask_type, core_mask); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +#ifdef CONFIG_MALI_VALHALL_DEVFREQ + +static int set_core_mask_gov(struct kbase_device *kbdev, u64 core_mask) +{ + if (!kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) { + dev_warn( + kbdev->dev, + "This function requires Kbase to have access to GOV_CORE_MASK register, cannot proceed\n"); + return -EIO; + } + + /* Requires a validity check to ensure we don't try to set cores we do not have */ + if ((core_mask & kbdev->gpu_props.shader_present) != core_mask) { + dev_err(kbdev->dev, + "core_mask (%llu) must be a subset of the shader present (%llu)", core_mask, + kbdev->gpu_props.shader_present); + return -EINVAL; + } + + kbase_pm_ca_set_gov_core_mask(kbdev, DEVFREQ_COREMASK, core_mask); + + return 0; +} + +static int set_core_mask_legacy(struct kbase_device *kbdev, u64 core_mask) +{ + struct kbase_pm_backend_data *pm_backend = &kbdev->pm.backend; + u64 old_core_mask = 0; + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); if (!(core_mask & kbdev->pm.debug_core_mask)) { dev_err(kbdev->dev, - "OPP core mask 0x%llX does not intersect with debug mask 0x%llX\n", + "OPP core mask 0x%llX does not intersect with sysfs debug mask 0x%llX\n", core_mask, kbdev->pm.debug_core_mask); - goto unlock; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + return -EINVAL; } old_core_mask = pm_backend->ca_cores_enabled; -#else - if (!(core_mask & kbdev->pm.debug_core_mask_all)) { - dev_err(kbdev->dev, - "OPP core mask 0x%llX does not intersect with debug mask 0x%llX\n", - core_mask, kbdev->pm.debug_core_mask_all); - goto unlock; - } - - if (kbase_dummy_job_wa_enabled(kbdev)) { - dev_err_once(kbdev->dev, - "Dynamic core scaling not supported as dummy job WA is enabled"); - goto unlock; - } -#endif /* MALI_USE_CSF */ pm_backend->ca_cores_enabled = core_mask; kbase_pm_update_state(kbdev); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#if MALI_USE_CSF /* Check if old_core_mask contained the undesired cores and wait * for those cores to get powered down */ @@ -114,33 +190,51 @@ void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask) } } + return 0; +} + +void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask) +{ + bool mmu_sync_needed = false; + int err; + + if (!IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI) && + kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_GPU2019_3901)) { + mmu_sync_needed = true; + down_write(&kbdev->csf.mmu_sync_sem); + } + + err = kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT) ? + set_core_mask_gov(kbdev, core_mask) : + set_core_mask_legacy(kbdev, core_mask); if (mmu_sync_needed) up_write(&kbdev->csf.mmu_sync_sem); -#endif - dev_dbg(kbdev->dev, "Devfreq policy : new core mask=%llX\n", pm_backend->ca_cores_enabled); - - return; -unlock: - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#if MALI_USE_CSF - if (mmu_sync_needed) - up_write(&kbdev->csf.mmu_sync_sem); -#endif + if (!err) + dev_dbg(kbdev->dev, "Devfreq policy : new core mask=%llX\n", core_mask); } KBASE_EXPORT_TEST_API(kbase_devfreq_set_core_mask); #endif u64 kbase_pm_ca_get_debug_core_mask(struct kbase_device *kbdev) { -#if MALI_USE_CSF return kbdev->pm.debug_core_mask; -#else - return kbdev->pm.debug_core_mask_all; -#endif } KBASE_EXPORT_TEST_API(kbase_pm_ca_get_debug_core_mask); +u64 kbase_pm_ca_get_sysfs_gov_core_mask(struct kbase_device *kbdev) +{ + if (!kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) { + dev_warn( + kbdev->dev, + "This function requires Kbase to have access to GOV_CORE_MASK register, cannot proceed\n"); + return 0; + } + + return kbdev->pm.sysfs_gov_core_mask; +} +KBASE_EXPORT_TEST_API(kbase_pm_ca_get_sysfs_gov_core_mask); + u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev) { u64 debug_core_mask = kbase_pm_ca_get_debug_core_mask(kbdev); @@ -160,18 +254,23 @@ u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev) return kbdev->gpu_props.curr_config.shader_present & debug_core_mask; #endif } - KBASE_EXPORT_TEST_API(kbase_pm_ca_get_core_mask); +u64 kbase_pm_ca_get_gov_core_mask(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + return kbdev->pm.backend.ca_gov_cores_enabled; +} +KBASE_EXPORT_TEST_API(kbase_pm_ca_get_gov_core_mask); + u64 kbase_pm_ca_get_instr_core_mask(struct kbase_device *kbdev) { lockdep_assert_held(&kbdev->hwaccess_lock); #if IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI) return (((1ull) << KBASE_DUMMY_MODEL_MAX_SHADER_CORES) - 1); -#elif MALI_USE_CSF - return kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER); #else - return kbdev->pm.backend.pm_shaders_core_mask; + return kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER); #endif } diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_ca.h b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_ca.h index 37d1020e28ff..8c55252a1a37 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_ca.h +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_ca.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2011-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -68,6 +68,26 @@ u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev); */ u64 kbase_pm_ca_get_debug_core_mask(struct kbase_device *kbdev); +/** + * kbase_pm_ca_get_sysfs_gov_core_mask - Get the value sysfs has requested to set shader core mask + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * This is used only when Kbase has access to the GOV_CORE_MASK register. + * Return: The bit mask of user-selected cores + */ +u64 kbase_pm_ca_get_sysfs_gov_core_mask(struct kbase_device *kbdev); + +/** + * kbase_pm_ca_get_gov_core_mask - Get currently available gov core mask. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * This is used only when Kbase has access to the GOV_CORE_MASK register. + * Return: a mask of the currently available shader cores. + */ +u64 kbase_pm_ca_get_gov_core_mask(struct kbase_device *kbdev); + /** * kbase_pm_ca_update_core_status - Update core status * diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_coarse_demand.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_coarse_demand.c index cd2d65b1e4bc..d9e2432b9ef7 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_coarse_demand.c +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_coarse_demand.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2012-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2012-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -59,9 +59,7 @@ const struct kbase_pm_policy kbase_pm_coarse_demand_policy_ops = { coarse_demand_get_core_active, /* get_core_active */ NULL, /* handle_event */ KBASE_PM_POLICY_ID_COARSE_DEMAND, /* id */ -#if MALI_USE_CSF COARSE_ON_DEMAND_PM_SCHED_FLAGS, /* pm_sched_flags */ -#endif }; KBASE_EXPORT_TEST_API(kbase_pm_coarse_demand_policy_ops); diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_defs.h b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_defs.h index 67ce9d48656b..0c9ec908d4b4 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_defs.h +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_defs.h @@ -31,10 +31,6 @@ #include -#if defined(CONFIG_PM_RUNTIME) || defined(CONFIG_PM) -#define KBASE_PM_RUNTIME 1 -#endif - /* Forward definition - see mali_kbase.h */ struct kbase_device; struct kbase_jd_atom; @@ -61,7 +57,6 @@ struct kbase_jd_atom; * * They specify which type of core should be acted on. */ -#if MALI_USE_CSF enum kbase_pm_core_type { KBASE_PM_CORE_L2 = HOST_POWER_ENUM(L2_PRESENT), KBASE_PM_CORE_SHADER = HOST_POWER_ENUM(SHADER_PRESENT), @@ -76,14 +71,6 @@ enum kbase_pm_core_type { */ KBASE_PM_CORE_BASE = HOST_POWER_ENUM(BASE_PRESENT) }; -#else -enum kbase_pm_core_type { - KBASE_PM_CORE_L2 = GPU_CONTROL_ENUM(L2_PRESENT), - KBASE_PM_CORE_SHADER = GPU_CONTROL_ENUM(SHADER_PRESENT), - KBASE_PM_CORE_TILER = GPU_CONTROL_ENUM(TILER_PRESENT), - KBASE_PM_CORE_STACK = GPU_CONTROL_ENUM(STACK_PRESENT) -}; -#endif /* * enum kbase_l2_core_state - The states used for the L2 cache & tiler power @@ -95,7 +82,6 @@ enum kbase_l2_core_state { #undef KBASEP_L2_STATE }; -#if MALI_USE_CSF /* * enum kbase_mcu_state - The states used for the MCU state machine. */ @@ -104,7 +90,6 @@ enum kbase_mcu_state { #include "mali_kbase_pm_mcu_states.h" #undef KBASEP_MCU_STATE }; -#endif /* * enum kbase_shader_core_state - The states used for the shaders' state machine. @@ -163,22 +148,11 @@ enum kbase_pm_runtime_suspend_abort_reason { * time_period_start timestamp, measured in units of 256ns. * @time_in_protm: The amount of time the GPU has spent in protected mode since * the time_period_start timestamp, measured in units of 256ns. - * @busy_cl: the amount of time the GPU was busy executing CL jobs. Note that - * if two CL jobs were active for 256ns, this value would be updated - * with 2 (2x256ns). - * @busy_gl: the amount of time the GPU was busy executing GL jobs. Note that - * if two GL jobs were active for 256ns, this value would be updated - * with 2 (2x256ns). */ struct kbasep_pm_metrics { u32 time_busy; u32 time_idle; -#if MALI_USE_CSF u32 time_in_protm; -#else - u32 busy_cl[2]; - u32 busy_gl; -#endif }; /** @@ -189,11 +163,6 @@ struct kbasep_pm_metrics { * kbase_ipa_control client * @skip_gpu_active_sanity_check: Decide whether to skip GPU_ACTIVE sanity * check in DVFS utilisation calculation - * @gpu_active: true when the GPU is executing jobs. false when - * not. Updated when the job scheduler informs us a job in submitted - * or removed from a GPU slot. - * @active_cl_ctx: number of CL jobs active on the GPU. Array is per-device. - * @active_gl_ctx: number of GL jobs active on the GPU. Array is per-slot. * @lock: spinlock protecting the kbasep_pm_metrics_state structure * @platform_data: pointer to data controlled by platform specific code * @kbdev: pointer to kbase device for which metrics are collected @@ -209,14 +178,8 @@ struct kbasep_pm_metrics { */ struct kbasep_pm_metrics_state { ktime_t time_period_start; -#if MALI_USE_CSF void *ipa_control_client; bool skip_gpu_active_sanity_check; -#else - bool gpu_active; - u32 active_cl_ctx[2]; - u32 active_gl_ctx[3]; -#endif spinlock_t lock; void *platform_data; @@ -336,6 +299,8 @@ union kbase_pm_policy_data { * called previously. * See &struct kbase_pm_callback_conf. * @ca_cores_enabled: Cores that are currently available + * @ca_gov_cores_enabled: Final value used for setting GOV_CORE_MASK register. + * Depends on sysfs-core-mask and devfreq-core-mask. * @apply_hw_issue_TITANHW_2938_wa: Indicates if the workaround for KBASE_HW_ISSUE_TITANHW_2938 * needs to be applied when unmapping memory from GPU. * @mcu_state: The current state of the micro-control unit, only applicable @@ -421,20 +386,6 @@ union kbase_pm_policy_data { * re-enabled whilst the flag is set. * @in_reset: True if a GPU is resetting and normal power manager operation is * suspended - * @partial_shaderoff: True if we want to partial power off shader cores, - * it indicates a partial shader core off case, - * do some special operation for such case like flush - * L2 cache because of GPU2017-861 - * @protected_entry_transition_override : True if GPU reset is being used - * before entering the protected mode and so - * the reset handling behaviour is being - * overridden. - * @protected_transition_override : True if a protected mode transition is in - * progress and is overriding power manager - * behaviour. - * @protected_l2_override : Non-zero if the L2 cache is required during a - * protected mode transition. Has no effect if not - * transitioning. * @hwcnt_desired: True if we want GPU hardware counters to be enabled. * @hwcnt_disabled: True if GPU hardware counters are not enabled. * @hwcnt_disable_work: Work item to disable GPU hardware counters, used if @@ -456,6 +407,7 @@ union kbase_pm_policy_data { * @gpu_clock_control_work: work item to set GPU clock during L2 power cycle * using gpu_clock_control * @reset_in_progress: Set if reset is ongoing, otherwise set to 0 + * @gpu_lost_pending: Set if GPU_LOST is detected and its handling is pending. * * This structure contains data for the power management framework. There is one * instance of this structure per device in the system. @@ -508,16 +460,14 @@ struct kbase_pm_backend_data { void (*callback_power_runtime_gpu_active)(struct kbase_device *kbdev); u64 ca_cores_enabled; + u64 ca_gov_cores_enabled; -#if MALI_USE_CSF bool apply_hw_issue_TITANHW_2938_wa; enum kbase_mcu_state mcu_state; -#endif enum kbase_l2_core_state l2_state; enum kbase_shader_core_state shaders_state; u64 shaders_avail; u64 shaders_desired_mask; -#if MALI_USE_CSF bool mcu_desired; bool policy_change_clamp_state_to_off; bool waiting_for_mmu_fault_handling; @@ -525,8 +475,6 @@ struct kbase_pm_backend_data { struct mutex policy_change_lock; struct workqueue_struct *core_idle_wq; struct work_struct core_idle_work; - -#ifdef KBASE_PM_RUNTIME unsigned long gpu_sleep_allowed; bool gpu_sleep_mode_active; bool exit_gpu_sleep_mode; @@ -534,7 +482,6 @@ struct kbase_pm_backend_data { bool gpu_wakeup_override; bool db_mirror_interrupt_enabled; enum kbase_pm_runtime_suspend_abort_reason runtime_suspend_abort_reason; -#endif /** * @has_host_pwr_iface: GPU supports the host power control interface. @@ -547,21 +494,12 @@ struct kbase_pm_backend_data { */ bool pwr_cntl_delegated; bool l2_force_off_after_mcu_halt; -#endif bool l2_desired; bool l2_always_on; bool shaders_desired; bool in_reset; -#if !MALI_USE_CSF - bool partial_shaderoff; - - bool protected_entry_transition_override; - bool protected_transition_override; - int protected_l2_override; -#endif - bool hwcnt_desired; bool hwcnt_disabled; struct work_struct hwcnt_disable_work; @@ -573,9 +511,10 @@ struct kbase_pm_backend_data { struct work_struct gpu_clock_control_work; atomic_t reset_in_progress; + + bool gpu_lost_pending; }; -#if MALI_USE_CSF /* CSF PM flag, signaling that the MCU shader Core should be kept on */ #define CSF_DYNAMIC_PM_CORE_KEEP_ON (1 << 0) /* CSF PM flag, signaling no scheduler suspension on idle groups */ @@ -591,7 +530,6 @@ struct kbase_pm_backend_data { #if !MALI_CUSTOMER_RELEASE #define ALWAYS_ON_DEMAND_PM_SCHED_FLAGS (CSF_DYNAMIC_PM_SCHED_IGNORE_IDLE) #endif -#endif /* List of policy IDs */ enum kbase_pm_policy_id { @@ -714,14 +652,12 @@ struct kbase_pm_policy { enum kbase_pm_policy_id id; -#if MALI_USE_CSF /* Policy associated with CSF PM scheduling operational flags. * There are pre-defined required flags exist for each of the * ARM released policies, such as 'always_on', 'coarse_demand' * and etc. */ unsigned int pm_sched_flags; -#endif }; #endif /* _KBASE_PM_HWACCESS_DEFS_H_ */ diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_driver.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_driver.c index 9547c4ca6b17..018d2699c1ac 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_driver.c +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_driver.c @@ -32,11 +32,7 @@ #include #include -#if MALI_USE_CSF #include -#else -#include -#endif /* !MALI_USE_CSF */ #include #include @@ -47,12 +43,9 @@ #include #include #include -#include #include -#if MALI_USE_CSF #include -#endif #include @@ -110,7 +103,6 @@ static u64 kbase_pm_get_state(struct kbase_device *kbdev, enum kbase_pm_core_typ static void kbase_pm_hw_issues_apply(struct kbase_device *kbdev); -#if MALI_USE_CSF static bool pwr_status_gpu_reset_allowed(struct kbase_device *kbdev) { @@ -173,107 +165,26 @@ bool kbase_pm_is_mcu_desired(struct kbase_device *kbdev) if (kbase_pm_no_mcu_core_pwroff(kbdev) && kbdev->pm.backend.l2_desired) return true; -#ifdef KBASE_PM_RUNTIME - if (kbdev->pm.backend.gpu_wakeup_override || - kbdev->pm.backend.runtime_suspend_abort_reason != ABORT_REASON_NONE) - return true; -#endif + if (IS_ENABLED(CONFIG_PM)) { + return kbdev->pm.backend.gpu_wakeup_override || + kbdev->pm.backend.runtime_suspend_abort_reason != ABORT_REASON_NONE; + } return false; } -#endif /* MALI_USE_CSF */ bool kbase_pm_is_l2_desired(struct kbase_device *kbdev) { -#if !MALI_USE_CSF - if (kbdev->pm.backend.protected_entry_transition_override) - return false; - - if (kbdev->pm.backend.protected_transition_override && - kbdev->pm.backend.protected_l2_override) - return true; - - if (kbdev->pm.backend.protected_transition_override && !kbdev->pm.backend.shaders_desired) - return false; -#else if (unlikely(kbdev->pm.backend.policy_change_clamp_state_to_off)) return false; /* Power up the L2 cache only when MCU is desired */ if (likely(kbdev->csf.firmware_inited)) return kbase_pm_is_mcu_desired(kbdev); -#endif return kbdev->pm.backend.l2_desired; } -#if !MALI_USE_CSF -void kbase_pm_protected_override_enable(struct kbase_device *kbdev) -{ - lockdep_assert_held(&kbdev->hwaccess_lock); - - kbdev->pm.backend.protected_transition_override = true; -} -void kbase_pm_protected_override_disable(struct kbase_device *kbdev) -{ - lockdep_assert_held(&kbdev->hwaccess_lock); - - kbdev->pm.backend.protected_transition_override = false; -} - -int kbase_pm_protected_entry_override_enable(struct kbase_device *kbdev) -{ - lockdep_assert_held(&kbdev->hwaccess_lock); - - WARN_ON(!kbdev->protected_mode_transition); - - if (kbdev->pm.backend.l2_always_on && (kbdev->system_coherency == COHERENCY_ACE)) { - WARN_ON(kbdev->pm.backend.protected_entry_transition_override); - - /* - * If there is already a GPU reset pending then wait for it to - * complete before initiating a special reset for protected - * mode entry. - */ - if (kbase_reset_gpu_silent(kbdev)) - return -EAGAIN; - - kbdev->pm.backend.protected_entry_transition_override = true; - } - - return 0; -} - -void kbase_pm_protected_entry_override_disable(struct kbase_device *kbdev) -{ - lockdep_assert_held(&kbdev->hwaccess_lock); - - WARN_ON(!kbdev->protected_mode_transition); - - if (kbdev->pm.backend.l2_always_on && (kbdev->system_coherency == COHERENCY_ACE)) { - WARN_ON(!kbdev->pm.backend.protected_entry_transition_override); - - kbdev->pm.backend.protected_entry_transition_override = false; - } -} - -void kbase_pm_protected_l2_override(struct kbase_device *kbdev, bool override) -{ - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (override) { - kbdev->pm.backend.protected_l2_override++; - WARN_ON(kbdev->pm.backend.protected_l2_override <= 0); - } else { - kbdev->pm.backend.protected_l2_override--; - WARN_ON(kbdev->pm.backend.protected_l2_override < 0); - } - - kbase_pm_update_state(kbdev); -} -#endif - -#if MALI_USE_CSF #define ACTION_TYPE_TO_REG_HOST_POWER(core_name) \ { \ switch (action) { \ @@ -299,7 +210,6 @@ void kbase_pm_protected_l2_override(struct kbase_device *kbdev, bool override) break; \ } \ } -#endif #define ACTION_TYPE_TO_REG_GPU_CONTROL(core_name) \ { \ @@ -348,7 +258,6 @@ static u32 core_type_to_reg(struct kbase_device *kbdev, enum kbase_pm_core_type { u32 reg = 0x0; -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface) { switch (core_type) { case KBASE_PM_CORE_L2: @@ -392,9 +301,7 @@ static u32 core_type_to_reg(struct kbase_device *kbdev, enum kbase_pm_core_type dev_err(kbdev->dev, "Invalid core type for host power control"); break; } - } else -#endif - { + } else { switch (core_type) { case KBASE_PM_CORE_L2: ACTION_TYPE_TO_REG_GPU_CONTROL(L2); @@ -438,75 +345,6 @@ static u32 core_type_to_reg(struct kbase_device *kbdev, enum kbase_pm_core_type return reg; } -#if !MALI_USE_CSF -/** - * map_core_type_to_tl_pm_state - Map core type to TL_PM_STATE. - * - * @kbdev: The kbase device for the core - * @core_type: The type of core - * - * Given a core_type (defined by kbase_pm_core_type) this function will return - * TL_PM_STATE_*, which is a mapping of core_type to respective core type timeline value. - * - * Return: Core type timeline value. - */ -__pure static u32 map_core_type_to_tl_pm_state(struct kbase_device *kbdev, - enum kbase_pm_core_type core_type) -{ - u32 map = 0; - - switch (core_type) { - case KBASE_PM_CORE_L2: - map = TL_PM_STATE_L2; - break; - case KBASE_PM_CORE_SHADER: - map = TL_PM_STATE_SHADER; - break; - case KBASE_PM_CORE_TILER: - map = TL_PM_STATE_TILER; - break; - case KBASE_PM_CORE_STACK: - if (corestack_driver_control) - map = TL_PM_STATE_STACK; - break; - default: - dev_err(kbdev->dev, "Invalid core type"); - } - - /* Core stack might not change default value */ - WARN_ON(!map); - return map; -} -#endif - -#if IS_ENABLED(CONFIG_ARM64) && !MALI_USE_CSF - -static void mali_cci_flush_l2(struct kbase_device *kbdev) -{ - u32 val; - const u32 mask = CLEAN_CACHES_COMPLETED | RESET_COMPLETED; - const u32 timeout_us = - kbase_get_timeout_ms(kbdev, KBASE_CLEAN_CACHE_TIMEOUT) * USEC_PER_MSEC; - - /* - * Note that we don't take the cache flush mutex here since - * we expect to be the last user of the L2, all other L2 users - * would have dropped their references, to initiate L2 power - * down, L2 power down being the only valid place for this - * to be called from. - */ - - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_COMMAND), GPU_COMMAND_CACHE_CLN_INV_L2); - - /* Wait for cache flush to complete before continuing, exit on - * gpu resets or loop expiry. - */ - kbase_reg_poll32_timeout(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_RAWSTAT), val, val & mask, 0, - timeout_us, false); -} -#endif - -#if MALI_USE_CSF /** * pwr_cmd_constructor - Construct an on or off power command for a core type. * @@ -563,7 +401,6 @@ static u64 pwr_cmd_constructor(struct kbase_device *kbdev, enum kbase_pm_core_ty return pwr_cmd; } -#endif /* MALI_USE_CSF */ /** * kbase_pm_invoke - Invokes an action on a core set @@ -585,20 +422,6 @@ static void kbase_pm_invoke(struct kbase_device *kbdev, enum kbase_pm_core_type reg = core_type_to_reg(kbdev, core_type, action); -#if !MALI_USE_CSF - if (cores) { - u64 state = kbase_pm_get_state(kbdev, core_type, ACTION_READY); - - if (action == ACTION_PWRON) - state |= cores; - else if (action == ACTION_PWROFF) - state &= ~cores; - - KBASE_TLSTREAM_AUX_PM_STATE(kbdev, map_core_type_to_tl_pm_state(kbdev, core_type), - state); - } - -#endif /* Tracing */ if (cores) { if (action == ACTION_PWRON) @@ -612,11 +435,9 @@ static void kbase_pm_invoke(struct kbase_device *kbdev, enum kbase_pm_core_type case KBASE_PM_CORE_L2: KBASE_KTRACE_ADD(kbdev, PM_PWRON_L2, NULL, cores); break; -#if MALI_USE_CSF case KBASE_PM_CORE_NEURAL: KBASE_KTRACE_ADD(kbdev, PM_PWRON_NEURAL, NULL, cores); break; -#endif default: break; } @@ -633,23 +454,15 @@ static void kbase_pm_invoke(struct kbase_device *kbdev, enum kbase_pm_core_type /* disable snoops before L2 is turned off */ kbase_pm_cache_snoop_disable(kbdev); break; -#if MALI_USE_CSF case KBASE_PM_CORE_NEURAL: KBASE_KTRACE_ADD(kbdev, PM_PWROFF_NEURAL, NULL, cores); break; -#endif default: break; } } - if (kbase_dummy_job_wa_enabled(kbdev) && action == ACTION_PWRON && - core_type == KBASE_PM_CORE_SHADER && - !(kbdev->dummy_job_wa.flags & KBASE_DUMMY_JOB_WA_FLAG_LOGICAL_SHADER_POWER)) { - kbase_dummy_job_wa_execute(kbdev, cores); - } -#if MALI_USE_CSF - else if (kbdev->pm.backend.has_host_pwr_iface) { + if (kbdev->pm.backend.has_host_pwr_iface) { kbase_reg_write64(kbdev, reg, cores); if (reg == HOST_POWER_ENUM(PWR_CMDARG)) { @@ -657,9 +470,7 @@ static void kbase_pm_invoke(struct kbase_device *kbdev, enum kbase_pm_core_type kbase_reg_write32(kbdev, HOST_POWER_ENUM(PWR_COMMAND), pwr_cmd); } - } -#endif /* MALI_USE_CSF */ - else + } else kbase_reg_write64(kbdev, reg, cores); } @@ -707,12 +518,10 @@ u64 kbase_pm_get_present_cores(struct kbase_device *kbdev, enum kbase_pm_core_ty return kbdev->gpu_props.tiler_present; case KBASE_PM_CORE_STACK: return kbdev->gpu_props.stack_present; -#if MALI_USE_CSF case KBASE_PM_CORE_NEURAL: return kbdev->gpu_props.neural_present; case KBASE_PM_CORE_BASE: return kbdev->gpu_props.base_present; -#endif default: break; } @@ -782,11 +591,9 @@ u64 kbase_pm_get_ready_cores(struct kbase_device *kbdev, enum kbase_pm_core_type case KBASE_PM_CORE_L2: KBASE_KTRACE_ADD(kbdev, PM_CORES_POWERED_L2, NULL, result); break; -#if MALI_USE_CSF case KBASE_PM_CORE_NEURAL: KBASE_KTRACE_ADD(kbdev, PM_CORES_POWERED_NEURAL, NULL, result); break; -#endif default: break; } @@ -823,13 +630,11 @@ static void kbase_pm_l2_config_override(struct kbase_device *kbdev) if (!kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_L2_CONFIG)) return; -#if MALI_USE_CSF if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_PBHA_HWU)) { val = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(L2_CONFIG)); kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(L2_CONFIG), L2_CONFIG_PBHA_HWU_SET(val, kbdev->pbha_propagate_bits)); } -#endif /* MALI_USE_CSF */ /* * Skip if size and hash are not given explicitly, @@ -851,7 +656,6 @@ static void kbase_pm_l2_config_override(struct kbase_device *kbdev) val &= ~L2_CONFIG_HASH_MASK; val |= (kbdev->l2_hash_override << L2_CONFIG_HASH_SHIFT); } else if (kbdev->l2_hash_values_override) { -#if MALI_USE_CSF uint i; WARN_ON(!kbase_hw_has_l2_slice_hash_feature(kbdev)); @@ -869,7 +673,6 @@ static void kbase_pm_l2_config_override(struct kbase_device *kbdev) kbase_reg_write32(kbdev, GPU_L2_SLICE_HASH_OFFSET(i), kbdev->l2_hash_values[i]); } -#endif /* MALI_USE_CSF */ } dev_dbg(kbdev->dev, "Program 0x%x to L2_CONFIG\n", val); @@ -885,7 +688,6 @@ static void kbase_pm_control_gpu_clock(struct kbase_device *kbdev) queue_work(system_wq, &backend->gpu_clock_control_work); } -#if MALI_USE_CSF static const char *kbase_mcu_state_to_string(enum kbase_mcu_state state) { const char *const strings[] = { @@ -951,8 +753,6 @@ bool kbase_pm_is_mcu_inactive(struct kbase_device *kbdev, enum kbase_mcu_state s return ((state == KBASE_MCU_OFF) || (state == KBASE_MCU_IN_SLEEP)); } -#ifdef KBASE_PM_RUNTIME - void kbase_pm_enable_mcu_db_notification(struct kbase_device *kbdev) { u32 val = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(MCU_CONTROL)); @@ -1052,7 +852,6 @@ static void wait_mcu_as_inactive(struct kbase_device *kbdev) if (kbase_prepare_to_reset_gpu(kbdev, 0)) kbase_reset_gpu(kbdev); } -#endif /** * kbasep_pm_toggle_power_interrupt - Toggles the IRQ mask for power interrupts @@ -1072,14 +871,12 @@ static void kbasep_pm_toggle_power_interrupt(struct kbase_device *kbdev, bool en lockdep_assert_held(&kbdev->hwaccess_lock); -#if MALI_USE_CSF /* No toggling is needed when Host control power interface is there, as PM actions * done by the firmware for Tiler, shader, neural won't generate the POWER_CHANGED * irq on Host side. */ if (kbdev->pm.backend.has_host_pwr_iface) return; -#endif irq_mask = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_MASK)); @@ -1093,7 +890,6 @@ static void kbasep_pm_toggle_power_interrupt(struct kbase_device *kbdev, bool en kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_MASK), irq_mask); } -#if MALI_USE_CSF /** * hctl_neural_engines_active - Check the neural engines are active * @@ -1212,9 +1008,7 @@ static bool hctl_base_power_down_done(struct kbase_device *kbdev, u64 shaders_re return (!base_trans && base_ready == shaders_ready); } -#endif /* MALI_USE_CSF */ -#if MALI_USE_CSF /** * hctl_shader_cores_active - Check the shader cores are active * @@ -1334,7 +1128,6 @@ static bool hctl_cores_power_down_done(struct kbase_device *kbdev, u64 cores_rea { return (!cores_trans && (cores_ready == cores_avail)); } -#endif /* MALI_USE_CSF */ static void disable_gpu_idle_timer_no_db(struct kbase_device *kbdev) { @@ -1392,16 +1185,20 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) bases_ready = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_BASE); } /* This check mirrors the check inside kbase_pm_l2_update_state(). */ + if (unlikely(!kbase_io_has_gpu(kbdev))) { - backend->hwcnt_desired = false; - if (!backend->hwcnt_disabled) { - backend->mcu_state = KBASE_MCU_ON_HWCNT_DISABLE; - KBASE_KTRACE_ADD(kbdev, PM_MCU_ON_HWCNT_DISABLE, NULL, - backend->mcu_state); - kbase_pm_trigger_hwcnt_disable(kbdev); + if (kbdev->csf.firmware_hctl_core_pwr) { + backend->hwcnt_desired = false; + if (!backend->hwcnt_disabled) { + backend->mcu_state = KBASE_MCU_ON_HWCNT_DISABLE; + KBASE_KTRACE_ADD(kbdev, PM_MCU_ON_HWCNT_DISABLE, NULL, + backend->mcu_state); + kbase_pm_trigger_hwcnt_disable(kbdev); + } } - if (backend->hwcnt_disabled && (backend->mcu_state != KBASE_MCU_OFF)) { + if (backend->mcu_state != KBASE_MCU_OFF && + (backend->hwcnt_disabled || !kbdev->csf.firmware_hctl_core_pwr)) { backend->mcu_state = KBASE_MCU_OFF; KBASE_KTRACE_ADD(kbdev, PM_MCU_OFF, NULL, backend->mcu_state); dev_dbg(kbdev->dev, "GPU lost has occurred - MCU off"); @@ -1520,20 +1317,22 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) break; case KBASE_MCU_ON_HWCNT_ENABLE: - backend->hwcnt_desired = true; - if (backend->hwcnt_disabled) { - unsigned long flags; + /* This state is only needed for the case of the shader power control + * under Host. + * HWCNT is left enabled throughout MCU power cycles in the opposite case. + */ + if (kbdev->csf.firmware_hctl_core_pwr) { + backend->hwcnt_desired = true; + if (backend->hwcnt_disabled) { + unsigned long flags; - kbase_csf_scheduler_spin_lock(kbdev, &flags); - kbase_hwcnt_backend_csf_set_hw_availability( - &kbdev->hwcnt_gpu_iface, - kbdev->gpu_props.curr_config.l2_slices, - kbdev->gpu_props.curr_config.shader_present, - kbdev->pm.debug_core_mask); - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); - kbase_csf_scheduler_spin_unlock(kbdev, flags); - backend->hwcnt_disabled = false; + kbase_csf_scheduler_spin_lock(kbdev, &flags); + kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); + kbase_csf_scheduler_spin_unlock(kbdev, flags); + backend->hwcnt_disabled = false; + } } + backend->mcu_state = KBASE_MCU_ON; break; @@ -1547,6 +1346,28 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) backend->hwcnt_desired = false; if (!backend->hwcnt_disabled) kbase_pm_trigger_hwcnt_disable(kbdev); + + /* WA to avoid a double call to + * kbase_hwcnt_backend_csf_on_after_mcu_on. + * If MCU reload hasn't been triggered yet, + * firmware_reloaded will be set to false. This will lead to + * a call to kbase_csf_firmware_reload_completed() on the + * following FW global IRQ. This call should be a no-op. + * However, HWC needs a MCU_OFF notification here to pair up + * with the first MCU_ON call made during initial FW boot. + * + * Note: On NO_MALI, there is no call to + * kbase_hwcnt_backend_csf_on_after_mcu_on during + * firmware init. First call is made on + * kbase_csf_firmware_reload_completed() during the first + * FW global IRQ. Hence, there is no need to pair up the + * MCU power state notifications to HWC. + */ + if (!IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI) && + !kbdev->csf.firmware_reloaded) + kbase_hwcnt_backend_csf_on_after_mcu_off( + &kbdev->hwcnt_gpu_iface); + backend->mcu_state = KBASE_MCU_HCTL_MCU_ON_RECHECK; } } else if (kbase_pm_handle_mcu_core_attr_update(kbdev)) @@ -1566,7 +1387,7 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) case KBASE_MCU_HCTL_MCU_ON_RECHECK: backend->shaders_desired_mask = kbase_pm_ca_get_core_mask(kbdev); - + WARN_ON(!kbdev->csf.firmware_hctl_core_pwr); if (!backend->hwcnt_disabled) { /* Wait for being disabled */ ; @@ -1640,17 +1461,16 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE; break; } + if (kbdev->csf.firmware_hctl_core_pwr) { + backend->hwcnt_desired = false; + if (!backend->hwcnt_disabled) + kbase_pm_trigger_hwcnt_disable(kbdev); + } - backend->hwcnt_desired = false; - if (!backend->hwcnt_disabled) - kbase_pm_trigger_hwcnt_disable(kbdev); - - if (backend->hwcnt_disabled) { -#ifdef KBASE_PM_RUNTIME - if (backend->gpu_sleep_mode_active) + if (backend->hwcnt_disabled || !kbdev->csf.firmware_hctl_core_pwr) { + if (IS_ENABLED(CONFIG_PM) && backend->gpu_sleep_mode_active) backend->mcu_state = KBASE_MCU_ON_SLEEP_INITIATE; else { -#endif backend->mcu_state = KBASE_MCU_ON_HALT; #if IS_ENABLED(CONFIG_MALI_VALHALL_CORESIGHT) kbase_debug_coresight_csf_state_request( @@ -1706,6 +1526,9 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) if (kbase_csf_firmware_mcu_halt_req_complete(kbdev)) { KBASE_KTRACE_ADD(kbdev, CSF_FIRMWARE_MCU_HALTED, NULL, kbase_csf_ktrace_gpu_cycle_cnt(kbdev)); + + kbase_hwcnt_backend_csf_on_after_mcu_off(&kbdev->hwcnt_gpu_iface); + if (kbdev->csf.firmware_hctl_core_pwr) backend->mcu_state = KBASE_MCU_HCTL_SHADERS_READY_OFF; else @@ -1766,7 +1589,7 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) kbasep_pm_toggle_power_interrupt(kbdev, true); backend->mcu_state = KBASE_MCU_OFF; break; -#ifdef KBASE_PM_RUNTIME + case KBASE_MCU_ON_SLEEP_INITIATE: if (!kbase_pm_is_mcu_desired(kbdev)) handle_sleep_initiate_state(kbdev); @@ -1778,6 +1601,7 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) if (kbase_pm_is_mcu_desired(kbdev)) { /* Assume the transition is complete and prepare to goto ON state */ WARN_ON_ONCE(backend->l2_state != KBASE_L2_ON); + kbase_hwcnt_backend_csf_on_after_mcu_off(&kbdev->hwcnt_gpu_iface); backend->mcu_state = KBASE_MCU_IN_SLEEP; break; } @@ -1787,6 +1611,7 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) if (kbase_csf_firmware_is_mcu_in_sleep(kbdev)) { KBASE_KTRACE_ADD(kbdev, CSF_FIRMWARE_MCU_SLEEP, NULL, kbase_csf_ktrace_gpu_cycle_cnt(kbdev)); + kbase_hwcnt_backend_csf_on_after_mcu_off(&kbdev->hwcnt_gpu_iface); backend->mcu_state = KBASE_MCU_IN_SLEEP; kbase_pm_enable_db_mirror_interrupt(kbdev); kbase_csf_scheduler_reval_idleness_post_sleep(kbdev); @@ -1827,15 +1652,20 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) */ if (!kbdev->csf.firmware_hctl_core_pwr) kbasep_pm_toggle_power_interrupt(kbdev, false); + backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE; kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); + kbase_hwcnt_backend_csf_on_after_mcu_on(&kbdev->hwcnt_gpu_iface); } break; -#endif /* KBASE_PM_RUNTIME */ + case KBASE_MCU_RESET_WAIT: /* Reset complete */ - if (!backend->in_reset) + if (!backend->in_reset) { + kbase_hwcnt_backend_csf_on_after_mcu_off_reset( + &kbdev->hwcnt_gpu_iface); backend->mcu_state = KBASE_MCU_OFF; + } #if IS_ENABLED(CONFIG_MALI_VALHALL_CORESIGHT) kbdev->csf.coresight.disable_on_pmode_enter = false; @@ -1886,7 +1716,6 @@ static void core_idle_worker(struct work_struct *work) spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } -#endif static const char *kbase_l2_core_state_to_string(enum kbase_l2_core_state state) { @@ -1919,23 +1748,6 @@ static void kbase_ktrace_log_l2_core_state(struct kbase_device *kbdev, #endif } -#if !MALI_USE_CSF -/* On powering on the L2, the tracked kctx becomes stale and can be cleared. - * This enables the backend to spare the START_FLUSH.INV_SHADER_OTHER - * operation on the first submitted katom after the L2 powering on. - */ -static void kbase_pm_l2_clear_backend_slot_submit_kctx(struct kbase_device *kbdev) -{ - int js; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - /* Clear the slots' last katom submission kctx */ - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) - kbdev->hwaccess.backend.slot_rb[js].last_kctx_tagged = SLOT_RB_NULL_TAG_VAL; -} -#endif - static bool can_power_down_l2(struct kbase_device *kbdev) { lockdep_assert_held(&kbdev->hwaccess_lock); @@ -1954,18 +1766,12 @@ static bool can_power_up_l2(struct kbase_device *kbdev) static bool need_tiler_control(struct kbase_device *kbdev) { -#if MALI_USE_CSF if (kbase_pm_no_mcu_core_pwroff(kbdev)) return true; else return false; -#else - CSTD_UNUSED(kbdev); - return true; -#endif } -#if MALI_USE_CSF /** * hctl_l2_power_down - Initiate power down of L2 cache * @@ -1994,7 +1800,6 @@ static void hctl_l2_power_down(struct kbase_device *kbdev) kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2, kbdev->gpu_props.curr_config.l2_present, ACTION_PWROFF); } -#endif /** * hctl_tiler_power_up_done - Check and/or initiate power up of Tiler @@ -2018,12 +1823,10 @@ static bool hctl_tiler_power_up_done(struct kbase_device *kbdev) return false; if (!tiler_ready) { -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface) { kbase_pm_invoke(kbdev, KBASE_PM_CORE_TILER, kbdev->gpu_props.tiler_present, ACTION_PWRON); } -#endif return false; } @@ -2031,8 +1834,11 @@ static bool hctl_tiler_power_up_done(struct kbase_device *kbdev) return true; } -#if MALI_USE_CSF +#if MALI_UNIT_TEST +int delegate_pm_domain_control_to_fw(struct kbase_device *kbdev, u32 pm_domain) +#else static int delegate_pm_domain_control_to_fw(struct kbase_device *kbdev, u32 pm_domain) +#endif { u64 val; int err; @@ -2058,10 +1864,17 @@ static int delegate_pm_domain_control_to_fw(struct kbase_device *kbdev, u32 pm_d if (kbase_prepare_to_reset_gpu(kbdev, RESET_FLAGS_HWC_UNRECOVERABLE_ERROR)) kbase_reset_gpu(kbdev); - return -ETIMEDOUT; + return err; } +#if MALI_UNIT_TEST +KBASE_EXPORT_TEST_API(delegate_pm_domain_control_to_fw); +#endif +#if MALI_UNIT_TEST +int retract_pm_domain_control_from_fw(struct kbase_device *kbdev, u32 pm_domain) +#else static int retract_pm_domain_control_from_fw(struct kbase_device *kbdev, u32 pm_domain) +#endif { u64 val; int err; @@ -2085,8 +1898,11 @@ static int retract_pm_domain_control_from_fw(struct kbase_device *kbdev, u32 pm_ if (kbase_prepare_to_reset_gpu(kbdev, RESET_FLAGS_HWC_UNRECOVERABLE_ERROR)) kbase_reset_gpu(kbdev); - return -ETIMEDOUT; + return err; } +#if MALI_UNIT_TEST +KBASE_EXPORT_TEST_API(retract_pm_domain_control_from_fw); +#endif /** * delegate_pm_domains_control_to_fw - Delegate power control for Tiler, Shading engine and @@ -2206,16 +2022,14 @@ int kbase_pm_get_domain_status(struct kbase_device *kbdev, u32 pm_domain, u32 en dev_err(kbdev->dev, "Inspect command for pm domain %u failed", pm_domain); return err; } -#endif /* MALI_USE_CSF */ +KBASE_EXPORT_TEST_API(kbase_pm_get_domain_status); static int kbase_pm_l2_update_state(struct kbase_device *kbdev) { struct kbase_pm_backend_data *backend = &kbdev->pm.backend; u64 l2_present = kbdev->gpu_props.curr_config.l2_present; u64 tiler_present = kbdev->gpu_props.tiler_present; -#if MALI_USE_CSF const bool has_host_pwr_iface = backend->has_host_pwr_iface; -#endif bool l2_power_up_done; enum kbase_l2_core_state prev_state; @@ -2232,22 +2046,25 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) */ if (!kbase_io_has_gpu(kbdev)) { backend->shaders_state = KBASE_SHADERS_OFF_CORESTACK_OFF; - backend->hwcnt_desired = false; - if (!backend->hwcnt_disabled) { - /* Don't progress until hw counters are disabled - * This may involve waiting for a worker to complete. - * The HW counters backend disable code checks for the - * GPU removed case and will error out without touching - * the hardware. This step is needed to keep the HW - * counters in a consistent state after a GPU lost. - */ - backend->l2_state = KBASE_L2_ON_HWCNT_DISABLE; - KBASE_KTRACE_ADD(kbdev, PM_L2_ON_HWCNT_DISABLE, NULL, - backend->l2_state); - kbase_pm_trigger_hwcnt_disable(kbdev); + if (kbdev->csf.firmware_hctl_core_pwr) { + backend->hwcnt_desired = false; + if (!backend->hwcnt_disabled) { + /* Don't progress until hw counters are disabled + * This may involve waiting for a worker to complete. + * The HW counters backend disable code checks for the + * GPU removed case and will error out without touching + * the hardware. This step is needed to keep the HW + * counters in a consistent state after a GPU lost. + */ + backend->l2_state = KBASE_L2_ON_HWCNT_DISABLE; + KBASE_KTRACE_ADD(kbdev, PM_L2_ON_HWCNT_DISABLE, NULL, + backend->l2_state); + kbase_pm_trigger_hwcnt_disable(kbdev); + } } - if (backend->hwcnt_disabled && (backend->l2_state != KBASE_L2_OFF)) { + if ((backend->hwcnt_disabled || !kbdev->csf.firmware_hctl_core_pwr) && + (backend->l2_state != KBASE_L2_OFF)) { backend->l2_state = KBASE_L2_OFF; KBASE_KTRACE_ADD(kbdev, PM_L2_OFF, NULL, backend->l2_state); dev_dbg(kbdev->dev, "GPU lost has occurred - L2 off\n"); @@ -2265,14 +2082,14 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) switch (backend->l2_state) { case KBASE_L2_OFF: if (kbase_pm_is_l2_desired(kbdev) && can_power_up_l2(kbdev)) { -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) - /* Enable HW timer of IPA control before - * L2 cache is powered-up. - */ - { - kbase_ipa_control_handle_gpu_sleep_exit(kbdev); + if (IS_ENABLED(CONFIG_PM)) { + /* Enable HW timer of IPA control before + * L2 cache is powered-up. + */ + { + kbase_ipa_control_handle_gpu_sleep_exit(kbdev); + } } -#endif /* * Set the desired config for L2 before * powering it on @@ -2287,7 +2104,6 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) * L2 cache. */ if (need_tiler_control(kbdev)) { -#if MALI_USE_CSF if (backend->pwr_cntl_delegated) { retract_pm_domains_control_from_fw(kbdev); backend->pwr_cntl_delegated = false; @@ -2300,30 +2116,17 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2, l2_present, ACTION_PWRON); } else -#endif kbase_pm_invoke(kbdev, KBASE_PM_CORE_TILER, tiler_present, ACTION_PWRON); } else { -#if MALI_USE_CSF if (has_host_pwr_iface && !backend->pwr_cntl_delegated) { delegate_pm_domains_control_to_fw(kbdev); backend->pwr_cntl_delegated = true; } -#endif kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2, l2_present, ACTION_PWRON); } -#if !MALI_USE_CSF - /* If we have more than one L2 cache then we - * must power them on explicitly. - */ - if (l2_present != 1) - kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2, l2_present & ~1ULL, - ACTION_PWRON); - /* Clear backend slot submission kctx */ - kbase_pm_l2_clear_backend_slot_submit_kctx(kbdev); -#endif backend->l2_state = KBASE_L2_PEND_ON; } break; @@ -2395,30 +2198,15 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) break; case KBASE_L2_ON_HWCNT_ENABLE: -#if !MALI_USE_CSF - backend->hwcnt_desired = true; - if (backend->hwcnt_disabled) { - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); - backend->hwcnt_disabled = false; - } -#endif backend->l2_state = KBASE_L2_ON; break; case KBASE_L2_ON: if (!kbase_pm_is_l2_desired(kbdev)) { -#if !MALI_USE_CSF - /* Do not power off L2 until the shaders and - * core stacks are off. - */ - if (backend->shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF) - break; -#else /* Do not power off L2 until the MCU has been stopped */ if ((backend->mcu_state != KBASE_MCU_OFF) && (backend->mcu_state != KBASE_MCU_IN_SLEEP)) break; -#endif /* We need to make sure hardware counters are * disabled before powering down the L2, to @@ -2435,38 +2223,7 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) break; case KBASE_L2_ON_HWCNT_DISABLE: -#if !MALI_USE_CSF - /* If the L2 became desired while we were waiting on the - * worker to do the actual hwcnt disable (which might - * happen if some work was submitted immediately after - * the shaders powered off), then we need to early-out - * of this state and re-enable hwcnt. - * - * If we get lucky, the hwcnt disable might not have - * actually started yet, and the logic in the hwcnt - * enable state will prevent the worker from - * performing the disable entirely, preventing loss of - * any hardware counter data. - * - * If the hwcnt disable has started, then we'll lose - * a tiny amount of hardware counter data between the - * disable and the re-enable occurring. - * - * This loss of data is preferable to the alternative, - * which is to block the shader cores from doing any - * work until we're sure hwcnt has been re-enabled. - */ - if (kbase_pm_is_l2_desired(kbdev)) { - backend->l2_state = KBASE_L2_ON_HWCNT_ENABLE; - break; - } - - backend->hwcnt_desired = false; - if (!backend->hwcnt_disabled) - kbase_pm_trigger_hwcnt_disable(kbdev); -#endif - - if (backend->hwcnt_disabled) { + if (backend->hwcnt_disabled || !kbdev->csf.firmware_hctl_core_pwr) { if (kbdev->pm.backend.gpu_clock_slow_down_wa) backend->l2_state = KBASE_L2_SLOW_DOWN_CLOCKS; else @@ -2503,7 +2260,6 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) backend->l2_state = KBASE_L2_PEND_ON; else if (can_power_down_l2(kbdev)) { if (!backend->l2_always_on) { -#if MALI_USE_CSF if (need_tiler_control(kbdev) && has_host_pwr_iface) { /* If using host power control, * the tiler must be explicitly turned off @@ -2512,7 +2268,6 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) kbase_pm_invoke(kbdev, KBASE_PM_CORE_TILER, tiler_present, ACTION_PWROFF); } else -#endif /* Powering off the L2 will also power off the tiler. */ kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2, l2_present, ACTION_PWROFF); @@ -2524,36 +2279,31 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) */ kbase_gpu_start_cache_clean_nolock( kbdev, GPU_COMMAND_CACHE_CLN_INV_L2); -#if !MALI_USE_CSF - KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL, 0u); -#else KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_L2, NULL, 0u); -#endif backend->l2_state = KBASE_L2_PEND_OFF; } break; case KBASE_L2_PEND_OFF: if (likely(!backend->l2_always_on)) { -#if MALI_USE_CSF if (need_tiler_control(kbdev) && l2_ready) { hctl_l2_power_down(kbdev); break; } -#endif if (l2_trans || l2_ready) break; } else if (kbdev->cache_clean_in_progress) break; -#if MALI_USE_CSF -#if defined(KBASE_PM_RUNTIME) + + if (IS_ENABLED(CONFIG_PM)) { /* Allow clock gating within the GPU and prevent it - * from being seen as active during sleep. - */ - { - kbase_ipa_control_handle_gpu_sleep_enter(kbdev); + * from being seen as active during sleep. + */ + { + kbase_ipa_control_handle_gpu_sleep_enter(kbdev); + } } -#endif + /* Disabling MCU after L2 cache power down is to address * KBASE_HW_ISSUE_TITANHW_2922 hardware issue. */ @@ -2562,7 +2312,6 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) WARN_ON_ONCE(backend->mcu_state != KBASE_MCU_OFF); backend->l2_force_off_after_mcu_halt = false; } -#endif /* L2 is now powered off */ backend->l2_state = KBASE_L2_OFF; @@ -2571,9 +2320,7 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) case KBASE_L2_RESET_WAIT: /* Reset complete */ if (!backend->in_reset) { -#if MALI_USE_CSF backend->l2_force_off_after_mcu_halt = false; -#endif backend->l2_state = KBASE_L2_OFF; } @@ -2597,12 +2344,7 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) if (kbdev->pm.backend.invoke_poweroff_wait_wq_when_l2_off && backend->l2_state == KBASE_L2_OFF) { kbdev->pm.backend.invoke_poweroff_wait_wq_when_l2_off = false; -#if MALI_USE_CSF kbase_csf_scheduler_enqueue_power_off_work(kbdev); -#else /* !MALI_USE_CSF */ - queue_work(kbdev->pm.backend.gpu_poweroff_wait_wq, - &kbdev->pm.backend.gpu_poweroff_wait_work); -#endif /* MALI_USE_CSF */ } return 0; @@ -2666,350 +2408,6 @@ static void shader_poweroff_timer_queue_cancel(struct kbase_device *kbdev) } } -#if !MALI_USE_CSF -static const char *kbase_shader_core_state_to_string(enum kbase_shader_core_state state) -{ - const char *const strings[] = { -#define KBASEP_SHADER_STATE(n) #n, -#include "mali_kbase_pm_shader_states.h" -#undef KBASEP_SHADER_STATE - }; - if (WARN_ON((size_t)state >= ARRAY_SIZE(strings))) - return "Bad shader core state"; - else - return strings[state]; -} - -static int kbase_pm_shaders_update_state(struct kbase_device *kbdev) -{ - struct kbase_pm_backend_data *backend = &kbdev->pm.backend; - struct kbasep_pm_tick_timer_state *stt = &kbdev->pm.backend.shader_tick_timer; - enum kbase_shader_core_state prev_state; - u64 stacks_avail = 0; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (corestack_driver_control) - /* Always power on all the corestacks. Disabling certain - * corestacks when their respective shaders are not in the - * available bitmap is not currently supported. - */ - stacks_avail = kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_STACK); - - do { - u64 shaders_trans = kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_SHADER); - u64 shaders_ready = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER); - u64 stacks_trans = 0; - u64 stacks_ready = 0; - - if (corestack_driver_control) { - stacks_trans = kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_STACK); - stacks_ready = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_STACK); - } - - /* - * kbase_pm_get_ready_cores and kbase_pm_get_trans_cores - * are vulnerable to corruption if gpu is lost - */ - if (!kbase_io_has_gpu(kbdev)) { - backend->shaders_state = KBASE_SHADERS_OFF_CORESTACK_OFF; - dev_dbg(kbdev->dev, "GPU lost has occurred - shaders off\n"); - break; - } - - /* mask off ready from trans in case transitions finished - * between the register reads - */ - shaders_trans &= ~shaders_ready; - stacks_trans &= ~stacks_ready; - - prev_state = backend->shaders_state; - - switch (backend->shaders_state) { - case KBASE_SHADERS_OFF_CORESTACK_OFF: - /* Ignore changes to the shader core availability - * except at certain points where we can handle it, - * i.e. off and SHADERS_ON_CORESTACK_ON. - */ - backend->shaders_desired_mask = kbase_pm_ca_get_core_mask(kbdev); - backend->pm_shaders_core_mask = 0; - - if (backend->shaders_desired && backend->l2_state == KBASE_L2_ON) { - if (backend->hwcnt_desired && !backend->hwcnt_disabled) { - /* Trigger a hwcounter dump */ - backend->hwcnt_desired = false; - kbase_pm_trigger_hwcnt_disable(kbdev); - } - - if (backend->hwcnt_disabled) { - if (corestack_driver_control) { - kbase_pm_invoke(kbdev, KBASE_PM_CORE_STACK, - stacks_avail, ACTION_PWRON); - } - backend->shaders_state = - KBASE_SHADERS_OFF_CORESTACK_PEND_ON; - } - } - break; - - case KBASE_SHADERS_OFF_CORESTACK_PEND_ON: - if (!stacks_trans && stacks_ready == stacks_avail) { - backend->shaders_avail = backend->shaders_desired_mask; - kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER, backend->shaders_avail, - ACTION_PWRON); - - if (backend->pm_current_policy && - backend->pm_current_policy->handle_event) - backend->pm_current_policy->handle_event( - kbdev, KBASE_PM_POLICY_EVENT_POWER_ON); - - backend->shaders_state = KBASE_SHADERS_PEND_ON_CORESTACK_ON; - } - break; - - case KBASE_SHADERS_PEND_ON_CORESTACK_ON: - if (!shaders_trans && shaders_ready == backend->shaders_avail) { - KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, - shaders_ready); - backend->pm_shaders_core_mask = shaders_ready; - backend->hwcnt_desired = true; - if (backend->hwcnt_disabled) { -#if MALI_USE_CSF - unsigned long flags; - - kbase_csf_scheduler_spin_lock(kbdev, &flags); -#endif - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); -#if MALI_USE_CSF - kbase_csf_scheduler_spin_unlock(kbdev, flags); -#endif - backend->hwcnt_disabled = false; - } - - backend->shaders_state = KBASE_SHADERS_ON_CORESTACK_ON; - } - break; - - case KBASE_SHADERS_ON_CORESTACK_ON: - backend->shaders_desired_mask = kbase_pm_ca_get_core_mask(kbdev); - - /* If shaders to change state, trigger a counter dump */ - if (!backend->shaders_desired || - (backend->shaders_desired_mask != shaders_ready)) { - backend->hwcnt_desired = false; - if (!backend->hwcnt_disabled) - kbase_pm_trigger_hwcnt_disable(kbdev); - backend->shaders_state = KBASE_SHADERS_ON_CORESTACK_ON_RECHECK; - } - break; - - case KBASE_SHADERS_ON_CORESTACK_ON_RECHECK: - backend->shaders_desired_mask = kbase_pm_ca_get_core_mask(kbdev); - - if (!backend->hwcnt_disabled) { - /* Wait for being disabled */ - ; - } else if (!backend->shaders_desired) { - if (backend->pm_current_policy && - backend->pm_current_policy->handle_event) - backend->pm_current_policy->handle_event( - kbdev, KBASE_PM_POLICY_EVENT_IDLE); - - if (kbdev->pm.backend.protected_transition_override || - (kbase_has_arbiter(kbdev) && (kbase_pm_is_suspending(kbdev) || - kbase_io_is_aw_removed(kbdev))) || - !stt->configured_ticks || WARN_ON(stt->cancel_queued)) { - backend->shaders_state = - KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; - } else { - stt->remaining_ticks = stt->configured_ticks; - stt->needed = true; - - /* The shader hysteresis timer is not - * done the obvious way, which would be - * to start an hrtimer when the shader - * power off is requested. Instead, - * use a 'tick' timer, and set the - * remaining number of ticks on a power - * off request. This avoids the - * latency of starting, then - * immediately cancelling an hrtimer - * when the shaders are re-requested - * before the timeout expires. - */ - if (!hrtimer_active(&stt->timer)) - hrtimer_start(&stt->timer, stt->configured_interval, - HRTIMER_MODE_REL); - - backend->shaders_state = - KBASE_SHADERS_WAIT_OFF_CORESTACK_ON; - } - } else if (backend->shaders_desired_mask & ~shaders_ready) { - /* set cores ready but not available to - * meet KBASE_SHADERS_PEND_ON_CORESTACK_ON - * check pass - */ - backend->shaders_avail = - (backend->shaders_desired_mask | shaders_ready); - - kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER, - backend->shaders_avail & ~shaders_ready, - ACTION_PWRON); - backend->shaders_state = KBASE_SHADERS_PEND_ON_CORESTACK_ON; - } else if (shaders_ready & ~backend->shaders_desired_mask) { - backend->shaders_state = KBASE_SHADERS_WAIT_GPU_IDLE; - } else { - backend->shaders_state = KBASE_SHADERS_PEND_ON_CORESTACK_ON; - } - break; - - case KBASE_SHADERS_WAIT_OFF_CORESTACK_ON: - if (WARN_ON(!hrtimer_active(&stt->timer))) { - stt->remaining_ticks = 0; - backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; - } - - if (backend->shaders_desired) { - if (backend->pm_current_policy && - backend->pm_current_policy->handle_event) - backend->pm_current_policy->handle_event( - kbdev, KBASE_PM_POLICY_EVENT_TIMER_HIT); - - stt->remaining_ticks = 0; - backend->shaders_state = KBASE_SHADERS_ON_CORESTACK_ON_RECHECK; - } else if (stt->remaining_ticks == 0) { - if (backend->pm_current_policy && - backend->pm_current_policy->handle_event) - backend->pm_current_policy->handle_event( - kbdev, KBASE_PM_POLICY_EVENT_TIMER_MISS); - - backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; - } else if (kbase_has_arbiter(kbdev) && (kbase_pm_is_suspending(kbdev) || - kbase_io_is_aw_removed(kbdev))) { - backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; - } - break; - - case KBASE_SHADERS_WAIT_GPU_IDLE: - /* If partial shader core off need to wait the job in - * running and next register finished then flush L2 - * or it might hit GPU2017-861 - */ - if (!kbase_gpu_atoms_submitted_any(kbdev)) { - backend->partial_shaderoff = true; - backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; - } - break; - - case KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON: - if (!backend->partial_shaderoff) - shader_poweroff_timer_queue_cancel(kbdev); - - if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TTRX_921)) { - kbase_gpu_start_cache_clean_nolock(kbdev, - GPU_COMMAND_CACHE_CLN_INV_L2); - backend->shaders_state = KBASE_SHADERS_L2_FLUSHING_CORESTACK_ON; - } else { - backend->shaders_state = KBASE_SHADERS_READY_OFF_CORESTACK_ON; - } - break; - - case KBASE_SHADERS_L2_FLUSHING_CORESTACK_ON: - if (!kbdev->cache_clean_in_progress) - backend->shaders_state = KBASE_SHADERS_READY_OFF_CORESTACK_ON; - - break; - - case KBASE_SHADERS_READY_OFF_CORESTACK_ON: - if (backend->partial_shaderoff) { - backend->partial_shaderoff = false; - /* remove cores available but not ready to - * meet KBASE_SHADERS_PEND_ON_CORESTACK_ON - * check pass - */ - - /* shaders_desired_mask shall be a subset of - * shaders_ready - */ - WARN_ON(backend->shaders_desired_mask & ~shaders_ready); - WARN_ON(!(backend->shaders_desired_mask & shaders_ready)); - - backend->shaders_avail = backend->shaders_desired_mask; - kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER, - shaders_ready & ~backend->shaders_avail, - ACTION_PWROFF); - backend->shaders_state = KBASE_SHADERS_PEND_ON_CORESTACK_ON; - KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, - (shaders_ready & ~backend->shaders_avail)); - } else { - kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER, shaders_ready, - ACTION_PWROFF); - - KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, 0u); - - backend->shaders_state = KBASE_SHADERS_PEND_OFF_CORESTACK_ON; - } - break; - - case KBASE_SHADERS_PEND_OFF_CORESTACK_ON: - if (!shaders_trans && !shaders_ready) { - if (corestack_driver_control) - kbase_pm_invoke(kbdev, KBASE_PM_CORE_STACK, stacks_avail, - ACTION_PWROFF); - - backend->shaders_state = KBASE_SHADERS_OFF_CORESTACK_PEND_OFF; - } - break; - - case KBASE_SHADERS_OFF_CORESTACK_PEND_OFF: - if (!stacks_trans && !stacks_ready) { - /* On powered off, re-enable the hwcnt */ - backend->pm_shaders_core_mask = 0; - backend->hwcnt_desired = true; - if (backend->hwcnt_disabled) { -#if MALI_USE_CSF - unsigned long flags; - - kbase_csf_scheduler_spin_lock(kbdev, &flags); -#endif - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); -#if MALI_USE_CSF - kbase_csf_scheduler_spin_unlock(kbdev, flags); -#endif - backend->hwcnt_disabled = false; - } - backend->shaders_state = - KBASE_SHADERS_OFF_CORESTACK_OFF_TIMER_PEND_OFF; - } - break; - - case KBASE_SHADERS_OFF_CORESTACK_OFF_TIMER_PEND_OFF: - if (!hrtimer_active(&stt->timer) && !stt->cancel_queued) - backend->shaders_state = KBASE_SHADERS_OFF_CORESTACK_OFF; - break; - - case KBASE_SHADERS_RESET_WAIT: - /* Reset complete */ - if (!backend->in_reset) - backend->shaders_state = - KBASE_SHADERS_OFF_CORESTACK_OFF_TIMER_PEND_OFF; - break; - } - - if (backend->shaders_state != prev_state) { - dev_dbg(kbdev->dev, "Shader state transition: %s to %s\n", - kbase_shader_core_state_to_string(prev_state), - kbase_shader_core_state_to_string(backend->shaders_state)); - KBASE_TLSTREAM_TL_KBASE_DEVICE_SHADER_CORE_STATE(kbdev, kbdev->id, - backend->shaders_state); - } - - } while (backend->shaders_state != prev_state); - - return 0; -} -#endif /* !MALI_USE_CSF */ - static bool kbase_pm_is_in_desired_state_nolock(struct kbase_device *kbdev) { bool in_desired_state = true; @@ -3018,16 +2416,7 @@ static bool kbase_pm_is_in_desired_state_nolock(struct kbase_device *kbdev) in_desired_state = kbase_pm_l2_is_in_desired_state(kbdev); -#if !MALI_USE_CSF - if (kbdev->pm.backend.shaders_desired && - kbdev->pm.backend.shaders_state != KBASE_SHADERS_ON_CORESTACK_ON) - in_desired_state = false; - else if (!kbdev->pm.backend.shaders_desired && - kbdev->pm.backend.shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF) - in_desired_state = false; -#else in_desired_state &= kbase_pm_mcu_is_in_desired_state(kbdev); -#endif return in_desired_state; } @@ -3058,31 +2447,9 @@ static bool kbase_pm_is_in_desired_state_with_l2_powered(struct kbase_device *kb return in_desired_state; } -#if !MALI_USE_CSF -static void kbase_pm_trace_power_state(struct kbase_device *kbdev) -{ - lockdep_assert_held(&kbdev->hwaccess_lock); - - KBASE_TLSTREAM_AUX_PM_STATE(kbdev, TL_PM_STATE_L2, - kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2)); - KBASE_TLSTREAM_AUX_PM_STATE(kbdev, TL_PM_STATE_SHADER, - kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER)); - KBASE_TLSTREAM_AUX_PM_STATE(kbdev, TL_PM_STATE_TILER, - kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_TILER)); - - if (corestack_driver_control) - KBASE_TLSTREAM_AUX_PM_STATE(kbdev, TL_PM_STATE_STACK, - kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_STACK)); -} -#endif - void kbase_pm_update_state(struct kbase_device *kbdev) { -#if !MALI_USE_CSF - enum kbase_shader_core_state prev_shaders_state = kbdev->pm.backend.shaders_state; -#else enum kbase_mcu_state prev_mcu_state = kbdev->pm.backend.mcu_state; -#endif lockdep_assert_held(&kbdev->hwaccess_lock); @@ -3092,20 +2459,6 @@ void kbase_pm_update_state(struct kbase_device *kbdev) if (kbase_pm_l2_update_state(kbdev)) return; -#if !MALI_USE_CSF - if (kbase_pm_shaders_update_state(kbdev)) - return; - - /* If the shaders just turned off, re-invoke the L2 state machine, in - * case it was waiting for the shaders to turn off before powering down - * the L2. - */ - if (prev_shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF && - kbdev->pm.backend.shaders_state == KBASE_SHADERS_OFF_CORESTACK_OFF) { - if (kbase_pm_l2_update_state(kbdev)) - return; - } -#else if (kbase_pm_mcu_update_state(kbdev)) return; @@ -3114,15 +2467,10 @@ void kbase_pm_update_state(struct kbase_device *kbdev) if (kbase_pm_l2_update_state(kbdev)) return; } -#endif if (kbase_pm_is_in_desired_state_nolock(kbdev)) { KBASE_KTRACE_ADD(kbdev, PM_DESIRED_REACHED, NULL, kbdev->pm.backend.shaders_avail); -#if !MALI_USE_CSF - kbase_pm_trace_power_state(kbdev); -#endif - KBASE_KTRACE_ADD(kbdev, PM_WAKE_WAITERS, NULL, 0); wake_up(&kbdev->pm.backend.gpu_in_desired_state_wait); } @@ -3177,7 +2525,6 @@ int kbase_pm_state_machine_init(struct kbase_device *kbdev) stt->default_ticks = DEFAULT_PM_POWEROFF_TICK_SHADER; stt->configured_ticks = stt->default_ticks; -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface) { const u64 neural_present = kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_NEURAL); const u64 shader_present = kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_SHADER); @@ -3216,16 +2563,13 @@ int kbase_pm_state_machine_init(struct kbase_device *kbdev) } INIT_WORK(&kbdev->pm.backend.core_idle_work, core_idle_worker); -#endif return 0; } void kbase_pm_state_machine_term(struct kbase_device *kbdev) { -#if MALI_USE_CSF destroy_workqueue(kbdev->pm.backend.core_idle_wq); -#endif hrtimer_cancel(&kbdev->pm.backend.shader_tick_timer.timer); destroy_workqueue(kbdev->pm.backend.shader_tick_timer.wq); } @@ -3239,23 +2583,19 @@ void kbase_pm_reset_start_locked(struct kbase_device *kbdev) backend->in_reset = true; backend->l2_state = KBASE_L2_RESET_WAIT; KBASE_KTRACE_ADD(kbdev, PM_L2_RESET_WAIT, NULL, backend->l2_state); -#if !MALI_USE_CSF - backend->shaders_state = KBASE_SHADERS_RESET_WAIT; -#else /* MCU state machine is exercised only after the initial load/boot * of the firmware. */ if (likely(kbdev->csf.firmware_inited)) { backend->mcu_state = KBASE_MCU_RESET_WAIT; KBASE_KTRACE_ADD(kbdev, PM_MCU_RESET_WAIT, NULL, backend->mcu_state); -#ifdef KBASE_PM_RUNTIME - backend->exit_gpu_sleep_mode = true; -#endif + if (IS_ENABLED(CONFIG_PM)) + backend->exit_gpu_sleep_mode = true; + kbdev->csf.firmware_reload_needed = true; } else { WARN_ON(backend->mcu_state != KBASE_MCU_OFF); } -#endif /* We're in a reset, so hwcnt will have been synchronously disabled by * this function's caller as part of the reset process. We therefore @@ -3288,42 +2628,39 @@ void kbase_pm_reset_complete(struct kbase_device *kbdev) */ kbase_gpu_cache_clean_wait_complete(kbdev); backend->in_reset = false; -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) - backend->gpu_wakeup_override = false; - backend->db_mirror_interrupt_enabled = false; - backend->gpu_sleep_mode_active = false; - backend->exit_gpu_sleep_mode = false; -#endif + if (IS_ENABLED(CONFIG_PM)) { + backend->gpu_wakeup_override = false; + backend->db_mirror_interrupt_enabled = false; + backend->gpu_sleep_mode_active = false; + backend->exit_gpu_sleep_mode = false; + } kbase_pm_update_state(kbdev); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } -#if !MALI_USE_CSF -/* Timeout in milliseconds for GPU Power Management to reach the desired - * Shader and L2 state. If the time spent waiting has exceeded this threshold - * then there is most likely a hardware issue. - */ -#define PM_TIMEOUT_MS (5000) /* 5s */ -#endif +#if MALI_UNIT_TEST +void kbase_pm_timed_out(struct kbase_device *kbdev, const char *timeout_msg); +void kbase_pm_timed_out(struct kbase_device *kbdev, const char *timeout_msg) +#else static void kbase_pm_timed_out(struct kbase_device *kbdev, const char *timeout_msg) +#endif { unsigned long flags; dev_err(kbdev->dev, "%s", timeout_msg); -#if !MALI_USE_CSF - CSTD_UNUSED(flags); - dev_err(kbdev->dev, "Desired state :\n"); - dev_err(kbdev->dev, "\tShader=%016llx\n", - kbdev->pm.backend.shaders_desired ? kbdev->pm.backend.shaders_avail : 0); -#else spin_lock_irqsave(&kbdev->hwaccess_lock, flags); dev_err(kbdev->dev, "\tMCU desired = %d\n", kbase_pm_is_mcu_desired(kbdev)); dev_err(kbdev->dev, "\tMCU sw state = %d\n", kbdev->pm.backend.mcu_state); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#endif dev_err(kbdev->dev, "Current state :\n"); + + if (kbase_io_is_aw_removed(kbdev)) { + dev_err(kbdev->dev, "\tGPU lost"); + return; + } + dev_err(kbdev->dev, "\tStack =%016llx\n", kbase_reg_read64(kbdev, GPU_CONTROL_ENUM(STACK_READY))); dev_err(kbdev->dev, "\tShader=%016llx\n", @@ -3332,10 +2669,8 @@ static void kbase_pm_timed_out(struct kbase_device *kbdev, const char *timeout_m kbase_reg_read64(kbdev, GPU_CONTROL_ENUM(TILER_READY))); dev_err(kbdev->dev, "\tL2 =%016llx\n", kbase_reg_read64(kbdev, GPU_CONTROL_ENUM(L2_READY))); -#if MALI_USE_CSF dev_err(kbdev->dev, "\tMCU status = %d\n", kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(MCU_STATUS))); -#endif dev_err(kbdev->dev, "Cores transitioning :\n"); dev_err(kbdev->dev, "\tStack =%016llx\n", kbase_reg_read64(kbdev, GPU_CONTROL_ENUM(STACK_PWRTRANS))); @@ -3351,6 +2686,9 @@ static void kbase_pm_timed_out(struct kbase_device *kbdev, const char *timeout_m kbase_reset_gpu(kbdev); } } +#if MALI_UNIT_TEST +KBASE_EXPORT_TEST_API(kbase_pm_timed_out); +#endif int kbase_pm_wait_for_l2_powered(struct kbase_device *kbdev) { @@ -3363,12 +2701,8 @@ int kbase_pm_wait_for_l2_powered(struct kbase_device *kbdev) kbase_pm_update_state(kbdev); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#if MALI_USE_CSF timeout = (unsigned long)kbase_csf_timeout_in_jiffies( kbase_get_timeout_ms(kbdev, CSF_PM_TIMEOUT)); -#else - timeout = msecs_to_jiffies(PM_TIMEOUT_MS); -#endif /* Wait for cores */ #if KERNEL_VERSION(4, 13, 1) <= LINUX_VERSION_CODE @@ -3396,11 +2730,7 @@ static int pm_wait_for_desired_state(struct kbase_device *kbdev, bool killable_w { unsigned long flags; long remaining; -#if MALI_USE_CSF long timeout = kbase_csf_timeout_in_jiffies(kbase_get_timeout_ms(kbdev, CSF_PM_TIMEOUT)); -#else - long timeout = (long)msecs_to_jiffies(PM_TIMEOUT_MS); -#endif int err = 0; /* Let the state machine latch the most recent desired state. */ @@ -3412,14 +2742,17 @@ static int pm_wait_for_desired_state(struct kbase_device *kbdev, bool killable_w #if KERNEL_VERSION(4, 13, 1) <= LINUX_VERSION_CODE if (killable_wait) remaining = wait_event_killable_timeout(kbdev->pm.backend.gpu_in_desired_state_wait, - kbase_pm_is_in_desired_state(kbdev), + kbase_pm_is_in_desired_state(kbdev) || + kbase_io_is_aw_removed(kbdev), timeout); #else killable_wait = false; #endif if (!killable_wait) remaining = wait_event_timeout(kbdev->pm.backend.gpu_in_desired_state_wait, - kbase_pm_is_in_desired_state(kbdev), timeout); + kbase_pm_is_in_desired_state(kbdev) || + kbase_io_is_aw_removed(kbdev), + timeout); if (!remaining) { kbase_pm_timed_out(kbdev, "Wait for power transition timed out"); err = -ETIMEDOUT; @@ -3427,6 +2760,9 @@ static int pm_wait_for_desired_state(struct kbase_device *kbdev, bool killable_w WARN_ON_ONCE(!killable_wait); dev_info(kbdev->dev, "Wait for power transition got interrupted"); err = (int)remaining; + } else if (kbase_io_is_aw_removed(kbdev) && !kbase_pm_is_in_desired_state(kbdev)) { + dev_warn(kbdev->dev, "%s(): aborting, AW is no longer connected", __func__); + err = -ETIMEDOUT; } return err; @@ -3443,7 +2779,6 @@ int kbase_pm_wait_for_desired_state(struct kbase_device *kbdev) } KBASE_EXPORT_TEST_API(kbase_pm_wait_for_desired_state); -#if MALI_USE_CSF /** * core_mask_update_done - Check if downscaling of shader cores is done * @@ -3497,7 +2832,6 @@ int kbase_pm_wait_for_cores_down_scale(struct kbase_device *kbdev) return err; } -#endif static bool is_poweroff_wait_in_progress(struct kbase_device *kbdev) { @@ -3515,7 +2849,6 @@ static int pm_wait_for_poweroff_work_complete(struct kbase_device *kbdev, bool k { long remaining; /* Indefinite wait needed when arbitration is enabled */ -#if MALI_USE_CSF /* gpu_poweroff_wait_work would be subjected to the kernel scheduling * and so the wait time can't only be the function of GPU frequency. */ @@ -3524,16 +2857,22 @@ static int pm_wait_for_poweroff_work_complete(struct kbase_device *kbdev, bool k kbase_get_timeout_ms(kbdev, CSF_PM_TIMEOUT) + extra_wait_time_ms); const long timeout = kbase_has_arbiter(kbdev) ? MAX_SCHEDULE_TIMEOUT : timeout_extra_wait_time; -#else - const long timeout = kbase_has_arbiter(kbdev) ? MAX_SCHEDULE_TIMEOUT : - (long)msecs_to_jiffies(PM_TIMEOUT_MS); -#endif int err = 0; + unsigned long flags; + + if (kbase_io_is_aw_removed(kbdev)) { + dev_info(kbdev->dev, "%s: AW disconnected, exiting early", __func__); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbdev->pm.backend.poweroff_wait_in_progress = false; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + return err; + } #if KERNEL_VERSION(4, 13, 1) <= LINUX_VERSION_CODE if (killable_wait) remaining = wait_event_killable_timeout(kbdev->pm.backend.poweroff_wait, - !is_poweroff_wait_in_progress(kbdev), + !is_poweroff_wait_in_progress(kbdev) || + kbase_io_is_aw_removed(kbdev), timeout); #else killable_wait = false; @@ -3541,7 +2880,8 @@ static int pm_wait_for_poweroff_work_complete(struct kbase_device *kbdev, bool k if (!killable_wait) remaining = wait_event_timeout(kbdev->pm.backend.poweroff_wait, - !is_poweroff_wait_in_progress(kbdev), timeout); + !is_poweroff_wait_in_progress(kbdev), timeout) || + kbase_io_is_aw_removed(kbdev); if (!remaining) { kbase_pm_timed_out(kbdev, "Wait for poweroff work timed out"); err = -ETIMEDOUT; @@ -3571,11 +2911,7 @@ void kbase_pm_enable_interrupts(struct kbase_device *kbdev) bool is_legacy_gpu_irq_mask; KBASE_DEBUG_ASSERT(kbdev != NULL); -#if MALI_USE_CSF is_legacy_gpu_irq_mask = !kbdev->pm.backend.has_host_pwr_iface; -#else - is_legacy_gpu_irq_mask = true; -#endif /* * Clear all interrupts, @@ -3593,19 +2929,13 @@ void kbase_pm_enable_interrupts(struct kbase_device *kbdev) kbase_reg_write32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_MASK), 0xFFFFFFFF); kbase_reg_write32(kbdev, MMU_CONTROL_ENUM(IRQ_CLEAR), 0xFFFFFFFF); -#if MALI_USE_CSF /* Enable only the Page fault bits part */ kbase_reg_write32(kbdev, MMU_CONTROL_ENUM(IRQ_MASK), 0xFFFF); -#else - kbase_reg_write32(kbdev, MMU_CONTROL_ENUM(IRQ_MASK), 0xFFFFFFFF); -#endif -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface) { kbase_reg_write32(kbdev, HOST_POWER_ENUM(PWR_IRQ_CLEAR), PWR_IRQ_REG_ALL); kbase_reg_write32(kbdev, HOST_POWER_ENUM(PWR_IRQ_MASK), PWR_IRQ_REG_ALL); } -#endif } KBASE_EXPORT_TEST_API(kbase_pm_enable_interrupts); @@ -3615,11 +2945,7 @@ void kbase_pm_disable_interrupts_nolock(struct kbase_device *kbdev) bool is_legacy_gpu_irq_mask; KBASE_DEBUG_ASSERT(kbdev != NULL); -#if MALI_USE_CSF is_legacy_gpu_irq_mask = !kbdev->pm.backend.has_host_pwr_iface; -#else - is_legacy_gpu_irq_mask = true; -#endif /* * Mask all interrupts, * and clear them all. @@ -3636,12 +2962,10 @@ void kbase_pm_disable_interrupts_nolock(struct kbase_device *kbdev) kbase_reg_write32(kbdev, MMU_CONTROL_ENUM(IRQ_MASK), 0); kbase_reg_write32(kbdev, MMU_CONTROL_ENUM(IRQ_CLEAR), 0xFFFFFFFF); -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface) { kbase_reg_write32(kbdev, HOST_POWER_ENUM(PWR_IRQ_MASK), 0); kbase_reg_write32(kbdev, HOST_POWER_ENUM(PWR_IRQ_CLEAR), PWR_IRQ_REG_ALL); } -#endif } void kbase_pm_disable_interrupts(struct kbase_device *kbdev) @@ -3655,7 +2979,6 @@ void kbase_pm_disable_interrupts(struct kbase_device *kbdev) KBASE_EXPORT_TEST_API(kbase_pm_disable_interrupts); -#if MALI_USE_CSF /** * update_user_reg_page_mapping - Update the mapping for USER Register page * @@ -3686,7 +3009,6 @@ static void update_user_reg_page_mapping(struct kbase_device *kbdev) } mutex_unlock(&kbdev->csf.reg_lock); } -#endif /* * pmu layout: @@ -3701,9 +3023,6 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) unsigned long flags; KBASE_DEBUG_ASSERT(kbdev != NULL); -#if !MALI_USE_CSF - lockdep_assert_held(&kbdev->js_data.runpool_mutex); -#endif /* !MALI_USE_CSF */ lockdep_assert_held(&kbdev->pm.lock); if (WARN_ON(kbase_io_is_aw_removed(kbdev))) { @@ -3712,12 +3031,11 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) } if (kbase_io_is_gpu_powered(kbdev)) { -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) - if (backend->gpu_idled) { + if (IS_ENABLED(CONFIG_PM) && backend->gpu_idled) { backend->callback_power_runtime_gpu_active(kbdev); backend->gpu_idled = false; } -#endif + /* Already turned on */ if (kbdev->poweroff_pending) kbase_pm_enable_interrupts(kbdev); @@ -3741,10 +3059,8 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) kbase_io_clear_status(kbdev->io, KBASE_IO_STATUS_GPU_OFF); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#if MALI_USE_CSF /* GPU has been turned on, can switch to actual register page */ update_user_reg_page_mapping(kbdev); -#endif if (ret == GPU_STATE_IN_RESET) { @@ -3783,13 +3099,6 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); mutex_unlock(&kbdev->mmu_hw_mutex); - if (kbdev->dummy_job_wa.flags & KBASE_DUMMY_JOB_WA_FLAG_LOGICAL_SHADER_POWER) { - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_dummy_job_wa_execute(kbdev, - kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_SHADER)); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - } - /* Enable the interrupts */ kbase_pm_enable_interrupts(kbdev); @@ -3797,7 +3106,6 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) spin_lock_irqsave(&kbdev->hwaccess_lock, flags); backend->gpu_ready = true; backend->l2_desired = true; -#if MALI_USE_CSF { if (ret != GPU_STATE_INTACT) { /* GPU reset was done after the power on, so send the post @@ -3809,19 +3117,16 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) kbase_ipa_control_handle_gpu_power_on(kbdev); } } -#endif kbase_pm_update_state(kbdev); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) /* GPU is now powered up. Invoke the GPU active callback as GPU idle * callback would have been invoked before the power down. */ - if (backend->gpu_idled) { + if (IS_ENABLED(CONFIG_PM) && backend->gpu_idled) { backend->callback_power_runtime_gpu_active(kbdev); backend->gpu_idled = false; } -#endif } KBASE_EXPORT_TEST_API(kbase_pm_clock_on); @@ -3861,11 +3166,9 @@ bool kbase_pm_clock_off(struct kbase_device *kbdev) } kbase_pm_cache_snoop_disable(kbdev); -#if MALI_USE_CSF { kbase_ipa_control_handle_gpu_power_off(kbdev); } -#endif if (kbase_io_is_aw_removed(kbdev)) { /* Ensure we unblock any threads that are stuck waiting @@ -3881,10 +3184,8 @@ bool kbase_pm_clock_off(struct kbase_device *kbdev) spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#if MALI_USE_CSF /* GPU is about to be turned off, switch to dummy page */ update_user_reg_page_mapping(kbdev); -#endif kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_IDLE_EVENT); if (kbdev->pm.backend.callback_power_off) @@ -3940,7 +3241,6 @@ static enum hrtimer_restart kbasep_reset_timeout(struct hrtimer *timer) return HRTIMER_NORESTART; } -#if MALI_USE_CSF /** * kbase_ne_control_apply - Apply DT values to NEURAL_CONTROL register * @@ -3979,37 +3279,11 @@ static int kbase_ne_control_apply(struct kbase_device *kbdev) return 0; } -#endif static int kbase_set_gpu_quirks(struct kbase_device *kbdev) { -#if MALI_USE_CSF if (kbase_reg_is_valid(kbdev, GPU_CONTROL_ENUM(CSF_CONFIG))) kbdev->hw_quirks_gpu = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(CSF_CONFIG)); -#else - u32 hw_quirks_gpu = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(JM_CONFIG)); - - if (kbdev->gpu_props.gpu_id.product_model == GPU_ID_PRODUCT_TMIX) { - /* Only for tMIx */ - u32 coherency_features; - - coherency_features = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(COHERENCY_FEATURES)); - - /* (COHERENCY_ACE_LITE | COHERENCY_ACE) was incorrectly - * documented for tMIx so force correct value here. - */ - if (coherency_features == COHERENCY_FEATURE_BIT(COHERENCY_ACE)) { - hw_quirks_gpu |= (COHERENCY_ACE_LITE | COHERENCY_ACE) - << JM_FORCE_COHERENCY_FEATURES_SHIFT; - } - } - - if (!kbase_io_has_gpu(kbdev)) - return -EIO; - - kbdev->hw_quirks_gpu = hw_quirks_gpu; - -#endif /* !MALI_USE_CSF */ if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_IDVS_GROUP_SIZE)) { u32 default_idvs_group_size = 0xF; u32 group_size = 0; @@ -4074,7 +3348,6 @@ static int kbase_set_tiler_quirks(struct kbase_device *kbdev) return 0; } -#if MALI_USE_CSF static int kbase_set_ne_quirks(struct kbase_device *kbdev) { u32 hw_quirks_ne = 0; @@ -4089,7 +3362,6 @@ static int kbase_set_ne_quirks(struct kbase_device *kbdev) return 0; } -#endif static int kbase_pm_hw_issues_detect(struct kbase_device *kbdev) { @@ -4143,7 +3415,6 @@ static int kbase_pm_hw_issues_detect(struct kbase_device *kbdev) error = kbase_set_mmu_quirks(kbdev); } -#if MALI_USE_CSF if (!of_property_read_u32(np, "quirks-ne", &kbdev->hw_quirks_ne)) { dev_info(kbdev->dev, "Found NE quirks = [0x%x] in Devicetree\n", kbdev->hw_quirks_ne); @@ -4152,7 +3423,6 @@ static int kbase_pm_hw_issues_detect(struct kbase_device *kbdev) if (error) return error; } -#endif return error; } @@ -4167,64 +3437,21 @@ static void kbase_pm_hw_issues_apply(struct kbase_device *kbdev) if (kbase_reg_is_valid(kbdev, GPU_CONTROL_ENUM(L2_MMU_CONFIG))) kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(L2_MMU_CONFIG), kbdev->hw_quirks_mmu); -#if MALI_USE_CSF if (kbase_reg_is_valid(kbdev, GPU_CONTROL_ENUM(CSF_CONFIG))) kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(CSF_CONFIG), kbdev->hw_quirks_gpu); -#else - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(JM_CONFIG), kbdev->hw_quirks_gpu); -#endif -#if MALI_USE_CSF if (kbase_reg_is_valid(kbdev, GPU_CONTROL_ENUM(NEURAL_CONFIG))) kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(NEURAL_CONFIG), kbdev->hw_quirks_ne); -#endif } void kbase_pm_cache_snoop_enable(struct kbase_device *kbdev) { -#if !MALI_USE_CSF - if ((kbdev->current_gpu_coherency_mode == COHERENCY_ACE) && !kbdev->cci_snoop_enabled) { -#if IS_ENABLED(CONFIG_ARM64) - if (kbdev->snoop_enable_smc != 0) - kbase_invoke_smc_fid(kbdev->snoop_enable_smc, 0, 0, 0); -#endif /* CONFIG_ARM64 */ - dev_dbg(kbdev->dev, "MALI - CCI Snoops - Enabled\n"); - kbdev->cci_snoop_enabled = true; - } -#endif /* !MALI_USE_CSF */ } void kbase_pm_cache_snoop_disable(struct kbase_device *kbdev) { -#if !MALI_USE_CSF - if (kbdev->cci_snoop_enabled) { -#if IS_ENABLED(CONFIG_ARM64) - if (kbdev->snoop_disable_smc != 0) { - mali_cci_flush_l2(kbdev); - kbase_invoke_smc_fid(kbdev->snoop_disable_smc, 0, 0, 0); - } -#endif /* CONFIG_ARM64 */ - dev_dbg(kbdev->dev, "MALI - CCI Snoops Disabled\n"); - kbdev->cci_snoop_enabled = false; - } -#endif /* !MALI_USE_CSF */ } -#if !MALI_USE_CSF -static void reenable_protected_mode_hwcnt(struct kbase_device *kbdev) -{ - unsigned long irq_flags; - - spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); - kbdev->protected_mode_hwcnt_desired = true; - if (kbdev->protected_mode_hwcnt_disabled) { - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); - kbdev->protected_mode_hwcnt_disabled = false; - } - spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); -} -#endif - static int kbase_pm_do_reset_soft(struct kbase_device *kbdev) { int ret; @@ -4236,14 +3463,11 @@ static int kbase_pm_do_reset_soft(struct kbase_device *kbdev) else if (ret > 0) return 0; } else { -#if MALI_USE_CSF /* Check if soft reset needs to be issued via PWR_COMMAND */ if (kbdev->pm.backend.has_host_pwr_iface) kbase_reg_write32(kbdev, HOST_POWER_ENUM(PWR_COMMAND), PWR_COMMAND_COMMAND_RESET_SOFT); - else -#endif - { + else { kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_COMMAND), GPU_COMMAND_SOFT_RESET); } @@ -4272,12 +3496,10 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev) reg_val = RESET_COMPLETED; /* Unmask the reset complete interrupt only */ -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface) { reg_offset = HOST_POWER_ENUM(PWR_IRQ_MASK); reg_val = PWR_IRQ_RESET_COMPLETED; } -#endif kbase_reg_write32(kbdev, reg_offset, reg_val); } @@ -4286,14 +3508,27 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev) rtdata.timed_out = false; /* Create a timer to use as a timeout on the reset */ +#if KERNEL_VERSION(6, 13, 0) <= LINUX_VERSION_CODE + hrtimer_setup_on_stack(&rtdata.timer, kbasep_reset_timeout, CLOCK_MONOTONIC, + HRTIMER_MODE_REL); +#else hrtimer_init_on_stack(&rtdata.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); rtdata.timer.function = kbasep_reset_timeout; +#endif hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT), HRTIMER_MODE_REL); /* Wait for the RESET_COMPLETED interrupt to be raised */ kbase_pm_wait_for_reset(kbdev); + if (kbase_io_is_aw_removed(kbdev)) { + dev_warn(kbdev->dev, "%s: AW removed, exiting early", __func__); + hrtimer_cancel(&rtdata.timer); + destroy_hrtimer_on_stack(&rtdata.timer); + + return -ENODEV; + } + if (!rtdata.timed_out) { /* GPU has been reset */ @@ -4305,12 +3540,10 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev) reg_offset = GPU_CONTROL_ENUM(GPU_IRQ_RAWSTAT); reg_val = RESET_COMPLETED; -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface) { reg_offset = HOST_POWER_ENUM(PWR_IRQ_RAWSTAT); reg_val = PWR_IRQ_RESET_COMPLETED; } -#endif /* No interrupt has been received - check if the RAWSTAT register says * the reset has completed. @@ -4344,13 +3577,10 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev) atomic_set(&kbdev->pm.backend.reset_in_progress, 1); -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface) kbase_reg_write32(kbdev, HOST_POWER_ENUM(PWR_COMMAND), PWR_COMMAND_COMMAND_RESET_HARD); - else -#endif - { + else { kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_COMMAND), GPU_COMMAND_HARD_RESET); } @@ -4408,10 +3638,8 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) kbase_io_clear_status(kbdev->io, KBASE_IO_STATUS_GPU_OFF); } -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface && !pwr_status_gpu_reset_allowed(kbdev)) return -EINVAL; -#endif /* Ensure interrupts are off to begin with, this also clears any * outstanding interrupts @@ -4434,7 +3662,6 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) err = kbdev->protected_ops->protected_mode_disable(kbdev->protected_dev); spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); -#if MALI_USE_CSF if (kbdev->protected_mode) { unsigned long flags; @@ -4444,14 +3671,12 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) kbase_hwcnt_backend_csf_protm_exited(&kbdev->hwcnt_gpu_iface); kbase_csf_scheduler_spin_unlock(kbdev, flags); } -#endif kbdev->protected_mode = false; spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); if (err) goto exit; -#if MALI_USE_CSF /* Check power status register value after GPU reset */ if (kbdev->pm.backend.has_host_pwr_iface && !pwr_status_value_valid_post_reset(kbdev)) { err = -EINVAL; @@ -4469,7 +3694,6 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) if (err) goto exit; } -#endif if (flags & PM_HW_ISSUES_DETECT) { err = kbase_pm_hw_issues_detect(kbdev); if (err) @@ -4479,10 +3703,8 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) kbase_pm_hw_issues_apply(kbdev); kbase_cache_set_coherency_mode(kbdev, kbdev->system_coherency); kbase_amba_set_shareable_cache_support(kbdev); -#if MALI_USE_CSF kbase_backend_update_gpu_timestamp_offset(kbdev); kbdev->csf.compute_progress_timeout_cc = 0; -#endif /* Sanity check protected mode was left after reset */ WARN_ON(kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_STATUS)) & @@ -4492,14 +3714,6 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) kbase_pm_enable_interrupts(kbdev); exit: -#if !MALI_USE_CSF - if (!kbdev->pm.backend.protected_entry_transition_override) { - /* Re-enable GPU hardware counters if we're resetting from - * protected mode. - */ - reenable_protected_mode_hwcnt(kbdev); - } -#endif return err; } @@ -4532,13 +3746,6 @@ static void kbase_pm_request_gpu_cycle_counter_do_request(struct kbase_device *k /* This might happen after GPU reset. * Then counter needs to be kicked. */ -#if !IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI) && !MALI_USE_CSF - if (!(kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_STATUS)) & - GPU_STATUS_CYCLE_COUNT_ACTIVE)) { - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_COMMAND), - GPU_COMMAND_CYCLE_COUNT_START); - } -#endif } spin_unlock_irqrestore(&kbdev->pm.backend.gpu_cycle_counter_requests_lock, flags); diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_internal.h b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_internal.h index b6e0311308e1..88ae28192851 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_internal.h +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_internal.h @@ -31,6 +31,19 @@ #include "backend/gpu/mali_kbase_pm_ca.h" #include "mali_kbase_pm_policy.h" +/** + * enum mask_type - used to determine which mask is being updated + * + * @SYSFS_COREMASK: core mask requested via sysfs + * @DEVFREQ_COREMASK: core mask requested via devfreq + */ +enum mask_type { + SYSFS_COREMASK, +#ifdef CONFIG_MALI_VALHALL_DEVFREQ + DEVFREQ_COREMASK +#endif +}; + /** * kbase_pm_dev_idle - The GPU is idle. * @@ -200,7 +213,6 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags); */ void kbase_pm_reset_done(struct kbase_device *kbdev); -#if MALI_USE_CSF /** * kbase_pm_wait_for_desired_state - Wait for the desired power state to be * reached @@ -222,30 +234,6 @@ void kbase_pm_reset_done(struct kbase_device *kbdev); * Return: 0 on success, or -ETIMEDOUT code on timeout error. */ int kbase_pm_wait_for_desired_state(struct kbase_device *kbdev); -#else -/** - * kbase_pm_wait_for_desired_state - Wait for the desired power state to be - * reached - * @kbdev: The kbase device structure for the device (must be a valid pointer) - * - * Wait for the L2 and shader power state machines to reach the states - * corresponding to the values of 'l2_desired' and 'shaders_desired'. - * - * The usual use-case for this is to ensure cores are 'READY' after performing - * a GPU Reset. - * - * Unlike kbase_pm_update_state(), the caller must not hold hwaccess_lock, - * because this function will take that lock itself. - * - * NOTE: This may not wait until the correct state is reached if there is a - * power off in progress. To correctly wait for the desired state the caller - * must ensure that this is not the case by, for example, calling - * kbase_pm_wait_for_poweroff_work_complete() - * - * Return: 0 on success, or -ETIMEDOUT error code on timeout error. - */ -int kbase_pm_wait_for_desired_state(struct kbase_device *kbdev); -#endif /** * kbase_pm_killable_wait_for_desired_state - Wait for the desired power state to be @@ -279,7 +267,6 @@ int kbase_pm_killable_wait_for_desired_state(struct kbase_device *kbdev); */ int kbase_pm_wait_for_l2_powered(struct kbase_device *kbdev); -#if MALI_USE_CSF /** * kbase_pm_wait_for_cores_down_scale - Wait for the downscaling of shader cores * @@ -308,7 +295,6 @@ int kbase_pm_wait_for_l2_powered(struct kbase_device *kbdev); * Return: 0 on success, error code on error or remaining jiffies on timeout. */ int kbase_pm_wait_for_cores_down_scale(struct kbase_device *kbdev); -#endif /** * kbase_pm_update_dynamic_cores_onoff - Update the L2 and shader power state @@ -534,14 +520,11 @@ void kbase_pm_runtime_term(struct kbase_device *kbdev); * @kbdev: The kbase device structure for the device (must be a valid pointer) * * Enables access to the GPU registers before power management has powered up - * the GPU with kbase_pm_powerup(). + * the GPU. * * This results in the power management callbacks provided in the driver * configuration to get called to turn on power and/or clocks to the GPU. See * kbase_pm_callback_conf. - * - * This should only be used before power management is powered up with - * kbase_pm_powerup() */ void kbase_pm_register_access_enable(struct kbase_device *kbdev); @@ -555,9 +538,6 @@ void kbase_pm_register_access_enable(struct kbase_device *kbdev); * This results in the power management callbacks provided in the driver * configuration to get called to turn off power and/or clocks to the GPU. See * kbase_pm_callback_conf - * - * This should only be used before power management is powered up with - * kbase_pm_powerup() */ void kbase_pm_register_access_disable(struct kbase_device *kbdev); @@ -606,7 +586,6 @@ void kbase_pm_get_dvfs_metrics(struct kbase_device *kbdev, struct kbasep_pm_metr #ifdef CONFIG_MALI_VALHALL_DVFS -#if MALI_USE_CSF /** * kbase_platform_dvfs_event - Report utilisation to DVFS code for CSF GPU * @@ -620,24 +599,6 @@ void kbase_pm_get_dvfs_metrics(struct kbase_device *kbdev, struct kbasep_pm_metr * Return: Returns 0 on failure and non zero on success. */ int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation); -#else -/** - * kbase_platform_dvfs_event - Report utilisation to DVFS code for JM GPU - * - * @kbdev: The kbase device structure for the device (must be a - * valid pointer) - * @utilisation: The current calculated utilisation by the metrics system. - * @util_gl_share: The current calculated gl share of utilisation. - * @util_cl_share: The current calculated cl share of utilisation per core - * group. - * Function provided by platform specific code when DVFS is enabled to allow - * the power management metrics system to report utilisation. - * - * Return: Returns 0 on failure and non zero on success. - */ -int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation, u32 util_gl_share, - u32 util_cl_share[2]); -#endif #endif /* CONFIG_MALI_VALHALL_DVFS */ @@ -672,6 +633,28 @@ void kbase_pm_cache_snoop_enable(struct kbase_device *kbdev); */ void kbase_pm_cache_snoop_disable(struct kbase_device *kbdev); +/** + * kbase_pm_ca_set_gov_core_mask - Set governor core mask + * @kbdev: Device pointer. + * @core_mask_type: which mask is being used to update register. + * @core_mask: New core mask. + * + * This function is used to change the available core mask as defined via either sysfs or devfreq. + */ +void kbase_pm_ca_set_gov_core_mask(struct kbase_device *kbdev, enum mask_type core_mask_type, + u64 core_mask); + +/** + * kbase_pm_ca_set_gov_core_mask_nolock - Set governor core mask with lock already taken + * @kbdev: Device pointer. + * @core_mask_type: which mask is being used to update register. + * @core_mask: New core mask. + * + * This function is used to change the available core mask as defined via either sysfs or devfreq. + */ +void kbase_pm_ca_set_gov_core_mask_nolock(struct kbase_device *kbdev, enum mask_type core_mask_type, + u64 core_mask); + #ifdef CONFIG_MALI_VALHALL_DEVFREQ /** * kbase_devfreq_set_core_mask - Set devfreq core mask @@ -704,81 +687,6 @@ void kbase_pm_reset_start_locked(struct kbase_device *kbdev); */ void kbase_pm_reset_complete(struct kbase_device *kbdev); -#if !MALI_USE_CSF -/** - * kbase_pm_protected_override_enable - Enable the protected mode override - * @kbdev: Device pointer - * - * When the protected mode override is enabled, all shader cores are requested - * to power down, and the L2 power state can be controlled by - * kbase_pm_protected_l2_override(). - * - * Caller must hold hwaccess_lock. - */ -void kbase_pm_protected_override_enable(struct kbase_device *kbdev); - -/** - * kbase_pm_protected_override_disable - Disable the protected mode override - * @kbdev: Device pointer - * - * Caller must hold hwaccess_lock. - */ -void kbase_pm_protected_override_disable(struct kbase_device *kbdev); - -/** - * kbase_pm_protected_l2_override - Control the protected mode L2 override - * @kbdev: Device pointer - * @override: true to enable the override, false to disable - * - * When the driver is transitioning in or out of protected mode, the L2 cache is - * forced to power off. This can be overridden to force the L2 cache to power - * on. This is required to change coherency settings on some GPUs. - */ -void kbase_pm_protected_l2_override(struct kbase_device *kbdev, bool override); - -/** - * kbase_pm_protected_entry_override_enable - Enable the protected mode entry - * override - * @kbdev: Device pointer - * - * Initiate a GPU reset and enable the protected mode entry override flag if - * l2_always_on WA is enabled and platform is fully coherent. If the GPU - * reset is already ongoing then protected mode entry override flag will not - * be enabled and function will have to be called again. - * - * When protected mode entry override flag is enabled to power down L2 via GPU - * reset, the GPU reset handling behavior gets changed. For example call to - * kbase_backend_reset() is skipped, Hw counters are not re-enabled and L2 - * isn't powered up again post reset. - * This is needed only as a workaround for a Hw issue where explicit power down - * of L2 causes a glitch. For entering protected mode on fully coherent - * platforms L2 needs to be powered down to switch to IO coherency mode, so to - * avoid the glitch GPU reset is used to power down L2. Hence, this function - * does nothing on systems where the glitch issue isn't present. - * - * Caller must hold hwaccess_lock. Should be only called during the transition - * to enter protected mode. - * - * Return: -EAGAIN if a GPU reset was required for the glitch workaround but - * was already ongoing, otherwise 0. - */ -int kbase_pm_protected_entry_override_enable(struct kbase_device *kbdev); - -/** - * kbase_pm_protected_entry_override_disable - Disable the protected mode entry - * override - * @kbdev: Device pointer - * - * This shall be called once L2 has powered down and switch to IO coherency - * mode has been made. As with kbase_pm_protected_entry_override_enable(), - * this function does nothing on systems where the glitch issue isn't present. - * - * Caller must hold hwaccess_lock. Should be only called during the transition - * to enter protected mode. - */ -void kbase_pm_protected_entry_override_disable(struct kbase_device *kbdev); -#endif - /* If true, the driver should explicitly control corestack power management, * instead of relying on the Power Domain Controller. */ @@ -795,7 +703,6 @@ extern bool corestack_driver_control; */ bool kbase_pm_is_l2_desired(struct kbase_device *kbdev); -#if MALI_USE_CSF /** * kbase_pm_is_mcu_desired - Check whether MCU is desired * @@ -821,8 +728,6 @@ bool kbase_pm_is_mcu_desired(struct kbase_device *kbdev); */ bool kbase_pm_is_mcu_inactive(struct kbase_device *kbdev, enum kbase_mcu_state state); -#ifdef KBASE_PM_RUNTIME - /** * kbase_pm_enable_mcu_db_notification - Enable the Doorbell notification on * MCU side @@ -834,8 +739,6 @@ bool kbase_pm_is_mcu_inactive(struct kbase_device *kbdev, enum kbase_mcu_state s */ void kbase_pm_enable_mcu_db_notification(struct kbase_device *kbdev); -#endif /* KBASE_PM_RUNTIME */ - /** * kbase_pm_idle_groups_sched_suspendable - Check whether the scheduler can be * suspended to low power state when all @@ -907,8 +810,6 @@ static inline bool kbase_pm_mcu_is_in_desired_state(struct kbase_device *kbdev) return in_desired_state; } -#endif - /** * kbase_pm_l2_is_in_desired_state - Check if L2 is in stable ON/OFF state. * @@ -937,9 +838,6 @@ static inline bool kbase_pm_l2_is_in_desired_state(struct kbase_device *kbdev) */ static inline void kbase_pm_lock(struct kbase_device *kbdev) { -#if !MALI_USE_CSF - mutex_lock(&kbdev->js_data.runpool_mutex); -#endif /* !MALI_USE_CSF */ mutex_lock(&kbdev->pm.lock); } @@ -951,12 +849,8 @@ static inline void kbase_pm_lock(struct kbase_device *kbdev) static inline void kbase_pm_unlock(struct kbase_device *kbdev) { mutex_unlock(&kbdev->pm.lock); -#if !MALI_USE_CSF - mutex_unlock(&kbdev->js_data.runpool_mutex); -#endif /* !MALI_USE_CSF */ } -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) /** * kbase_pm_gpu_sleep_allowed - Check if the GPU is allowed to be put in sleep * @@ -978,8 +872,10 @@ static inline bool kbase_pm_gpu_sleep_allowed(struct kbase_device *kbdev) * A high positive value of autosuspend_delay can be used to keep the * GPU in sleep state for a long time. */ +#if IS_ENABLED(CONFIG_PM) if (unlikely(kbdev->dev->power.autosuspend_delay <= 0)) return false; +#endif return test_bit(KBASE_GPU_SUPPORTS_GPU_SLEEP, &kbdev->pm.backend.gpu_sleep_allowed); } @@ -997,8 +893,10 @@ static inline bool kbase_pm_gpu_sleep_allowed(struct kbase_device *kbdev) */ static inline bool kbase_pm_fw_sleep_on_idle_allowed(struct kbase_device *kbdev) { +#if IS_ENABLED(CONFIG_PM) if (unlikely(kbdev->dev->power.autosuspend_delay <= 0)) return false; +#endif return kbdev->pm.backend.gpu_sleep_allowed == KBASE_GPU_FW_SLEEP_ON_IDLE_ALLOWED; } @@ -1049,7 +947,6 @@ static inline void kbase_pm_disable_db_mirror_interrupt(struct kbase_device *kbd kbdev->pm.backend.db_mirror_interrupt_enabled = false; } } -#endif /** * kbase_pm_l2_allow_mmu_page_migration - L2 state allows MMU page migration or not @@ -1074,7 +971,12 @@ static inline bool kbase_pm_l2_allow_mmu_page_migration(struct kbase_device *kbd return (backend->l2_state != KBASE_L2_PEND_ON && backend->l2_state != KBASE_L2_PEND_OFF); } -#if MALI_USE_CSF +#if MALI_UNIT_TEST +int delegate_pm_domain_control_to_fw(struct kbase_device *kbdev, u32 pm_domain); + +int retract_pm_domain_control_from_fw(struct kbase_device *kbdev, u32 pm_domain); +#endif + /** * kbase_pm_get_domain_status - get pm domain status for particular endpoint * @@ -1089,6 +991,5 @@ static inline bool kbase_pm_l2_allow_mmu_page_migration(struct kbase_device *kbd */ int kbase_pm_get_domain_status(struct kbase_device *kbdev, u32 pm_domain, u32 endpoint, u32 *domain_status); -#endif /* MALI_USE_CSF */ #endif /* _KBASE_BACKEND_PM_INTERNAL_H_ */ diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_metrics.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_metrics.c index 4822e64f645f..2966a94efe65 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_metrics.c +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_metrics.c @@ -28,17 +28,13 @@ #include #include -#if MALI_USE_CSF #include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h" #include -#else -#include -#endif /* !MALI_USE_CSF */ #include #include -#if defined(CONFIG_MALI_VALHALL_DEVFREQ) || defined(CONFIG_MALI_VALHALL_DVFS) || !MALI_USE_CSF +#if defined(CONFIG_MALI_VALHALL_DEVFREQ) || defined(CONFIG_MALI_VALHALL_DVFS) /* Shift used for kbasep_pm_metrics_data.time_busy/idle - units of (1 << 8) ns * This gives a maximum period between samples of 2^(32+8)/100 ns = slightly * under 11s. Exceeding this will cause overflow @@ -46,10 +42,8 @@ #define KBASE_PM_TIME_SHIFT 8 #endif -#if MALI_USE_CSF /* To get the GPU_ACTIVE value in nano seconds unit */ #define GPU_ACTIVE_SCALING_FACTOR ((u64)1E9) -#endif /* * Possible state transitions @@ -101,7 +95,6 @@ static enum hrtimer_restart dvfs_callback(struct hrtimer *timer) int kbasep_pm_metrics_init(struct kbase_device *kbdev) { -#if MALI_USE_CSF struct kbase_ipa_control_perf_counter perf_counter; int err; @@ -129,11 +122,6 @@ int kbasep_pm_metrics_init(struct kbase_device *kbdev) dev_err(kbdev->dev, "Failed to register IPA with kbase_ipa_control: err=%d", err); return -1; } -#else - KBASE_DEBUG_ASSERT(kbdev != NULL); - kbdev->pm.backend.metrics.kbdev = kbdev; - kbdev->pm.backend.metrics.time_period_start = ktime_get_raw(); -#endif spin_lock_init(&kbdev->pm.backend.metrics.lock); #ifdef CONFIG_MALI_VALHALL_DVFS @@ -144,13 +132,11 @@ int kbasep_pm_metrics_init(struct kbase_device *kbdev) kbase_pm_metrics_start(kbdev); #endif /* CONFIG_MALI_VALHALL_DVFS */ -#if MALI_USE_CSF /* The sanity check on the GPU_ACTIVE performance counter * is skipped for Juno platforms that have timing problems. */ kbdev->pm.backend.metrics.skip_gpu_active_sanity_check = (kbdev->gpu_props.impl_tech >= THREAD_FEATURES_IMPLEMENTATION_TECHNOLOGY_FPGA); -#endif return 0; } @@ -167,11 +153,7 @@ void kbasep_pm_metrics_term(struct kbase_device *kbdev) hrtimer_cancel(&kbdev->pm.backend.metrics.timer); #endif /* CONFIG_MALI_VALHALL_DVFS */ -#if MALI_USE_CSF kbase_ipa_control_unregister(kbdev, kbdev->pm.backend.metrics.ipa_control_client); -#else - CSTD_UNUSED(kbdev); -#endif } KBASE_EXPORT_TEST_API(kbasep_pm_metrics_term); @@ -179,7 +161,6 @@ KBASE_EXPORT_TEST_API(kbasep_pm_metrics_term); /* caller needs to hold kbdev->pm.backend.metrics.lock before calling this * function */ -#if MALI_USE_CSF #if defined(CONFIG_MALI_VALHALL_DEVFREQ) || defined(CONFIG_MALI_VALHALL_DVFS) static void kbase_pm_get_dvfs_utilisation_calc(struct kbase_device *kbdev) { @@ -284,39 +265,6 @@ static void kbase_pm_get_dvfs_utilisation_calc(struct kbase_device *kbdev) kbdev->pm.backend.metrics.time_period_start = now; } #endif /* defined(CONFIG_MALI_VALHALL_DEVFREQ) || defined(CONFIG_MALI_VALHALL_DVFS) */ -#else -static void kbase_pm_get_dvfs_utilisation_calc(struct kbase_device *kbdev, ktime_t now) -{ - ktime_t diff; - - lockdep_assert_held(&kbdev->pm.backend.metrics.lock); - - diff = ktime_sub(now, kbdev->pm.backend.metrics.time_period_start); - if (ktime_to_ns(diff) < 0) - return; - - if (kbdev->pm.backend.metrics.gpu_active) { - u32 ns_time = (u32)(ktime_to_ns(diff) >> KBASE_PM_TIME_SHIFT); - - kbdev->pm.backend.metrics.values.time_busy += ns_time; - if (kbdev->pm.backend.metrics.active_cl_ctx[0]) - kbdev->pm.backend.metrics.values.busy_cl[0] += ns_time; - if (kbdev->pm.backend.metrics.active_cl_ctx[1]) - kbdev->pm.backend.metrics.values.busy_cl[1] += ns_time; - if (kbdev->pm.backend.metrics.active_gl_ctx[0]) - kbdev->pm.backend.metrics.values.busy_gl += ns_time; - if (kbdev->pm.backend.metrics.active_gl_ctx[1]) - kbdev->pm.backend.metrics.values.busy_gl += ns_time; - if (kbdev->pm.backend.metrics.active_gl_ctx[2]) - kbdev->pm.backend.metrics.values.busy_gl += ns_time; - } else { - kbdev->pm.backend.metrics.values.time_idle += - (u32)(ktime_to_ns(diff) >> KBASE_PM_TIME_SHIFT); - } - - kbdev->pm.backend.metrics.time_period_start = now; -} -#endif /* MALI_USE_CSF */ #if defined(CONFIG_MALI_VALHALL_DEVFREQ) || defined(CONFIG_MALI_VALHALL_DVFS) void kbase_pm_get_dvfs_metrics(struct kbase_device *kbdev, struct kbasep_pm_metrics *last, @@ -326,23 +274,13 @@ void kbase_pm_get_dvfs_metrics(struct kbase_device *kbdev, struct kbasep_pm_metr unsigned long flags; spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); -#if MALI_USE_CSF kbase_pm_get_dvfs_utilisation_calc(kbdev); -#else - kbase_pm_get_dvfs_utilisation_calc(kbdev, ktime_get_raw()); -#endif memset(diff, 0, sizeof(*diff)); diff->time_busy = cur->time_busy - last->time_busy; diff->time_idle = cur->time_idle - last->time_idle; -#if MALI_USE_CSF diff->time_in_protm = cur->time_in_protm - last->time_in_protm; -#else - diff->busy_cl[0] = cur->busy_cl[0] - last->busy_cl[0]; - diff->busy_cl[1] = cur->busy_cl[1] - last->busy_cl[1]; - diff->busy_gl = cur->busy_gl - last->busy_gl; -#endif *last = *cur; @@ -356,11 +294,6 @@ void kbase_pm_get_dvfs_action(struct kbase_device *kbdev) { int utilisation; struct kbasep_pm_metrics *diff; -#if !MALI_USE_CSF - int busy; - int util_gl_share; - int util_cl_share[2]; -#endif KBASE_DEBUG_ASSERT(kbdev != NULL); @@ -370,15 +303,6 @@ void kbase_pm_get_dvfs_action(struct kbase_device *kbdev) utilisation = (100 * diff->time_busy) / max(diff->time_busy + diff->time_idle, 1u); -#if !MALI_USE_CSF - busy = max(diff->busy_gl + diff->busy_cl[0] + diff->busy_cl[1], 1u); - - util_gl_share = (100 * diff->busy_gl) / busy; - util_cl_share[0] = (100 * diff->busy_cl[0]) / busy; - util_cl_share[1] = (100 * diff->busy_cl[1]) / busy; - - kbase_platform_dvfs_event(kbdev, utilisation, util_gl_share, util_cl_share); -#else /* Note that, at present, we don't pass protected-mode time to the * platform here. It's unlikely to be useful, however, as the platform * probably just cares whether the GPU is busy or not; time in @@ -386,7 +310,6 @@ void kbase_pm_get_dvfs_action(struct kbase_device *kbdev) * so we should be good. */ kbase_platform_dvfs_event(kbdev, utilisation); -#endif } bool kbase_pm_metrics_is_active(struct kbase_device *kbdev) @@ -421,77 +344,3 @@ void kbase_pm_metrics_stop(struct kbase_device *kbdev) } #endif /* CONFIG_MALI_VALHALL_DVFS */ - -#if !MALI_USE_CSF -/** - * kbase_pm_metrics_active_calc - Update PM active counts based on currently - * running atoms - * @kbdev: Device pointer - * - * The caller must hold kbdev->pm.backend.metrics.lock - */ -static void kbase_pm_metrics_active_calc(struct kbase_device *kbdev) -{ - unsigned int js; - - lockdep_assert_held(&kbdev->pm.backend.metrics.lock); - - kbdev->pm.backend.metrics.active_gl_ctx[0] = 0; - kbdev->pm.backend.metrics.active_gl_ctx[1] = 0; - kbdev->pm.backend.metrics.active_gl_ctx[2] = 0; - kbdev->pm.backend.metrics.active_cl_ctx[0] = 0; - kbdev->pm.backend.metrics.active_cl_ctx[1] = 0; - kbdev->pm.backend.metrics.gpu_active = false; - - for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { - struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0); - - /* Head atom may have just completed, so if it isn't running - * then try the next atom - */ - if (katom && katom->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) - katom = kbase_gpu_inspect(kbdev, js, 1); - - if (katom && katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) { - if (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) { - u32 device_nr = - (katom->core_req & BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) ? - katom->device_nr : - 0; - if (!WARN_ON(device_nr >= 2)) - kbdev->pm.backend.metrics.active_cl_ctx[device_nr] = 1; - } else { - kbdev->pm.backend.metrics.active_gl_ctx[js] = 1; - trace_sysgraph(SGR_ACTIVE, 0, js); - } - kbdev->pm.backend.metrics.gpu_active = true; - } else { - trace_sysgraph(SGR_INACTIVE, 0, js); - } - } -} - -/* called when job is submitted to or removed from a GPU slot */ -void kbase_pm_metrics_update(struct kbase_device *kbdev, ktime_t *timestamp) -{ - unsigned long flags; - ktime_t now; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); - - if (!timestamp) { - now = ktime_get_raw(); - timestamp = &now; - } - - /* Track how much of time has been spent busy or idle. For JM GPUs, - * this also evaluates how long CL and/or GL jobs have been busy for. - */ - kbase_pm_get_dvfs_utilisation_calc(kbdev, *timestamp); - - kbase_pm_metrics_active_calc(kbdev); - spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); -} -#endif /* !MALI_USE_CSF */ diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_policy.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_policy.c index 4ad6f703ad67..78e3ebef0ecc 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_policy.c +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_policy.c @@ -30,7 +30,7 @@ #include #include -#if MALI_USE_CSF && defined CONFIG_MALI_VALHALL_DEBUG +#if defined CONFIG_MALI_VALHALL_DEBUG #include #endif @@ -64,7 +64,7 @@ void kbase_pm_policy_init(struct kbase_device *kbdev) } } -#if MALI_USE_CSF && defined(CONFIG_MALI_VALHALL_DEBUG) +#if defined(CONFIG_MALI_VALHALL_DEBUG) /* Use always_on policy if module param fw_debug=1 is * passed, to aid firmware debugging. */ @@ -74,24 +74,19 @@ void kbase_pm_policy_init(struct kbase_device *kbdev) default_policy->init(kbdev); -#if MALI_USE_CSF spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbdev->pm.backend.pm_current_policy = default_policy; kbdev->pm.backend.csf_pm_sched_flags = default_policy->pm_sched_flags; -#ifdef KBASE_PM_RUNTIME - if (kbase_pm_idle_groups_sched_suspendable(kbdev)) - clear_bit(KBASE_GPU_IGNORE_IDLE_EVENT, &kbdev->pm.backend.gpu_sleep_allowed); - else - set_bit(KBASE_GPU_IGNORE_IDLE_EVENT, &kbdev->pm.backend.gpu_sleep_allowed); -#endif /* KBASE_PM_RUNTIME */ + if (IS_ENABLED(CONFIG_PM)) { + if (kbase_pm_idle_groups_sched_suspendable(kbdev)) + clear_bit(KBASE_GPU_IGNORE_IDLE_EVENT, + &kbdev->pm.backend.gpu_sleep_allowed); + else + set_bit(KBASE_GPU_IGNORE_IDLE_EVENT, &kbdev->pm.backend.gpu_sleep_allowed); + } spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - -#else - CSTD_UNUSED(flags); - kbdev->pm.backend.pm_current_policy = default_policy; -#endif } void kbase_pm_policy_term(struct kbase_device *kbdev) @@ -123,7 +118,6 @@ void kbase_pm_update_active(struct kbase_device *kbdev) if (!pm->backend.invoke_poweroff_wait_wq_when_l2_off && pm->backend.poweroff_wait_in_progress) { KBASE_DEBUG_ASSERT(kbase_io_is_gpu_powered(kbdev)); -#if MALI_USE_CSF if (likely(!pm->backend.waiting_for_mmu_fault_handling)) { /* L2 has been powered off. Invoke the state machine to power * up the L2 cache and also effectively cancel the GPU power off @@ -137,7 +131,6 @@ void kbase_pm_update_active(struct kbase_device *kbdev) wake_up(&kbdev->pm.backend.poweroff_wait); return; } -#endif pm->backend.poweron_required = true; spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } else { @@ -151,9 +144,7 @@ void kbase_pm_update_active(struct kbase_device *kbdev) pm->backend.invoke_poweroff_wait_wq_when_l2_off = false; pm->backend.poweroff_wait_in_progress = false; pm->backend.l2_desired = true; -#if MALI_USE_CSF pm->backend.mcu_desired = true; -#endif spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); kbase_pm_do_poweron(kbdev, false); @@ -190,7 +181,6 @@ void kbase_pm_update_dynamic_cores_onoff(struct kbase_device *kbdev) if (kbdev->pm.backend.poweroff_wait_in_progress) return; -#if MALI_USE_CSF CSTD_UNUSED(shaders_desired); /* Invoke the MCU state machine to send a request to FW for updating * the mask of shader cores that can be used for allocation of @@ -198,18 +188,6 @@ void kbase_pm_update_dynamic_cores_onoff(struct kbase_device *kbdev) */ if (kbase_pm_is_mcu_desired(kbdev)) kbase_pm_update_state(kbdev); -#else - /* In protected transition, don't allow outside shader core request - * affect transition, return directly - */ - if (kbdev->pm.backend.protected_transition_override) - return; - - shaders_desired = kbdev->pm.backend.pm_current_policy->shaders_needed(kbdev); - - if (shaders_desired && kbase_pm_is_l2_desired(kbdev)) - kbase_pm_update_state(kbdev); -#endif } void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev) @@ -223,16 +201,6 @@ void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev) if (kbdev->pm.backend.poweroff_wait_in_progress) return; -#if !MALI_USE_CSF - if (kbdev->pm.backend.protected_transition_override) - /* We are trying to change in/out of protected mode - force all - * cores off so that the L2 powers down - */ - shaders_desired = false; - else - shaders_desired = kbdev->pm.backend.pm_current_policy->shaders_needed(kbdev); -#endif - if (kbdev->pm.backend.shaders_desired != shaders_desired) { KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_DESIRED, NULL, kbdev->pm.backend.shaders_desired); @@ -274,7 +242,6 @@ const struct kbase_pm_policy *kbase_pm_get_policy(struct kbase_device *kbdev) KBASE_EXPORT_TEST_API(kbase_pm_get_policy); -#if MALI_USE_CSF static int policy_change_wait_for_L2_off(struct kbase_device *kbdev) { long remaining; @@ -308,13 +275,11 @@ static int policy_change_wait_for_L2_off(struct kbase_device *kbdev) return err; } -#endif void kbase_pm_set_policy(struct kbase_device *kbdev, const struct kbase_pm_policy *new_policy) { const struct kbase_pm_policy *old_policy; unsigned long flags; -#if MALI_USE_CSF unsigned int new_policy_csf_pm_sched_flags; bool sched_suspend; bool reset_gpu = false; @@ -322,14 +287,12 @@ void kbase_pm_set_policy(struct kbase_device *kbdev, const struct kbase_pm_polic struct kbase_csf_scheduler *scheduler = NULL; u64 pwroff_ns; bool switching_to_always_on; -#endif KBASE_DEBUG_ASSERT(kbdev != NULL); KBASE_DEBUG_ASSERT(new_policy != NULL); KBASE_KTRACE_ADD(kbdev, PM_SET_POLICY, NULL, new_policy->id); -#if MALI_USE_CSF pwroff_ns = kbase_csf_firmware_get_mcu_core_pwroff_time(kbdev); switching_to_always_on = new_policy == &kbase_pm_always_on_policy_ops; if (pwroff_ns == 0 && !switching_to_always_on) { @@ -392,7 +355,6 @@ void kbase_pm_set_policy(struct kbase_device *kbdev, const struct kbase_pm_polic if (sched_suspend) reset_gpu = policy_change_wait_for_L2_off(kbdev); -#endif kbase_pm_lock(kbdev); @@ -424,19 +386,19 @@ void kbase_pm_set_policy(struct kbase_device *kbdev, const struct kbase_pm_polic spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbdev->pm.backend.pm_current_policy = new_policy; -#if MALI_USE_CSF kbdev->pm.backend.csf_pm_sched_flags = new_policy_csf_pm_sched_flags; /* New policy in place, release the clamping on mcu/L2 off state */ kbdev->pm.backend.policy_change_clamp_state_to_off = false; kbase_pm_update_state(kbdev); -#ifdef KBASE_PM_RUNTIME - if (kbase_pm_idle_groups_sched_suspendable(kbdev)) - clear_bit(KBASE_GPU_IGNORE_IDLE_EVENT, &kbdev->pm.backend.gpu_sleep_allowed); - else - set_bit(KBASE_GPU_IGNORE_IDLE_EVENT, &kbdev->pm.backend.gpu_sleep_allowed); -#endif /* KBASE_PM_RUNTIME */ -#endif + if (IS_ENABLED(CONFIG_PM)) { + if (kbase_pm_idle_groups_sched_suspendable(kbdev)) + clear_bit(KBASE_GPU_IGNORE_IDLE_EVENT, + &kbdev->pm.backend.gpu_sleep_allowed); + else + set_bit(KBASE_GPU_IGNORE_IDLE_EVENT, &kbdev->pm.backend.gpu_sleep_allowed); + } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); /* If any core power state changes were previously attempted, but @@ -452,7 +414,6 @@ void kbase_pm_set_policy(struct kbase_device *kbdev, const struct kbase_pm_polic kbase_pm_context_idle_locked(kbdev); kbase_pm_unlock(kbdev); -#if MALI_USE_CSF /* Reverse the suspension done */ if (sched_suspend) kbase_csf_scheduler_pm_resume_no_lock(kbdev); @@ -469,7 +430,6 @@ void kbase_pm_set_policy(struct kbase_device *kbdev, const struct kbase_pm_polic } mutex_unlock(&kbdev->pm.backend.policy_change_lock); -#endif } KBASE_EXPORT_TEST_API(kbase_pm_set_policy); diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_shader_states.h b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_shader_states.h index 8622ef78d363..84f1cbbcc95b 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_shader_states.h +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_shader_states.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -42,11 +42,6 @@ * @WAIT_OFF_CORESTACK_ON: The shaders have been requested to power * off, but they remain on for the duration * of the hysteresis timer - * @WAIT_GPU_IDLE: The shaders partial poweroff needs to - * reach a state where jobs on the GPU are - * finished including jobs currently running - * and in the GPU queue because of - * GPU2017-861 * @WAIT_FINISHED_CORESTACK_ON: The hysteresis timer has expired * @L2_FLUSHING_CORESTACK_ON: The core stacks are on and the level 2 * cache is being flushed. @@ -67,9 +62,6 @@ KBASEP_SHADER_STATE(PEND_ON_CORESTACK_ON) KBASEP_SHADER_STATE(ON_CORESTACK_ON) KBASEP_SHADER_STATE(ON_CORESTACK_ON_RECHECK) KBASEP_SHADER_STATE(WAIT_OFF_CORESTACK_ON) -#if !MALI_USE_CSF -KBASEP_SHADER_STATE(WAIT_GPU_IDLE) -#endif /* !MALI_USE_CSF */ KBASEP_SHADER_STATE(WAIT_FINISHED_CORESTACK_ON) KBASEP_SHADER_STATE(L2_FLUSHING_CORESTACK_ON) KBASEP_SHADER_STATE(READY_OFF_CORESTACK_ON) diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_time.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_time.c index 0d00c2e7b5c0..5c7ec2b9c47d 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_time.c +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_time.c @@ -310,6 +310,9 @@ KBASE_EXPORT_TEST_API(kbase_get_timeout_ms); u64 kbase_backend_get_cycle_cnt(struct kbase_device *kbdev) { + if (kbase_io_is_aw_removed(kbdev)) + return 0; + return kbase_reg_read64_coherent(kbdev, GPU_CONTROL_ENUM(CYCLE_COUNT)); } @@ -330,6 +333,36 @@ u64 kbase_arch_timer_get_cntfrq(struct kbase_device *kbdev) return freq; } +static int kbase_gpu_timestamp_offset_read(void *data, u64 *val) +{ + struct kbase_device *kbdev = (struct kbase_device *)data; + + if (kbdev->backend_time.gpu_timestamp_offset == GPU_TIMESTAMP_OFFSET_INVALID) + return -EINVAL; + + *val = kbdev->backend_time.gpu_timestamp_offset; + + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(timestamp_offset_debugfs_fops, kbase_gpu_timestamp_offset_read, NULL, + "%lld\n"); + +void kbase_gpu_timestamp_offset_debugfs_init(struct kbase_device *kbdev) +{ + struct dentry *timestamp_offset_file; + + if (unlikely(!kbdev)) { + pr_warn("%s: kbdev is NULL\n", __func__); + return; + } + + timestamp_offset_file = debugfs_create_file("gpu_timestamp_offset", 0400, + kbdev->mali_debugfs_directory, kbdev, + ×tamp_offset_debugfs_fops); + if (IS_ERR_OR_NULL(timestamp_offset_file)) + dev_warn(kbdev->dev, "Failed to create gpu_timestamp_offset debugfs entry"); +} int kbase_backend_time_init(struct kbase_device *kbdev) { diff --git a/drivers/gpu/arm/valhall/build.bp b/drivers/gpu/arm/valhall/build.bp index 139632595381..5b72e1b17bc6 100644 --- a/drivers/gpu/arm/valhall/build.bp +++ b/drivers/gpu/arm/valhall/build.bp @@ -117,7 +117,6 @@ bob_defaults { "CONFIG_MALI_VALHALL_PLATFORM_NAME={{.mali_platform_name}}", "MALI_CUSTOMER_RELEASE={{.release}}", "MALI_UNIT_TEST={{.unit_test_code}}", - "MALI_USE_CSF={{.gpu_has_csf}}", "MALI_JIT_PRESSURE_LIMIT_BASE={{.jit_pressure_limit_base}}", // Start of CS experimental features definitions. @@ -133,7 +132,6 @@ bob_defaults { // is an umbrella feature that would be open for inappropriate use // (catch-all for experimental CS code without separating it into // different features). - "MALI_BASE_CSF_PERFORMANCE_TESTS={{.base_csf_performance_tests}}", ], } diff --git a/drivers/gpu/arm/valhall/context/Kbuild b/drivers/gpu/arm/valhall/context/Kbuild index 057693c40a30..cfc152edd81a 100644 --- a/drivers/gpu/arm/valhall/context/Kbuild +++ b/drivers/gpu/arm/valhall/context/Kbuild @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2012-2013, 2016-2017, 2020-2021 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2012-2013, 2016-2017, 2020-2021, 2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,6 @@ # # -valhall_kbase-y += context/mali_kbase_context.o - -ifeq ($(CONFIG_MALI_VALHALL_CSF_SUPPORT),y) - valhall_kbase-y += context/backend/mali_kbase_context_csf.o -else - valhall_kbase-y += context/backend/mali_kbase_context_jm.o -endif +valhall_kbase-y += \ + context/mali_kbase_context.o \ + context/backend/mali_kbase_context_csf.o diff --git a/drivers/gpu/arm/valhall/context/backend/mali_kbase_context_csf.c b/drivers/gpu/arm/valhall/context/backend/mali_kbase_context_csf.c index fe1dbfaca872..44cf8aba9b8f 100644 --- a/drivers/gpu/arm/valhall/context/backend/mali_kbase_context_csf.c +++ b/drivers/gpu/arm/valhall/context/backend/mali_kbase_context_csf.c @@ -91,6 +91,8 @@ static const struct kbase_context_init context_init[] = { "Common context initialization failed" }, { kbase_context_mem_pool_group_init, kbase_context_mem_pool_group_term, "Memory pool group initialization failed" }, + { kbase_context_pgd_mem_pool_init, kbase_context_pgd_mem_pool_term, + "pgd memory pool initialization failed" }, { kbase_mem_evictable_init, kbase_mem_evictable_deinit, "Memory evictable initialization failed" }, { kbase_ctx_sched_init_ctx, NULL, NULL }, @@ -207,6 +209,7 @@ void kbase_destroy_context(struct kbase_context *kctx) wait_event(kbdev->pm.resume_wait, !kbase_pm_is_resuming(kbdev)); kbase_mem_pool_group_mark_dying(&kctx->mem_pools); + kbase_mem_pool_mark_dying(&kctx->pgd_mem_pool); kbase_context_term_partial(kctx, ARRAY_SIZE(context_init)); diff --git a/drivers/gpu/arm/valhall/context/backend/mali_kbase_context_jm.c b/drivers/gpu/arm/valhall/context/backend/mali_kbase_context_jm.c deleted file mode 100644 index ef474f625f63..000000000000 --- a/drivers/gpu/arm/valhall/context/backend/mali_kbase_context_jm.c +++ /dev/null @@ -1,266 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Base kernel context APIs for Job Manager GPUs - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#if IS_ENABLED(CONFIG_DEBUG_FS) -#include -#include -#include -#include - -void kbase_context_debugfs_init(struct kbase_context *const kctx) -{ - kbase_debug_mem_view_init(kctx); - kbase_debug_mem_zones_init(kctx); - kbase_debug_mem_allocs_init(kctx); - kbase_mem_pool_debugfs_init(kctx->kctx_dentry, kctx); - kbase_jit_debugfs_init(kctx); - kbasep_jd_debugfs_ctx_init(kctx); -} -KBASE_EXPORT_SYMBOL(kbase_context_debugfs_init); - -void kbase_context_debugfs_term(struct kbase_context *const kctx) -{ - debugfs_remove_recursive(kctx->kctx_dentry); -} -KBASE_EXPORT_SYMBOL(kbase_context_debugfs_term); -#else -void kbase_context_debugfs_init(struct kbase_context *const kctx) -{ - CSTD_UNUSED(kctx); -} -KBASE_EXPORT_SYMBOL(kbase_context_debugfs_init); - -void kbase_context_debugfs_term(struct kbase_context *const kctx) -{ - CSTD_UNUSED(kctx); -} -KBASE_EXPORT_SYMBOL(kbase_context_debugfs_term); -#endif /* CONFIG_DEBUG_FS */ - -static int kbase_context_kbase_kinstr_jm_init(struct kbase_context *kctx) -{ - return kbase_kinstr_jm_init(&kctx->kinstr_jm); -} - -static void kbase_context_kbase_kinstr_jm_term(struct kbase_context *kctx) -{ - kbase_kinstr_jm_term(kctx->kinstr_jm); -} - -static int kbase_context_kbase_timer_setup(struct kbase_context *kctx) -{ - kbase_timer_setup(&kctx->soft_job_timeout, kbasep_soft_job_timeout_worker); - - return 0; -} - -static int kbase_context_submit_check(struct kbase_context *kctx) -{ - struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info; - unsigned long irq_flags = 0; - - base_context_create_flags const flags = kctx->create_flags; - - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, irq_flags); - - /* Translate the flags */ - if ((flags & BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED) == 0) - kbase_ctx_flag_clear(kctx, KCTX_SUBMIT_DISABLED); - - spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, irq_flags); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - - return 0; -} - -static void kbase_context_flush_jobs(struct kbase_context *kctx) -{ - kbase_jd_zap_context(kctx); - flush_workqueue(kctx->jctx.job_done_wq); -} - -/** - * kbase_context_free - Free kcontext at its destruction - * - * @kctx: kcontext to be freed - */ -static void kbase_context_free(struct kbase_context *kctx) -{ - kbase_timeline_post_kbase_context_destroy(kctx); - - vfree(kctx); -} - -static const struct kbase_context_init context_init[] = { - { NULL, kbase_context_free, NULL }, - { kbase_context_common_init, kbase_context_common_term, - "Common context initialization failed" }, - { kbase_context_mem_pool_group_init, kbase_context_mem_pool_group_term, - "Memory pool group initialization failed" }, - { kbase_mem_evictable_init, kbase_mem_evictable_deinit, - "Memory evictable initialization failed" }, - { kbase_ctx_sched_init_ctx, NULL, NULL }, - { kbase_context_mmu_init, kbase_context_mmu_term, "MMU initialization failed" }, - { kbase_context_mem_alloc_page, kbase_context_mem_pool_free, "Memory alloc page failed" }, - { kbase_region_tracker_init, kbase_region_tracker_term, - "Region tracker initialization failed" }, - { kbase_sticky_resource_init, kbase_context_sticky_resource_term, - "Sticky resource initialization failed" }, - { kbase_jit_init, kbase_jit_term, "JIT initialization failed" }, - { kbase_context_kbase_kinstr_jm_init, kbase_context_kbase_kinstr_jm_term, - "JM instrumentation initialization failed" }, - { kbase_context_kbase_timer_setup, NULL, "Timers initialization failed" }, - { kbase_event_init, kbase_event_cleanup, "Event initialization failed" }, - { kbasep_js_kctx_init, kbasep_js_kctx_term, "JS kctx initialization failed" }, - { kbase_jd_init, kbase_jd_exit, "JD initialization failed" }, - { kbase_context_submit_check, NULL, "Enabling job submission failed" }, -#if IS_ENABLED(CONFIG_DEBUG_FS) - { kbase_debug_job_fault_context_init, kbase_debug_job_fault_context_term, - "Job fault context initialization failed" }, -#endif - { kbasep_platform_context_init, kbasep_platform_context_term, - "Platform callback for kctx initialization failed" }, - { NULL, kbase_context_flush_jobs, NULL }, - { kbase_context_add_to_dev_list, kbase_context_remove_from_dev_list, - "Adding kctx to device failed" }, -}; - -static void kbase_context_term_partial(struct kbase_context *kctx, unsigned int i) -{ - while (i-- > 0) { - if (context_init[i].term) - context_init[i].term(kctx); - } -} - -struct kbase_context *kbase_create_context(struct kbase_device *kbdev, bool is_compat, - base_context_create_flags const flags, - unsigned long const api_version, struct file *const filp) -{ - struct kbase_context *kctx; - unsigned int i = 0; - - if (WARN_ON(!kbdev)) - return NULL; - - /* Validate flags */ - if (WARN_ON(flags != (flags & BASEP_CONTEXT_CREATE_KERNEL_FLAGS))) - return NULL; - - /* zero-inited as lot of code assume it's zero'ed out on create */ - kctx = vzalloc(sizeof(*kctx)); - if (WARN_ON(!kctx)) - return NULL; - - kctx->kbdev = kbdev; - kctx->api_version = api_version; - kctx->filp = filp; - kctx->create_flags = flags; - - if (is_compat) - kbase_ctx_flag_set(kctx, KCTX_COMPAT); -#if defined(CONFIG_64BIT) - else - kbase_ctx_flag_set(kctx, KCTX_FORCE_SAME_VA); -#endif /* defined(CONFIG_64BIT) */ - - for (i = 0; i < ARRAY_SIZE(context_init); i++) { - int err = 0; - - if (context_init[i].init) - err = context_init[i].init(kctx); - - if (err) { - dev_err(kbdev->dev, "%s error = %d\n", context_init[i].err_mes, err); - - /* kctx should be freed by kbase_context_free(). - * Otherwise it will result in memory leak. - */ - WARN_ON(i == 0); - - kbase_context_term_partial(kctx, i); - return NULL; - } - } - - return kctx; -} -KBASE_EXPORT_SYMBOL(kbase_create_context); - -void kbase_destroy_context(struct kbase_context *kctx) -{ - struct kbase_device *kbdev; - - if (WARN_ON(!kctx)) - return; - - kbdev = kctx->kbdev; - if (WARN_ON(!kbdev)) - return; - - /* Context termination could happen whilst the system suspend of - * the GPU device is ongoing or has completed. It has been seen on - * Customer side that a hang could occur if context termination is - * not blocked until the resume of GPU device. - */ - if (kbase_has_arbiter(kbdev)) - atomic_inc(&kbdev->pm.gpu_users_waiting); - while (kbase_pm_context_active_handle_suspend(kbdev, - KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE)) { - dev_dbg(kbdev->dev, "Suspend in progress when destroying context"); - wait_event(kbdev->pm.resume_wait, !kbase_pm_is_suspending(kbdev)); - } - - /* Have synchronized against the System suspend and incremented the - * pm.active_count. So any subsequent invocation of System suspend - * callback would get blocked. - * If System suspend callback was already in progress then the above loop - * would have waited till the System resume callback has begun. - * So wait for the System resume callback to also complete as we want to - * avoid context termination during System resume also. - */ - wait_event(kbdev->pm.resume_wait, !kbase_pm_is_resuming(kbdev)); - - if (kbase_has_arbiter(kbdev)) - atomic_dec(&kbdev->pm.gpu_users_waiting); - - kbase_mem_pool_group_mark_dying(&kctx->mem_pools); - - kbase_context_term_partial(kctx, ARRAY_SIZE(context_init)); - - kbase_pm_context_idle(kbdev); -} -KBASE_EXPORT_SYMBOL(kbase_destroy_context); diff --git a/drivers/gpu/arm/valhall/context/mali_kbase_context.c b/drivers/gpu/arm/valhall/context/mali_kbase_context.c index 5126f3f6d82a..34f40073779e 100644 --- a/drivers/gpu/arm/valhall/context/mali_kbase_context.c +++ b/drivers/gpu/arm/valhall/context/mali_kbase_context.c @@ -203,6 +203,7 @@ int kbase_context_common_init(struct kbase_context *kctx) } } + kctx->offslot_ts = 0; return err; } @@ -296,8 +297,11 @@ void kbase_context_common_term(struct kbase_context *kctx) int kbase_context_mem_pool_group_init(struct kbase_context *kctx) { - return kbase_mem_pool_group_init(&kctx->mem_pools, kctx->kbdev, - &kctx->kbdev->mem_pool_defaults, &kctx->kbdev->mem_pools); + size_t const small_max_size = KBASE_MEM_POOL_MAX_SIZE_KCTX; + size_t const large_max_size = small_max_size >> (KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER - + KBASE_MEM_POOL_SMALL_PAGE_TABLE_ORDER); + return kbase_mem_pool_group_init(&kctx->mem_pools, kctx->kbdev, small_max_size, + large_max_size, NULL); } void kbase_context_mem_pool_group_term(struct kbase_context *kctx) @@ -305,6 +309,20 @@ void kbase_context_mem_pool_group_term(struct kbase_context *kctx) kbase_mem_pool_group_term(&kctx->mem_pools); } +int kbase_context_pgd_mem_pool_init(struct kbase_context *kctx) +{ + int err = kbase_mem_pool_init_no_reclaim(&kctx->pgd_mem_pool, + BASE_PGD_MEM_POOL_MAX_SIZE_KBDEV, + KBASE_MEM_POOL_SMALL_PAGE_TABLE_ORDER, 0, + kctx->kbdev); + return err; +} + +void kbase_context_pgd_mem_pool_term(struct kbase_context *kctx) +{ + kbase_mem_pool_term(&kctx->pgd_mem_pool); +} + int kbase_context_mmu_init(struct kbase_context *kctx) { return kbase_mmu_init(kctx->kbdev, &kctx->mmu, kctx, diff --git a/drivers/gpu/arm/valhall/context/mali_kbase_context_internal.h b/drivers/gpu/arm/valhall/context/mali_kbase_context_internal.h index 8d9b39419586..5552aa3e036e 100644 --- a/drivers/gpu/arm/valhall/context/mali_kbase_context_internal.h +++ b/drivers/gpu/arm/valhall/context/mali_kbase_context_internal.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -50,6 +50,9 @@ void kbase_context_common_term(struct kbase_context *kctx); int kbase_context_mem_pool_group_init(struct kbase_context *kctx); void kbase_context_mem_pool_group_term(struct kbase_context *kctx); +int kbase_context_pgd_mem_pool_init(struct kbase_context *kctx); +void kbase_context_pgd_mem_pool_term(struct kbase_context *kctx); + int kbase_context_mmu_init(struct kbase_context *kctx); void kbase_context_mmu_term(struct kbase_context *kctx); diff --git a/drivers/gpu/arm/valhall/csf/ipa_control/mali_kbase_csf_ipa_control.c b/drivers/gpu/arm/valhall/csf/ipa_control/mali_kbase_csf_ipa_control.c index ecf19001c345..257d64b609c1 100644 --- a/drivers/gpu/arm/valhall/csf/ipa_control/mali_kbase_csf_ipa_control.c +++ b/drivers/gpu/arm/valhall/csf/ipa_control/mali_kbase_csf_ipa_control.c @@ -897,7 +897,6 @@ void kbase_ipa_control_handle_gpu_reset_post(struct kbase_device *kbdev) } KBASE_EXPORT_TEST_API(kbase_ipa_control_handle_gpu_reset_post); -#ifdef KBASE_PM_RUNTIME void kbase_ipa_control_handle_gpu_sleep_enter(struct kbase_device *kbdev) { lockdep_assert_held(&kbdev->hwaccess_lock); @@ -931,7 +930,6 @@ void kbase_ipa_control_handle_gpu_sleep_exit(struct kbase_device *kbdev) } } KBASE_EXPORT_TEST_API(kbase_ipa_control_handle_gpu_sleep_exit); -#endif #if MALI_UNIT_TEST void kbase_ipa_control_rate_change_notify_test(struct kbase_device *kbdev, u32 clk_index, diff --git a/drivers/gpu/arm/valhall/csf/ipa_control/mali_kbase_csf_ipa_control.h b/drivers/gpu/arm/valhall/csf/ipa_control/mali_kbase_csf_ipa_control.h index 80aabc1022c5..0d1fe664d9a5 100644 --- a/drivers/gpu/arm/valhall/csf/ipa_control/mali_kbase_csf_ipa_control.h +++ b/drivers/gpu/arm/valhall/csf/ipa_control/mali_kbase_csf_ipa_control.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -197,7 +197,6 @@ void kbase_ipa_control_handle_gpu_reset_pre(struct kbase_device *kbdev); */ void kbase_ipa_control_handle_gpu_reset_post(struct kbase_device *kbdev); -#ifdef KBASE_PM_RUNTIME /** * kbase_ipa_control_handle_gpu_sleep_enter - Handle the pre GPU Sleep event * @@ -222,7 +221,6 @@ void kbase_ipa_control_handle_gpu_sleep_enter(struct kbase_device *kbdev); * was called previously. */ void kbase_ipa_control_handle_gpu_sleep_exit(struct kbase_device *kbdev); -#endif #if MALI_UNIT_TEST /** diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf.c b/drivers/gpu/arm/valhall/csf/mali_kbase_csf.c index c0231abe8d5f..22214f5b95e7 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf.c +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2025 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -319,10 +319,9 @@ void kbase_csf_free_command_stream_user_pages(struct kbase_context *kctx, struct * This condition is only true when mali_kbase_supports_csg_cs_user_page_allocation() * is disable. */ - if (!mali_kbase_supports_csg_cs_user_page_allocation(kctx->api_version)) { + if (!mali_kbase_supports_csg_cs_user_page_allocation(kctx->api_version)) kernel_free_user_io_pages(kctx, queue->phys, queue->user_io_addr); - queue->user_io_addr = NULL; - } + queue->user_io_addr = NULL; /* The user_io_gpu_va should have been unmapped inside the scheduler */ WARN_ONCE(queue->user_io_gpu_va, "Userio pages appears still have mapping"); @@ -773,6 +772,12 @@ int kbase_csf_queue_bind(struct kbase_context *kctx, union kbase_ioctl_cs_queue_ if (bind->in.csi_index >= max_streams) goto out; + if (queue->user_io_addr != NULL) { + dev_err(kctx->kbdev->dev, "Queue with stale user_io address: %pK", + (void *)queue->user_io_addr); + goto out; + } + if (group->run_state == KBASE_CSF_GROUP_TERMINATED) goto out; @@ -1018,6 +1023,13 @@ static void unbind_stopped_queue(struct kbase_context *kctx, struct kbase_queue queue->group = NULL; kbase_csf_scheduler_spin_unlock(kctx->kbdev, flags); + /* Ensure that the user I/O pages are no longer accessible */ + mutex_lock(&kctx->kbdev->csf.reg_lock); + unmap_mapping_range(kctx->kbdev->csf.db_filp->f_inode->i_mapping, + (loff_t)(queue->db_file_offset << PAGE_SHIFT), + BASEP_QUEUE_NR_MMAP_USER_PAGES * PAGE_SIZE, 1); + mutex_unlock(&kctx->kbdev->csf.reg_lock); + put_user_pages_mmap_handle(kctx, queue); WARN_ON_ONCE(queue->doorbell_nr != KBASEP_USER_DB_NR_INVALID); queue->bind_state = KBASE_CSF_QUEUE_UNBOUND; @@ -1852,8 +1864,6 @@ void kbase_csf_ctx_handle_fault(struct kbase_context *kctx, struct kbase_fault * void kbase_csf_ctx_term(struct kbase_context *kctx) { struct kbase_device *kbdev = kctx->kbdev; - struct kbase_as *as = NULL; - unsigned long flags; u32 i; int err; bool reset_prevented = false; @@ -1922,15 +1932,57 @@ void kbase_csf_ctx_term(struct kbase_context *kctx) flush_work(&kctx->kbdev->csf.glb_fatal_work); /* A work item to handle page_fault/bus_fault/gpu_fault could be - * pending for the outgoing context. Flush the workqueue that will - * execute that work item. + * pending for the outgoing context which we no longer care about. + * Ensure that the context won't be accessed anymore by the fault + * workers. */ - spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags); - if (kctx->as_nr != KBASEP_AS_NR_INVALID) - as = &kctx->kbdev->as[kctx->as_nr]; - spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); - if (as) - flush_workqueue(as->pf_wq); + while (true) { + unsigned long flags; + int refcount; + + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + refcount = atomic_read(&kctx->refcount); + if ((refcount != 0) && !WARN_ON_ONCE(kctx->as_nr == KBASEP_AS_NR_INVALID)) { + struct kbase_as *as = &kctx->kbdev->as[kctx->as_nr]; + int new_refcount; + + dev_dbg(kbdev->dev, + "Waiting for pending fault worker to complete when terminating context (%d_%d)", + kctx->tgid, kctx->id); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + flush_workqueue(as->pf_wq); + + new_refcount = atomic_read(&kctx->refcount); + if (refcount != new_refcount) { + /* Fault workers executed and released some references, re-check */ + continue; + } else { + /* Waiting for pending fault workers to execute was not effective, + * we're going to forcefully de-assign the AS from this context + * because nothing else should still be accessing the context at + * this point. + * + * This should never happen and a WARN_ON() would be printed by + * kbase_ctx_sched_remove_ctx() if the refcount is non-zero. + */ + dev_warn( + kbdev->dev, + "No fault workers executed, %d refs remain for terminating context (%d_%d)", + new_refcount, kctx->tgid, kctx->id); + kbase_ctx_sched_remove_ctx(kctx); + } + } else { + kbase_ctx_sched_remove_ctx_nolock(kctx); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + } + + break; + } mutex_lock(&kctx->csf.lock); @@ -2439,27 +2491,33 @@ static void handle_progress_timer_events(struct kbase_device *const kbdev, unsig u32 max_csg_slots = kbdev->csf.global_iface.group_num; u32 csg_nr; struct kbase_queue_group *group = NULL; + bool aw_removed; kbase_csf_scheduler_spin_lock_assert_held(kbdev); if (likely(bitmap_empty(slot_mask, BASEP_QUEUE_GROUP_MAX))) return; - /* Log each timeout and Update timestamp of compute progress timeout */ - for_each_set_bit(csg_nr, slot_mask, max_csg_slots) { - group = kbdev->csf.scheduler.csg_slots[csg_nr].resident_group; - group->progress_timer_state = kbase_csf_fw_io_group_read(&kbdev->csf.fw_io, csg_nr, - CSG_PROGRESS_TIMER_STATE); + aw_removed = kbase_io_is_aw_removed(kbdev); - dev_info( - kbdev->dev, - "[%llu] Iterator PROGRESS_TIMER timeout notification received for group %u of ctx %d_%d on slot %u with state %x", - kbase_backend_get_cycle_cnt(kbdev), group->handle, group->kctx->tgid, - group->kctx->id, csg_nr, group->progress_timer_state); + if (!aw_removed) + /* Log each timeout and Update timestamp of compute progress timeout */ + for_each_set_bit(csg_nr, slot_mask, max_csg_slots) { + group = kbdev->csf.scheduler.csg_slots[csg_nr].resident_group; + group->progress_timer_state = kbase_csf_fw_io_group_read( + &kbdev->csf.fw_io, csg_nr, CSG_PROGRESS_TIMER_STATE); - if (CSG_PROGRESS_TIMER_STATE_GET(group->progress_timer_state) == - CSG_PROGRESS_TIMER_STATE_COMPUTE) - kbdev->csf.compute_progress_timeout_cc = kbase_backend_get_cycle_cnt(kbdev); - } + dev_info( + kbdev->dev, + "[%llu] Iterator PROGRESS_TIMER timeout notification received for group %u of ctx %d_%d on slot %u with state %x", + kbase_backend_get_cycle_cnt(kbdev), group->handle, + group->kctx->tgid, group->kctx->id, csg_nr, + group->progress_timer_state); + + if (CSG_PROGRESS_TIMER_STATE_GET(group->progress_timer_state) == + CSG_PROGRESS_TIMER_STATE_COMPUTE) + kbdev->csf.compute_progress_timeout_cc = + kbase_backend_get_cycle_cnt(kbdev); + } /* Ignore fragment timeout if it is following a compute timeout. * Otherwise, terminate the command stream group. @@ -2470,8 +2528,8 @@ static void handle_progress_timer_events(struct kbase_device *const kbdev, unsig /* Check if it is a fragment timeout right after another compute timeout. * In such case, kill compute CSG and give fragment CSG a second chance */ - if (CSG_PROGRESS_TIMER_STATE_GET(group->progress_timer_state) == - CSG_PROGRESS_TIMER_STATE_FRAGMENT) { + if (!aw_removed && CSG_PROGRESS_TIMER_STATE_GET(group->progress_timer_state) == + CSG_PROGRESS_TIMER_STATE_FRAGMENT) { u64 cycle_counter = kbase_backend_get_cycle_cnt(kbdev); u64 compute_progress_timeout_cc = kbdev->csf.compute_progress_timeout_cc; @@ -3005,7 +3063,27 @@ static int process_cs_interrupts(struct kbase_queue_group *const group, u32 grou kbase_csf_fw_io_close(fw_io, fw_io_flags); } - /* PROTM_PEND and TILER_OOM can be safely ignored + /* PROTM_PEND request should be handled if the group + * is assigned a CSG slot in the future. We set + * protm_pending_bitmap here in case we skip saving + * the CS slots' state due to the group becoming active + * shortly after the suspension request is made. + */ + if ((cs_req & CS_REQ_PROTM_PEND_MASK) ^ (cs_ack & CS_ACK_PROTM_PEND_MASK)) { + KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_INTERRUPT_PROTM_PEND, group, + queue, cs_req ^ cs_ack); + + dev_dbg(kbdev->dev, + "Protected mode entry request for queue on csi %d bound to group-%d on slot %d", + queue->csi_index, group->handle, group->csg_nr); + + bitmap_set(group->protm_pending_bitmap, i, 1); + KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_PROTM_PEND_SET, group, queue, + group->protm_pending_bitmap[0]); + protm_pend = true; + } + + /* TILER_OOM can be safely ignored * because they will be raised again if the group * is assigned a CSG slot in future. */ @@ -3026,20 +3104,6 @@ static int process_cs_interrupts(struct kbase_queue_group *const group, u32 grou kbase_csf_handle_pending_oom_interrupt(queue, group_id); } - - if ((cs_req & CS_REQ_PROTM_PEND_MASK) ^ (cs_ack & CS_ACK_PROTM_PEND_MASK)) { - KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_INTERRUPT_PROTM_PEND, group, - queue, cs_req ^ cs_ack); - - dev_dbg(kbdev->dev, - "Protected mode entry request for queue on csi %d bound to group-%d on slot %d", - queue->csi_index, group->handle, group->csg_nr); - - bitmap_set(group->protm_pending_bitmap, i, 1); - KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_PROTM_PEND_SET, group, queue, - group->protm_pending_bitmap[0]); - protm_pend = true; - } } } @@ -3757,7 +3821,7 @@ void kbase_csf_doorbell_mapping_term(struct kbase_device *kbdev) * module unload path, so the page can be left uncleared before returning it * back to kbdev memory pool. */ - kbase_mem_pool_free(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], page, false); + kbase_mem_pool_free(&kbdev->fw_mem_pools.small, page, false); fput(kbdev->csf.db_filp); } @@ -3773,8 +3837,7 @@ int kbase_csf_doorbell_mapping_init(struct kbase_device *kbdev) if (IS_ERR(filp)) return PTR_ERR(filp); - ret = kbase_mem_pool_alloc_pages(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], 1, &phys, - false, NULL); + ret = kbase_mem_pool_alloc_pages(&kbdev->fw_mem_pools.small, 1, &phys, false, NULL); if (ret <= 0) { fput(filp); @@ -3821,7 +3884,7 @@ void kbase_csf_free_dummy_user_reg_page(struct kbase_device *kbdev) * path, so the page can be left uncleared before returning it back to kbdev * memory pool. */ - kbase_mem_pool_free(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], page, false); + kbase_mem_pool_free(&kbdev->fw_mem_pools.small, page, false); fput(kbdev->csf.user_reg.filp); } } @@ -3841,8 +3904,7 @@ int kbase_csf_setup_dummy_user_reg_page(struct kbase_device *kbdev) return PTR_ERR(filp); } - if (kbase_mem_pool_alloc_pages(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], 1, &phys, - false, NULL) <= 0) { + if (kbase_mem_pool_alloc_pages(&kbdev->fw_mem_pools.small, 1, &phys, false, NULL) <= 0) { fput(filp); return -ENOMEM; } diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_csg_debugfs.c b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_csg_debugfs.c index 3d614514e8bd..7a53bb7ad556 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_csg_debugfs.c +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_csg_debugfs.c @@ -50,10 +50,8 @@ static const char *scheduler_state_to_string(struct kbase_device *kbdev, return "INACTIVE"; case SCHED_SUSPENDED: return "SUSPENDED"; -#ifdef KBASE_PM_RUNTIME case SCHED_SLEEPING: return "SLEEPING"; -#endif default: dev_warn(kbdev->dev, "Unknown Scheduler state %d", sched_state); return NULL; @@ -249,10 +247,8 @@ static ssize_t kbase_csf_debugfs_scheduler_state_set(struct file *file, const ch if (sysfs_streq(buf, "SUSPENDED")) kbase_csf_scheduler_pm_suspend(kbdev); -#ifdef KBASE_PM_RUNTIME - else if (sysfs_streq(buf, "SLEEPING")) + else if (IS_ENABLED(CONFIG_PM) && sysfs_streq(buf, "SLEEPING")) kbase_csf_scheduler_force_sleep(kbdev); -#endif else if (sysfs_streq(buf, "INACTIVE")) kbase_csf_scheduler_force_wakeup(kbdev); else { diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_defs.h b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_defs.h index 9f62587ba2cc..48569836e320 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_defs.h +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_defs.h @@ -1043,6 +1043,17 @@ struct kbase_csf_csg_slot { u8 priority; }; +/** + * struct kbase_csf_heap_reclaim_offslot - Setting for reclaiming offslot CSG's heap. + * + * @timeout_ms: Reclaim from CSGs being offslot longer than this. + * @pages: Max number of pages for each reclaim. + */ +struct kbase_csf_heap_reclaim_offslot { + u32 timeout_ms; + u32 pages; +}; + /** * struct kbase_csf_sched_heap_reclaim_mgr - Object for managing tiler heap reclaim * kctx lists inside the CSF device's scheduler. @@ -1052,12 +1063,14 @@ struct kbase_csf_csg_slot { * lists track the kctxs attached to the reclaim manager. * @unused_pages: Estimated number of unused pages from the @ctxlist array. The * number is indicative for use with reclaim shrinker's count method. + * @offslot_setting: Setting for reclaiming offslot CSG's heap. */ struct kbase_csf_sched_heap_reclaim_mgr { DEFINE_KBASE_SHRINKER heap_reclaim; struct list_head ctx_lists[KBASE_QUEUE_GROUP_PRIORITY_COUNT]; atomic_t unused_pages; + struct kbase_csf_heap_reclaim_offslot offslot_setting; }; /** @@ -1177,6 +1190,9 @@ struct kbase_csf_mcu_shared_regions { * perform a scheduling tock. * @pending_gpu_idle_work: Indicates that kbase_csf_scheduler_kthread() should * handle the GPU IDLE event. + * @pending_runtime_suspend_work: Indicates that kbase_csf_scheduler_kthread() + * should proceed to suspend the GPU as part of + * handling the runtime suspend event. * @pending_power_off_work: Indicates that kbase_csf_scheduler_kthread() should * proceed to power off the GPU. * @ping_work: Work item that would ping the firmware at regular @@ -1228,10 +1244,13 @@ struct kbase_csf_mcu_shared_regions { * @mcu_regs_data: Scheduler MCU shared regions data for managing the * shared interface mappings for on-slot queues and * CSG suspend buffers. - * @kthread_signal: Used to wake up the GPU queue submission - * thread when a queue needs attention. - * @kthread_running: Whether the GPU queue submission thread should keep - * executing. + * @kthread_signal: Used to wake up the main CSF scheduler thread + * to handle pending work items. + * @kthread_running: Set to true to indicate that the CSF scheduler + * thread will handle work items. Work items that + * are handled by this thread all require the schduler + * mutex lock, thus are serialised and executed in a + * predefined order. * @gpuq_kthread: Dedicated thread primarily used to handle * latency-sensitive tasks such as GPU queue * submissions. @@ -1272,6 +1291,7 @@ struct kbase_csf_scheduler { atomic_t pending_tick_work; atomic_t pending_tock_work; atomic_t pending_gpu_idle_work; + atomic_t pending_runtime_suspend_work; atomic_t pending_power_off_work; struct delayed_work ping_work; struct kbase_context *top_kctx; @@ -1813,7 +1833,8 @@ struct kbase_csf_user_reg { * @fw_io: Firmware I/O interface. * @compute_progress_timeout_cc: Value of GPU cycle count register when progress * timer timeout is reported for the compute iterator. - * @num_doorbells: Number of doorbells supported by the GPU. + * @neural_allowed_mask: A mask for optionally disabling neural cores across all CSGs + * @num_doorbells: Number of doorbells supported by the GPU. */ struct kbase_csf_device { struct kbase_mmu_table mcu_mmu; @@ -1877,6 +1898,7 @@ struct kbase_csf_device { u32 page_fault_cnt; struct kbase_csf_fw_io fw_io; u64 compute_progress_timeout_cc; + u64 neural_allowed_mask; u32 num_doorbells; }; diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_event.c b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_event.c index 08faace8ef06..a91d99816a9c 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_event.c +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_event.c @@ -94,11 +94,11 @@ static void sync_update_notify_gpu(struct kbase_context *kctx) spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags); can_notify_gpu = kbase_io_is_gpu_powered(kctx->kbdev); -#ifdef KBASE_PM_RUNTIME - if (kctx->kbdev->pm.backend.db_mirror_interrupt_enabled || - kctx->kbdev->pm.backend.gpu_sleep_mode_active) - can_notify_gpu = false; -#endif + if (IS_ENABLED(CONFIG_PM)) { + if (kctx->kbdev->pm.backend.db_mirror_interrupt_enabled || + kctx->kbdev->pm.backend.gpu_sleep_mode_active) + can_notify_gpu = false; + } if (can_notify_gpu) { diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_firmware.c b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_firmware.c index d1d53a075cbb..5680d2c21116 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_firmware.c +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_firmware.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2025 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -377,6 +377,15 @@ void kbase_csf_stop_firmware_and_wait(struct kbase_device *kbdev) kbase_csf_firmware_disable_mcu_wait(kbdev); } +static void kbasep_hwcnt_init_on_boot(struct kbase_device *kbdev) +{ + kbase_hwcnt_backend_csf_on_before_mcu_cold_boot(&kbdev->hwcnt_gpu_iface); + kbase_hwcnt_backend_csf_set_hw_availability(&kbdev->hwcnt_gpu_iface, + kbdev->gpu_props.curr_config.l2_slices, + kbdev->gpu_props.curr_config.shader_present, + kbdev->pm.debug_core_mask); +} + static void wait_for_firmware_boot(struct kbase_device *kbdev) { long wait_timeout; @@ -400,6 +409,8 @@ static void wait_for_firmware_boot(struct kbase_device *kbdev) wait_timeout); if (!remaining) dev_err(kbdev->dev, "Timed out waiting for fw boot completion"); + kbase_hwcnt_backend_csf_on_after_mcu_on(&kbdev->hwcnt_gpu_iface); + kbdev->csf.interrupt_received = false; } @@ -414,6 +425,8 @@ static void enable_mcu(struct kbase_device *kbdev) static void boot_csf_firmware(struct kbase_device *kbdev) { + kbasep_hwcnt_init_on_boot(kbdev); + enable_mcu(kbdev); wait_for_firmware_boot(kbdev); @@ -440,6 +453,12 @@ static int wait_ready(struct kbase_device *kbdev) if (!err) return 0; + /* Bailout on GPU_LOST without RESET */ + if (err == -ENODEV) { + dev_warn(kbdev->dev, "%s: AW removed, bailing out", __func__); + return 0; + } + dev_err(kbdev->dev, "AS_ACTIVE bit stuck for MCU AS. Might be caused by unstable GPU clk/pwr or faulty system"); @@ -835,10 +854,10 @@ retry_alloc: } } else { if (!reuse_pages) { - ret = kbase_mem_pool_alloc_pages( - kbase_mem_pool_group_select(kbdev, KBASE_MEM_GROUP_CSF_FW, - is_small_page), - num_pages_aligned, phys, false, NULL); + ret = kbase_mem_pool_alloc_pages(is_small_page ? + &kbdev->fw_mem_pools.small : + &kbdev->fw_mem_pools.large, + num_pages_aligned, phys, false, NULL); } } @@ -971,10 +990,9 @@ out: kbase_csf_protected_memory_free(kbdev, pma, num_pages_aligned, is_small_page); } else { - kbase_mem_pool_free_pages( - kbase_mem_pool_group_select(kbdev, KBASE_MEM_GROUP_CSF_FW, - is_small_page), - num_pages_aligned, phys, false, false); + kbase_mem_pool_free_pages(is_small_page ? &kbdev->fw_mem_pools.small : + &kbdev->fw_mem_pools.large, + num_pages_aligned, phys, false, false); } kfree(phys); } @@ -1386,6 +1404,11 @@ static int parse_capabilities(struct kbase_device *kbdev) else iface->instr_features = 0; + if (iface->version >= kbase_csf_interface_version(1, 1, 0)) + iface->prfcnt_features = shared_info[GLB_PRFCNT_FEATURES / 4]; + else + iface->prfcnt_features = 0; + if ((GROUP_CONTROL_0 + (unsigned long)iface->group_num * iface->group_stride) > (interface->num_pages * PAGE_SIZE)) { dev_err(kbdev->dev, @@ -1600,9 +1623,7 @@ static void set_global_request(struct kbase_csf_fw_io *fw_io, u32 const req_mask kbase_csf_fw_io_global_write_mask(fw_io, GLB_REQ, glb_req, req_mask); } -static void enable_endpoints_global(struct kbase_csf_fw_io *fw_io, - - u64 const shader_core_mask) +static void enable_endpoints_global(struct kbase_csf_fw_io *fw_io, u64 const shader_core_mask) { kbase_csf_fw_io_assert_opened(fw_io); @@ -1905,6 +1926,11 @@ static void global_init(struct kbase_device *const kbdev, u64 core_mask) kbasep_enable_rtu(kbdev); /* Update shader core allocation enable mask */ + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) + if (kbase_io_is_gpu_powered(kbdev)) + kbase_reg_write64(kbdev, GPU_GOVERNOR_ENUM(GOV_CORE_MASK), + kbase_pm_ca_get_gov_core_mask(kbdev)); + enable_endpoints_global(fw_io, core_mask); set_shader_poweroff_timer(fw_io); @@ -1935,6 +1961,13 @@ static void global_init(struct kbase_device *const kbdev, u64 core_mask) if (kbdev->pm.backend.has_host_pwr_iface) ack_irq_mask |= GLB_ACK_IRQ_MASK_STATE_MASK; + + /* Do not modify PRFCNT bits of GLB_ACK_IRQ_MASK. */ + ack_irq_mask |= + (kbase_csf_fw_io_global_input_read(&kbdev->csf.fw_io, GLB_ACK_IRQ_MASK) & + (GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_MASK | GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_MASK | + GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_MASK | GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_MASK)); + /* Unmask the interrupts */ kbase_csf_fw_io_global_write(fw_io, GLB_ACK_IRQ_MASK, ack_irq_mask); @@ -1965,14 +1998,32 @@ exit: */ static int global_init_on_boot(struct kbase_device *const kbdev) { - unsigned long flags; + unsigned long flags, scheduler_lock_flags; u64 core_mask; int ret = 0; u32 request_mask = CSF_GLB_REQ_CFG_MASK; spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + core_mask = kbase_pm_ca_get_core_mask(kbdev); + kbdev->csf.firmware_hctl_core_pwr = kbase_pm_no_mcu_core_pwroff(kbdev); + + /* Enable the HWC context for the case of MCU controlling the shader core power. + * HWC context will be left enabled throughout the MCU power cycles, unless + * there is a GPU reset. + * + * In case of GPU reset, HWC needs to be disabled to ensure + * the correct HWC backend functionality. The context then gets reenabled during + * the reboot. + */ + if (!kbdev->csf.firmware_hctl_core_pwr) { + kbase_csf_scheduler_spin_lock(kbdev, &scheduler_lock_flags); + kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); + kbase_csf_scheduler_spin_unlock(kbdev, scheduler_lock_flags); + kbdev->pm.backend.hwcnt_disabled = false; + kbdev->pm.backend.hwcnt_desired = true; + } spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); global_init(kbdev, core_mask); @@ -1996,10 +2047,25 @@ static int global_init_on_boot(struct kbase_device *const kbdev) void kbase_csf_firmware_global_reinit(struct kbase_device *kbdev, u64 core_mask) { + unsigned long flags; lockdep_assert_held(&kbdev->hwaccess_lock); kbdev->csf.glb_init_request_pending = true; kbdev->csf.firmware_hctl_core_pwr = kbase_pm_no_mcu_core_pwroff(kbdev); + if (!kbdev->csf.firmware_hctl_core_pwr) { + /* The HWC context reenabling is needed if: + * - shader core power control was previously under host. + * - context was disabled when preparing for a GPU reset. + */ + kbdev->pm.backend.hwcnt_desired = true; + if (kbdev->pm.backend.hwcnt_disabled) { + kbase_csf_scheduler_spin_lock(kbdev, &flags); + kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); + kbase_csf_scheduler_spin_unlock(kbdev, flags); + kbdev->pm.backend.hwcnt_disabled = false; + } + } + global_init(kbdev, core_mask); } @@ -2020,9 +2086,6 @@ void kbase_csf_firmware_update_core_attr(struct kbase_device *kbdev, bool update struct kbase_csf_fw_io *fw_io = &kbdev->csf.fw_io; unsigned long flags, fw_io_flags; - if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) - core_mask = U64_MAX; - lockdep_assert_held(&kbdev->hwaccess_lock); kbase_csf_scheduler_spin_lock(kbdev, &flags); @@ -2083,6 +2146,7 @@ static void kbase_csf_firmware_reload_worker(struct work_struct *work) return; /* Reboot the firmware */ + kbase_hwcnt_backend_csf_on_before_mcu_cold_boot(&kbdev->hwcnt_gpu_iface); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbase_csf_firmware_enable_mcu(kbdev); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); @@ -2096,9 +2160,13 @@ void kbase_csf_firmware_trigger_reload(struct kbase_device *kbdev) if (kbdev->csf.firmware_reload_needed) { kbdev->csf.firmware_reload_needed = false; + /* MCU cold boot requested. */ queue_work(system_wq, &kbdev->csf.firmware_reload_work); } else { - kbase_csf_firmware_enable_mcu(kbdev); + /* MCU shall not boot while reset is in progress */ + if (likely(!kbdev->pm.backend.in_reset)) + /* MCU warm boot requested. */ + kbase_csf_firmware_enable_mcu(kbdev); } } @@ -2129,6 +2197,11 @@ void kbase_csf_firmware_reload_completed(struct kbase_device *kbdev) dev_err(kbdev->dev, "Version check failed in firmware reboot."); KBASE_KTRACE_ADD(kbdev, CSF_FIRMWARE_REBOOT, NULL, 0u); + kbase_hwcnt_backend_csf_set_hw_availability(&kbdev->hwcnt_gpu_iface, + kbdev->gpu_props.curr_config.l2_slices, + kbdev->gpu_props.curr_config.shader_present, + kbdev->pm.debug_core_mask); + kbase_hwcnt_backend_csf_on_after_mcu_on(&kbdev->hwcnt_gpu_iface); /* Tell MCU state machine to transit to next state */ kbdev->csf.firmware_reloaded = true; @@ -2419,10 +2492,9 @@ int kbase_csf_firmware_late_init(struct kbase_device *kbdev) kbdev->csf.gpu_idle_hysteresis_ns = FIRMWARE_IDLE_HYSTERESIS_TIME_NS; -#ifdef KBASE_PM_RUNTIME - if (kbase_pm_gpu_sleep_allowed(kbdev)) + if (IS_ENABLED(CONFIG_PM) && kbase_pm_gpu_sleep_allowed(kbdev)) kbdev->csf.gpu_idle_hysteresis_ns /= FIRMWARE_IDLE_HYSTERESIS_GPU_SLEEP_SCALER; -#endif + WARN_ON(!kbdev->csf.gpu_idle_hysteresis_ns); kbdev->csf.gpu_idle_dur_count = convert_dur_to_idle_count(kbdev, kbdev->csf.gpu_idle_hysteresis_ns, &no_modifier); @@ -2751,10 +2823,11 @@ void kbase_csf_firmware_unload_term(struct kbase_device *kbdev) interface->num_pages_aligned, interface->is_small_page); } else { - kbase_mem_pool_free_pages( - kbase_mem_pool_group_select(kbdev, KBASE_MEM_GROUP_CSF_FW, - interface->is_small_page), - interface->num_pages_aligned, interface->phys, true, false); + kbase_mem_pool_free_pages(interface->is_small_page ? + &kbdev->fw_mem_pools.small : + &kbdev->fw_mem_pools.large, + interface->num_pages_aligned, + interface->phys, true, false); } kfree(interface->phys); @@ -3128,7 +3201,6 @@ void kbase_csf_firmware_enable_mcu(struct kbase_device *kbdev) enable_mcu(kbdev); } -#ifdef KBASE_PM_RUNTIME void kbase_csf_firmware_trigger_mcu_sleep(struct kbase_device *kbdev) { struct kbase_csf_fw_io *fw_io = &kbdev->csf.fw_io; @@ -3169,7 +3241,6 @@ bool kbase_csf_firmware_is_mcu_in_sleep(struct kbase_device *kbdev) } KBASE_EXPORT_TEST_API(kbase_csf_firmware_is_mcu_in_sleep); -#endif /* KBASE_PM_RUNTIME */ bool kbase_csf_firmware_mcu_halt_req_complete(struct kbase_device *kbdev) { @@ -3359,8 +3430,7 @@ int kbase_csf_firmware_mcu_shared_mapping_init(struct kbase_device *kbdev, unsig if (!page_list) goto page_list_alloc_error; - ret = kbase_mem_pool_alloc_pages(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], num_pages, - phys, false, NULL); + ret = kbase_mem_pool_alloc_pages(&kbdev->fw_mem_pools.small, num_pages, phys, false, NULL); if (ret <= 0) goto phys_mem_pool_alloc_error; @@ -3408,8 +3478,7 @@ va_region_add_error: va_region_alloc_error: vunmap(cpu_addr); vmap_error: - kbase_mem_pool_free_pages(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], num_pages, phys, - false, false); + kbase_mem_pool_free_pages(&kbdev->fw_mem_pools.small, num_pages, phys, false, false); phys_mem_pool_alloc_error: kfree(page_list); @@ -3441,16 +3510,14 @@ void kbase_csf_firmware_mcu_shared_mapping_term(struct kbase_device *kbdev, /* This is on module unload path, so the pages can be left uncleared before * returning them back to kbdev memory pool. */ - kbase_mem_pool_free_pages(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], - csf_mapping->num_pages, csf_mapping->phys, false, false); + kbase_mem_pool_free_pages(&kbdev->fw_mem_pools.small, csf_mapping->num_pages, + csf_mapping->phys, false, false); } vunmap(csf_mapping->cpu_addr); kfree(csf_mapping->phys); } -#ifdef KBASE_PM_RUNTIME - void kbase_csf_firmware_soi_update(struct kbase_device *kbdev) { struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; @@ -3592,5 +3659,3 @@ int kbase_csf_firmware_soi_disable_on_scheduler_suspend(struct kbase_device *kbd return 0; } - -#endif /* KBASE_PM_RUNTIME */ diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_firmware.h b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_firmware.h index e2c627902097..6762bd22ec3e 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_firmware.h +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_firmware.h @@ -161,6 +161,7 @@ struct kbase_csf_cmd_stream_group_info { * CSG capability structures. * @prfcnt_size: Performance counters size. * @instr_features: Instrumentation features. (csf >= 1.1.0) + * @prfcnt_features: Performance Counter features. * @groups: Address of an array of CSG capability structures. */ struct kbase_csf_global_iface { @@ -171,6 +172,7 @@ struct kbase_csf_global_iface { u32 group_stride; u32 prfcnt_size; u32 instr_features; + u32 prfcnt_features; struct kbase_csf_cmd_stream_group_info *groups; }; @@ -477,7 +479,6 @@ void kbase_csf_firmware_disable_mcu_wait(struct kbase_device *kbdev); */ void kbase_csf_stop_firmware_and_wait(struct kbase_device *kbdev); -#ifdef KBASE_PM_RUNTIME /** * kbase_csf_firmware_trigger_mcu_sleep - Send the command to put MCU in sleep * state. @@ -496,7 +497,6 @@ void kbase_csf_firmware_trigger_mcu_sleep(struct kbase_device *kbdev); */ bool kbase_csf_firmware_is_mcu_in_sleep(struct kbase_device *kbdev); -#endif /** * kbase_csf_firmware_trigger_reload() - Trigger the reboot of MCU firmware, for @@ -812,8 +812,6 @@ int kbase_csf_trigger_firmware_config_update(struct kbase_device *kbdev); */ int kbase_csf_firmware_req_core_dump(struct kbase_device *const kbdev); -#ifdef KBASE_PM_RUNTIME - /** * kbase_csf_firmware_soi_update - Update FW Sleep-on-Idle config * @@ -842,7 +840,5 @@ void kbase_csf_firmware_glb_idle_timer_update(struct kbase_device *kbdev); */ int kbase_csf_firmware_soi_disable_on_scheduler_suspend(struct kbase_device *kbdev); -#endif /* KBASE_PM_RUNTIME */ - -#endif +#endif /* _KBASE_CSF_FIRMWARE_H_ */ diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_firmware_no_mali.c b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_firmware_no_mali.c index 0ac6ad394b2d..c8c57862a295 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_firmware_no_mali.c +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_firmware_no_mali.c @@ -221,6 +221,12 @@ static int invent_capabilities(struct kbase_device *kbdev) iface->group_num = ARRAY_SIZE(interface->csg); iface->group_stride = 0; + iface->prfcnt_features = 0; + + /* Set METATA_SIZE in GLB_PRFCNT_FEATURES. Always equal to one block. */ + WARN_ON((KBASE_DUMMY_MODEL_BLOCK_SIZE >> 8) > 0xf); + iface->prfcnt_features |= ((KBASE_DUMMY_MODEL_BLOCK_SIZE >> 8) & 0xf); + iface->groups = kcalloc(iface->group_num, sizeof(*iface->groups), GFP_KERNEL); if (iface->groups == NULL) return -ENOMEM; @@ -584,11 +590,22 @@ static void global_init(struct kbase_device *const kbdev, u64 core_mask) } /* Update shader core allocation enable mask */ + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) + if (kbase_io_is_gpu_powered(kbdev)) + kbase_reg_write64(kbdev, GPU_GOVERNOR_ENUM(GOV_CORE_MASK), + kbase_pm_ca_get_gov_core_mask(kbdev)); + enable_endpoints_global(fw_io, core_mask); set_shader_poweroff_timer(fw_io); set_timeout_global(fw_io, kbase_csf_timeout_get(kbdev)); + /* Do not modify PRFCNT bits of GLB_ACK_IRQ_MASK. */ + ack_irq_mask |= + (kbase_csf_fw_io_global_input_read(&kbdev->csf.fw_io, GLB_ACK_IRQ_MASK) & + (GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_MASK | GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_MASK | + GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_MASK | GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_MASK)); + /* Unmask the interrupts */ kbase_csf_fw_io_global_write(fw_io, GLB_ACK_IRQ_MASK, ack_irq_mask); @@ -610,14 +627,32 @@ exit: */ static int global_init_on_boot(struct kbase_device *const kbdev) { - unsigned long flags; + unsigned long flags, scheduler_lock_flags; u64 core_mask; int ret = 0; u32 request_mask = CSF_GLB_REQ_CFG_MASK; spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + core_mask = kbase_pm_ca_get_core_mask(kbdev); + kbdev->csf.firmware_hctl_core_pwr = kbase_pm_no_mcu_core_pwroff(kbdev); + + /* Enable the HWC context for the case of MCU controlling the shader core power. + * HWC context will be left enabled throughout the MCU power cycles, unless + * there is a GPU reset. + * + * In case of GPU reset, HWC needs to be disabled to ensure + * the correct HWC backend functionality. The context then gets reenabled during + * the reboot. + */ + if (!kbdev->csf.firmware_hctl_core_pwr) { + kbase_csf_scheduler_spin_lock(kbdev, &scheduler_lock_flags); + kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); + kbase_csf_scheduler_spin_unlock(kbdev, scheduler_lock_flags); + kbdev->pm.backend.hwcnt_disabled = false; + kbdev->pm.backend.hwcnt_desired = true; + } spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); global_init(kbdev, core_mask); @@ -629,10 +664,25 @@ static int global_init_on_boot(struct kbase_device *const kbdev) void kbase_csf_firmware_global_reinit(struct kbase_device *kbdev, u64 core_mask) { + unsigned long flags; + lockdep_assert_held(&kbdev->hwaccess_lock); kbdev->csf.glb_init_request_pending = true; kbdev->csf.firmware_hctl_core_pwr = kbase_pm_no_mcu_core_pwroff(kbdev); + if (!kbdev->csf.firmware_hctl_core_pwr) { + /* The HWC context reenabling is needed if: + * - shader core power control was previously under host. + * - context was disabled when preparing for a GPU reset. + */ + kbdev->pm.backend.hwcnt_desired = true; + if (kbdev->pm.backend.hwcnt_disabled) { + kbase_csf_scheduler_spin_lock(kbdev, &flags); + kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); + kbase_csf_scheduler_spin_unlock(kbdev, flags); + kbdev->pm.backend.hwcnt_disabled = false; + } + } global_init(kbdev, core_mask); } @@ -653,9 +703,6 @@ void kbase_csf_firmware_update_core_attr(struct kbase_device *kbdev, bool update struct kbase_csf_fw_io *fw_io = &kbdev->csf.fw_io; unsigned long flags, fw_io_flags; - if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) - core_mask = U64_MAX; - lockdep_assert_held(&kbdev->hwaccess_lock); kbase_csf_scheduler_spin_lock(kbdev, &flags); @@ -688,10 +735,16 @@ static void kbase_csf_firmware_reload_worker(struct work_struct *work) container_of(work, struct kbase_device, csf.firmware_reload_work); unsigned long flags; + kbase_hwcnt_backend_csf_on_before_mcu_cold_boot(&kbdev->hwcnt_gpu_iface); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); /* Reboot the firmware */ kbase_csf_firmware_enable_mcu(kbdev); + kbase_hwcnt_backend_csf_set_hw_availability(&kbdev->hwcnt_gpu_iface, + kbdev->gpu_props.curr_config.l2_slices, + kbdev->gpu_props.curr_config.shader_present, + kbdev->pm.debug_core_mask); + kbase_hwcnt_backend_csf_on_after_mcu_on(&kbdev->hwcnt_gpu_iface); /* Tell MCU state machine to transit to next state */ kbdev->csf.firmware_reloaded = true; kbase_pm_update_state(kbdev); @@ -709,6 +762,10 @@ void kbase_csf_firmware_trigger_reload(struct kbase_device *kbdev) queue_work(system_wq, &kbdev->csf.firmware_reload_work); } else { kbase_csf_firmware_enable_mcu(kbdev); + kbase_hwcnt_backend_csf_set_hw_availability( + &kbdev->hwcnt_gpu_iface, kbdev->gpu_props.curr_config.l2_slices, + kbdev->gpu_props.curr_config.shader_present, kbdev->pm.debug_core_mask); + kbase_hwcnt_backend_csf_on_after_mcu_on(&kbdev->hwcnt_gpu_iface); kbdev->csf.firmware_reloaded = true; } } @@ -721,7 +778,11 @@ void kbase_csf_firmware_reload_completed(struct kbase_device *kbdev) if (unlikely(!kbdev->csf.firmware_inited)) return; - + kbase_hwcnt_backend_csf_set_hw_availability(&kbdev->hwcnt_gpu_iface, + kbdev->gpu_props.curr_config.l2_slices, + kbdev->gpu_props.curr_config.shader_present, + kbdev->pm.debug_core_mask); + kbase_hwcnt_backend_csf_on_after_mcu_on(&kbdev->hwcnt_gpu_iface); /* Tell MCU state machine to transit to next state */ kbdev->csf.firmware_reloaded = true; kbase_pm_update_state(kbdev); @@ -955,6 +1016,15 @@ u32 kbase_csf_firmware_reset_mcu_core_pwroff_time(struct kbase_device *kbdev) return kbase_csf_firmware_set_mcu_core_pwroff_time(kbdev, DEFAULT_GLB_PWROFF_TIMEOUT_NS); } +static void kbasep_hwcnt_init_on_boot(struct kbase_device *kbdev) +{ + kbase_hwcnt_backend_csf_on_before_mcu_cold_boot(&kbdev->hwcnt_gpu_iface); + kbase_hwcnt_backend_csf_set_hw_availability(&kbdev->hwcnt_gpu_iface, + kbdev->gpu_props.curr_config.l2_slices, + kbdev->gpu_props.curr_config.shader_present, + kbdev->pm.debug_core_mask); +} + int kbase_csf_firmware_early_init(struct kbase_device *kbdev) { kbdev->csf.num_doorbells = CSF_NUM_DOORBELL_MAX; @@ -987,10 +1057,9 @@ int kbase_csf_firmware_late_init(struct kbase_device *kbdev) u32 no_modifier = 0; kbdev->csf.gpu_idle_hysteresis_ns = FIRMWARE_IDLE_HYSTERESIS_TIME_NS; -#ifdef KBASE_PM_RUNTIME - if (kbase_pm_gpu_sleep_allowed(kbdev)) + if (IS_ENABLED(CONFIG_PM) && kbase_pm_gpu_sleep_allowed(kbdev)) kbdev->csf.gpu_idle_hysteresis_ns /= FIRMWARE_IDLE_HYSTERESIS_GPU_SLEEP_SCALER; -#endif + WARN_ON(!kbdev->csf.gpu_idle_hysteresis_ns); kbdev->csf.gpu_idle_dur_count = convert_dur_to_idle_count(kbdev, kbdev->csf.gpu_idle_hysteresis_ns, &no_modifier); @@ -1041,6 +1110,8 @@ int kbase_csf_firmware_load_init(struct kbase_device *kbdev) kbase_csf_fw_io_init(&kbdev->csf.fw_io, kbdev); + kbasep_hwcnt_init_on_boot(kbdev); + ret = invent_capabilities(kbdev); if (ret != 0) goto error; @@ -1273,7 +1344,6 @@ void kbase_csf_firmware_enable_mcu(struct kbase_device *kbdev) kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(MCU_CONTROL), MCU_CONTROL_REQ_AUTO); } -#ifdef KBASE_PM_RUNTIME void kbase_csf_firmware_trigger_mcu_sleep(struct kbase_device *kbdev) { struct kbase_csf_fw_io *fw_io = &kbdev->csf.fw_io; @@ -1299,7 +1369,6 @@ bool kbase_csf_firmware_is_mcu_in_sleep(struct kbase_device *kbdev) kbase_csf_firmware_mcu_halted(kbdev)); } -#endif bool kbase_csf_firmware_mcu_halt_req_complete(struct kbase_device *kbdev) { @@ -1467,8 +1536,7 @@ int kbase_csf_firmware_mcu_shared_mapping_init(struct kbase_device *kbdev, unsig if (!page_list) goto page_list_alloc_error; - ret = kbase_mem_pool_alloc_pages(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], num_pages, - phys, false, NULL); + ret = kbase_mem_pool_alloc_pages(&kbdev->fw_mem_pools.small, num_pages, phys, false, NULL); if (ret <= 0) goto phys_mem_pool_alloc_error; @@ -1516,8 +1584,7 @@ va_region_add_error: va_region_alloc_error: vunmap(cpu_addr); vmap_error: - kbase_mem_pool_free_pages(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], num_pages, phys, - false, false); + kbase_mem_pool_free_pages(&kbdev->fw_mem_pools.small, num_pages, phys, false, false); phys_mem_pool_alloc_error: kfree(page_list); @@ -1546,16 +1613,14 @@ void kbase_csf_firmware_mcu_shared_mapping_term(struct kbase_device *kbdev, } if (csf_mapping->phys) { - kbase_mem_pool_free_pages(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], - csf_mapping->num_pages, csf_mapping->phys, false, false); + kbase_mem_pool_free_pages(&kbdev->fw_mem_pools.small, csf_mapping->num_pages, + csf_mapping->phys, false, false); } vunmap(csf_mapping->cpu_addr); kfree(csf_mapping->phys); } -#ifdef KBASE_PM_RUNTIME - void kbase_csf_firmware_soi_update(struct kbase_device *kbdev) { } @@ -1568,5 +1633,3 @@ int kbase_csf_firmware_soi_disable_on_scheduler_suspend(struct kbase_device *kbd { return 0; } - -#endif /* KBASE_PM_RUNTIME */ diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_fw_io.c b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_fw_io.c index 6e20f9dfe302..a2ece518bd4a 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_fw_io.c +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_fw_io.c @@ -22,7 +22,6 @@ #include "mali_kbase.h" #include "mali_kbase_csf_fw_io.h" #include -#include "mali_kbase_io.h" #include static inline u32 input_page_read(const u32 *const input, const u32 offset) @@ -89,7 +88,7 @@ static inline void input_page_partial_write64(u64 *const input, const u32 offset void kbase_csf_fw_io_init(struct kbase_csf_fw_io *fw_io, struct kbase_device *kbdev) { spin_lock_init(&fw_io->lock); - kbase_io_clear_status(kbdev->io, KBASE_IO_STATUS_GPU_SUSPENDED); + bitmap_zero(fw_io->status, KBASEP_FW_IO_STATUS_NUM_BITS); fw_io->kbdev = kbdev; } KBASE_EXPORT_TEST_API(kbase_csf_fw_io_init); @@ -358,19 +357,19 @@ KBASE_EXPORT_TEST_API(kbase_csf_fw_io_stream_read); void kbase_csf_fw_io_set_status_gpu_suspended(struct kbase_csf_fw_io *fw_io) { - kbase_io_set_status(fw_io->kbdev->io, KBASE_IO_STATUS_GPU_SUSPENDED); + set_bit(KBASEP_FW_IO_STATUS_GPU_SUSPENDED, fw_io->status); } KBASE_EXPORT_TEST_API(kbase_csf_fw_io_set_status_gpu_suspended); void kbase_csf_fw_io_clear_status_gpu_suspended(struct kbase_csf_fw_io *fw_io) { - kbase_io_clear_status(fw_io->kbdev->io, KBASE_IO_STATUS_GPU_SUSPENDED); + clear_bit(KBASEP_FW_IO_STATUS_GPU_SUSPENDED, fw_io->status); } KBASE_EXPORT_TEST_API(kbase_csf_fw_io_clear_status_gpu_suspended); bool kbase_csf_fw_io_check_status_gpu_suspended(struct kbase_csf_fw_io *fw_io) { - return kbase_io_test_status(fw_io->kbdev, KBASE_IO_STATUS_GPU_SUSPENDED); + return test_bit(KBASEP_FW_IO_STATUS_GPU_SUSPENDED, fw_io->status); } KBASE_EXPORT_TEST_API(kbase_csf_fw_io_check_status_gpu_suspended); diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_fw_io.h b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_fw_io.h index a46400484840..ba6b6bcc2800 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_fw_io.h +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_fw_io.h @@ -31,6 +31,17 @@ /** The wait completed because the GPU was lost. */ #define KBASE_CSF_FW_IO_WAIT_GPU_LOST 1 +/** + * enum kbasep_csf_fw_io_status_bits - Status bits for firmware I/O interface. + * + * @KBASEP_FW_IO_STATUS_GPU_SUSPENDED: The GPU is suspended. + * @KBASEP_FW_IO_STATUS_NUM_BITS: Number of bits used to encode the status. + */ +enum kbasep_csf_fw_io_status_bits { + KBASEP_FW_IO_STATUS_GPU_SUSPENDED = 0, + KBASEP_FW_IO_STATUS_NUM_BITS, +}; + /** * struct kbasep_csf_fw_io_stream_pages - Addresses to CS I/O pages. * @@ -72,11 +83,13 @@ struct kbasep_csf_fw_io_pages { * struct kbase_csf_fw_io - Manager of firmware input/output interface. * * @lock: Mutex to serialize access to the interface. + * @status: Internal status of the MCU interface. * @pages: Addresses to FW I/O pages * @kbdev: Pointer to the instance of a GPU platform device that implements a CSF interface. */ struct kbase_csf_fw_io { spinlock_t lock; + DECLARE_BITMAP(status, KBASEP_FW_IO_STATUS_NUM_BITS); struct kbasep_csf_fw_io_pages pages; struct kbase_device *kbdev; }; @@ -169,7 +182,8 @@ void kbase_csf_fw_io_pages_term(struct kbase_csf_fw_io *fw_io, u32 group_num); */ static inline int kbase_csf_fw_io_open(struct kbase_csf_fw_io *fw_io, unsigned long *flags) { - if (kbase_io_test_status(fw_io->kbdev, KBASE_IO_STATUS_GPU_SUSPENDED)) + if (test_bit(KBASEP_FW_IO_STATUS_GPU_SUSPENDED, fw_io->status) || + kbase_io_is_aw_removed(fw_io->kbdev)) return -KBASE_CSF_FW_IO_WAIT_GPU_LOST; spin_lock_irqsave(&fw_io->lock, *flags); @@ -410,8 +424,8 @@ bool kbase_csf_fw_io_check_status_gpu_suspended(struct kbase_csf_fw_io *fw_io); /** * kbase_csf_fw_io_wait_event_timeout() - Wait until condition gets true, timeout - * occurs or a GPU_SUSPENDED FW I/O status bit is set. The rest of the functionalities is equal - * to wait_event_timeout(). + * occurs, GPU_SUSPENDED FW I/O status bit is set, or AW was removed. + * The rest of the functionalities is equal to wait_event_timeout(). * * @fw_io: Firmware I/O manager. * @wq_head: The waitqueue to wait on. @@ -420,16 +434,20 @@ bool kbase_csf_fw_io_check_status_gpu_suspended(struct kbase_csf_fw_io *fw_io); * * Return: Remaining jiffies (at least 1) on success, * 0 on timeout, - * negative KBASE_CSF_FW_IO_WAIT_LOST error if GPU_SUSPENDED FW I/O status bit is set. + * negative KBASE_CSF_FW_IO_WAIT_LOST error + * if GPU_SUSPENDED FW I/O status bit is set + * or AW was removed. */ -#define kbase_csf_fw_io_wait_event_timeout(fw_io, wq_head, condition, timeout) \ - ({ \ - int __ret; \ - int __wait_remaining = wait_event_timeout( \ - wq_head, (condition) || kbase_csf_fw_io_check_status_gpu_suspended(fw_io), \ - timeout); \ - __ret = kbasep_csf_fw_io_handle_wait_result(fw_io, __wait_remaining); \ - __ret; \ +#define kbase_csf_fw_io_wait_event_timeout(fw_io, wq_head, condition, timeout) \ + ({ \ + int __ret; \ + int __wait_remaining = wait_event_timeout( \ + wq_head, \ + (condition) || kbase_csf_fw_io_check_status_gpu_suspended(fw_io) || \ + kbase_io_is_aw_removed((fw_io)->kbdev), \ + timeout); \ + __ret = kbasep_csf_fw_io_handle_wait_result(fw_io, __wait_remaining); \ + __ret; \ }) /** @@ -441,13 +459,17 @@ bool kbase_csf_fw_io_check_status_gpu_suspended(struct kbase_csf_fw_io *fw_io); * * Return: Remaining jiffies (at least 1) on success, * 0 on timeout, - * negative KBASE_CSF_FW_IO_WAIT_LOST error if GPU_SUSPENDED FW I/O status bit is set. + * negative KBASE_CSF_FW_IO_WAIT_LOST error + * if GPU_SUSPENDED FW I/O status bit is set + * or AW was removed. */ static inline int kbasep_csf_fw_io_handle_wait_result(struct kbase_csf_fw_io *fw_io, int wait_remaining) { - return kbase_csf_fw_io_check_status_gpu_suspended(fw_io) ? -KBASE_CSF_FW_IO_WAIT_GPU_LOST : - wait_remaining; + return (kbase_csf_fw_io_check_status_gpu_suspended(fw_io) || + kbase_io_is_aw_removed(fw_io->kbdev)) ? + -KBASE_CSF_FW_IO_WAIT_GPU_LOST : + wait_remaining; } #if IS_ENABLED(CONFIG_MALI_VALHALL_DEBUG) || IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI) diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_fw_io_no_mali.c b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_fw_io_no_mali.c index 9907c151df82..22da5bb78ae8 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_fw_io_no_mali.c +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_fw_io_no_mali.c @@ -87,7 +87,7 @@ static inline void input_page_partial_write64(u64 *const input, const u32 offset void kbase_csf_fw_io_init(struct kbase_csf_fw_io *fw_io, struct kbase_device *kbdev) { spin_lock_init(&fw_io->lock); - kbase_io_clear_status(kbdev->io, KBASE_IO_STATUS_GPU_SUSPENDED); + bitmap_zero(fw_io->status, KBASEP_FW_IO_STATUS_NUM_BITS); fw_io->kbdev = kbdev; } KBASE_EXPORT_TEST_API(kbase_csf_fw_io_init); @@ -393,19 +393,19 @@ KBASE_EXPORT_TEST_API(kbase_csf_fw_io_stream_read); void kbase_csf_fw_io_set_status_gpu_suspended(struct kbase_csf_fw_io *fw_io) { - kbase_io_set_status(fw_io->kbdev->io, KBASE_IO_STATUS_GPU_SUSPENDED); + set_bit(KBASEP_FW_IO_STATUS_GPU_SUSPENDED, fw_io->status); } KBASE_EXPORT_TEST_API(kbase_csf_fw_io_set_status_gpu_suspended); void kbase_csf_fw_io_clear_status_gpu_suspended(struct kbase_csf_fw_io *fw_io) { - kbase_io_clear_status(fw_io->kbdev->io, KBASE_IO_STATUS_GPU_SUSPENDED); + clear_bit(KBASEP_FW_IO_STATUS_GPU_SUSPENDED, fw_io->status); } KBASE_EXPORT_TEST_API(kbase_csf_fw_io_clear_status_gpu_suspended); bool kbase_csf_fw_io_check_status_gpu_suspended(struct kbase_csf_fw_io *fw_io) { - return kbase_io_test_status(fw_io->kbdev, KBASE_IO_STATUS_GPU_SUSPENDED); + return test_bit(KBASEP_FW_IO_STATUS_GPU_SUSPENDED, fw_io->status); } KBASE_EXPORT_TEST_API(kbase_csf_fw_io_check_status_gpu_suspended); diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_kcpu.c b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_kcpu.c index 65db914b895d..e30fb051a43e 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_kcpu.c +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_kcpu.c @@ -1664,7 +1664,7 @@ static int kbase_kcpu_fence_force_signal_process(struct kbase_kcpu_command_queue #endif /* dma_fence refcount needs to be decreased to release it. */ - dma_fence_put(fence_info->fence); + kbase_fence_put(fence_info->fence); fence_info->fence = NULL; return ret; @@ -1700,6 +1700,7 @@ static void kcpu_force_signal_fence(struct kbase_kcpu_command_queue *kcpu_queue) /* set ETIMEDOUT error flag before signal the fence*/ dma_fence_set_error_helper(fence, -ETIMEDOUT); + kbase_fence_put(fence); /* force signal fence */ status = @@ -1860,6 +1861,8 @@ static int kbasep_kcpu_fence_signal_init(struct kbase_kcpu_command_queue *kcpu_q goto fd_flags_fail; } + __module_get(THIS_MODULE); + kcpu_fence->module = THIS_MODULE; fence->basep.fd = *fd; current_command->type = BASE_KCPU_COMMAND_TYPE_FENCE_SIGNAL; diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_kcpu.h b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_kcpu.h index e9aa7b1f4801..625ac06ef9c4 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_kcpu.h +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_kcpu.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_mcu_shared_reg.c b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_mcu_shared_reg.c index c22bc1ac396d..1a8c7cedef75 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_mcu_shared_reg.c +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_mcu_shared_reg.c @@ -850,8 +850,8 @@ int kbase_csf_mcu_shared_regs_data_init(struct kbase_device *kbdev) if (!shared_regs->dummy_phys) return -ENOMEM; - if (kbase_mem_pool_alloc_pages(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], 1, - &shared_regs->dummy_phys[0], false, NULL) <= 0) + if (kbase_mem_pool_alloc_pages(&kbdev->fw_mem_pools.small, 1, &shared_regs->dummy_phys[0], + false, NULL) <= 0) return -ENOMEM; shared_regs->dummy_phys_allocated = true; @@ -920,7 +920,7 @@ void kbase_csf_mcu_shared_regs_data_term(struct kbase_device *kbdev) if (shared_regs->dummy_phys_allocated) { struct page *page = as_page(shared_regs->dummy_phys[0]); - kbase_mem_pool_free(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], page, false); + kbase_mem_pool_free(&kbdev->fw_mem_pools.small, page, false); } kfree(shared_regs->dummy_phys); diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_registers.h b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_registers.h index deb1878a0e6b..c08f48f48ec7 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_registers.h +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_registers.h @@ -204,6 +204,7 @@ #define GLB_GROUP_STRIDE 0x0014 /* () Stride between CSG interfaces */ #define GLB_PRFCNT_SIZE 0x0018 /* () Size of CSF performance counters */ #define GLB_INSTR_FEATURES 0x001C /* () TRACE_POINT instrumentation. (csf >= 1.1.0) */ +#define GLB_PRFCNT_FEATURES 0x0020 /* () Performance counter features */ #define GROUP_CONTROL_0 0x1000 /* () CSG control and capabilities */ #define GROUP_CONTROL(n) (GROUP_CONTROL_0 + (n)*256) #define GROUP_CONTROL_REG(n, r) (GROUP_CONTROL(n) + GROUP_CONTROL_BLOCK_REG(r)) @@ -1982,6 +1983,16 @@ #define GLB_PRFCNT_CONFIG_SET_SELECT_SET(reg_val, value) \ (((reg_val) & ~GLB_PRFCNT_CONFIG_SET_SELECT_MASK) | \ (((value) << GLB_PRFCNT_CONFIG_SET_SELECT_SHIFT) & GLB_PRFCNT_CONFIG_SET_SELECT_MASK)) +#define GLB_PRFCNT_CONFIG_METADATA_ENABLE_SHIFT GPU_U(10) +#define GLB_PRFCNT_CONFIG_METADATA_ENABLE_MASK \ + (GPU_U(0x1) << GLB_PRFCNT_CONFIG_METADATA_ENABLE_SHIFT) +#define GLB_PRFCNT_CONFIG_METADATA_ENABLE_GET(reg_val) \ + (((reg_val)&GLB_PRFCNT_CONFIG_METADATA_ENABLE_MASK) >> \ + GLB_PRFCNT_CONFIG_METADATA_ENABLE_SHIFT) +#define GLB_PRFCNT_CONFIG_METADATA_ENABLE_SET(reg_val, value) \ + (~(~(reg_val) | GLB_PRFCNT_CONFIG_METADATA_ENABLE_MASK) | \ + (((value) << GLB_PRFCNT_CONFIG_METADATA_ENABLE_SHIFT) & \ + GLB_PRFCNT_CONFIG_METADATA_ENABLE_MASK)) /* GLB_PRFCNT_SIZE register */ #define GLB_PRFCNT_SIZE_HARDWARE_SIZE_SET_MOD(value) ((value) >> 8) @@ -2007,6 +2018,16 @@ ((GLB_PRFCNT_SIZE_FIRMWARE_SIZE_SET_MOD(value) << GLB_PRFCNT_SIZE_FIRMWARE_SIZE_SHIFT) & \ GLB_PRFCNT_SIZE_FIRMWARE_SIZE_MASK)) +/* GLB_PRFCNT_FEATURES register */ +#define GLB_PRFCNT_FEATURES_METADATA_SIZE_GET_MOD(value) (value << 8) +#define GLB_PRFCNT_FEATURES_METADATA_SIZE_SHIFT GPU_U(0) +#define GLB_PRFCNT_FEATURES_METADATA_SIZE_MASK \ + (GPU_U(0xF) << GLB_PRFCNT_FEATURES_METADATA_SIZE_SHIFT) +#define GLB_PRFCNT_FEATURES_METADATA_SIZE_GET(reg_val) \ + (GLB_PRFCNT_FEATURES_METADATA_SIZE_GET_MOD( \ + ((reg_val)&GLB_PRFCNT_FEATURES_METADATA_SIZE_MASK) >> \ + GLB_PRFCNT_FEATURES_METADATA_SIZE_SHIFT)) + /* GLB_DEBUG_REQ register */ #define GLB_DEBUG_REQ_DEBUG_RUN_SHIFT GPU_U(23) #define GLB_DEBUG_REQ_DEBUG_RUN_MASK (GPU_U(0x1) << GLB_DEBUG_REQ_DEBUG_RUN_SHIFT) diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_reset_gpu.c b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_reset_gpu.c index 03fdf60ce1b5..1fa97035a8dd 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_reset_gpu.c +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_reset_gpu.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2025 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -243,7 +243,6 @@ static void kbase_csf_debug_dump_registers(struct kbase_device *kbdev) kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_MASK)), kbase_reg_read32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_MASK)), kbase_reg_read32(kbdev, MMU_CONTROL_ENUM(IRQ_MASK))); -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface) { dev_err(kbdev->dev, " PWR_IRQ_RAWSTAT=0x%08x", kbase_reg_read32(kbdev, HOST_POWER_ENUM(PWR_IRQ_RAWSTAT))); @@ -252,7 +251,6 @@ static void kbase_csf_debug_dump_registers(struct kbase_device *kbdev) dev_err(kbdev->dev, " PWR_STATUS=0x%016llx", kbase_reg_read64(kbdev, HOST_POWER_ENUM(PWR_STATUS))); } -#endif if (kbdev->gpu_props.gpu_id.arch_id < GPU_ID_ARCH_MAKE(14, 10, 0)) { dev_err(kbdev->dev, " PWR_OVERRIDE0=0x%08x PWR_OVERRIDE1=0x%08x", kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(PWR_OVERRIDE0)), @@ -264,7 +262,6 @@ static void kbase_csf_debug_dump_registers(struct kbase_device *kbdev) kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(TILER_CONFIG))); } -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface) { u32 domain_status; @@ -276,7 +273,6 @@ static void kbase_csf_debug_dump_registers(struct kbase_device *kbdev) dev_err(kbdev->dev, " L2_PWR_STATUS=0x%05llx", kbase_reg_read64(kbdev, HOST_POWER_ENUM(PWR_CMDARG))); } -#endif spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } @@ -426,13 +422,17 @@ static int kbase_csf_reset_gpu_now(struct kbase_device *kbdev, bool firmware_ini spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbdev->csf.firmware_reload_needed = false; + /* Prevent MCU enable before FW data is reloaded */ + kbdev->pm.backend.in_reset = true; spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); cancel_work_sync(&kbdev->csf.firmware_reload_work); - dev_dbg(kbdev->dev, "Disable GPU hardware counters.\n"); - /* This call will block until counters are disabled. */ - kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); + { + dev_dbg(kbdev->dev, "Disable GPU hardware counters.\n"); + /* This call will block until counters are disabled. */ + kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); + } ret = kbase_csf_reset_gpu_once(kbdev, firmware_inited, silent); @@ -488,13 +488,12 @@ static void kbase_csf_reset_gpu_worker(struct work_struct *data) spin_lock_irqsave(&kbdev->hwaccess_lock, flags); firmware_inited = kbdev->csf.firmware_inited; -#ifdef KBASE_PM_RUNTIME - gpu_sleep_mode_active = kbdev->pm.backend.gpu_sleep_mode_active; -#endif + if (IS_ENABLED(CONFIG_PM)) + gpu_sleep_mode_active = kbdev->pm.backend.gpu_sleep_mode_active; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - if (unlikely(gpu_sleep_mode_active)) { -#ifdef KBASE_PM_RUNTIME + if (unlikely(gpu_sleep_mode_active) && IS_ENABLED(CONFIG_PM)) { /* As prior to GPU reset all on-slot groups are suspended, * need to wake up the MCU from sleep. * No pm active reference is taken here since GPU is in sleep @@ -508,7 +507,6 @@ static void kbase_csf_reset_gpu_worker(struct work_struct *data) kbase_pm_unlock(kbdev); err = kbase_csf_reset_gpu_now(kbdev, firmware_inited, silent); -#endif } else if (!kbase_pm_context_active_handle_suspend( kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { err = kbase_csf_reset_gpu_now(kbdev, firmware_inited, silent); diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_scheduler.c b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_scheduler.c index bda9bf1ceafa..ca7db6074bab 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_scheduler.c +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_scheduler.c @@ -277,6 +277,9 @@ static u64 drain_gpu_metrics_trace_buffer(struct kbase_device *kbdev) lockdep_assert_held(&kbdev->csf.scheduler.gpu_metrics_lock); + if (kbase_io_is_aw_removed(kbdev)) + return 0; + kbase_backend_get_gpu_time_norequest(kbdev, NULL, &system_time, NULL); /* CPU time value that was used to derive the parameters for time conversion, * was retrieved from ktime_get_raw_ts64(). But the tracing subsystem would use @@ -503,7 +506,6 @@ static void schedule_actions_trigger_df(struct kbase_device *kbdev, struct kbase #endif } -#ifdef KBASE_PM_RUNTIME /** * wait_for_scheduler_to_exit_sleep() - Wait for Scheduler to exit the * sleeping state. @@ -519,7 +521,11 @@ static void schedule_actions_trigger_df(struct kbase_device *kbdev, struct kbase static int wait_for_scheduler_to_exit_sleep(struct kbase_device *kbdev) { struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; +#if IS_ENABLED(CONFIG_PM) int autosuspend_delay = kbdev->dev->power.autosuspend_delay; +#else + int autosuspend_delay = 0; +#endif unsigned int sleep_exit_wait_time; long remaining; int ret = 0; @@ -578,7 +584,7 @@ static int force_scheduler_to_exit_sleep(struct kbase_device *kbdev) lockdep_assert_held(&scheduler->lock); WARN_ON(scheduler->state != SCHED_SLEEPING); - WARN_ON(!kbdev->pm.backend.gpu_sleep_mode_active); + WARN_ON(!kbdev->pm.backend.gpu_sleep_mode_active && !kbase_io_is_aw_removed(kbdev)); kbase_pm_lock(kbdev); ret = kbase_pm_force_mcu_wakeup_after_sleep(kbdev); @@ -626,7 +632,6 @@ out: return ret; } -#endif /** * tick_timer_callback() - Callback function for the scheduling tick hrtimer @@ -1061,7 +1066,6 @@ static int scheduler_pm_active_handle_suspend(struct kbase_device *kbdev, return ret; } -#ifdef KBASE_PM_RUNTIME /** * scheduler_pm_active_after_sleep() - Acquire the PM reference count for * Scheduler @@ -1084,7 +1088,6 @@ static int scheduler_pm_active_after_sleep(struct kbase_device *kbdev) return scheduler_pm_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE, true); } -#endif /** * scheduler_pm_idle() - Release the PM reference count held by Scheduler @@ -1102,7 +1105,6 @@ static inline void scheduler_pm_idle(struct kbase_device *kbdev) kbase_pm_context_idle(kbdev); } -#ifdef KBASE_PM_RUNTIME /** * scheduler_pm_idle_before_sleep() - Release the PM reference count and * trigger the transition to sleep state. @@ -1126,7 +1128,6 @@ static void scheduler_pm_idle_before_sleep(struct kbase_device *kbdev) scheduler_pm_idle(kbdev); } -#endif static void scheduler_wakeup(struct kbase_device *kbdev, bool kick) { @@ -1149,11 +1150,9 @@ static void scheduler_wakeup(struct kbase_device *kbdev, bool kick) HRTIMER_MODE_REL_SOFT); } #endif - } else { -#ifdef KBASE_PM_RUNTIME + } else if (IS_ENABLED(CONFIG_PM)) { dev_dbg(kbdev->dev, "Re-activating the Scheduler out of sleep"); ret = scheduler_pm_active_after_sleep(kbdev); -#endif } if (ret) { @@ -1178,19 +1177,21 @@ static int scheduler_suspend(struct kbase_device *kbdev) lockdep_assert_held(&scheduler->lock); if (!WARN_ON(scheduler->state == SCHED_SUSPENDED)) { -#if KBASE_PM_RUNTIME - int ret; + if (IS_ENABLED(CONFIG_PM)) { + int ret; - /* If GPU-level suspend is not supported then all on-slot CSGs - * will be suspended one at a time. Disable Sleep-on-Idle to - * avoid repeated cycles of FW going to sleep and resuming. - * GLB_IDLE timer (and Sleep-on-Idle) would otherwise be - * disabled just before the MCU transitions out of HALT state. - */ - ret = kbase_csf_firmware_soi_disable_on_scheduler_suspend(kbdev); - if (ret) - return ret; -#endif /* KBASE_PM_RUNTIME */ + /* If GPU-level suspend is not supported then all on-slot CSGs + * will be suspended one at a time. Disable Sleep-on-Idle to + * avoid repeated cycles of FW going to sleep and resuming. + * GLB_IDLE timer (and Sleep-on-Idle) would otherwise be + * disabled just before the MCU transitions out of HALT state. + */ + { + ret = kbase_csf_firmware_soi_disable_on_scheduler_suspend(kbdev); + if (ret) + return ret; + } + } dev_dbg(kbdev->dev, "Suspending the Scheduler"); scheduler_pm_idle(kbdev); scheduler->state = SCHED_SUSPENDED; @@ -2321,7 +2322,7 @@ static void process_cs_pending_events(struct kbase_csf_fw_io *fw_io, u32 group_i } /** - * save_slot_cs() - Save the state for blocked GPU command queue. + * save_cs_wait_state() - Save the state for blocked GPU command queue. * * @fw_io: Pointer to FW I/O manager. * @group_id: GPU on-slot CSG index. @@ -2334,7 +2335,8 @@ static void process_cs_pending_events(struct kbase_csf_fw_io *fw_io, u32 group_i * * Return: true if the queue is blocked on a sync wait operation. */ -static bool save_slot_cs(struct kbase_csf_fw_io *fw_io, u32 group_id, struct kbase_queue *queue) +static bool save_cs_wait_state(struct kbase_csf_fw_io *fw_io, u32 group_id, + struct kbase_queue *queue) { u32 stream_id = queue->csi_index; u32 status; @@ -2388,8 +2390,6 @@ static bool save_slot_cs(struct kbase_csf_fw_io *fw_io, u32 group_id, struct kba queue->status_wait = 0; } - process_cs_pending_events(fw_io, group_id, queue); - return is_waiting; } @@ -2792,31 +2792,35 @@ static void save_csg_slot(struct kbase_queue_group *group) for (i = 0; i < max_streams; i++) update_hw_active(group->bound_queues[i], false); #endif /* CONFIG_MALI_VALHALL_NO_MALI */ - for (i = 0; idle && i < max_streams; i++) { + for (i = 0; i < max_streams; i++) { struct kbase_queue *const queue = group->bound_queues[i]; if (!queue || !queue->enabled) continue; - if (save_slot_cs(&kbdev->csf.fw_io, group->csg_nr, queue)) { - /* sync_wait is only true if the queue is blocked on - * a CQS and not a scoreboard. - */ - if (queue->blocked_reason != CS_STATUS_BLOCKED_ON_SB_WAIT) - sync_wait = true; - } else { - /* Need to confirm if ringbuffer of the GPU - * queue is empty or not. A race can arise - * between the flush of GPU queue and suspend - * of CSG. If a queue is flushed after FW has - * set the IDLE bit in CSG_STATUS_STATE, then - * Scheduler will incorrectly consider CSG - * as idle. And there may not be any further - * flush call for the GPU queue, which would - * have de-idled the CSG. - */ - idle = confirm_cmd_buf_empty(queue); + if (idle) { + if (save_cs_wait_state(&kbdev->csf.fw_io, group->csg_nr, queue)) { + /* sync_wait is only true if the queue is blocked on + * a CQS and not a scoreboard. + */ + if (queue->blocked_reason != CS_STATUS_BLOCKED_ON_SB_WAIT) + sync_wait = true; + } else { + /* Need to confirm if ringbuffer of the GPU + * queue is empty or not. A race can arise + * between the flush of GPU queue and suspend + * of CSG. If a queue is flushed after FW has + * set the IDLE bit in CSG_STATUS_STATE, then + * Scheduler will incorrectly consider CSG + * as idle. And there may not be any further + * flush call for the GPU queue, which would + * have de-idled the CSG. + */ + idle = confirm_cmd_buf_empty(queue); + } } + + process_cs_pending_events(&kbdev->csf.fw_io, group->csg_nr, queue); } if (idle) { @@ -3028,7 +3032,8 @@ static void program_csg_slot(struct kbase_queue_group *group, s8 slot, u8 prio) const u64 compute_mask = shader_core_mask & group->compute_mask; const u64 fragment_mask = shader_core_mask & group->fragment_mask; const u64 tiler_mask = tiler_core_mask & group->tiler_mask; - const u64 neural_mask = shader_core_mask & group->neural_mask; + const u64 neural_mask = shader_core_mask & group->neural_mask & + kbdev->csf.neural_allowed_mask; const u8 neural_max = min(kbdev->gpu_props.num_cores, group->neural_max); const u8 comp_pri_threshold = min_t(u8, group->comp_pri_threshold, (u8)COMP_PRI_THRESHOLD_MAX); @@ -3116,6 +3121,13 @@ static void program_csg_slot(struct kbase_queue_group *group, s8 slot, u8 prio) goto skip_fw; } + if (kbdev->csf.neural_allowed_mask != UINT64_MAX) + dev_warn_once( + kbdev->dev, + "neural_allowed_mask module param passed in as 0x%llx. Writing restricted neural_mask 0x%llx to CSG_ALLOW_NEURAL. shader_core_mask 0x%llx group->neural_mask 0x%llx", + kbdev->csf.neural_allowed_mask, neural_mask, shader_core_mask, + group->neural_mask); + /* Endpoint programming for CSG */ kbase_csf_fw_io_group_write(fw_io, slot, CSG_ALLOW_COMPUTE_LO, compute_mask & U32_MAX); kbase_csf_fw_io_group_write(fw_io, slot, CSG_ALLOW_COMPUTE_HI, compute_mask >> 32); @@ -3356,7 +3368,6 @@ void kbase_csf_scheduler_group_deschedule(struct kbase_queue_group *group) on_slot = kbasep_csf_scheduler_group_is_on_slot_locked(group); -#ifdef KBASE_PM_RUNTIME /* If the queue group is on slot and Scheduler is in SLEEPING state, * then we need to wake up the Scheduler to exit the sleep state rather * than waiting for the runtime suspend or power down of GPU. @@ -3364,7 +3375,7 @@ void kbase_csf_scheduler_group_deschedule(struct kbase_queue_group *group) * thread and it has been seen that certain Apps can destroy groups at * random points and not necessarily when the App is exiting. */ - if (on_slot && (scheduler->state == SCHED_SLEEPING)) { + if (IS_ENABLED(CONFIG_PM) && on_slot && (scheduler->state == SCHED_SLEEPING)) { scheduler_wakeup(kbdev, true); /* Wait for MCU firmware to start running */ @@ -3380,7 +3391,7 @@ void kbase_csf_scheduler_group_deschedule(struct kbase_queue_group *group) wait_for_termination = false; } } -#endif + if (!on_slot) { sched_evict_group(group, false, true); } else { @@ -5144,7 +5155,6 @@ static bool scheduler_idle_suspendable(struct kbase_device *kbdev) return suspend; } -#ifdef KBASE_PM_RUNTIME /** * scheduler_sleep_on_idle - Put the Scheduler in sleeping state on GPU * becoming idle. @@ -5168,7 +5178,6 @@ static void scheduler_sleep_on_idle(struct kbase_device *kbdev) scheduler->state = SCHED_SLEEPING; KBASE_KTRACE_ADD(kbdev, SCHED_SLEEPING, NULL, scheduler->state); } -#endif /** * scheduler_suspend_on_idle - Put the Scheduler in suspended state on GPU @@ -5213,6 +5222,11 @@ static void gpu_idle_worker(struct kbase_device *kbdev) WARN_ON_ONCE(atomic_read(&scheduler->pending_gpu_idle_work) == 0); + if (kbase_io_is_aw_removed(kbdev)) { + dev_info(kbdev->dev, "%s(): aborting as AW is no longer connected", __func__); + goto exit; + } + KBASE_KTRACE_ADD(kbdev, SCHEDULER_GPU_IDLE_WORKER_START, NULL, 0u); #define __ENCODE_KTRACE_INFO(reset, idle, all_suspend) \ @@ -5239,12 +5253,11 @@ static void gpu_idle_worker(struct kbase_device *kbdev) if (scheduler_is_idle_suspendable) { KBASE_KTRACE_ADD(kbdev, SCHEDULER_GPU_IDLE_WORKER_HANDLING_START, NULL, kbase_csf_ktrace_gpu_cycle_cnt(kbdev)); -#ifdef KBASE_PM_RUNTIME - if (kbase_pm_gpu_sleep_allowed(kbdev) && + + if (IS_ENABLED(CONFIG_PM) && kbase_pm_gpu_sleep_allowed(kbdev) && kbase_csf_scheduler_get_nr_active_csgs(kbdev)) scheduler_sleep_on_idle(kbdev); else -#endif all_groups_suspended = scheduler_suspend_on_idle(kbdev); KBASE_KTRACE_ADD(kbdev, SCHEDULER_GPU_IDLE_WORKER_HANDLING_END, NULL, 0u); @@ -5758,8 +5771,16 @@ static bool can_skip_scheduling(struct kbase_device *kbdev) if (scheduler->state == SCHED_SUSPENDED) return true; -#ifdef KBASE_PM_RUNTIME - if (scheduler->state == SCHED_SLEEPING) { + /* + * Schduler should always be in SUSPENDED state when + * AW is removed. + */ + if (kbase_io_is_aw_removed(kbdev)) { + dev_warn(kbdev->dev, "Skipping scheduling as AW is no longer connected"); + return true; + } + + if (IS_ENABLED(CONFIG_PM) && scheduler->state == SCHED_SLEEPING) { unsigned long flags; spin_lock_irqsave(&kbdev->hwaccess_lock, flags); @@ -5780,7 +5801,6 @@ static bool can_skip_scheduling(struct kbase_device *kbdev) spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); return true; } -#endif return false; } @@ -5834,7 +5854,7 @@ exit_no_schedule_unlock: static void schedule_on_tick(struct kbase_device *kbdev) { struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; - + struct kbase_csf_sched_heap_reclaim_mgr *mgr = &kbdev->csf.scheduler.reclaim_mgr; int err = kbase_reset_gpu_try_prevent(kbdev); /* Regardless of whether reset failed or is currently happening, exit * early @@ -5871,6 +5891,8 @@ static void schedule_on_tick(struct kbase_device *kbdev) } scheduler->state = SCHED_INACTIVE; + if (atomic_read(&mgr->unused_pages)) + kbase_csf_tiler_heap_reclaim_unused_pages(kbdev, HEAP_RECLAIM_SCENARIO_SCHEDULER); mutex_unlock(&scheduler->lock); KBASE_KTRACE_ADD(kbdev, SCHED_INACTIVE, NULL, scheduler->state); kbase_reset_gpu_allow(kbdev); @@ -5903,8 +5925,10 @@ static int suspend_active_queue_groups(struct kbase_device *kbdev, unsigned long suspend_queue_group(group); set_bit(slot_num, slot_mask); } - ret = wait_csg_slots_suspend(kbdev, slot_mask); } + + ret = wait_csg_slots_suspend(kbdev, slot_mask); + return ret; } @@ -6083,15 +6107,14 @@ static void scheduler_inner_reset(struct kbase_device *kbdev) scheduler->num_active_address_spaces | (((u64)scheduler->total_runnable_grps) << 32)); -#ifdef KBASE_PM_RUNTIME - if (scheduler->state == SCHED_SLEEPING) { + if (IS_ENABLED(CONFIG_PM) && scheduler->state == SCHED_SLEEPING) { #if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) hrtimer_cancel(&scheduler->gpu_metrics_timer); #endif scheduler->state = SCHED_SUSPENDED; KBASE_KTRACE_ADD(kbdev, SCHED_SUSPENDED, NULL, scheduler->state); } -#endif + mutex_unlock(&scheduler->lock); } @@ -6241,8 +6264,7 @@ int kbase_csf_scheduler_group_copy_suspend_buf(struct kbase_queue_group *group, on_slot = kbasep_csf_scheduler_group_is_on_slot_locked(group); -#ifdef KBASE_PM_RUNTIME - if (on_slot && (scheduler->state == SCHED_SLEEPING)) { + if (IS_ENABLED(CONFIG_PM) && on_slot && (scheduler->state == SCHED_SLEEPING)) { if (wait_for_scheduler_to_exit_sleep(kbdev)) { dev_warn( kbdev->dev, @@ -6265,7 +6287,7 @@ int kbase_csf_scheduler_group_copy_suspend_buf(struct kbase_queue_group *group, */ on_slot = kbasep_csf_scheduler_group_is_on_slot_locked(group); } -#endif + if (on_slot) { DECLARE_BITMAP(slot_mask, BASEP_QUEUE_GROUP_MAX) = { 0 }; @@ -6812,7 +6834,6 @@ event_wait_add_failed: void kbase_csf_scheduler_context_term(struct kbase_context *kctx) { - kbase_ctx_sched_remove_ctx(kctx); #if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) gpu_metrics_ctx_term(kctx); #endif /* CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD */ @@ -6984,6 +7005,12 @@ static int kbase_csf_scheduler_kthread(void *data) while (atomic_read(&scheduler->pending_gpu_idle_work) > 0) gpu_idle_worker(kbdev); + /* Drain pending GPU suspend work */ + if (atomic_read(&scheduler->pending_runtime_suspend_work) == true) { + kbdev->pm.runtime_suspend_result = kbase_pm_handle_runtime_suspend(kbdev); + atomic_set(&scheduler->pending_runtime_suspend_work, false); + } + /* Drain pending GPU power off work */ if (atomic_cmpxchg(&scheduler->pending_power_off_work, true, false) == true) kbase_pm_handle_gpu_poweroff_wait_work(kbdev); @@ -7097,6 +7124,7 @@ int kbase_csf_scheduler_early_init(struct kbase_device *kbdev) atomic_set(&scheduler->pending_tick_work, false); atomic_set(&scheduler->pending_tock_work, false); atomic_set(&scheduler->pending_gpu_idle_work, 0); + atomic_set(&scheduler->pending_runtime_suspend_work, false); atomic_set(&scheduler->pending_power_off_work, false); return kbase_csf_tiler_heap_reclaim_mgr_init(kbdev); @@ -7240,11 +7268,22 @@ int kbase_csf_scheduler_pm_suspend_no_lock(struct kbase_device *kbdev) return -EBUSY; #endif -#ifdef KBASE_PM_RUNTIME + if (kbase_io_is_aw_removed(kbdev)) { + dev_warn(kbdev->dev, "Suspending scheduler(AW_REMOVED), state: %d", + scheduler->state); + if (scheduler->state != SCHED_SUSPENDED && scheduler->state != SCHED_SLEEPING) + scheduler_pm_idle(kbdev); + scheduler->state = SCHED_SUSPENDED; +#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) + hrtimer_cancel(&scheduler->gpu_metrics_timer); +#endif + return 0; + } + /* If scheduler is in sleeping state, then MCU needs to be activated * to suspend CSGs. */ - if (scheduler->state == SCHED_SLEEPING) { + if (IS_ENABLED(CONFIG_PM) && scheduler->state == SCHED_SLEEPING) { dev_info(kbdev->dev, "Activating MCU out of sleep on system suspend"); result = force_scheduler_to_exit_sleep(kbdev); if (result) { @@ -7252,7 +7291,7 @@ int kbase_csf_scheduler_pm_suspend_no_lock(struct kbase_device *kbdev) goto exit; } } -#endif + if (scheduler->state != SCHED_SUSPENDED) { result = suspend_active_groups_on_powerdown(kbdev, true); if (result) { @@ -7267,6 +7306,17 @@ int kbase_csf_scheduler_pm_suspend_no_lock(struct kbase_device *kbdev) } exit: + if (result && kbase_io_is_aw_removed(kbdev)) { + dev_warn(kbdev->dev, "Forcing scheduler suspend(AW_REMOVED), state: %d", + scheduler->state); + if (scheduler->state != SCHED_SUSPENDED && scheduler->state != SCHED_SLEEPING) + scheduler_pm_idle(kbdev); + scheduler->state = SCHED_SUSPENDED; +#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) + hrtimer_cancel(&scheduler->gpu_metrics_timer); +#endif + result = 0; + } return result; } @@ -7361,7 +7411,10 @@ static int scheduler_wait_mcu_active(struct kbase_device *kbdev, bool killable_w err = kbase_pm_wait_for_desired_state(kbdev); if (!err) { spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - WARN_ON(kbdev->pm.backend.mcu_state != KBASE_MCU_ON); + if (kbase_io_is_aw_removed(kbdev)) + err = -ENODEV; + else + WARN_ON(kbdev->pm.backend.mcu_state != KBASE_MCU_ON); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } @@ -7380,7 +7433,6 @@ int kbase_csf_scheduler_wait_mcu_active(struct kbase_device *kbdev) KBASE_EXPORT_TEST_API(kbase_csf_scheduler_wait_mcu_active); -#ifdef KBASE_PM_RUNTIME int kbase_csf_scheduler_handle_runtime_suspend(struct kbase_device *kbdev) { struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; @@ -7532,7 +7584,6 @@ void kbase_csf_scheduler_force_sleep(struct kbase_device *kbdev) scheduler_sleep_on_idle(kbdev); mutex_unlock(&scheduler->lock); } -#endif void kbase_csf_scheduler_force_wakeup(struct kbase_device *kbdev) { diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_scheduler.h b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_scheduler.h index 50d2f0fbd853..96e9e35b49db 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_scheduler.h +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_scheduler.h @@ -610,7 +610,6 @@ static inline bool kbase_csf_scheduler_queue_has_trace(struct kbase_queue *queue return (queue->trace_buffer_size && queue->trace_buffer_base); } -#ifdef KBASE_PM_RUNTIME /** * kbase_csf_scheduler_reval_idleness_post_sleep() - Check GPU's idleness after * putting MCU to sleep state @@ -645,7 +644,6 @@ void kbase_csf_scheduler_reval_idleness_post_sleep(struct kbase_device *kbdev); * Return: 0 if all the CSGs were suspended, otherwise an error code. */ int kbase_csf_scheduler_handle_runtime_suspend(struct kbase_device *kbdev); -#endif /** * kbase_csf_scheduler_process_gpu_idle_event() - Process GPU idle IRQ @@ -697,7 +695,6 @@ u32 kbase_csf_scheduler_get_nr_active_csgs_locked(struct kbase_device *kbdev); */ void kbase_csf_scheduler_force_wakeup(struct kbase_device *kbdev); -#ifdef KBASE_PM_RUNTIME /** * kbase_csf_scheduler_force_sleep() - Forcefully put the Scheduler to sleeping * state. @@ -710,7 +707,6 @@ void kbase_csf_scheduler_force_wakeup(struct kbase_device *kbdev); * This function is only used for testing purpose. */ void kbase_csf_scheduler_force_sleep(struct kbase_device *kbdev); -#endif #endif /* _KBASE_CSF_SCHEDULER_H_ */ diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_tiler_heap_reclaim.c b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_tiler_heap_reclaim.c index ce0ecea13d16..8496d145f6fd 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_tiler_heap_reclaim.c +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_tiler_heap_reclaim.c @@ -34,6 +34,13 @@ /* Tiler heap reclaim scan (free) method size for limiting a scan run length */ #define HEAP_RECLAIM_SCAN_BATCH_SIZE (HEAP_SHRINKER_BATCH << 7) +/* + * Default setting for reclaiming offslot CSG's heap. + * Disabling reclaim with pages being 0. + */ +#define HEAP_RECLAIM_OFFSLOT_TIMEOUT_MS (30000) +#define HEAP_RECLAIM_OFFSLOT_PAGES (0) + static u8 get_kctx_highest_csg_priority(struct kbase_context *kctx) { u8 prio; @@ -76,6 +83,9 @@ static void detach_ctx_from_heap_reclaim_mgr(struct kbase_context *kctx) "Reclaim_mgr_detach: ctx_%d_%d, est_pages=0%u, freed_pages=%u", kctx->tgid, kctx->id, info->nr_est_unused_pages, info->nr_freed_pages); } + + /* 0 indicates that the kctx may have CSG on slot */ + kctx->offslot_ts = 0; } static void attach_ctx_to_heap_reclaim_mgr(struct kbase_context *kctx) @@ -98,8 +108,9 @@ static void attach_ctx_to_heap_reclaim_mgr(struct kbase_context *kctx) /* Accumulate the estimated pages to the manager total field */ atomic_add((int)info->nr_est_unused_pages, &scheduler->reclaim_mgr.unused_pages); - dev_dbg(kctx->kbdev->dev, "Reclaim_mgr_attach: ctx_%d_%d, est_count_pages=%u", kctx->tgid, - kctx->id, info->nr_est_unused_pages); + kctx->offslot_ts = ktime_get_raw_ns(); + dev_dbg(kctx->kbdev->dev, "Reclaim_mgr_attach [%llu]: ctx_%d_%d, est_count_pages=%u", + kctx->offslot_ts, kctx->tgid, kctx->id, info->nr_est_unused_pages); } void kbase_csf_tiler_heap_reclaim_sched_notify_grp_active(struct kbase_queue_group *group) @@ -187,15 +198,43 @@ void kbase_csf_tiler_heap_reclaim_sched_notify_grp_suspend(struct kbase_queue_gr } } -static unsigned long reclaim_unused_heap_pages(struct kbase_device *kbdev) +unsigned long kbase_csf_tiler_heap_reclaim_unused_pages(struct kbase_device *kbdev, + enum heap_reclaim_scenario scenario) { struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; struct kbase_csf_sched_heap_reclaim_mgr *const mgr = &scheduler->reclaim_mgr; - unsigned long total_freed_pages = 0; - int prio; + unsigned long total_freed_pages = 0, max_pages; + int prio, min_prio; + u64 now = 0, offslot_ts = 0; lockdep_assert_held(&scheduler->lock); + if (scenario == HEAP_RECLAIM_SCENARIO_SHRINKER) { + /* triggered by shrinker */ + max_pages = HEAP_RECLAIM_SCAN_BATCH_SIZE; + min_prio = KBASE_QUEUE_GROUP_PRIORITY_REALTIME; + } else if (scenario == HEAP_RECLAIM_SCENARIO_SCHEDULER) { + /* triggered by schedule_on_tick + * reclaim heap from CSGs with below conditions: + * 1.being offslot for a period + * 2.limit page numbers for each reclaim + * 3.skip RT kctx + */ + if (mgr->offslot_setting.pages == 0) { + dev_dbg(kbdev->dev, "HEAP_RECLAIM_SCENARIO_SCHEDULER is disabled"); + return 0; + } + max_pages = mgr->offslot_setting.pages; + min_prio = KBASE_QUEUE_GROUP_PRIORITY_HIGH; + now = ktime_get_raw_ns(); + offslot_ts = now > mgr->offslot_setting.timeout_ms * 1000000ULL ? + now - mgr->offslot_setting.timeout_ms * 1000000ULL : + 0; + } else { + dev_err(kbdev->dev, "Unknown heap reclaim scenario %d\n", scenario); + return 0; + } + if (scheduler->state != SCHED_SUSPENDED) { /* Clean and invalidate the L2 cache before reading from the heap contexts, * headers of the individual chunks and buffer descriptors. @@ -219,9 +258,7 @@ static unsigned long reclaim_unused_heap_pages(struct kbase_device *kbdev) } for (prio = KBASE_QUEUE_GROUP_PRIORITY_LOW; - total_freed_pages < HEAP_RECLAIM_SCAN_BATCH_SIZE && - prio >= KBASE_QUEUE_GROUP_PRIORITY_REALTIME; - prio--) { + total_freed_pages < max_pages && prio >= min_prio; prio--) { struct kbase_csf_ctx_heap_reclaim_info *info, *tmp; u32 cnt_ctxs = 0; @@ -229,9 +266,26 @@ static unsigned long reclaim_unused_heap_pages(struct kbase_device *kbdev) mgr_link) { struct kbase_context *kctx = container_of(info, struct kbase_context, csf.sched.heap_info); - u32 freed_pages = kbase_csf_tiler_heap_scan_kctx_unused_pages( + u32 freed_pages; + + if (scenario == HEAP_RECLAIM_SCENARIO_SCHEDULER) { + WARN_ON(kctx->offslot_ts == 0); + if (kctx->offslot_ts > offslot_ts) { + dev_dbg(kbdev->dev, + "Reclaim aborts from ctx %d_%d, prio %d, current time %llu - offslot time %llu = %llu", + kctx->tgid, kctx->id, prio, now, kctx->offslot_ts, + now - kctx->offslot_ts); + /* Skip following ctx as they are attached later */ + break; + } + } + + freed_pages = kbase_csf_tiler_heap_scan_kctx_unused_pages( kctx, info->nr_est_unused_pages); + dev_dbg(kbdev->dev, "Reclaim free heap pages for ctx %d_%d freed pages %u", + kctx->tgid, kctx->id, freed_pages); + if (freed_pages) { /* Remove the freed pages from the manager retained estimate. The * accumulated removals from the kctx should not exceed the kctx @@ -260,7 +314,7 @@ static unsigned long reclaim_unused_heap_pages(struct kbase_device *kbdev) cnt_ctxs++; /* Enough has been freed, break to avoid holding the lock too long */ - if (total_freed_pages >= HEAP_RECLAIM_SCAN_BATCH_SIZE) + if (total_freed_pages >= max_pages) break; } @@ -313,7 +367,8 @@ static unsigned long kbase_csf_tiler_heap_reclaim_scan_free_pages(struct kbase_d avail = (unsigned long)atomic_read(&mgr->unused_pages); if (avail) - freed = reclaim_unused_heap_pages(kbdev); + freed = kbase_csf_tiler_heap_reclaim_unused_pages(kbdev, + HEAP_RECLAIM_SCENARIO_SHRINKER); mutex_unlock(&kbdev->csf.scheduler.lock); @@ -372,6 +427,9 @@ int kbase_csf_tiler_heap_reclaim_mgr_init(struct kbase_device *kbdev) prio++) INIT_LIST_HEAD(&scheduler->reclaim_mgr.ctx_lists[prio]); + scheduler->reclaim_mgr.offslot_setting.timeout_ms = HEAP_RECLAIM_OFFSLOT_TIMEOUT_MS; + scheduler->reclaim_mgr.offslot_setting.pages = HEAP_RECLAIM_OFFSLOT_PAGES; + reclaim->count_objects = kbase_csf_tiler_heap_reclaim_count_objects; reclaim->scan_objects = kbase_csf_tiler_heap_reclaim_scan_objects; reclaim->seeks = HEAP_SHRINKER_SEEKS; diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_tiler_heap_reclaim.h b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_tiler_heap_reclaim.h index d41b7baabd02..e74bd4f25926 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_tiler_heap_reclaim.h +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_tiler_heap_reclaim.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2022-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2022-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -79,4 +79,26 @@ int kbase_csf_tiler_heap_reclaim_mgr_init(struct kbase_device *kbdev); */ void kbase_csf_tiler_heap_reclaim_mgr_term(struct kbase_device *kbdev); +/** + * enum heap_reclaim_scenario - heap reclaim scenario list. + * + * @HEAP_RECLAIM_SCENARIO_SHRINKER: Reclaim through shrinker. + * @HEAP_RECLAIM_SCENARIO_SCHEDULER: Reclaim through scheduler. + * @HEAP_RECLAIM_SCENARIO_COUNT: The number of reclaim scenarios. + */ +enum heap_reclaim_scenario { + HEAP_RECLAIM_SCENARIO_SHRINKER = 0, + HEAP_RECLAIM_SCENARIO_SCHEDULER, + HEAP_RECLAIM_SCENARIO_COUNT +}; + +/** + * kbase_csf_tiler_heap_reclaim_unused_pages - Reclaim unused heap pages. + * @kbdev: Pointer to the device. + * @scenario: Reclaim scenario. + * + * Return: total freed pages. + */ +unsigned long kbase_csf_tiler_heap_reclaim_unused_pages(struct kbase_device *kbdev, + enum heap_reclaim_scenario scenario); #endif diff --git a/drivers/gpu/arm/valhall/debug/Kbuild b/drivers/gpu/arm/valhall/debug/Kbuild index a27458a51d4d..74b20f5ea3dc 100644 --- a/drivers/gpu/arm/valhall/debug/Kbuild +++ b/drivers/gpu/arm/valhall/debug/Kbuild @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2021-2022 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2021-2022, 2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -18,11 +18,7 @@ # # -valhall_kbase-y += debug/mali_kbase_debug_ktrace.o - -ifeq ($(CONFIG_MALI_VALHALL_CSF_SUPPORT),y) - valhall_kbase-y += debug/backend/mali_kbase_debug_ktrace_csf.o - valhall_kbase-$(CONFIG_MALI_VALHALL_CORESIGHT) += debug/backend/mali_kbase_debug_coresight_csf.o -else - valhall_kbase-y += debug/backend/mali_kbase_debug_ktrace_jm.o -endif +valhall_kbase-y += \ + debug/mali_kbase_debug_ktrace.o \ + debug/backend/mali_kbase_debug_ktrace_csf.o +valhall_kbase-$(CONFIG_MALI_VALHALL_CORESIGHT) += debug/backend/mali_kbase_debug_coresight_csf.o diff --git a/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_ktrace_codes_jm.h b/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_ktrace_codes_jm.h deleted file mode 100644 index 14dcc265bc90..000000000000 --- a/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_ktrace_codes_jm.h +++ /dev/null @@ -1,203 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2011-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * ***** IMPORTANT: THIS IS NOT A NORMAL HEADER FILE ***** - * ***** DO NOT INCLUDE DIRECTLY ***** - * ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** - */ - -/* - * The purpose of this header file is just to contain a list of trace code - * identifiers - * - * When updating this file, also remember to update - * mali_kbase_debug_linux_ktrace_jm.h - * - * IMPORTANT: THIS FILE MUST NOT BE USED FOR ANY OTHER PURPOSE OTHER THAN THAT - * DESCRIBED IN mali_kbase_debug_ktrace_codes.h - */ - -#if 0 /* Dummy section to avoid breaking formatting */ -int dummy_array[] = { -#endif - -/* - * Job Slot management events - */ -/* info_val==irq rawstat at start */ -KBASE_KTRACE_CODE_MAKE_CODE(JM_IRQ), - /* info_val==jobs processed */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_IRQ_END), - /* In the following: - * - * - ctx is set if a corresponding job found (NULL otherwise, e.g. some - * soft-stop cases) - * - uatom==kernel-side mapped uatom address (for correlation with - * user-side) - */ - /* info_val==exit code; gpu_addr==chain gpuaddr */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_JOB_DONE), - /* gpu_addr==JS_HEAD read - * info_val==event code - */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_RETURN_ATOM_TO_JS), - /* gpu_addr==JS_HEAD read - * info_val==event code - */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_MARK_FOR_RETURN_TO_JS), - /* gpu_addr==JS_HEAD_NEXT written, info_val==lower 32 bits of - * affinity - */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_SUBMIT), - /* gpu_addr is as follows: - * - If JS_STATUS active after soft-stop, val==gpu addr written to - * JS_HEAD on submit - * - otherwise gpu_addr==0 - */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_SOFTSTOP), KBASE_KTRACE_CODE_MAKE_CODE(JM_SOFTSTOP_0), - KBASE_KTRACE_CODE_MAKE_CODE(JM_SOFTSTOP_1), - /* gpu_addr==JS_HEAD read */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_HARDSTOP), - /* gpu_addr==JS_HEAD read */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_HARDSTOP_0), - /* gpu_addr==JS_HEAD read */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_HARDSTOP_1), - /* gpu_addr==JS_TAIL read */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_UPDATE_HEAD), - /* gpu_addr is as follows: - * - If JS_STATUS active before soft-stop, val==JS_HEAD - * - otherwise gpu_addr==0 - */ - /* gpu_addr==JS_HEAD read */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_CHECK_HEAD), KBASE_KTRACE_CODE_MAKE_CODE(JM_FLUSH_WORKQS), - KBASE_KTRACE_CODE_MAKE_CODE(JM_FLUSH_WORKQS_DONE), - /* info_val == is_scheduled */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_ZAP_NON_SCHEDULED), - /* info_val == is_scheduled */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_ZAP_SCHEDULED), KBASE_KTRACE_CODE_MAKE_CODE(JM_ZAP_DONE), - /* info_val == nr jobs submitted */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_SLOT_SOFT_OR_HARD_STOP), - /* gpu_addr==JS_HEAD_NEXT last written */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_SLOT_EVICT), - KBASE_KTRACE_CODE_MAKE_CODE(JM_SUBMIT_AFTER_RESET), - KBASE_KTRACE_CODE_MAKE_CODE(JM_BEGIN_RESET_WORKER), - KBASE_KTRACE_CODE_MAKE_CODE(JM_END_RESET_WORKER), - /* - * Job dispatch events - */ - /* gpu_addr==value to write into JS_HEAD */ - KBASE_KTRACE_CODE_MAKE_CODE(JD_DONE), - /* gpu_addr==value to write into JS_HEAD */ - KBASE_KTRACE_CODE_MAKE_CODE(JD_DONE_WORKER), - /* gpu_addr==value to write into JS_HEAD */ - KBASE_KTRACE_CODE_MAKE_CODE(JD_DONE_WORKER_END), - /* gpu_addr==value to write into JS_HEAD */ - KBASE_KTRACE_CODE_MAKE_CODE(JD_DONE_TRY_RUN_NEXT_JOB), - /* gpu_addr==0, info_val==0, uatom==0 */ - KBASE_KTRACE_CODE_MAKE_CODE(JD_ZAP_CONTEXT), - /* gpu_addr==value to write into JS_HEAD */ - KBASE_KTRACE_CODE_MAKE_CODE(JD_CANCEL), - /* gpu_addr==value to write into JS_HEAD */ - KBASE_KTRACE_CODE_MAKE_CODE(JD_CANCEL_WORKER), - /* - * Scheduler Core events - */ - /* gpu_addr==value to write into JS_HEAD */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_ADD_JOB), - /* gpu_addr==last value written/would be written to JS_HEAD */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_REMOVE_JOB), - /* gpu_addr==value to write into JS_HEAD - * info_val==priority of atom as a KBASE_JS_ATOM_SCHED_PRIO_<...> value - * (0 highest) - */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_PULL_JOB), - /* gpu_addr==value that would be written to JS_HEAD if run again */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_UNPULL_JOB), - KBASE_KTRACE_CODE_MAKE_CODE(JS_TRY_SCHEDULE_HEAD_CTX), - /* gpu_addr==value to write into JS_HEAD */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_JOB_DONE_TRY_RUN_NEXT_JOB), - /* gpu_addr==value to write into JS_HEAD */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_JOB_DONE_RETRY_NEEDED), - KBASE_KTRACE_CODE_MAKE_CODE(JS_AFFINITY_SUBMIT_TO_BLOCKED), - /* info_val == lower 32 bits of affinity */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_AFFINITY_CURRENT), - /* info_val == lower 32 bits of affinity */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_REQUEST_CORES_FAILED), - /* info_val == lower 32 bits of affinity */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_REGISTER_INUSE_FAILED), - /* info_val == lower 32 bits of rechecked affinity */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_REQUEST_ON_RECHECK_FAILED), - /* info_val == lower 32 bits of rechecked affinity */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_REGISTER_ON_RECHECK_FAILED), - /* info_val == lower 32 bits of affinity */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_AFFINITY_WOULD_VIOLATE), - /* info_val == the ctx attribute now on ctx */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_ON_CTX), - /* info_val == the ctx attribute now on runpool */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_ON_RUNPOOL), - /* info_val == the ctx attribute now off ctx */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_OFF_CTX), - /* info_val == the ctx attribute now off runpool */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_OFF_RUNPOOL), - /* gpu_addr==value to write into JS_HEAD */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_RETURN_WORKER), - /* gpu_addr==value to write into JS_HEAD */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_RETURN_WORKER_END), - /* info_val==priority level blocked (0 highest) */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_SLOT_PRIO_BLOCKED), - /* info_val==priority level unblocked (0 highest) - * note that the priority level may still be blocked on higher levels - */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_SLOT_PRIO_UNBLOCKED), - /* gpu_addr==value to write into JS_HEAD - * info_val==priority level unblocked - priorities at this and higher - * are unblocked (0 highest) - */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_SLOT_PRIO_AND_HIGHER_UNBLOCKED), - /* gpu_addr==value to write into JS_HEAD - * info_val==priority level blocked (0 highest) - */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_SLOT_PRIO_IS_BLOCKED), - /* - * Scheduler Policy events - */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_INIT_CTX), - KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_TERM_CTX), - /* info_val == whether it was evicted */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_TRY_EVICT_CTX), - KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_FOREACH_CTX_JOBS), - KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_ENQUEUE_CTX), - KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_HEAD_CTX), - KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_RUNPOOL_ADD_CTX), - KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_RUNPOOL_REMOVE_CTX), - KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_JOB), - KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_JOB_IRQ), - /* gpu_addr==JS_HEAD to write if the job were run */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_ENQUEUE_JOB), - KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_TIMER_START), - KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_TIMER_END), - -#if 0 /* Dummy section to avoid breaking formatting */ -}; -#endif - - /* ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */ diff --git a/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_ktrace_defs_jm.h b/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_ktrace_defs_jm.h deleted file mode 100644 index 9978e4939d14..000000000000 --- a/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_ktrace_defs_jm.h +++ /dev/null @@ -1,111 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#ifndef _KBASE_DEBUG_KTRACE_DEFS_JM_H_ -#define _KBASE_DEBUG_KTRACE_DEFS_JM_H_ - -#if KBASE_KTRACE_TARGET_RBUF -/** - * DOC: KTrace version history, JM variant - * - * 1.0: - * Original version (implicit, header did not carry version information). - * - * 2.0: - * Introduced version information into the header. - * - * Some changes of parameter names in header. - * - * Trace now uses all 64-bits of info_val. - * - * Non-JM specific parts moved to using info_val instead of refcount/gpu_addr. - * - * 2.1: - * kctx field is no longer a pointer, and is now an ID of the format %d_%u as - * used by kctx directories in mali debugfs entries: (tgid creating the kctx), - * (unique kctx id). - * - * ftrace backend now outputs kctx field (as %d_%u format). - * - * 2.2: - * Add tracing codes for pulling, unpulling, and returns atoms to JS for - * diagnosing soft-stop path and preemption problems - */ -#define KBASE_KTRACE_VERSION_MAJOR 2 -#define KBASE_KTRACE_VERSION_MINOR 2 -#endif /* KBASE_KTRACE_TARGET_RBUF */ - -/* - * Note: mali_kbase_debug_ktrace_jm.h needs these value even if the RBUF target - * is disabled (they get discarded with CSTD_UNUSED(), but they're still - * referenced) - */ - -/* indicates if the trace message has a valid refcount member */ -#define KBASE_KTRACE_FLAG_JM_REFCOUNT (((kbase_ktrace_flag_t)1) << 0) -/* indicates if the trace message has a valid jobslot member */ -#define KBASE_KTRACE_FLAG_JM_JOBSLOT (((kbase_ktrace_flag_t)1) << 1) -/* indicates if the trace message has valid atom related info. */ -#define KBASE_KTRACE_FLAG_JM_ATOM (((kbase_ktrace_flag_t)1) << 2) - -#if KBASE_KTRACE_TARGET_RBUF -/* Collect all the flags together for debug checking */ -#define KBASE_KTRACE_FLAG_BACKEND_ALL \ - (KBASE_KTRACE_FLAG_JM_REFCOUNT | KBASE_KTRACE_FLAG_JM_JOBSLOT | KBASE_KTRACE_FLAG_JM_ATOM) - -/** - * union kbase_ktrace_backend - backend specific part of a trace message - * Contains only a struct but is a union such that it is compatible with - * generic JM and CSF KTrace calls. - * - * @gpu: gpu union member - * @gpu.atom_udata: Copy of the user data sent for the atom in base_jd_submit. - * Only valid if KBASE_KTRACE_FLAG_JM_ATOM is set in @flags - * @gpu.gpu_addr: GPU address, usually of the job-chain represented by an - * atom. - * @gpu.atom_number: id of the atom for which trace message was added. Only - * valid if KBASE_KTRACE_FLAG_JM_ATOM is set in @flags - * @gpu.code: Identifies the event, refer to enum kbase_ktrace_code. - * @gpu.flags: indicates information about the trace message itself. Used - * during dumping of the message. - * @gpu.jobslot: job-slot for which trace message was added, valid only for - * job-slot management events. - * @gpu.refcount: reference count for the context, valid for certain events - * related to scheduler core and policy. - */ -union kbase_ktrace_backend { - struct { - /* Place 64 and 32-bit members together */ - u64 atom_udata[2]; /* Only valid for - * KBASE_KTRACE_FLAG_JM_ATOM - */ - u64 gpu_addr; - unsigned int atom_number; /* Only valid for KBASE_KTRACE_FLAG_JM_ATOM */ - /* Pack smaller members together */ - kbase_ktrace_code_t code; - kbase_ktrace_flag_t flags; - u8 jobslot; - u8 refcount; - } gpu; -}; -#endif /* KBASE_KTRACE_TARGET_RBUF */ - -#endif /* _KBASE_DEBUG_KTRACE_DEFS_JM_H_ */ diff --git a/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_ktrace_jm.c b/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_ktrace_jm.c deleted file mode 100644 index 39306e7d45e3..000000000000 --- a/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_ktrace_jm.c +++ /dev/null @@ -1,115 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include -#include "debug/mali_kbase_debug_ktrace_internal.h" -#include "debug/backend/mali_kbase_debug_ktrace_jm.h" - -#if KBASE_KTRACE_TARGET_RBUF - -void kbasep_ktrace_backend_format_header(char *buffer, int sz, s32 *written) -{ - *written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), - "katom,gpu_addr,jobslot,refcount"), - 0); -} - -void kbasep_ktrace_backend_format_msg(struct kbase_ktrace_msg *trace_msg, char *buffer, int sz, - s32 *written) -{ - /* katom */ - if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_JM_ATOM) - *written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), - "atom %u (ud: 0x%llx 0x%llx)", - trace_msg->backend.gpu.atom_number, - trace_msg->backend.gpu.atom_udata[0], - trace_msg->backend.gpu.atom_udata[1]), - 0); - - /* gpu_addr */ - if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_BACKEND) - *written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), - ",%.8llx,", trace_msg->backend.gpu.gpu_addr), - 0); - else - *written += - MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), ",,"), 0); - - /* jobslot */ - if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_JM_JOBSLOT) - *written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), "%d", - trace_msg->backend.gpu.jobslot), - 0); - - *written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), ","), 0); - - /* refcount */ - if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_JM_REFCOUNT) - *written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), "%d", - trace_msg->backend.gpu.refcount), - 0); -} - -void kbasep_ktrace_add_jm(struct kbase_device *kbdev, enum kbase_ktrace_code code, - struct kbase_context *kctx, const struct kbase_jd_atom *katom, - u64 gpu_addr, kbase_ktrace_flag_t flags, int refcount, - unsigned int jobslot, u64 info_val) -{ - unsigned long irqflags; - struct kbase_ktrace_msg *trace_msg; - - if (unlikely(!kbasep_ktrace_initialized(&kbdev->ktrace))) - return; - - spin_lock_irqsave(&kbdev->ktrace.lock, irqflags); - - /* Reserve and update indices */ - trace_msg = kbasep_ktrace_reserve(&kbdev->ktrace); - - /* Fill the common part of the message (including backend.gpu.flags) */ - kbasep_ktrace_msg_init(&kbdev->ktrace, trace_msg, code, kctx, flags, info_val); - - /* Indicate to the common code that backend-specific parts will be - * valid - */ - trace_msg->backend.gpu.flags |= KBASE_KTRACE_FLAG_BACKEND; - - /* Fill the JM-specific parts of the message */ - if (katom) { - trace_msg->backend.gpu.flags |= KBASE_KTRACE_FLAG_JM_ATOM; - - trace_msg->backend.gpu.atom_number = kbase_jd_atom_id(katom->kctx, katom); - trace_msg->backend.gpu.atom_udata[0] = katom->udata.blob[0]; - trace_msg->backend.gpu.atom_udata[1] = katom->udata.blob[1]; - } - - trace_msg->backend.gpu.gpu_addr = gpu_addr; - trace_msg->backend.gpu.jobslot = jobslot; - /* Clamp refcount */ - trace_msg->backend.gpu.refcount = MIN((unsigned int)refcount, 0xFF); - - WARN_ON((trace_msg->backend.gpu.flags & ~KBASE_KTRACE_FLAG_ALL)); - - /* Done */ - spin_unlock_irqrestore(&kbdev->ktrace.lock, irqflags); -} - -#endif /* KBASE_KTRACE_TARGET_RBUF */ diff --git a/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_ktrace_jm.h b/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_ktrace_jm.h deleted file mode 100644 index 59ad23d7677c..000000000000 --- a/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_ktrace_jm.h +++ /dev/null @@ -1,297 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#ifndef _KBASE_DEBUG_KTRACE_JM_H_ -#define _KBASE_DEBUG_KTRACE_JM_H_ - -/* - * KTrace target for internal ringbuffer - */ -#if KBASE_KTRACE_TARGET_RBUF -/** - * kbasep_ktrace_add_jm - internal function to add trace about Job Management - * @kbdev: kbase device - * @code: trace code - * @kctx: kbase context, or NULL if no context - * @katom: kbase atom, or NULL if no atom - * @gpu_addr: GPU address, usually related to @katom - * @flags: flags about the message - * @refcount: reference count information to add to the trace - * @jobslot: jobslot information to add to the trace - * @info_val: generic information about @code to add to the trace - * - * PRIVATE: do not use directly. Use KBASE_KTRACE_ADD_JM() instead. - */ -void kbasep_ktrace_add_jm(struct kbase_device *kbdev, enum kbase_ktrace_code code, - struct kbase_context *kctx, const struct kbase_jd_atom *katom, - u64 gpu_addr, kbase_ktrace_flag_t flags, int refcount, - unsigned int jobslot, u64 info_val); - -#define KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, gpu_addr, flags, refcount, jobslot, \ - info_val) \ - kbasep_ktrace_add_jm(kbdev, KBASE_KTRACE_CODE(code), kctx, katom, gpu_addr, flags, \ - refcount, jobslot, info_val) - -#else /* KBASE_KTRACE_TARGET_RBUF */ - -#define KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, gpu_addr, flags, refcount, jobslot, \ - info_val) \ - do { \ - CSTD_UNUSED(kbdev); \ - CSTD_NOP(code); \ - CSTD_UNUSED(kctx); \ - CSTD_UNUSED(katom); \ - CSTD_UNUSED(gpu_addr); \ - CSTD_UNUSED(flags); \ - CSTD_UNUSED(refcount); \ - CSTD_UNUSED(jobslot); \ - CSTD_UNUSED(info_val); \ - CSTD_NOP(0); \ - } while (0) -#endif /* KBASE_KTRACE_TARGET_RBUF */ - -/* - * KTrace target for Linux's ftrace - * - * Note: the header file(s) that define the trace_mali_<...> tracepoints are - * included by the parent header file - */ -#if KBASE_KTRACE_TARGET_FTRACE -#define KBASE_KTRACE_FTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, jobslot) \ - trace_mali_##code(kctx, jobslot, 0) - -#define KBASE_KTRACE_FTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, gpu_addr, jobslot, \ - info_val) \ - trace_mali_##code(kctx, jobslot, info_val) - -#define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, gpu_addr, refcount) \ - trace_mali_##code(kctx, refcount, 0) - -#define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, gpu_addr, refcount, \ - info_val) \ - trace_mali_##code(kctx, refcount, info_val) - -#define KBASE_KTRACE_FTRACE_ADD_JM(kbdev, code, kctx, katom, gpu_addr, info_val) \ - trace_mali_##code(kctx, gpu_addr, info_val) -#else /* KBASE_KTRACE_TARGET_FTRACE */ -#define KBASE_KTRACE_FTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, jobslot) \ - do { \ - CSTD_UNUSED(kbdev); \ - CSTD_NOP(code); \ - CSTD_UNUSED(kctx); \ - CSTD_UNUSED(katom); \ - CSTD_UNUSED(gpu_addr); \ - CSTD_UNUSED(jobslot); \ - CSTD_NOP(0); \ - } while (0) - -#define KBASE_KTRACE_FTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, gpu_addr, jobslot, \ - info_val) \ - do { \ - CSTD_UNUSED(kbdev); \ - CSTD_NOP(code); \ - CSTD_UNUSED(kctx); \ - CSTD_UNUSED(katom); \ - CSTD_UNUSED(gpu_addr); \ - CSTD_UNUSED(jobslot); \ - CSTD_UNUSED(info_val); \ - CSTD_NOP(0); \ - } while (0) - -#define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, gpu_addr, refcount) \ - do { \ - CSTD_UNUSED(kbdev); \ - CSTD_NOP(code); \ - CSTD_UNUSED(kctx); \ - CSTD_UNUSED(katom); \ - CSTD_UNUSED(gpu_addr); \ - CSTD_UNUSED(refcount); \ - CSTD_NOP(0); \ - } while (0) - -#define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, gpu_addr, refcount, \ - info_val) \ - do { \ - CSTD_UNUSED(kbdev); \ - CSTD_NOP(code); \ - CSTD_UNUSED(kctx); \ - CSTD_UNUSED(katom); \ - CSTD_UNUSED(gpu_addr); \ - CSTD_UNUSED(info_val); \ - CSTD_NOP(0); \ - } while (0) - -#define KBASE_KTRACE_FTRACE_ADD_JM(kbdev, code, kctx, katom, gpu_addr, info_val) \ - do { \ - CSTD_UNUSED(kbdev); \ - CSTD_NOP(code); \ - CSTD_UNUSED(kctx); \ - CSTD_UNUSED(katom); \ - CSTD_UNUSED(gpu_addr); \ - CSTD_UNUSED(info_val); \ - CSTD_NOP(0); \ - } while (0) -#endif /* KBASE_KTRACE_TARGET_FTRACE */ - -/* - * Master set of macros to route KTrace to any of the targets - */ - -/** - * KBASE_KTRACE_ADD_JM_SLOT - Add trace values about a job-slot - * @kbdev: kbase device - * @code: trace code - * @kctx: kbase context, or NULL if no context - * @katom: kbase atom, or NULL if no atom - * @gpu_addr: GPU address, usually related to @katom - * @jobslot: jobslot information to add to the trace - * - * Note: Any functions called through this macro will still be evaluated in - * Release builds (CONFIG_MALI_VALHALL_DEBUG not defined). Therefore, when - * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied - * to this macro must: - * a) be static or static inline, and - * b) just return 0 and have no other statements present in the body. - */ -#define KBASE_KTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, jobslot) \ - do { \ - /* capture values that could come from non-pure function calls */ \ - u64 __gpu_addr = gpu_addr; \ - unsigned int __jobslot = jobslot; \ - KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, \ - KBASE_KTRACE_FLAG_JM_JOBSLOT, 0, __jobslot, 0); \ - KBASE_KTRACE_FTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, __gpu_addr, __jobslot); \ - } while (0) - -/** - * KBASE_KTRACE_ADD_JM_SLOT_INFO - Add trace values about a job-slot, with info - * @kbdev: kbase device - * @code: trace code - * @kctx: kbase context, or NULL if no context - * @katom: kbase atom, or NULL if no atom - * @gpu_addr: GPU address, usually related to @katom - * @jobslot: jobslot information to add to the trace - * @info_val: generic information about @code to add to the trace - * - * Note: Any functions called through this macro will still be evaluated in - * Release builds (CONFIG_MALI_VALHALL_DEBUG not defined). Therefore, when - * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied - * to this macro must: - * a) be static or static inline, and - * b) just return 0 and have no other statements present in the body. - */ -#define KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, gpu_addr, jobslot, info_val) \ - do { \ - /* capture values that could come from non-pure function calls */ \ - u64 __gpu_addr = gpu_addr; \ - unsigned int __jobslot = jobslot; \ - u64 __info_val = info_val; \ - KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, \ - KBASE_KTRACE_FLAG_JM_JOBSLOT, 0, __jobslot, __info_val); \ - KBASE_KTRACE_FTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, __gpu_addr, \ - __jobslot, __info_val); \ - } while (0) - -/** - * KBASE_KTRACE_ADD_JM_REFCOUNT - Add trace values about a kctx refcount - * @kbdev: kbase device - * @code: trace code - * @kctx: kbase context, or NULL if no context - * @katom: kbase atom, or NULL if no atom - * @gpu_addr: GPU address, usually related to @katom - * @refcount: reference count information to add to the trace - * - * Note: Any functions called through this macro will still be evaluated in - * Release builds (CONFIG_MALI_VALHALL_DEBUG not defined). Therefore, when - * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied - * to this macro must: - * a) be static or static inline, and - * b) just return 0 and have no other statements present in the body. - */ -#define KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, gpu_addr, refcount) \ - do { \ - /* capture values that could come from non-pure function calls */ \ - u64 __gpu_addr = gpu_addr; \ - int __refcount = refcount; \ - KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, \ - KBASE_KTRACE_FLAG_JM_REFCOUNT, __refcount, 0, 0u); \ - KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, __gpu_addr, \ - (unsigned int)__refcount); \ - } while (0) - -/** - * KBASE_KTRACE_ADD_JM_REFCOUNT_INFO - Add trace values about a kctx refcount, - * and info - * @kbdev: kbase device - * @code: trace code - * @kctx: kbase context, or NULL if no context - * @katom: kbase atom, or NULL if no atom - * @gpu_addr: GPU address, usually related to @katom - * @refcount: reference count information to add to the trace - * @info_val: generic information about @code to add to the trace - * - * Note: Any functions called through this macro will still be evaluated in - * Release builds (CONFIG_MALI_VALHALL_DEBUG not defined). Therefore, when - * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied - * to this macro must: - * a) be static or static inline, and - * b) just return 0 and have no other statements present in the body. - */ -#define KBASE_KTRACE_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, gpu_addr, refcount, info_val) \ - do { \ - /* capture values that could come from non-pure function calls */ \ - u64 __gpu_addr = gpu_addr; \ - int __refcount = refcount; \ - u64 __info_val = info_val; \ - KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, \ - KBASE_KTRACE_FLAG_JM_REFCOUNT, __refcount, 0, \ - __info_val); \ - KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, __gpu_addr, \ - (unsigned int)__refcount, __info_val); \ - } while (0) - -/** - * KBASE_KTRACE_ADD_JM - Add trace values (no slot or refcount) - * @kbdev: kbase device - * @code: trace code - * @kctx: kbase context, or NULL if no context - * @katom: kbase atom, or NULL if no atom - * @gpu_addr: GPU address, usually related to @katom - * @info_val: generic information about @code to add to the trace - * - * Note: Any functions called through this macro will still be evaluated in - * Release builds (CONFIG_MALI_VALHALL_DEBUG not defined). Therefore, when - * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied - * to this macro must: - * a) be static or static inline, and - * b) just return 0 and have no other statements present in the body. - */ -#define KBASE_KTRACE_ADD_JM(kbdev, code, kctx, katom, gpu_addr, info_val) \ - do { \ - /* capture values that could come from non-pure function calls */ \ - u64 __gpu_addr = gpu_addr; \ - u64 __info_val = info_val; \ - KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, 0u, 0, 0, \ - __info_val); \ - KBASE_KTRACE_FTRACE_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, __info_val); \ - } while (0) - -#endif /* _KBASE_DEBUG_KTRACE_JM_H_ */ diff --git a/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_linux_ktrace_jm.h b/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_linux_ktrace_jm.h deleted file mode 100644 index 3e5b0ec89758..000000000000 --- a/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_linux_ktrace_jm.h +++ /dev/null @@ -1,151 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * NOTE: This must **only** be included through mali_linux_trace.h, - * otherwise it will fail to setup tracepoints correctly - */ - -#if !defined(_KBASE_DEBUG_LINUX_KTRACE_JM_H_) || defined(TRACE_HEADER_MULTI_READ) -#define _KBASE_DEBUG_LINUX_KTRACE_JM_H_ - -DECLARE_EVENT_CLASS(mali_jm_slot_template, - TP_PROTO(struct kbase_context *kctx, unsigned int jobslot, u64 info_val), - TP_ARGS(kctx, jobslot, info_val), - TP_STRUCT__entry(__field(pid_t, kctx_tgid) __field(u32, kctx_id) - __field(unsigned int, jobslot) __field(u64, info_val)), - TP_fast_assign(__entry->kctx_id = (kctx) ? kctx->id : 0u; - __entry->kctx_tgid = (kctx) ? kctx->tgid : 0; - __entry->jobslot = jobslot; __entry->info_val = info_val;), - TP_printk("kctx=%d_%u jobslot=%u info=0x%llx", __entry->kctx_tgid, - __entry->kctx_id, __entry->jobslot, __entry->info_val)); - -#define DEFINE_MALI_JM_SLOT_EVENT(name) \ - DEFINE_EVENT(mali_jm_slot_template, mali_##name, \ - TP_PROTO(struct kbase_context *kctx, unsigned int jobslot, u64 info_val), \ - TP_ARGS(kctx, jobslot, info_val)) -DEFINE_MALI_JM_SLOT_EVENT(JM_RETURN_ATOM_TO_JS); -DEFINE_MALI_JM_SLOT_EVENT(JM_MARK_FOR_RETURN_TO_JS); -DEFINE_MALI_JM_SLOT_EVENT(JM_SUBMIT); -DEFINE_MALI_JM_SLOT_EVENT(JM_JOB_DONE); -DEFINE_MALI_JM_SLOT_EVENT(JM_UPDATE_HEAD); -DEFINE_MALI_JM_SLOT_EVENT(JM_CHECK_HEAD); -DEFINE_MALI_JM_SLOT_EVENT(JM_SOFTSTOP); -DEFINE_MALI_JM_SLOT_EVENT(JM_SOFTSTOP_0); -DEFINE_MALI_JM_SLOT_EVENT(JM_SOFTSTOP_1); -DEFINE_MALI_JM_SLOT_EVENT(JM_HARDSTOP); -DEFINE_MALI_JM_SLOT_EVENT(JM_HARDSTOP_0); -DEFINE_MALI_JM_SLOT_EVENT(JM_HARDSTOP_1); -DEFINE_MALI_JM_SLOT_EVENT(JM_SLOT_SOFT_OR_HARD_STOP); -DEFINE_MALI_JM_SLOT_EVENT(JM_SLOT_EVICT); -DEFINE_MALI_JM_SLOT_EVENT(JM_BEGIN_RESET_WORKER); -DEFINE_MALI_JM_SLOT_EVENT(JM_END_RESET_WORKER); -DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_REGISTER_ON_RECHECK_FAILED); -DEFINE_MALI_JM_SLOT_EVENT(JS_AFFINITY_SUBMIT_TO_BLOCKED); -DEFINE_MALI_JM_SLOT_EVENT(JS_AFFINITY_CURRENT); -DEFINE_MALI_JM_SLOT_EVENT(JD_DONE_TRY_RUN_NEXT_JOB); -DEFINE_MALI_JM_SLOT_EVENT(JS_PULL_JOB); -DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_REQUEST_CORES_FAILED); -DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_REGISTER_INUSE_FAILED); -DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_REQUEST_ON_RECHECK_FAILED); -DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_AFFINITY_WOULD_VIOLATE); -DEFINE_MALI_JM_SLOT_EVENT(JS_JOB_DONE_TRY_RUN_NEXT_JOB); -DEFINE_MALI_JM_SLOT_EVENT(JS_JOB_DONE_RETRY_NEEDED); -DEFINE_MALI_JM_SLOT_EVENT(JS_POLICY_DEQUEUE_JOB); -DEFINE_MALI_JM_SLOT_EVENT(JS_POLICY_DEQUEUE_JOB_IRQ); -DEFINE_MALI_JM_SLOT_EVENT(JS_SLOT_PRIO_BLOCKED); -DEFINE_MALI_JM_SLOT_EVENT(JS_SLOT_PRIO_UNBLOCKED); -DEFINE_MALI_JM_SLOT_EVENT(JS_SLOT_PRIO_AND_HIGHER_UNBLOCKED); -DEFINE_MALI_JM_SLOT_EVENT(JS_SLOT_PRIO_IS_BLOCKED); -#undef DEFINE_MALI_JM_SLOT_EVENT - -DECLARE_EVENT_CLASS(mali_jm_refcount_template, - TP_PROTO(struct kbase_context *kctx, unsigned int refcount, u64 info_val), - TP_ARGS(kctx, refcount, info_val), - TP_STRUCT__entry(__field(pid_t, kctx_tgid) __field(u32, kctx_id) - __field(unsigned int, refcount) - __field(u64, info_val)), - TP_fast_assign(__entry->kctx_id = (kctx) ? kctx->id : 0u; - __entry->kctx_tgid = (kctx) ? kctx->tgid : 0; - __entry->refcount = refcount; __entry->info_val = info_val;), - TP_printk("kctx=%d_%u refcount=%u info=0x%llx", __entry->kctx_tgid, - __entry->kctx_id, __entry->refcount, __entry->info_val)); - -#define DEFINE_MALI_JM_REFCOUNT_EVENT(name) \ - DEFINE_EVENT(mali_jm_refcount_template, mali_##name, \ - TP_PROTO(struct kbase_context *kctx, unsigned int refcount, u64 info_val), \ - TP_ARGS(kctx, refcount, info_val)) -DEFINE_MALI_JM_REFCOUNT_EVENT(JS_ADD_JOB); -DEFINE_MALI_JM_REFCOUNT_EVENT(JS_REMOVE_JOB); -DEFINE_MALI_JM_REFCOUNT_EVENT(JS_TRY_SCHEDULE_HEAD_CTX); -DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_INIT_CTX); -DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_TERM_CTX); -DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_ENQUEUE_CTX); -DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_DEQUEUE_HEAD_CTX); -DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_TRY_EVICT_CTX); -DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_RUNPOOL_ADD_CTX); -DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_RUNPOOL_REMOVE_CTX); -DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_FOREACH_CTX_JOBS); -#undef DEFINE_MALI_JM_REFCOUNT_EVENT - -DECLARE_EVENT_CLASS(mali_jm_add_template, - TP_PROTO(struct kbase_context *kctx, u64 gpu_addr, u64 info_val), - TP_ARGS(kctx, gpu_addr, info_val), - TP_STRUCT__entry(__field(pid_t, kctx_tgid) __field(u32, kctx_id) - __field(u64, gpu_addr) __field(u64, info_val)), - TP_fast_assign(__entry->kctx_id = (kctx) ? kctx->id : 0u; - __entry->kctx_tgid = (kctx) ? kctx->tgid : 0; - __entry->gpu_addr = gpu_addr; __entry->info_val = info_val;), - TP_printk("kctx=%d_%u gpu_addr=0x%llx info=0x%llx", __entry->kctx_tgid, - __entry->kctx_id, __entry->gpu_addr, __entry->info_val)); - -#define DEFINE_MALI_JM_ADD_EVENT(name) \ - DEFINE_EVENT(mali_jm_add_template, mali_##name, \ - TP_PROTO(struct kbase_context *kctx, u64 gpu_addr, u64 info_val), \ - TP_ARGS(kctx, gpu_addr, info_val)) -DEFINE_MALI_JM_ADD_EVENT(JD_DONE_WORKER); -DEFINE_MALI_JM_ADD_EVENT(JD_DONE_WORKER_END); -DEFINE_MALI_JM_ADD_EVENT(JD_CANCEL_WORKER); -DEFINE_MALI_JM_ADD_EVENT(JD_DONE); -DEFINE_MALI_JM_ADD_EVENT(JD_CANCEL); -DEFINE_MALI_JM_ADD_EVENT(JD_ZAP_CONTEXT); -DEFINE_MALI_JM_ADD_EVENT(JM_IRQ); -DEFINE_MALI_JM_ADD_EVENT(JM_IRQ_END); -DEFINE_MALI_JM_ADD_EVENT(JM_FLUSH_WORKQS); -DEFINE_MALI_JM_ADD_EVENT(JM_FLUSH_WORKQS_DONE); -DEFINE_MALI_JM_ADD_EVENT(JM_ZAP_NON_SCHEDULED); -DEFINE_MALI_JM_ADD_EVENT(JM_ZAP_SCHEDULED); -DEFINE_MALI_JM_ADD_EVENT(JM_ZAP_DONE); -DEFINE_MALI_JM_ADD_EVENT(JM_SUBMIT_AFTER_RESET); -DEFINE_MALI_JM_ADD_EVENT(JM_JOB_COMPLETE); -DEFINE_MALI_JM_ADD_EVENT(JS_UNPULL_JOB); -DEFINE_MALI_JM_ADD_EVENT(JS_CTX_ATTR_NOW_ON_RUNPOOL); -DEFINE_MALI_JM_ADD_EVENT(JS_CTX_ATTR_NOW_OFF_RUNPOOL); -DEFINE_MALI_JM_ADD_EVENT(JS_CTX_ATTR_NOW_ON_CTX); -DEFINE_MALI_JM_ADD_EVENT(JS_CTX_ATTR_NOW_OFF_CTX); -DEFINE_MALI_JM_ADD_EVENT(JS_RETURN_WORKER); -DEFINE_MALI_JM_ADD_EVENT(JS_RETURN_WORKER_END); -DEFINE_MALI_JM_ADD_EVENT(JS_POLICY_TIMER_END); -DEFINE_MALI_JM_ADD_EVENT(JS_POLICY_TIMER_START); -DEFINE_MALI_JM_ADD_EVENT(JS_POLICY_ENQUEUE_JOB); -#undef DEFINE_MALI_JM_ADD_EVENT - -#endif /* !defined(_KBASE_DEBUG_LINUX_KTRACE_JM_H_) || defined(TRACE_HEADER_MULTI_READ)*/ diff --git a/drivers/gpu/arm/valhall/debug/mali_kbase_debug_ktrace.h b/drivers/gpu/arm/valhall/debug/mali_kbase_debug_ktrace.h index 9d12a000c5bb..53c1debc840c 100644 --- a/drivers/gpu/arm/valhall/debug/mali_kbase_debug_ktrace.h +++ b/drivers/gpu/arm/valhall/debug/mali_kbase_debug_ktrace.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -40,11 +40,7 @@ #include "mali_linux_trace.h" #endif -#if MALI_USE_CSF #include "debug/backend/mali_kbase_debug_ktrace_csf.h" -#else -#include "debug/backend/mali_kbase_debug_ktrace_jm.h" -#endif /** * kbase_ktrace_init - initialize kbase ktrace. diff --git a/drivers/gpu/arm/valhall/debug/mali_kbase_debug_ktrace_codes.h b/drivers/gpu/arm/valhall/debug/mali_kbase_debug_ktrace_codes.h index f5810d477c83..94c5ae0dd290 100644 --- a/drivers/gpu/arm/valhall/debug/mali_kbase_debug_ktrace_codes.h +++ b/drivers/gpu/arm/valhall/debug/mali_kbase_debug_ktrace_codes.h @@ -165,11 +165,7 @@ KBASE_KTRACE_CODE_MAKE_CODE(CORE_CTX_DESTROY), KBASE_KTRACE_CODE_MAKE_CODE(ARB_GPU_STOPPED), KBASE_KTRACE_CODE_MAKE_CODE(ARB_GPU_REQUESTED), -#if MALI_USE_CSF #include "debug/backend/mali_kbase_debug_ktrace_codes_csf.h" -#else -#include "debug/backend/mali_kbase_debug_ktrace_codes_jm.h" -#endif /* * Unused code just to make it easier to not have a comma at the end. * All other codes MUST come before this diff --git a/drivers/gpu/arm/valhall/debug/mali_kbase_debug_ktrace_defs.h b/drivers/gpu/arm/valhall/debug/mali_kbase_debug_ktrace_defs.h index be91473b81ec..9907c74efcc0 100644 --- a/drivers/gpu/arm/valhall/debug/mali_kbase_debug_ktrace_defs.h +++ b/drivers/gpu/arm/valhall/debug/mali_kbase_debug_ktrace_defs.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -91,11 +91,7 @@ union kbase_ktrace_backend; #endif /* KBASE_KTRACE_TARGET_RBUF */ -#if MALI_USE_CSF #include "debug/backend/mali_kbase_debug_ktrace_defs_csf.h" -#else -#include "debug/backend/mali_kbase_debug_ktrace_defs_jm.h" -#endif #if KBASE_KTRACE_TARGET_RBUF /* Indicates if the trace message has backend related info. diff --git a/drivers/gpu/arm/valhall/debug/mali_kbase_debug_linux_ktrace.h b/drivers/gpu/arm/valhall/debug/mali_kbase_debug_linux_ktrace.h index 76ac6a6c429b..224a758d2c2c 100644 --- a/drivers/gpu/arm/valhall/debug/mali_kbase_debug_linux_ktrace.h +++ b/drivers/gpu/arm/valhall/debug/mali_kbase_debug_linux_ktrace.h @@ -111,11 +111,7 @@ DEFINE_MALI_ADD_EVENT(ARB_GPU_STOP_REQUESTED); DEFINE_MALI_ADD_EVENT(ARB_GPU_STOPPED); DEFINE_MALI_ADD_EVENT(ARB_GPU_REQUESTED); -#if MALI_USE_CSF #include "backend/mali_kbase_debug_linux_ktrace_csf.h" -#else -#include "backend/mali_kbase_debug_linux_ktrace_jm.h" -#endif #undef DEFINE_MALI_ADD_EVENT diff --git a/drivers/gpu/arm/valhall/device/Kbuild b/drivers/gpu/arm/valhall/device/Kbuild index 7b4057023574..5a50d974c080 100644 --- a/drivers/gpu/arm/valhall/device/Kbuild +++ b/drivers/gpu/arm/valhall/device/Kbuild @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2021 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2021, 2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -20,14 +20,6 @@ valhall_kbase-y += \ device/mali_kbase_device.o \ - device/mali_kbase_device_hw.o - -ifeq ($(CONFIG_MALI_VALHALL_CSF_SUPPORT),y) - valhall_kbase-y += \ - device/backend/mali_kbase_device_csf.o \ - device/backend/mali_kbase_device_hw_csf.o -else - valhall_kbase-y += \ - device/backend/mali_kbase_device_jm.o \ - device/backend/mali_kbase_device_hw_jm.o -endif + device/mali_kbase_device_hw.o \ + device/backend/mali_kbase_device_csf.o \ + device/backend/mali_kbase_device_hw_csf.o diff --git a/drivers/gpu/arm/valhall/device/backend/mali_kbase_device_csf.c b/drivers/gpu/arm/valhall/device/backend/mali_kbase_device_csf.c index 4362dd1de561..539590737d90 100644 --- a/drivers/gpu/arm/valhall/device/backend/mali_kbase_device_csf.c +++ b/drivers/gpu/arm/valhall/device/backend/mali_kbase_device_csf.c @@ -61,6 +61,7 @@ static void kbase_device_firmware_hwcnt_term(struct kbase_device *kbdev) kbase_kinstr_prfcnt_term(kbdev->kinstr_prfcnt_ctx); kbase_hwcnt_virtualizer_term(kbdev->hwcnt_gpu_virt); kbase_hwcnt_backend_csf_metadata_term(&kbdev->hwcnt_gpu_iface); + kbase_hwcnt_backend_csf_ring_buf_term(&kbdev->hwcnt_gpu_iface); kbase_csf_firmware_unload_term(kbdev); } } diff --git a/drivers/gpu/arm/valhall/device/backend/mali_kbase_device_hw_csf.c b/drivers/gpu/arm/valhall/device/backend/mali_kbase_device_hw_csf.c index 792d2efa7706..e9019425f688 100644 --- a/drivers/gpu/arm/valhall/device/backend/mali_kbase_device_hw_csf.c +++ b/drivers/gpu/arm/valhall/device/backend/mali_kbase_device_hw_csf.c @@ -82,12 +82,10 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; bool is_legacy_gpu_irq_mask = true; -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface) { power_changed_mask = MCU_STATUS_GPU_IRQ; is_legacy_gpu_irq_mask = false; } -#endif KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ, NULL, val); @@ -136,15 +134,10 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) val &= ~GPU_PROTECTED_FAULT; } -#if MALI_USE_CSF if (!kbdev->pm.backend.has_host_pwr_iface) { if (val & RESET_COMPLETED) kbase_pm_reset_done(kbdev); } -#else - if (val & RESET_COMPLETED) - kbase_pm_reset_done(kbdev); -#endif /* Defer clearing CLEAN_CACHES_COMPLETED to kbase_clean_caches_done. * We need to acquire hwaccess_lock to avoid a race condition with @@ -153,8 +146,7 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, val & ~CLEAN_CACHES_COMPLETED); kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_CLEAR), val & ~CLEAN_CACHES_COMPLETED); -#ifdef KBASE_PM_RUNTIME - if (val & DOORBELL_MIRROR) { + if (IS_ENABLED(CONFIG_PM) && (val & DOORBELL_MIRROR)) { unsigned long flags; dev_dbg(kbdev->dev, "Doorbell mirror interrupt received"); @@ -213,7 +205,6 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) } spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } -#endif /* kbase_pm_check_transitions (called by kbase_pm_power_changed) must * be called after the IRQ has been cleared. This is because it might diff --git a/drivers/gpu/arm/valhall/device/backend/mali_kbase_device_hw_jm.c b/drivers/gpu/arm/valhall/device/backend/mali_kbase_device_hw_jm.c deleted file mode 100644 index 368ffe257771..000000000000 --- a/drivers/gpu/arm/valhall/device/backend/mali_kbase_device_hw_jm.c +++ /dev/null @@ -1,104 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -/** - * kbase_report_gpu_fault - Report a GPU fault. - * @kbdev: Kbase device pointer - * @multiple: Zero if only GPU_FAULT was raised, non-zero if MULTIPLE_GPU_FAULTS - * was also set - * - * This function is called from the interrupt handler when a GPU fault occurs. - * It reports the details of the fault using dev_warn(). - */ -static void kbase_report_gpu_fault(struct kbase_device *kbdev, int multiple) -{ - u32 status = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_FAULTSTATUS)); - uintptr_t phys_addr = kbase_reg_read64(kbdev, GPU_CONTROL_ENUM(GPU_FAULTADDRESS)); - - dev_warn(kbdev->dev, "GPU Fault 0x%08x (%s) at PA 0x%pK", status, - kbase_gpu_exception_name(status & 0xFF), (void *)phys_addr); - if (multiple) - dev_warn(kbdev->dev, - "There were multiple GPU faults - some have not been reported\n"); - -} - -void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) -{ - KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ, NULL, val); - if (val & GPU_FAULT) - kbase_report_gpu_fault(kbdev, val & MULTIPLE_GPU_FAULTS); - - if (val & RESET_COMPLETED) - kbase_pm_reset_done(kbdev); - - /* Defer clearing CLEAN_CACHES_COMPLETED to kbase_clean_caches_done. - * We need to acquire hwaccess_lock to avoid a race condition with - * kbase_gpu_cache_flush_and_busy_wait - */ - KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, val & ~CLEAN_CACHES_COMPLETED); - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_CLEAR), val & ~CLEAN_CACHES_COMPLETED); - - /* kbase_instr_hwcnt_sample_done frees the HWCNT pipeline to request another - * sample. Therefore this must be called after clearing the IRQ to avoid a - * race between clearing and the next sample raising the IRQ again. - */ - if (val & PRFCNT_SAMPLE_COMPLETED) - kbase_instr_hwcnt_sample_done(kbdev); - - /* kbase_pm_check_transitions (called by kbase_pm_power_changed) must - * be called after the IRQ has been cleared. This is because it might - * trigger further power transitions and we don't want to miss the - * interrupt raised to notify us that these further transitions have - * finished. The same applies to kbase_clean_caches_done() - if another - * clean was queued, it might trigger another clean, which might - * generate another interrupt which shouldn't be missed. - */ - - if (val & CLEAN_CACHES_COMPLETED) - kbase_clean_caches_done(kbdev); - - if (val & POWER_CHANGED_ALL) { - kbase_pm_power_changed(kbdev); - } else if (val & CLEAN_CACHES_COMPLETED) { - /* If cache line evict messages can be lost when shader cores - * power down then we need to flush the L2 cache before powering - * down cores. When the flush completes, the shaders' state - * machine needs to be re-invoked to proceed with powering down - * cores. - */ - if (kbdev->pm.backend.l2_always_on || - kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TTRX_921)) - kbase_pm_power_changed(kbdev); - } - - KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_DONE, NULL, val); -} -KBASE_EXPORT_TEST_API(kbase_gpu_interrupt); diff --git a/drivers/gpu/arm/valhall/device/backend/mali_kbase_device_jm.c b/drivers/gpu/arm/valhall/device/backend/mali_kbase_device_jm.c deleted file mode 100644 index 0c96b2ba3eba..000000000000 --- a/drivers/gpu/arm/valhall/device/backend/mali_kbase_device_jm.c +++ /dev/null @@ -1,344 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) -#include -#endif - -/** - * kbase_backend_late_init - Perform any backend-specific initialization. - * @kbdev: Device pointer - * - * Return: 0 on success, or an error code on failure. - */ -static int kbase_backend_late_init(struct kbase_device *kbdev) -{ - int err; - - err = kbase_hwaccess_pm_init(kbdev); - if (err) - return err; - - err = kbase_reset_gpu_init(kbdev); - if (err) - goto fail_reset_gpu_init; - - err = kbase_hwaccess_pm_powerup(kbdev, PM_HW_ISSUES_DETECT); - if (err) - goto fail_pm_powerup; - - err = kbase_backend_timer_init(kbdev); - if (err) - goto fail_timer; - -#ifdef CONFIG_MALI_VALHALL_DEBUG -#if IS_ENABLED(CONFIG_MALI_VALHALL_REAL_HW) - if (kbase_validate_interrupts(kbdev) != 0) { - dev_err(kbdev->dev, "Interrupt validation failed.\n"); - err = -EINVAL; - goto fail_interrupt_test; - } -#endif /* IS_ENABLED(CONFIG_MALI_VALHALL_REAL_HW) */ -#endif /* CONFIG_MALI_VALHALL_DEBUG */ - - err = kbase_job_slot_init(kbdev); - if (err) - goto fail_job_slot; - - /* Do the initialisation of devfreq. - * Devfreq needs backend_timer_init() for completion of its - * initialisation and it also needs to catch the first callback - * occurrence of the runtime_suspend event for maintaining state - * coherence with the backend power management, hence needs to be - * placed before the kbase_pm_context_idle(). - */ - err = kbase_backend_devfreq_init(kbdev); - if (err) - goto fail_devfreq_init; - - /* Update gpuprops with L2_FEATURES if applicable */ - err = kbase_gpuprops_update_l2_features(kbdev); - if (err) - goto fail_update_l2_features; - - init_waitqueue_head(&kbdev->hwaccess.backend.reset_wait); - - /* Idle the GPU and/or cores, if the policy wants it to */ - kbase_pm_context_idle(kbdev); - - mutex_init(&kbdev->fw_load_lock); - - return 0; - -fail_update_l2_features: - kbase_backend_devfreq_term(kbdev); -fail_devfreq_init: - kbase_job_slot_term(kbdev); -fail_job_slot: - -#ifdef CONFIG_MALI_VALHALL_DEBUG -#if IS_ENABLED(CONFIG_MALI_VALHALL_REAL_HW) -fail_interrupt_test: -#endif /* IS_ENABLED(CONFIG_MALI_VALHALL_REAL_HW) */ -#endif /* CONFIG_MALI_VALHALL_DEBUG */ - - kbase_backend_timer_term(kbdev); -fail_timer: - kbase_pm_context_idle(kbdev); - kbase_hwaccess_pm_halt(kbdev); -fail_pm_powerup: - kbase_reset_gpu_term(kbdev); -fail_reset_gpu_init: - kbase_hwaccess_pm_term(kbdev); - - return err; -} - -/** - * kbase_backend_late_term - Perform any backend-specific termination. - * @kbdev: Device pointer - */ -static void kbase_backend_late_term(struct kbase_device *kbdev) -{ - kbase_backend_devfreq_term(kbdev); - kbase_job_slot_halt(kbdev); - kbase_job_slot_term(kbdev); - kbase_backend_timer_term(kbdev); - kbase_hwaccess_pm_halt(kbdev); - kbase_reset_gpu_term(kbdev); - kbase_hwaccess_pm_term(kbdev); -} - -/** - * kbase_device_hwcnt_watchdog_if_init - Create hardware counter watchdog - * interface. - * @kbdev: Device pointer - * Return: 0 on success, or an error code on failure. - */ -static int kbase_device_hwcnt_watchdog_if_init(struct kbase_device *kbdev) -{ - return kbase_hwcnt_watchdog_if_timer_create(&kbdev->hwcnt_watchdog_timer); -} - -/** - * kbase_device_hwcnt_watchdog_if_term - Terminate hardware counter watchdog - * interface. - * @kbdev: Device pointer - */ -static void kbase_device_hwcnt_watchdog_if_term(struct kbase_device *kbdev) -{ - kbase_hwcnt_watchdog_if_timer_destroy(&kbdev->hwcnt_watchdog_timer); -} - -/** - * kbase_device_hwcnt_backend_jm_init - Create hardware counter backend. - * @kbdev: Device pointer - * Return: 0 on success, or an error code on failure. - */ -static int kbase_device_hwcnt_backend_jm_init(struct kbase_device *kbdev) -{ - return kbase_hwcnt_backend_jm_create(kbdev, &kbdev->hwcnt_gpu_jm_backend); -} - -/** - * kbase_device_hwcnt_backend_jm_term - Terminate hardware counter backend. - * @kbdev: Device pointer - */ -static void kbase_device_hwcnt_backend_jm_term(struct kbase_device *kbdev) -{ - kbase_hwcnt_backend_jm_destroy(&kbdev->hwcnt_gpu_jm_backend); -} - -/** - * kbase_device_hwcnt_backend_jm_watchdog_init - Create hardware counter watchdog backend. - * @kbdev: Device pointer - * Return: 0 on success, or an error code on failure. - */ -static int kbase_device_hwcnt_backend_jm_watchdog_init(struct kbase_device *kbdev) -{ - const u32 timer_interval = - (kbdev->gpu_props.impl_tech == THREAD_FEATURES_IMPLEMENTATION_TECHNOLOGY_FPGA) || - (kbdev->gpu_props.impl_tech == - THREAD_FEATURES_IMPLEMENTATION_TECHNOLOGY_SOFTWARE) ? - HWCNT_BACKEND_WATCHDOG_TIMER_INTERVAL_FPGA_MS : - HWCNT_BACKEND_WATCHDOG_TIMER_INTERVAL_MS; - return kbase_hwcnt_backend_jm_watchdog_create(&kbdev->hwcnt_gpu_jm_backend, - &kbdev->hwcnt_watchdog_timer, - &kbdev->hwcnt_gpu_iface, timer_interval); -} - -/** - * kbase_device_hwcnt_backend_jm_watchdog_term - Terminate hardware counter watchdog backend. - * @kbdev: Device pointer - */ -static void kbase_device_hwcnt_backend_jm_watchdog_term(struct kbase_device *kbdev) -{ - kbase_hwcnt_backend_jm_watchdog_destroy(&kbdev->hwcnt_gpu_iface); -} - -static const struct kbase_device_init dev_init[] = { -#if !IS_ENABLED(CONFIG_MALI_VALHALL_REAL_HW) - { kbase_gpu_device_create, kbase_gpu_device_destroy, "Dummy model initialization failed" }, -#else /* !IS_ENABLED(CONFIG_MALI_VALHALL_REAL_HW) */ - { kbase_get_irqs, NULL, "IRQ search failed" }, - { registers_map, registers_unmap, "Register map failed" }, -#endif /* !IS_ENABLED(CONFIG_MALI_VALHALL_REAL_HW) */ -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) - { kbase_gpu_metrics_init, kbase_gpu_metrics_term, "GPU metrics initialization failed" }, -#endif /* IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) */ - { power_control_init, power_control_term, "Power control initialization failed" }, - { kbase_io_init, kbase_io_term, "Kbase IO initialization failed" }, - { kbase_device_io_history_init, kbase_device_io_history_term, - "Register access history initialization failed" }, - { kbase_device_early_init, kbase_device_early_term, "Early device initialization failed" }, - { kbase_backend_time_init, NULL, "Time backend initialization failed" }, - { kbase_device_misc_init, kbase_device_misc_term, - "Miscellaneous device initialization failed" }, - { kbase_device_pcm_dev_init, kbase_device_pcm_dev_term, - "Priority control manager initialization failed" }, - { kbase_ctx_sched_init, kbase_ctx_sched_term, "Context scheduler initialization failed" }, - { kbase_mem_init, kbase_mem_term, "Memory subsystem initialization failed" }, - { kbase_device_coherency_init, NULL, "Device coherency init failed" }, - { kbase_protected_mode_init, kbase_protected_mode_term, - "Protected mode subsystem initialization failed" }, - { kbase_device_list_init, kbase_device_list_term, "Device list setup failed" }, - { kbasep_js_devdata_init, kbasep_js_devdata_term, "Job JS devdata initialization failed" }, - { kbase_device_timeline_init, kbase_device_timeline_term, - "Timeline stream initialization failed" }, - { kbase_clk_rate_trace_manager_init, kbase_clk_rate_trace_manager_term, - "Clock rate trace manager initialization failed" }, - { kbase_instr_backend_init, kbase_instr_backend_term, - "Instrumentation backend initialization failed" }, - { kbase_device_hwcnt_watchdog_if_init, kbase_device_hwcnt_watchdog_if_term, - "GPU hwcnt backend watchdog interface creation failed" }, - { kbase_device_hwcnt_backend_jm_init, kbase_device_hwcnt_backend_jm_term, - "GPU hwcnt backend creation failed" }, - { kbase_device_hwcnt_backend_jm_watchdog_init, kbase_device_hwcnt_backend_jm_watchdog_term, - "GPU hwcnt watchdog backend creation failed" }, - { kbase_device_hwcnt_context_init, kbase_device_hwcnt_context_term, - "GPU hwcnt context initialization failed" }, - { kbase_device_hwcnt_virtualizer_init, kbase_device_hwcnt_virtualizer_term, - "GPU hwcnt virtualizer initialization failed" }, - { kbase_device_kinstr_prfcnt_init, kbase_device_kinstr_prfcnt_term, - "Performance counter instrumentation initialization failed" }, - { kbase_backend_late_init, kbase_backend_late_term, "Late backend initialization failed" }, - { kbase_debug_job_fault_dev_init, kbase_debug_job_fault_dev_term, - "Job fault debug initialization failed" }, - { kbase_device_debugfs_init, kbase_device_debugfs_term, "DebugFS initialization failed" }, - /* Sysfs init needs to happen before registering the device with - * misc_register(), otherwise it causes a race condition between - * registering the device and a uevent event being generated for - * userspace, causing udev rules to run which might expect certain - * sysfs attributes present. As a result of the race condition - * we avoid, some Mali sysfs entries may have appeared to udev - * to not exist. - * For more information, see - * https://www.kernel.org/doc/Documentation/driver-model/device.txt, the - * paragraph that starts with "Word of warning", currently the - * second-last paragraph. - */ - { kbase_sysfs_init, kbase_sysfs_term, "SysFS group creation failed" }, - { kbase_device_misc_register, kbase_device_misc_deregister, - "Misc device registration failed" }, - { kbase_gpuprops_populate_user_buffer, kbase_gpuprops_free_user_buffer, - "GPU property population failed" }, - { NULL, kbase_dummy_job_wa_cleanup, NULL }, - { kbase_device_late_init, kbase_device_late_term, "Late device initialization failed" }, -}; - -static void kbase_device_term_partial(struct kbase_device *kbdev, unsigned int i) -{ - while (i-- > 0) { - if (dev_init[i].term) - dev_init[i].term(kbdev); - } -} - -void kbase_device_term(struct kbase_device *kbdev) -{ - kbase_device_term_partial(kbdev, ARRAY_SIZE(dev_init)); - kbasep_js_devdata_halt(kbdev); - kbase_mem_halt(kbdev); -} - -int kbase_device_init(struct kbase_device *kbdev) -{ - int err = 0; - unsigned int i = 0; - - dev_info(kbdev->dev, "Kernel DDK version %s", MALI_RELEASE_NAME); - - kbase_device_id_init(kbdev); - kbase_disjoint_init(kbdev); - - for (i = 0; i < ARRAY_SIZE(dev_init); i++) { - if (dev_init[i].init) { - err = dev_init[i].init(kbdev); - if (err) { - if (err != -EPROBE_DEFER) - dev_err(kbdev->dev, "%s error = %d\n", dev_init[i].err_mes, - err); - kbase_device_term_partial(kbdev, i); - break; - } - } - } - - return err; -} - -int kbase_device_firmware_init_once(struct kbase_device *kbdev) -{ - int ret = 0; - - mutex_lock(&kbdev->fw_load_lock); - - if (!kbdev->dummy_job_wa_loaded) { - ret = kbase_dummy_job_wa_load(kbdev); - if (!ret) - kbdev->dummy_job_wa_loaded = true; - } - - mutex_unlock(&kbdev->fw_load_lock); - - return ret; -} diff --git a/drivers/gpu/arm/valhall/device/mali_kbase_device.c b/drivers/gpu/arm/valhall/device/mali_kbase_device.c index 8de24066f2ee..78eab00ea46b 100644 --- a/drivers/gpu/arm/valhall/device/mali_kbase_device.c +++ b/drivers/gpu/arm/valhall/device/mali_kbase_device.c @@ -53,6 +53,12 @@ #include "arbiter/mali_kbase_arbiter_pm.h" #include +static uint64_t neural_allowed_mask = UINT64_MAX; +module_param(neural_allowed_mask, ullong, 0444); +MODULE_PARM_DESC( + neural_allowed_mask, + "Additional optional bitmask to restrict which neural engine cores any CSG can enable"); + #if defined(CONFIG_DEBUG_FS) && !IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI) /* Number of register accesses for the buffer that we allocate during @@ -123,8 +129,6 @@ static void kbase_device_all_as_term(struct kbase_device *kbdev) static int pcm_prioritized_process_cb(struct notifier_block *nb, unsigned long action, void *data) { -#if MALI_USE_CSF - struct kbase_device *const kbdev = container_of(nb, struct kbase_device, pcm_prioritized_process_nb); struct pcm_prioritized_process_notifier_data *const notifier_data = data; @@ -143,8 +147,6 @@ static int pcm_prioritized_process_cb(struct notifier_block *nb, unsigned long a return ret; -#endif /* MALI_USE_CSF */ - return 0; } @@ -351,12 +353,7 @@ int kbase_device_misc_init(struct kbase_device *const kbdev) kbdev->pm.dvfs_period = DEFAULT_PM_DVFS_PERIOD; -#if MALI_USE_CSF kbdev->reset_timeout_ms = kbase_get_timeout_ms(kbdev, CSF_GPU_RESET_TIMEOUT); -#else /* MALI_USE_CSF */ - kbdev->reset_timeout_ms = JM_DEFAULT_RESET_TIMEOUT_MS; -#endif /* !MALI_USE_CSF */ - kbdev->mmu_mode = kbase_mmu_mode_get_aarch64(); mutex_init(&kbdev->kctx_list_lock); @@ -372,15 +369,9 @@ int kbase_device_misc_init(struct kbase_device *const kbdev) kbdev->oom_notifier_block.notifier_call = NULL; } -#if !MALI_USE_CSF - spin_lock_init(&kbdev->quick_reset_lock); - kbdev->quick_reset_enabled = true; - kbdev->num_of_atoms_hw_completed = 0; -#endif - -#if MALI_USE_CSF atomic_set(&kbdev->fence_signal_timeout_enabled, 1); -#endif + + kbdev->csf.neural_allowed_mask = neural_allowed_mask; return 0; @@ -405,43 +396,12 @@ void kbase_device_misc_term(struct kbase_device *kbdev) if (kbdev->oom_notifier_block.notifier_call) unregister_oom_notifier(&kbdev->oom_notifier_block); -#if MALI_USE_CSF && IS_ENABLED(CONFIG_SYNC_FILE) +#if IS_ENABLED(CONFIG_SYNC_FILE) if (atomic_read(&kbdev->live_fence_metadata) > 0) dev_warn(kbdev->dev, "Terminating Kbase device with live fence metadata!"); #endif } -#if !MALI_USE_CSF -void kbase_enable_quick_reset(struct kbase_device *kbdev) -{ - unsigned long flags; - - spin_lock_irqsave(&kbdev->quick_reset_lock, flags); - - kbdev->quick_reset_enabled = true; - kbdev->num_of_atoms_hw_completed = 0; - - spin_unlock_irqrestore(&kbdev->quick_reset_lock, flags); -} - -void kbase_disable_quick_reset(struct kbase_device *kbdev) -{ - unsigned long flags; - - spin_lock_irqsave(&kbdev->quick_reset_lock, flags); - - kbdev->quick_reset_enabled = false; - kbdev->num_of_atoms_hw_completed = 0; - - spin_unlock_irqrestore(&kbdev->quick_reset_lock, flags); -} - -bool kbase_is_quick_reset_enabled(struct kbase_device *kbdev) -{ - return kbdev->quick_reset_enabled; -} -#endif - void kbase_device_free(struct kbase_device *kbdev) { vfree(kbdev); diff --git a/drivers/gpu/arm/valhall/device/mali_kbase_device.h b/drivers/gpu/arm/valhall/device/mali_kbase_device.h index 32dff8c2ed1e..2c40fefffc04 100644 --- a/drivers/gpu/arm/valhall/device/mali_kbase_device.h +++ b/drivers/gpu/arm/valhall/device/mali_kbase_device.h @@ -102,10 +102,8 @@ void kbase_device_term(struct kbase_device *kbdev); * * Return: 0 if successful or a negative error code on failure. */ -#if MALI_USE_CSF int kbase_gpu_cache_flush_pa_range_and_busy_wait(struct kbase_device *kbdev, phys_addr_t phys, size_t nr_bytes, u32 flush_op); -#endif /* MALI_USE_CSF */ /** * kbase_gpu_cache_flush_and_busy_wait - Start a cache flush and busy wait @@ -196,7 +194,6 @@ void kbase_clean_caches_done(struct kbase_device *kbdev); */ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val); -#if MALI_USE_CSF /** * kbase_pwr_interrupt - GPU power interrupt handler * @kbdev: Kbase device pointer @@ -206,5 +203,4 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val); * handled. */ void kbase_pwr_interrupt(struct kbase_device *kbdev, u32 val); -#endif /* MALI_USE_CSF */ #endif /* _MALI_KBASE_DEVICE_H_ */ diff --git a/drivers/gpu/arm/valhall/device/mali_kbase_device_hw.c b/drivers/gpu/arm/valhall/device/mali_kbase_device_hw.c index 3bfcc5e8f54e..5cfd10afa597 100644 --- a/drivers/gpu/arm/valhall/device/mali_kbase_device_hw.c +++ b/drivers/gpu/arm/valhall/device/mali_kbase_device_hw.c @@ -44,9 +44,7 @@ static int busy_wait_cache_operation(struct kbase_device *kbdev, u32 irq_bit) bool completed = false; s64 diff; u32 irq_bits_to_check = irq_bit; -#if MALI_USE_CSF const bool has_host_pwr_iface = kbdev->pm.backend.has_host_pwr_iface; -#endif /* MALI_USE_CSF */ /* hwaccess_lock must be held to prevent concurrent threads from * cleaning the IRQ bits, otherwise it could be possible for this thread @@ -60,12 +58,8 @@ static int busy_wait_cache_operation(struct kbase_device *kbdev, u32 irq_bit) * been reset which implies that any cache flush operation has been * completed, too. */ -#if MALI_USE_CSF if (!has_host_pwr_iface) -#endif /* MALI_USE_CSF */ - { irq_bits_to_check |= RESET_COMPLETED; - } do { unsigned int i; @@ -76,7 +70,6 @@ static int busy_wait_cache_operation(struct kbase_device *kbdev, u32 irq_bit) completed = true; break; } -#if MALI_USE_CSF /* Check whether the GPU has been reset, which implies that any * cache flush operation has been completed. */ @@ -87,11 +80,8 @@ static int busy_wait_cache_operation(struct kbase_device *kbdev, u32 irq_bit) break; } } -#endif /* MALI_USE_CSF */ } - if (kbase_io_is_aw_removed(kbdev)) - return -ENODEV; diff = ktime_to_ms(ktime_sub(ktime_get_raw(), wait_loop_start)); } while ((diff < wait_time_ms) && !completed); @@ -127,8 +117,6 @@ static int busy_wait_cache_operation(struct kbase_device *kbdev, u32 irq_bit) return 0; } -#if MALI_USE_CSF - int kbase_gpu_cache_flush_pa_range_and_busy_wait(struct kbase_device *kbdev, phys_addr_t phys, size_t nr_bytes, u32 flush_op) { @@ -137,6 +125,7 @@ int kbase_gpu_cache_flush_pa_range_and_busy_wait(struct kbase_device *kbdev, phy lockdep_assert_held(&kbdev->hwaccess_lock); + /* 1. Clear the interrupt FLUSH_PA_RANGE_COMPLETED bit. */ kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_CLEAR), FLUSH_PA_RANGE_COMPLETED); @@ -153,7 +142,6 @@ int kbase_gpu_cache_flush_pa_range_and_busy_wait(struct kbase_device *kbdev, phy return ret; } -#endif /* MALI_USE_CSF */ int kbase_gpu_cache_flush_and_busy_wait(struct kbase_device *kbdev, u32 flush_op) { @@ -165,6 +153,7 @@ int kbase_gpu_cache_flush_and_busy_wait(struct kbase_device *kbdev, u32 flush_op */ lockdep_assert_held(&kbdev->hwaccess_lock); + /* 1. Check if kbdev->cache_clean_in_progress is set. * If it is set, it means there are threads waiting for * CLEAN_CACHES_COMPLETED irq to be raised and that the @@ -218,6 +207,7 @@ void kbase_gpu_start_cache_clean_nolock(struct kbase_device *kbdev, u32 flush_op lockdep_assert_held(&kbdev->hwaccess_lock); + if (kbdev->cache_clean_in_progress) { /* If this is called while another clean is in progress, we * can't rely on the current one to flush any new changes in diff --git a/drivers/gpu/arm/valhall/gpu/Kbuild b/drivers/gpu/arm/valhall/gpu/Kbuild index 772182586a24..cffa4edad1d8 100644 --- a/drivers/gpu/arm/valhall/gpu/Kbuild +++ b/drivers/gpu/arm/valhall/gpu/Kbuild @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2012, 2020-2021 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2012, 2020-2021, 2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -19,9 +19,4 @@ # valhall_kbase-y += gpu/mali_kbase_gpu.o - -ifeq ($(CONFIG_MALI_VALHALL_CSF_SUPPORT),y) - valhall_kbase-y += gpu/backend/mali_kbase_gpu_fault_csf.o -else - valhall_kbase-y += gpu/backend/mali_kbase_gpu_fault_jm.o -endif +valhall_kbase-y += gpu/backend/mali_kbase_gpu_fault_csf.o diff --git a/drivers/gpu/arm/valhall/gpu/backend/mali_kbase_gpu_fault_jm.c b/drivers/gpu/arm/valhall/gpu/backend/mali_kbase_gpu_fault_jm.c deleted file mode 100644 index 7f3743ca6432..000000000000 --- a/drivers/gpu/arm/valhall/gpu/backend/mali_kbase_gpu_fault_jm.c +++ /dev/null @@ -1,176 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2019-2022 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include - -#include - -const char *kbase_gpu_exception_name(u32 const exception_code) -{ - const char *e; - - switch (exception_code) { - /* Non-Fault Status code */ - case 0x00: - e = "NOT_STARTED/IDLE/OK"; - break; - case 0x01: - e = "DONE"; - break; - case 0x02: - e = "INTERRUPTED"; - break; - case 0x03: - e = "STOPPED"; - break; - case 0x04: - e = "TERMINATED"; - break; - case 0x08: - e = "ACTIVE"; - break; - /* Job exceptions */ - case 0x40: - e = "JOB_CONFIG_FAULT"; - break; - case 0x41: - e = "JOB_POWER_FAULT"; - break; - case 0x42: - e = "JOB_READ_FAULT"; - break; - case 0x43: - e = "JOB_WRITE_FAULT"; - break; - case 0x44: - e = "JOB_AFFINITY_FAULT"; - break; - case 0x48: - e = "JOB_BUS_FAULT"; - break; - case 0x50: - e = "INSTR_INVALID_PC"; - break; - case 0x51: - e = "INSTR_INVALID_ENC"; - break; - case 0x52: - e = "INSTR_TYPE_MISMATCH"; - break; - case 0x53: - e = "INSTR_OPERAND_FAULT"; - break; - case 0x54: - e = "INSTR_TLS_FAULT"; - break; - case 0x55: - e = "INSTR_BARRIER_FAULT"; - break; - case 0x56: - e = "INSTR_ALIGN_FAULT"; - break; - case 0x58: - e = "DATA_INVALID_FAULT"; - break; - case 0x59: - e = "TILE_RANGE_FAULT"; - break; - case 0x5A: - e = "ADDR_RANGE_FAULT"; - break; - case 0x60: - e = "OUT_OF_MEMORY"; - break; - /* GPU exceptions */ - case 0x80: - e = "DELAYED_BUS_FAULT"; - break; - case 0x88: - e = "SHAREABILITY_FAULT"; - break; - /* MMU exceptions */ - case 0xC0: - case 0xC1: - case 0xC2: - case 0xC3: - case 0xC4: - case 0xC5: - case 0xC6: - case 0xC7: - e = "TRANSLATION_FAULT"; - break; - case 0xC8: - case 0xC9: - case 0xCA: - case 0xCB: - case 0xCC: - case 0xCD: - case 0xCE: - case 0xCF: - e = "PERMISSION_FAULT"; - break; - case 0xD0: - case 0xD1: - case 0xD2: - case 0xD3: - case 0xD4: - case 0xD5: - case 0xD6: - case 0xD7: - e = "TRANSTAB_BUS_FAULT"; - break; - case 0xD8: - case 0xD9: - case 0xDA: - case 0xDB: - case 0xDC: - case 0xDD: - case 0xDE: - case 0xDF: - e = "ACCESS_FLAG"; - break; - case 0xE0: - case 0xE1: - case 0xE2: - case 0xE3: - case 0xE4: - case 0xE5: - case 0xE6: - case 0xE7: - e = "ADDRESS_SIZE_FAULT"; - break; - case 0xE8: - case 0xE9: - case 0xEA: - case 0xEB: - case 0xEC: - case 0xED: - case 0xEE: - case 0xEF: - e = "MEMORY_ATTRIBUTES_FAULT"; - break; - default: - e = "UNKNOWN"; - break; - } - - return e; -} diff --git a/drivers/gpu/arm/valhall/hw_access/Kbuild b/drivers/gpu/arm/valhall/hw_access/Kbuild index f3bbfc8bafe2..0d3d45fb0900 100644 --- a/drivers/gpu/arm/valhall/hw_access/Kbuild +++ b/drivers/gpu/arm/valhall/hw_access/Kbuild @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2023 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -19,12 +19,7 @@ # valhall_kbase-y += hw_access/mali_kbase_hw_access.o - -ifeq ($(CONFIG_MALI_VALHALL_CSF_SUPPORT),y) - valhall_kbase-y += hw_access/regmap/mali_kbase_regmap_csf.o -else - valhall_kbase-y += hw_access/regmap/mali_kbase_regmap_jm.o -endif +valhall_kbase-y += hw_access/regmap/mali_kbase_regmap_csf.o ifeq ($(CONFIG_MALI_VALHALL_REAL_HW), y) valhall_kbase-y += hw_access/backend/mali_kbase_hw_access_real_hw.o diff --git a/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access.c b/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access.c index f06a62b607c6..45fb2e65640d 100644 --- a/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access.c +++ b/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access.c @@ -33,19 +33,6 @@ static u32 always_powered_regs[] = { -#if !MALI_USE_CSF - PTM_AW_IRQ_CLEAR, - PTM_AW_IRQ_INJECTION, - PTM_AW_IRQ_MASK, - PTM_AW_IRQ_RAWSTAT, - PTM_AW_IRQ_STATUS, - PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE0, - PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE1, - PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE0, - PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE1, - PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE_STATUS, - PTM_ID, -#endif /* !MALI_USE_CSF */ }; static void kbasep_reg_setup_always_powered_registers(struct kbase_device *kbdev) @@ -53,11 +40,6 @@ static void kbasep_reg_setup_always_powered_registers(struct kbase_device *kbdev u32 i; -#if !MALI_USE_CSF - if (kbdev->gpu_props.gpu_id.arch_id < GPU_ID_ARCH_MAKE(9, 14, 0)) - return; -#endif /* MALI_USE_CSF */ - for (i = 0; i < ARRAY_SIZE(always_powered_regs); i++) { u32 reg_enum = always_powered_regs[i]; diff --git a/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access.h b/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access.h index 1e9f016e2969..3aa50b0770a3 100644 --- a/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access.h +++ b/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access.h @@ -270,9 +270,7 @@ static inline u32 kbase_reg_gpu_irq_all(bool is_legacy) u32 mask = GPU_IRQ_REG_COMMON; if (is_legacy) { -#if MALI_USE_CSF mask |= (RESET_COMPLETED | POWER_CHANGED_ALL); -#endif /* MALI_USE_CSF */ /* Include POWER_CHANGED_SINGLE in debug builds for use in irq latency test. */ if (IS_ENABLED(CONFIG_MALI_VALHALL_DEBUG)) mask |= POWER_CHANGED_SINGLE; diff --git a/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access_regmap.h b/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access_regmap.h index c10a32de1e91..bf00919e720a 100644 --- a/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access_regmap.h +++ b/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access_regmap.h @@ -22,13 +22,8 @@ #ifndef _MALI_KBASE_HW_ACCESS_REGMAP_H_ #define _MALI_KBASE_HW_ACCESS_REGMAP_H_ -#if MALI_USE_CSF #include "regmap/mali_kbase_regmap_csf_enums.h" #include "regmap/mali_kbase_regmap_csf_macros.h" -#else -#include "regmap/mali_kbase_regmap_jm_enums.h" -#include "regmap/mali_kbase_regmap_jm_macros.h" -#endif /* GPU_U definition */ #ifdef __ASSEMBLER__ diff --git a/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access_regmap_legacy.h b/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access_regmap_legacy.h index 1722428a7904..d4b0b6ee86d3 100644 --- a/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access_regmap_legacy.h +++ b/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access_regmap_legacy.h @@ -22,11 +22,7 @@ #ifndef _MALI_KBASE_REGMAP_LEGACY_H_ #define _MALI_KBASE_REGMAP_LEGACY_H_ -#if MALI_USE_CSF #include "regmap/mali_kbase_regmap_legacy_csf.h" -#else -#include "regmap/mali_kbase_regmap_legacy_jm.h" -#endif /* Begin Register Offsets */ diff --git a/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_csf_enums.h b/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_csf_enums.h index a912c1fd095a..300781384fe9 100644 --- a/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_csf_enums.h +++ b/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_csf_enums.h @@ -27,10 +27,6 @@ #ifndef _MALI_KBASE_REGMAP_CSF_ENUMS_H_ #define _MALI_KBASE_REGMAP_CSF_ENUMS_H_ -#if !MALI_USE_CSF -#error "Cannot be compiled with JM" -#endif - enum kbase_regmap_enum_v10_8 { GPU_CONTROL__GPU_ID = 0, /* (RO) 32-bit 0x0 */ GPU_CONTROL__L2_FEATURES, /* (RO) 32-bit 0x4 */ diff --git a/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_csf_macros.h b/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_csf_macros.h index afdd0dfd44c2..121312acef24 100644 --- a/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_csf_macros.h +++ b/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_csf_macros.h @@ -22,10 +22,6 @@ #ifndef _MALI_KBASE_REGMAP_CSF_MACROS_H_ #define _MALI_KBASE_REGMAP_CSF_MACROS_H_ -#if !MALI_USE_CSF -#error "Cannot be compiled with JM" -#endif - #define ENUM_OFFSET(_index, _base, _next) (_base + _index * (_next - _base)) diff --git a/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_jm.c b/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_jm.c deleted file mode 100644 index 819fae7ed398..000000000000 --- a/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_jm.c +++ /dev/null @@ -1,3041 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * This file is autogenerated. Avoid modifying this file unless absolutely - * necessary. - */ - -#include -#include - -#include -#include "mali_kbase_regmap_jm_enums.h" -#include - -static void kbase_regmap_v6_0_init(struct kbase_device *kbdev) -{ - if (kbdev->regmap.regs == NULL && kbdev->regmap.flags == NULL) { - kbdev->regmap.size = NR_V6_0_REGS; - kbdev->regmap.regs = - kcalloc(kbdev->regmap.size, sizeof(void __iomem *), GFP_KERNEL); - kbdev->regmap.flags = kcalloc(kbdev->regmap.size, sizeof(u32), GFP_KERNEL); - } - - if (WARN_ON(kbdev->regmap.regs == NULL)) - return; - if (WARN_ON(kbdev->regmap.flags == NULL)) - return; - - kbdev->regmap.flags[GPU_CONTROL__GPU_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__L2_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__TILER_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__MEM_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__MMU_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__AS_PRESENT] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS_PRESENT] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__GPU_IRQ_RAWSTAT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__GPU_IRQ_CLEAR] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__GPU_IRQ_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__GPU_IRQ_STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__GPU_COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__GPU_STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__LATEST_FLUSH] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__GPU_FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__GPU_FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__AFBC_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__PWR_KEY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__PWR_OVERRIDE0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__PWR_OVERRIDE1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__PRFCNT_BASE] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__PRFCNT_CONFIG] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__PRFCNT_JM_EN] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__PRFCNT_SHADER_EN] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__PRFCNT_TILER_EN] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__PRFCNT_MMU_L2_EN] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__CYCLE_COUNT] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__TIMESTAMP] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__THREAD_MAX_THREADS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__THREAD_MAX_WORKGROUP_SIZE] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__THREAD_MAX_BARRIER_SIZE] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__THREAD_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__TEXTURE_FEATURES_0] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__TEXTURE_FEATURES_1] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__TEXTURE_FEATURES_2] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS0_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS1_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS2_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS3_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS4_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS5_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS6_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS7_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS8_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS9_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS10_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS11_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS12_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS13_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS14_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS15_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__SHADER_PRESENT] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__TILER_PRESENT] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__L2_PRESENT] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__SHADER_READY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__TILER_READY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__L2_READY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__SHADER_PWRON] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__TILER_PWRON] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__L2_PWRON] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__SHADER_PWROFF] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__TILER_PWROFF] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__L2_PWROFF] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__SHADER_PWRTRANS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__TILER_PWRTRANS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__L2_PWRTRANS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__SHADER_PWRACTIVE] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__TILER_PWRACTIVE] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__L2_PWRACTIVE] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__COHERENCY_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__COHERENCY_ENABLE] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__GPU_USER_INn] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__GPU_USER_OUTn] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__JM_CONFIG] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__SHADER_CONFIG] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__TILER_CONFIG] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__L2_MMU_CONFIG] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JOB_IRQ_RAWSTAT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JOB_IRQ_CLEAR] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JOB_IRQ_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JOB_IRQ_STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JOB_IRQ_JS_STATE] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JOB_IRQ_THROTTLE] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__EVENT_IRQ_RAWSTAT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__EVENT_IRQ_CLEAR] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__EVENT_IRQ_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__EVENT_IRQ_STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS0__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS0__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS0__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS0__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS0__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS0__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS0__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS0__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS0__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS0__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS0__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS0__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS0__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS0__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS0__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS0__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS0__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS0__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS0__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS1__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS1__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS1__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS1__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS1__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS1__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS1__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS1__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS1__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS1__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS1__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS1__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS1__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS1__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS1__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS1__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS1__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS1__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS1__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS2__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS2__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS2__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS2__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS2__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS2__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS2__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS2__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS2__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS2__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS2__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS2__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS2__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS2__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS2__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS2__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS2__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS2__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS2__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS3__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS3__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS3__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS3__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS3__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS3__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS3__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS3__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS3__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS3__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS3__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS3__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS3__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS3__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS3__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS3__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS3__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS3__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS3__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS4__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS4__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS4__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS4__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS4__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS4__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS4__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS4__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS4__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS4__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS4__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS4__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS4__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS4__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS4__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS4__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS4__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS4__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS4__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS5__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS5__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS5__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS5__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS5__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS5__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS5__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS5__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS5__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS5__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS5__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS5__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS5__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS5__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS5__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS5__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS5__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS5__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS5__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS6__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS6__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS6__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS6__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS6__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS6__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS6__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS6__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS6__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS6__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS6__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS6__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS6__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS6__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS6__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS6__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS6__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS6__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS6__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS7__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS7__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS7__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS7__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS7__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS7__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS7__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS7__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS7__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS7__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS7__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS7__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS7__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS7__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS7__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS7__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS7__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS7__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS7__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS8__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS8__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS8__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS8__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS8__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS8__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS8__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS8__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS8__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS8__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS8__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS8__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS8__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS8__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS8__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS8__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS8__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS8__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS8__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS9__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS9__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS9__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS9__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS9__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS9__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS9__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS9__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS9__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS9__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS9__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS9__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS9__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS9__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS9__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS9__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS9__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS9__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS9__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS10__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS10__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS10__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS10__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS10__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS10__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS10__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS10__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS10__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS10__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS10__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS10__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS10__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS10__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS10__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS10__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS10__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS10__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS10__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS11__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS11__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS11__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS11__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS11__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS11__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS11__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS11__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS11__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS11__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS11__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS11__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS11__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS11__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS11__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS11__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS11__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS11__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS11__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS12__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS12__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS12__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS12__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS12__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS12__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS12__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS12__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS12__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS12__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS12__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS12__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS12__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS12__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS12__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS12__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS12__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS12__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS12__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS13__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS13__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS13__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS13__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS13__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS13__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS13__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS13__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS13__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS13__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS13__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS13__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS13__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS13__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS13__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS13__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS13__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS13__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS13__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS14__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS14__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS14__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS14__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS14__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS14__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS14__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS14__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS14__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS14__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS14__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS14__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS14__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS14__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS14__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS14__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS14__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS14__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS14__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS15__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS15__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS15__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS15__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS15__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS15__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS15__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS15__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS15__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS15__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS15__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS15__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS15__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS15__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS15__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS15__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS15__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS15__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS15__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__IRQ_RAWSTAT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__IRQ_CLEAR] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__IRQ_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__IRQ_STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS0__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS0__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS0__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS0__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS0__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS0__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS0__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS0__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS0__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS1__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS1__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS1__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS1__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS1__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS1__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS1__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS1__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS1__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS2__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS2__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS2__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS2__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS2__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS2__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS2__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS2__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS2__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS3__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS3__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS3__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS3__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS3__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS3__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS3__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS3__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS3__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS4__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS4__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS4__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS4__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS4__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS4__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS4__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS4__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS4__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS5__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS5__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS5__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS5__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS5__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS5__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS5__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS5__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS5__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS6__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS6__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS6__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS6__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS6__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS6__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS6__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS6__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS6__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS7__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS7__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS7__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS7__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS7__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS7__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS7__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS7__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS7__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS8__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS8__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS8__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS8__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS8__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS8__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS8__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS8__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS8__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS9__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS9__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS9__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS9__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS9__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS9__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS9__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS9__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS9__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS10__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS10__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS10__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS10__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS10__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS10__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS10__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS10__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS10__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS11__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS11__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS11__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS11__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS11__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS11__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS11__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS11__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS11__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS12__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS12__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS12__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS12__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS12__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS12__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS12__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS12__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS12__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS13__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS13__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS13__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS13__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS13__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS13__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS13__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS13__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS13__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS14__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS14__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS14__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS14__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS14__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS14__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS14__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS14__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS14__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS15__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS15__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS15__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS15__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS15__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS15__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS15__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS15__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS15__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__IRQ_RAWSTAT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__IRQ_CLEAR] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__IRQ_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__IRQ_STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS0__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS0__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS0__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS0__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS0__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS0__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS0__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS0__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS0__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS1__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS1__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS1__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS1__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS1__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS1__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS1__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS1__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS1__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS2__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS2__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS2__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS2__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS2__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS2__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS2__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS2__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS2__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS3__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS3__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS3__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS3__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS3__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS3__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS3__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS3__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS3__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS4__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS4__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS4__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS4__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS4__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS4__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS4__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS4__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS4__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS5__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS5__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS5__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS5__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS5__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS5__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS5__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS5__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS5__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS6__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS6__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS6__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS6__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS6__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS6__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS6__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS6__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS6__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS7__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS7__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS7__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS7__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS7__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS7__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS7__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS7__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS7__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS8__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS8__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS8__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS8__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS8__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS8__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS8__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS8__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS8__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS9__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS9__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS9__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS9__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS9__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS9__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS9__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS9__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS9__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS10__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS10__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS10__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS10__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS10__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS10__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS10__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS10__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS10__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS11__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS11__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS11__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS11__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS11__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS11__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS11__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS11__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS11__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS12__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS12__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS12__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS12__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS12__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS12__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS12__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS12__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS12__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS13__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS13__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS13__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS13__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS13__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS13__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS13__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS13__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS13__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS14__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS14__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS14__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS14__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS14__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS14__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS14__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS14__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS14__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS15__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS15__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS15__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS15__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS15__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS15__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS15__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS15__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS15__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - - kbdev->regmap.regs[GPU_CONTROL__GPU_ID] = kbdev->reg + 0x0; - kbdev->regmap.regs[GPU_CONTROL__L2_FEATURES] = kbdev->reg + 0x4; - kbdev->regmap.regs[GPU_CONTROL__TILER_FEATURES] = kbdev->reg + 0xc; - kbdev->regmap.regs[GPU_CONTROL__MEM_FEATURES] = kbdev->reg + 0x10; - kbdev->regmap.regs[GPU_CONTROL__MMU_FEATURES] = kbdev->reg + 0x14; - kbdev->regmap.regs[GPU_CONTROL__AS_PRESENT] = kbdev->reg + 0x18; - kbdev->regmap.regs[GPU_CONTROL__JS_PRESENT] = kbdev->reg + 0x1c; - kbdev->regmap.regs[GPU_CONTROL__GPU_IRQ_RAWSTAT] = kbdev->reg + 0x20; - kbdev->regmap.regs[GPU_CONTROL__GPU_IRQ_CLEAR] = kbdev->reg + 0x24; - kbdev->regmap.regs[GPU_CONTROL__GPU_IRQ_MASK] = kbdev->reg + 0x28; - kbdev->regmap.regs[GPU_CONTROL__GPU_IRQ_STATUS] = kbdev->reg + 0x2c; - kbdev->regmap.regs[GPU_CONTROL__GPU_COMMAND] = kbdev->reg + 0x30; - kbdev->regmap.regs[GPU_CONTROL__GPU_STATUS] = kbdev->reg + 0x34; - kbdev->regmap.regs[GPU_CONTROL__LATEST_FLUSH] = kbdev->reg + 0x38; - kbdev->regmap.regs[GPU_CONTROL__GPU_FAULTSTATUS] = kbdev->reg + 0x3c; - kbdev->regmap.regs[GPU_CONTROL__GPU_FAULTADDRESS] = kbdev->reg + 0x40; - kbdev->regmap.regs[GPU_CONTROL__AFBC_FEATURES] = kbdev->reg + 0x4c; - kbdev->regmap.regs[GPU_CONTROL__PWR_KEY] = kbdev->reg + 0x50; - kbdev->regmap.regs[GPU_CONTROL__PWR_OVERRIDE0] = kbdev->reg + 0x54; - kbdev->regmap.regs[GPU_CONTROL__PWR_OVERRIDE1] = kbdev->reg + 0x58; - kbdev->regmap.regs[GPU_CONTROL__PRFCNT_BASE] = kbdev->reg + 0x60; - kbdev->regmap.regs[GPU_CONTROL__PRFCNT_CONFIG] = kbdev->reg + 0x68; - kbdev->regmap.regs[GPU_CONTROL__PRFCNT_JM_EN] = kbdev->reg + 0x6c; - kbdev->regmap.regs[GPU_CONTROL__PRFCNT_SHADER_EN] = kbdev->reg + 0x70; - kbdev->regmap.regs[GPU_CONTROL__PRFCNT_TILER_EN] = kbdev->reg + 0x74; - kbdev->regmap.regs[GPU_CONTROL__PRFCNT_MMU_L2_EN] = kbdev->reg + 0x7c; - kbdev->regmap.regs[GPU_CONTROL__CYCLE_COUNT] = kbdev->reg + 0x90; - kbdev->regmap.regs[GPU_CONTROL__TIMESTAMP] = kbdev->reg + 0x98; - kbdev->regmap.regs[GPU_CONTROL__THREAD_MAX_THREADS] = kbdev->reg + 0xa0; - kbdev->regmap.regs[GPU_CONTROL__THREAD_MAX_WORKGROUP_SIZE] = kbdev->reg + 0xa4; - kbdev->regmap.regs[GPU_CONTROL__THREAD_MAX_BARRIER_SIZE] = kbdev->reg + 0xa8; - kbdev->regmap.regs[GPU_CONTROL__THREAD_FEATURES] = kbdev->reg + 0xac; - kbdev->regmap.regs[GPU_CONTROL__TEXTURE_FEATURES_0] = kbdev->reg + 0xb0; - kbdev->regmap.regs[GPU_CONTROL__TEXTURE_FEATURES_1] = kbdev->reg + 0xb4; - kbdev->regmap.regs[GPU_CONTROL__TEXTURE_FEATURES_2] = kbdev->reg + 0xb8; - kbdev->regmap.regs[GPU_CONTROL__JS0_FEATURES] = kbdev->reg + 0xc0; - kbdev->regmap.regs[GPU_CONTROL__JS1_FEATURES] = kbdev->reg + 0xc4; - kbdev->regmap.regs[GPU_CONTROL__JS2_FEATURES] = kbdev->reg + 0xc8; - kbdev->regmap.regs[GPU_CONTROL__JS3_FEATURES] = kbdev->reg + 0xcc; - kbdev->regmap.regs[GPU_CONTROL__JS4_FEATURES] = kbdev->reg + 0xd0; - kbdev->regmap.regs[GPU_CONTROL__JS5_FEATURES] = kbdev->reg + 0xd4; - kbdev->regmap.regs[GPU_CONTROL__JS6_FEATURES] = kbdev->reg + 0xd8; - kbdev->regmap.regs[GPU_CONTROL__JS7_FEATURES] = kbdev->reg + 0xdc; - kbdev->regmap.regs[GPU_CONTROL__JS8_FEATURES] = kbdev->reg + 0xe0; - kbdev->regmap.regs[GPU_CONTROL__JS9_FEATURES] = kbdev->reg + 0xe4; - kbdev->regmap.regs[GPU_CONTROL__JS10_FEATURES] = kbdev->reg + 0xe8; - kbdev->regmap.regs[GPU_CONTROL__JS11_FEATURES] = kbdev->reg + 0xec; - kbdev->regmap.regs[GPU_CONTROL__JS12_FEATURES] = kbdev->reg + 0xf0; - kbdev->regmap.regs[GPU_CONTROL__JS13_FEATURES] = kbdev->reg + 0xf4; - kbdev->regmap.regs[GPU_CONTROL__JS14_FEATURES] = kbdev->reg + 0xf8; - kbdev->regmap.regs[GPU_CONTROL__JS15_FEATURES] = kbdev->reg + 0xfc; - kbdev->regmap.regs[GPU_CONTROL__SHADER_PRESENT] = kbdev->reg + 0x100; - kbdev->regmap.regs[GPU_CONTROL__TILER_PRESENT] = kbdev->reg + 0x110; - kbdev->regmap.regs[GPU_CONTROL__L2_PRESENT] = kbdev->reg + 0x120; - kbdev->regmap.regs[GPU_CONTROL__SHADER_READY] = kbdev->reg + 0x140; - kbdev->regmap.regs[GPU_CONTROL__TILER_READY] = kbdev->reg + 0x150; - kbdev->regmap.regs[GPU_CONTROL__L2_READY] = kbdev->reg + 0x160; - kbdev->regmap.regs[GPU_CONTROL__SHADER_PWRON] = kbdev->reg + 0x180; - kbdev->regmap.regs[GPU_CONTROL__TILER_PWRON] = kbdev->reg + 0x190; - kbdev->regmap.regs[GPU_CONTROL__L2_PWRON] = kbdev->reg + 0x1a0; - kbdev->regmap.regs[GPU_CONTROL__SHADER_PWROFF] = kbdev->reg + 0x1c0; - kbdev->regmap.regs[GPU_CONTROL__TILER_PWROFF] = kbdev->reg + 0x1d0; - kbdev->regmap.regs[GPU_CONTROL__L2_PWROFF] = kbdev->reg + 0x1e0; - kbdev->regmap.regs[GPU_CONTROL__SHADER_PWRTRANS] = kbdev->reg + 0x200; - kbdev->regmap.regs[GPU_CONTROL__TILER_PWRTRANS] = kbdev->reg + 0x210; - kbdev->regmap.regs[GPU_CONTROL__L2_PWRTRANS] = kbdev->reg + 0x220; - kbdev->regmap.regs[GPU_CONTROL__SHADER_PWRACTIVE] = kbdev->reg + 0x240; - kbdev->regmap.regs[GPU_CONTROL__TILER_PWRACTIVE] = kbdev->reg + 0x250; - kbdev->regmap.regs[GPU_CONTROL__L2_PWRACTIVE] = kbdev->reg + 0x260; - kbdev->regmap.regs[GPU_CONTROL__COHERENCY_FEATURES] = kbdev->reg + 0x300; - kbdev->regmap.regs[GPU_CONTROL__COHERENCY_ENABLE] = kbdev->reg + 0x304; - kbdev->regmap.regs[GPU_CONTROL__GPU_USER_INn] = kbdev->reg + 0x400; - kbdev->regmap.regs[GPU_CONTROL__GPU_USER_OUTn] = kbdev->reg + 0x500; - kbdev->regmap.regs[GPU_CONTROL__JM_CONFIG] = kbdev->reg + 0xf00; - kbdev->regmap.regs[GPU_CONTROL__SHADER_CONFIG] = kbdev->reg + 0xf04; - kbdev->regmap.regs[GPU_CONTROL__TILER_CONFIG] = kbdev->reg + 0xf08; - kbdev->regmap.regs[GPU_CONTROL__L2_MMU_CONFIG] = kbdev->reg + 0xf0c; - kbdev->regmap.regs[JOB_CONTROL__JOB_IRQ_RAWSTAT] = kbdev->reg + 0x1000; - kbdev->regmap.regs[JOB_CONTROL__JOB_IRQ_CLEAR] = kbdev->reg + 0x1004; - kbdev->regmap.regs[JOB_CONTROL__JOB_IRQ_MASK] = kbdev->reg + 0x1008; - kbdev->regmap.regs[JOB_CONTROL__JOB_IRQ_STATUS] = kbdev->reg + 0x100c; - kbdev->regmap.regs[JOB_CONTROL__JOB_IRQ_JS_STATE] = kbdev->reg + 0x1010; - kbdev->regmap.regs[JOB_CONTROL__JOB_IRQ_THROTTLE] = kbdev->reg + 0x1014; - kbdev->regmap.regs[JOB_CONTROL__EVENT_IRQ_RAWSTAT] = kbdev->reg + 0x1020; - kbdev->regmap.regs[JOB_CONTROL__EVENT_IRQ_CLEAR] = kbdev->reg + 0x1024; - kbdev->regmap.regs[JOB_CONTROL__EVENT_IRQ_MASK] = kbdev->reg + 0x1028; - kbdev->regmap.regs[JOB_CONTROL__EVENT_IRQ_STATUS] = kbdev->reg + 0x102c; - kbdev->regmap.regs[JOB_CONTROL__JS0__HEAD] = kbdev->reg + 0x1800; - kbdev->regmap.regs[JOB_CONTROL__JS0__TAIL] = kbdev->reg + 0x1808; - kbdev->regmap.regs[JOB_CONTROL__JS0__AFFINITY] = kbdev->reg + 0x1810; - kbdev->regmap.regs[JOB_CONTROL__JS0__CONFIG] = kbdev->reg + 0x1818; - kbdev->regmap.regs[JOB_CONTROL__JS0__XAFFINITY] = kbdev->reg + 0x181c; - kbdev->regmap.regs[JOB_CONTROL__JS0__COMMAND] = kbdev->reg + 0x1820; - kbdev->regmap.regs[JOB_CONTROL__JS0__STATUS] = kbdev->reg + 0x1824; - kbdev->regmap.regs[JOB_CONTROL__JS0__FLUSH_ID] = kbdev->reg + 0x1830; - kbdev->regmap.regs[JOB_CONTROL__JS0__EVENT_0] = kbdev->reg + 0x1834; - kbdev->regmap.regs[JOB_CONTROL__JS0__EVENT_1] = kbdev->reg + 0x1838; - kbdev->regmap.regs[JOB_CONTROL__JS0__EVENT_MASK] = kbdev->reg + 0x183c; - kbdev->regmap.regs[JOB_CONTROL__JS0__HEAD_NEXT] = kbdev->reg + 0x1840; - kbdev->regmap.regs[JOB_CONTROL__JS0__TAIL_NEXT] = kbdev->reg + 0x1848; - kbdev->regmap.regs[JOB_CONTROL__JS0__AFFINITY_NEXT] = kbdev->reg + 0x1850; - kbdev->regmap.regs[JOB_CONTROL__JS0__CONFIG_NEXT] = kbdev->reg + 0x1858; - kbdev->regmap.regs[JOB_CONTROL__JS0__XAFFINITY_NEXT] = kbdev->reg + 0x185c; - kbdev->regmap.regs[JOB_CONTROL__JS0__COMMAND_NEXT] = kbdev->reg + 0x1860; - kbdev->regmap.regs[JOB_CONTROL__JS0__FLUSH_ID_NEXT] = kbdev->reg + 0x1870; - kbdev->regmap.regs[JOB_CONTROL__JS0__EVENT_MASK_NEXT] = kbdev->reg + 0x187c; - kbdev->regmap.regs[JOB_CONTROL__JS1__HEAD] = kbdev->reg + 0x1880; - kbdev->regmap.regs[JOB_CONTROL__JS1__TAIL] = kbdev->reg + 0x1888; - kbdev->regmap.regs[JOB_CONTROL__JS1__AFFINITY] = kbdev->reg + 0x1890; - kbdev->regmap.regs[JOB_CONTROL__JS1__CONFIG] = kbdev->reg + 0x1898; - kbdev->regmap.regs[JOB_CONTROL__JS1__XAFFINITY] = kbdev->reg + 0x189c; - kbdev->regmap.regs[JOB_CONTROL__JS1__COMMAND] = kbdev->reg + 0x18a0; - kbdev->regmap.regs[JOB_CONTROL__JS1__STATUS] = kbdev->reg + 0x18a4; - kbdev->regmap.regs[JOB_CONTROL__JS1__FLUSH_ID] = kbdev->reg + 0x18b0; - kbdev->regmap.regs[JOB_CONTROL__JS1__EVENT_0] = kbdev->reg + 0x18b4; - kbdev->regmap.regs[JOB_CONTROL__JS1__EVENT_1] = kbdev->reg + 0x18b8; - kbdev->regmap.regs[JOB_CONTROL__JS1__EVENT_MASK] = kbdev->reg + 0x18bc; - kbdev->regmap.regs[JOB_CONTROL__JS1__HEAD_NEXT] = kbdev->reg + 0x18c0; - kbdev->regmap.regs[JOB_CONTROL__JS1__TAIL_NEXT] = kbdev->reg + 0x18c8; - kbdev->regmap.regs[JOB_CONTROL__JS1__AFFINITY_NEXT] = kbdev->reg + 0x18d0; - kbdev->regmap.regs[JOB_CONTROL__JS1__CONFIG_NEXT] = kbdev->reg + 0x18d8; - kbdev->regmap.regs[JOB_CONTROL__JS1__XAFFINITY_NEXT] = kbdev->reg + 0x18dc; - kbdev->regmap.regs[JOB_CONTROL__JS1__COMMAND_NEXT] = kbdev->reg + 0x18e0; - kbdev->regmap.regs[JOB_CONTROL__JS1__FLUSH_ID_NEXT] = kbdev->reg + 0x18f0; - kbdev->regmap.regs[JOB_CONTROL__JS1__EVENT_MASK_NEXT] = kbdev->reg + 0x18fc; - kbdev->regmap.regs[JOB_CONTROL__JS2__HEAD] = kbdev->reg + 0x1900; - kbdev->regmap.regs[JOB_CONTROL__JS2__TAIL] = kbdev->reg + 0x1908; - kbdev->regmap.regs[JOB_CONTROL__JS2__AFFINITY] = kbdev->reg + 0x1910; - kbdev->regmap.regs[JOB_CONTROL__JS2__CONFIG] = kbdev->reg + 0x1918; - kbdev->regmap.regs[JOB_CONTROL__JS2__XAFFINITY] = kbdev->reg + 0x191c; - kbdev->regmap.regs[JOB_CONTROL__JS2__COMMAND] = kbdev->reg + 0x1920; - kbdev->regmap.regs[JOB_CONTROL__JS2__STATUS] = kbdev->reg + 0x1924; - kbdev->regmap.regs[JOB_CONTROL__JS2__FLUSH_ID] = kbdev->reg + 0x1930; - kbdev->regmap.regs[JOB_CONTROL__JS2__EVENT_0] = kbdev->reg + 0x1934; - kbdev->regmap.regs[JOB_CONTROL__JS2__EVENT_1] = kbdev->reg + 0x1938; - kbdev->regmap.regs[JOB_CONTROL__JS2__EVENT_MASK] = kbdev->reg + 0x193c; - kbdev->regmap.regs[JOB_CONTROL__JS2__HEAD_NEXT] = kbdev->reg + 0x1940; - kbdev->regmap.regs[JOB_CONTROL__JS2__TAIL_NEXT] = kbdev->reg + 0x1948; - kbdev->regmap.regs[JOB_CONTROL__JS2__AFFINITY_NEXT] = kbdev->reg + 0x1950; - kbdev->regmap.regs[JOB_CONTROL__JS2__CONFIG_NEXT] = kbdev->reg + 0x1958; - kbdev->regmap.regs[JOB_CONTROL__JS2__XAFFINITY_NEXT] = kbdev->reg + 0x195c; - kbdev->regmap.regs[JOB_CONTROL__JS2__COMMAND_NEXT] = kbdev->reg + 0x1960; - kbdev->regmap.regs[JOB_CONTROL__JS2__FLUSH_ID_NEXT] = kbdev->reg + 0x1970; - kbdev->regmap.regs[JOB_CONTROL__JS2__EVENT_MASK_NEXT] = kbdev->reg + 0x197c; - kbdev->regmap.regs[JOB_CONTROL__JS3__HEAD] = kbdev->reg + 0x1980; - kbdev->regmap.regs[JOB_CONTROL__JS3__TAIL] = kbdev->reg + 0x1988; - kbdev->regmap.regs[JOB_CONTROL__JS3__AFFINITY] = kbdev->reg + 0x1990; - kbdev->regmap.regs[JOB_CONTROL__JS3__CONFIG] = kbdev->reg + 0x1998; - kbdev->regmap.regs[JOB_CONTROL__JS3__XAFFINITY] = kbdev->reg + 0x199c; - kbdev->regmap.regs[JOB_CONTROL__JS3__COMMAND] = kbdev->reg + 0x19a0; - kbdev->regmap.regs[JOB_CONTROL__JS3__STATUS] = kbdev->reg + 0x19a4; - kbdev->regmap.regs[JOB_CONTROL__JS3__FLUSH_ID] = kbdev->reg + 0x19b0; - kbdev->regmap.regs[JOB_CONTROL__JS3__EVENT_0] = kbdev->reg + 0x19b4; - kbdev->regmap.regs[JOB_CONTROL__JS3__EVENT_1] = kbdev->reg + 0x19b8; - kbdev->regmap.regs[JOB_CONTROL__JS3__EVENT_MASK] = kbdev->reg + 0x19bc; - kbdev->regmap.regs[JOB_CONTROL__JS3__HEAD_NEXT] = kbdev->reg + 0x19c0; - kbdev->regmap.regs[JOB_CONTROL__JS3__TAIL_NEXT] = kbdev->reg + 0x19c8; - kbdev->regmap.regs[JOB_CONTROL__JS3__AFFINITY_NEXT] = kbdev->reg + 0x19d0; - kbdev->regmap.regs[JOB_CONTROL__JS3__CONFIG_NEXT] = kbdev->reg + 0x19d8; - kbdev->regmap.regs[JOB_CONTROL__JS3__XAFFINITY_NEXT] = kbdev->reg + 0x19dc; - kbdev->regmap.regs[JOB_CONTROL__JS3__COMMAND_NEXT] = kbdev->reg + 0x19e0; - kbdev->regmap.regs[JOB_CONTROL__JS3__FLUSH_ID_NEXT] = kbdev->reg + 0x19f0; - kbdev->regmap.regs[JOB_CONTROL__JS3__EVENT_MASK_NEXT] = kbdev->reg + 0x19fc; - kbdev->regmap.regs[JOB_CONTROL__JS4__HEAD] = kbdev->reg + 0x1a00; - kbdev->regmap.regs[JOB_CONTROL__JS4__TAIL] = kbdev->reg + 0x1a08; - kbdev->regmap.regs[JOB_CONTROL__JS4__AFFINITY] = kbdev->reg + 0x1a10; - kbdev->regmap.regs[JOB_CONTROL__JS4__CONFIG] = kbdev->reg + 0x1a18; - kbdev->regmap.regs[JOB_CONTROL__JS4__XAFFINITY] = kbdev->reg + 0x1a1c; - kbdev->regmap.regs[JOB_CONTROL__JS4__COMMAND] = kbdev->reg + 0x1a20; - kbdev->regmap.regs[JOB_CONTROL__JS4__STATUS] = kbdev->reg + 0x1a24; - kbdev->regmap.regs[JOB_CONTROL__JS4__FLUSH_ID] = kbdev->reg + 0x1a30; - kbdev->regmap.regs[JOB_CONTROL__JS4__EVENT_0] = kbdev->reg + 0x1a34; - kbdev->regmap.regs[JOB_CONTROL__JS4__EVENT_1] = kbdev->reg + 0x1a38; - kbdev->regmap.regs[JOB_CONTROL__JS4__EVENT_MASK] = kbdev->reg + 0x1a3c; - kbdev->regmap.regs[JOB_CONTROL__JS4__HEAD_NEXT] = kbdev->reg + 0x1a40; - kbdev->regmap.regs[JOB_CONTROL__JS4__TAIL_NEXT] = kbdev->reg + 0x1a48; - kbdev->regmap.regs[JOB_CONTROL__JS4__AFFINITY_NEXT] = kbdev->reg + 0x1a50; - kbdev->regmap.regs[JOB_CONTROL__JS4__CONFIG_NEXT] = kbdev->reg + 0x1a58; - kbdev->regmap.regs[JOB_CONTROL__JS4__XAFFINITY_NEXT] = kbdev->reg + 0x1a5c; - kbdev->regmap.regs[JOB_CONTROL__JS4__COMMAND_NEXT] = kbdev->reg + 0x1a60; - kbdev->regmap.regs[JOB_CONTROL__JS4__FLUSH_ID_NEXT] = kbdev->reg + 0x1a70; - kbdev->regmap.regs[JOB_CONTROL__JS4__EVENT_MASK_NEXT] = kbdev->reg + 0x1a7c; - kbdev->regmap.regs[JOB_CONTROL__JS5__HEAD] = kbdev->reg + 0x1a80; - kbdev->regmap.regs[JOB_CONTROL__JS5__TAIL] = kbdev->reg + 0x1a88; - kbdev->regmap.regs[JOB_CONTROL__JS5__AFFINITY] = kbdev->reg + 0x1a90; - kbdev->regmap.regs[JOB_CONTROL__JS5__CONFIG] = kbdev->reg + 0x1a98; - kbdev->regmap.regs[JOB_CONTROL__JS5__XAFFINITY] = kbdev->reg + 0x1a9c; - kbdev->regmap.regs[JOB_CONTROL__JS5__COMMAND] = kbdev->reg + 0x1aa0; - kbdev->regmap.regs[JOB_CONTROL__JS5__STATUS] = kbdev->reg + 0x1aa4; - kbdev->regmap.regs[JOB_CONTROL__JS5__FLUSH_ID] = kbdev->reg + 0x1ab0; - kbdev->regmap.regs[JOB_CONTROL__JS5__EVENT_0] = kbdev->reg + 0x1ab4; - kbdev->regmap.regs[JOB_CONTROL__JS5__EVENT_1] = kbdev->reg + 0x1ab8; - kbdev->regmap.regs[JOB_CONTROL__JS5__EVENT_MASK] = kbdev->reg + 0x1abc; - kbdev->regmap.regs[JOB_CONTROL__JS5__HEAD_NEXT] = kbdev->reg + 0x1ac0; - kbdev->regmap.regs[JOB_CONTROL__JS5__TAIL_NEXT] = kbdev->reg + 0x1ac8; - kbdev->regmap.regs[JOB_CONTROL__JS5__AFFINITY_NEXT] = kbdev->reg + 0x1ad0; - kbdev->regmap.regs[JOB_CONTROL__JS5__CONFIG_NEXT] = kbdev->reg + 0x1ad8; - kbdev->regmap.regs[JOB_CONTROL__JS5__XAFFINITY_NEXT] = kbdev->reg + 0x1adc; - kbdev->regmap.regs[JOB_CONTROL__JS5__COMMAND_NEXT] = kbdev->reg + 0x1ae0; - kbdev->regmap.regs[JOB_CONTROL__JS5__FLUSH_ID_NEXT] = kbdev->reg + 0x1af0; - kbdev->regmap.regs[JOB_CONTROL__JS5__EVENT_MASK_NEXT] = kbdev->reg + 0x1afc; - kbdev->regmap.regs[JOB_CONTROL__JS6__HEAD] = kbdev->reg + 0x1b00; - kbdev->regmap.regs[JOB_CONTROL__JS6__TAIL] = kbdev->reg + 0x1b08; - kbdev->regmap.regs[JOB_CONTROL__JS6__AFFINITY] = kbdev->reg + 0x1b10; - kbdev->regmap.regs[JOB_CONTROL__JS6__CONFIG] = kbdev->reg + 0x1b18; - kbdev->regmap.regs[JOB_CONTROL__JS6__XAFFINITY] = kbdev->reg + 0x1b1c; - kbdev->regmap.regs[JOB_CONTROL__JS6__COMMAND] = kbdev->reg + 0x1b20; - kbdev->regmap.regs[JOB_CONTROL__JS6__STATUS] = kbdev->reg + 0x1b24; - kbdev->regmap.regs[JOB_CONTROL__JS6__FLUSH_ID] = kbdev->reg + 0x1b30; - kbdev->regmap.regs[JOB_CONTROL__JS6__EVENT_0] = kbdev->reg + 0x1b34; - kbdev->regmap.regs[JOB_CONTROL__JS6__EVENT_1] = kbdev->reg + 0x1b38; - kbdev->regmap.regs[JOB_CONTROL__JS6__EVENT_MASK] = kbdev->reg + 0x1b3c; - kbdev->regmap.regs[JOB_CONTROL__JS6__HEAD_NEXT] = kbdev->reg + 0x1b40; - kbdev->regmap.regs[JOB_CONTROL__JS6__TAIL_NEXT] = kbdev->reg + 0x1b48; - kbdev->regmap.regs[JOB_CONTROL__JS6__AFFINITY_NEXT] = kbdev->reg + 0x1b50; - kbdev->regmap.regs[JOB_CONTROL__JS6__CONFIG_NEXT] = kbdev->reg + 0x1b58; - kbdev->regmap.regs[JOB_CONTROL__JS6__XAFFINITY_NEXT] = kbdev->reg + 0x1b5c; - kbdev->regmap.regs[JOB_CONTROL__JS6__COMMAND_NEXT] = kbdev->reg + 0x1b60; - kbdev->regmap.regs[JOB_CONTROL__JS6__FLUSH_ID_NEXT] = kbdev->reg + 0x1b70; - kbdev->regmap.regs[JOB_CONTROL__JS6__EVENT_MASK_NEXT] = kbdev->reg + 0x1b7c; - kbdev->regmap.regs[JOB_CONTROL__JS7__HEAD] = kbdev->reg + 0x1b80; - kbdev->regmap.regs[JOB_CONTROL__JS7__TAIL] = kbdev->reg + 0x1b88; - kbdev->regmap.regs[JOB_CONTROL__JS7__AFFINITY] = kbdev->reg + 0x1b90; - kbdev->regmap.regs[JOB_CONTROL__JS7__CONFIG] = kbdev->reg + 0x1b98; - kbdev->regmap.regs[JOB_CONTROL__JS7__XAFFINITY] = kbdev->reg + 0x1b9c; - kbdev->regmap.regs[JOB_CONTROL__JS7__COMMAND] = kbdev->reg + 0x1ba0; - kbdev->regmap.regs[JOB_CONTROL__JS7__STATUS] = kbdev->reg + 0x1ba4; - kbdev->regmap.regs[JOB_CONTROL__JS7__FLUSH_ID] = kbdev->reg + 0x1bb0; - kbdev->regmap.regs[JOB_CONTROL__JS7__EVENT_0] = kbdev->reg + 0x1bb4; - kbdev->regmap.regs[JOB_CONTROL__JS7__EVENT_1] = kbdev->reg + 0x1bb8; - kbdev->regmap.regs[JOB_CONTROL__JS7__EVENT_MASK] = kbdev->reg + 0x1bbc; - kbdev->regmap.regs[JOB_CONTROL__JS7__HEAD_NEXT] = kbdev->reg + 0x1bc0; - kbdev->regmap.regs[JOB_CONTROL__JS7__TAIL_NEXT] = kbdev->reg + 0x1bc8; - kbdev->regmap.regs[JOB_CONTROL__JS7__AFFINITY_NEXT] = kbdev->reg + 0x1bd0; - kbdev->regmap.regs[JOB_CONTROL__JS7__CONFIG_NEXT] = kbdev->reg + 0x1bd8; - kbdev->regmap.regs[JOB_CONTROL__JS7__XAFFINITY_NEXT] = kbdev->reg + 0x1bdc; - kbdev->regmap.regs[JOB_CONTROL__JS7__COMMAND_NEXT] = kbdev->reg + 0x1be0; - kbdev->regmap.regs[JOB_CONTROL__JS7__FLUSH_ID_NEXT] = kbdev->reg + 0x1bf0; - kbdev->regmap.regs[JOB_CONTROL__JS7__EVENT_MASK_NEXT] = kbdev->reg + 0x1bfc; - kbdev->regmap.regs[JOB_CONTROL__JS8__HEAD] = kbdev->reg + 0x1c00; - kbdev->regmap.regs[JOB_CONTROL__JS8__TAIL] = kbdev->reg + 0x1c08; - kbdev->regmap.regs[JOB_CONTROL__JS8__AFFINITY] = kbdev->reg + 0x1c10; - kbdev->regmap.regs[JOB_CONTROL__JS8__CONFIG] = kbdev->reg + 0x1c18; - kbdev->regmap.regs[JOB_CONTROL__JS8__XAFFINITY] = kbdev->reg + 0x1c1c; - kbdev->regmap.regs[JOB_CONTROL__JS8__COMMAND] = kbdev->reg + 0x1c20; - kbdev->regmap.regs[JOB_CONTROL__JS8__STATUS] = kbdev->reg + 0x1c24; - kbdev->regmap.regs[JOB_CONTROL__JS8__FLUSH_ID] = kbdev->reg + 0x1c30; - kbdev->regmap.regs[JOB_CONTROL__JS8__EVENT_0] = kbdev->reg + 0x1c34; - kbdev->regmap.regs[JOB_CONTROL__JS8__EVENT_1] = kbdev->reg + 0x1c38; - kbdev->regmap.regs[JOB_CONTROL__JS8__EVENT_MASK] = kbdev->reg + 0x1c3c; - kbdev->regmap.regs[JOB_CONTROL__JS8__HEAD_NEXT] = kbdev->reg + 0x1c40; - kbdev->regmap.regs[JOB_CONTROL__JS8__TAIL_NEXT] = kbdev->reg + 0x1c48; - kbdev->regmap.regs[JOB_CONTROL__JS8__AFFINITY_NEXT] = kbdev->reg + 0x1c50; - kbdev->regmap.regs[JOB_CONTROL__JS8__CONFIG_NEXT] = kbdev->reg + 0x1c58; - kbdev->regmap.regs[JOB_CONTROL__JS8__XAFFINITY_NEXT] = kbdev->reg + 0x1c5c; - kbdev->regmap.regs[JOB_CONTROL__JS8__COMMAND_NEXT] = kbdev->reg + 0x1c60; - kbdev->regmap.regs[JOB_CONTROL__JS8__FLUSH_ID_NEXT] = kbdev->reg + 0x1c70; - kbdev->regmap.regs[JOB_CONTROL__JS8__EVENT_MASK_NEXT] = kbdev->reg + 0x1c7c; - kbdev->regmap.regs[JOB_CONTROL__JS9__HEAD] = kbdev->reg + 0x1c80; - kbdev->regmap.regs[JOB_CONTROL__JS9__TAIL] = kbdev->reg + 0x1c88; - kbdev->regmap.regs[JOB_CONTROL__JS9__AFFINITY] = kbdev->reg + 0x1c90; - kbdev->regmap.regs[JOB_CONTROL__JS9__CONFIG] = kbdev->reg + 0x1c98; - kbdev->regmap.regs[JOB_CONTROL__JS9__XAFFINITY] = kbdev->reg + 0x1c9c; - kbdev->regmap.regs[JOB_CONTROL__JS9__COMMAND] = kbdev->reg + 0x1ca0; - kbdev->regmap.regs[JOB_CONTROL__JS9__STATUS] = kbdev->reg + 0x1ca4; - kbdev->regmap.regs[JOB_CONTROL__JS9__FLUSH_ID] = kbdev->reg + 0x1cb0; - kbdev->regmap.regs[JOB_CONTROL__JS9__EVENT_0] = kbdev->reg + 0x1cb4; - kbdev->regmap.regs[JOB_CONTROL__JS9__EVENT_1] = kbdev->reg + 0x1cb8; - kbdev->regmap.regs[JOB_CONTROL__JS9__EVENT_MASK] = kbdev->reg + 0x1cbc; - kbdev->regmap.regs[JOB_CONTROL__JS9__HEAD_NEXT] = kbdev->reg + 0x1cc0; - kbdev->regmap.regs[JOB_CONTROL__JS9__TAIL_NEXT] = kbdev->reg + 0x1cc8; - kbdev->regmap.regs[JOB_CONTROL__JS9__AFFINITY_NEXT] = kbdev->reg + 0x1cd0; - kbdev->regmap.regs[JOB_CONTROL__JS9__CONFIG_NEXT] = kbdev->reg + 0x1cd8; - kbdev->regmap.regs[JOB_CONTROL__JS9__XAFFINITY_NEXT] = kbdev->reg + 0x1cdc; - kbdev->regmap.regs[JOB_CONTROL__JS9__COMMAND_NEXT] = kbdev->reg + 0x1ce0; - kbdev->regmap.regs[JOB_CONTROL__JS9__FLUSH_ID_NEXT] = kbdev->reg + 0x1cf0; - kbdev->regmap.regs[JOB_CONTROL__JS9__EVENT_MASK_NEXT] = kbdev->reg + 0x1cfc; - kbdev->regmap.regs[JOB_CONTROL__JS10__HEAD] = kbdev->reg + 0x1d00; - kbdev->regmap.regs[JOB_CONTROL__JS10__TAIL] = kbdev->reg + 0x1d08; - kbdev->regmap.regs[JOB_CONTROL__JS10__AFFINITY] = kbdev->reg + 0x1d10; - kbdev->regmap.regs[JOB_CONTROL__JS10__CONFIG] = kbdev->reg + 0x1d18; - kbdev->regmap.regs[JOB_CONTROL__JS10__XAFFINITY] = kbdev->reg + 0x1d1c; - kbdev->regmap.regs[JOB_CONTROL__JS10__COMMAND] = kbdev->reg + 0x1d20; - kbdev->regmap.regs[JOB_CONTROL__JS10__STATUS] = kbdev->reg + 0x1d24; - kbdev->regmap.regs[JOB_CONTROL__JS10__FLUSH_ID] = kbdev->reg + 0x1d30; - kbdev->regmap.regs[JOB_CONTROL__JS10__EVENT_0] = kbdev->reg + 0x1d34; - kbdev->regmap.regs[JOB_CONTROL__JS10__EVENT_1] = kbdev->reg + 0x1d38; - kbdev->regmap.regs[JOB_CONTROL__JS10__EVENT_MASK] = kbdev->reg + 0x1d3c; - kbdev->regmap.regs[JOB_CONTROL__JS10__HEAD_NEXT] = kbdev->reg + 0x1d40; - kbdev->regmap.regs[JOB_CONTROL__JS10__TAIL_NEXT] = kbdev->reg + 0x1d48; - kbdev->regmap.regs[JOB_CONTROL__JS10__AFFINITY_NEXT] = kbdev->reg + 0x1d50; - kbdev->regmap.regs[JOB_CONTROL__JS10__CONFIG_NEXT] = kbdev->reg + 0x1d58; - kbdev->regmap.regs[JOB_CONTROL__JS10__XAFFINITY_NEXT] = kbdev->reg + 0x1d5c; - kbdev->regmap.regs[JOB_CONTROL__JS10__COMMAND_NEXT] = kbdev->reg + 0x1d60; - kbdev->regmap.regs[JOB_CONTROL__JS10__FLUSH_ID_NEXT] = kbdev->reg + 0x1d70; - kbdev->regmap.regs[JOB_CONTROL__JS10__EVENT_MASK_NEXT] = kbdev->reg + 0x1d7c; - kbdev->regmap.regs[JOB_CONTROL__JS11__HEAD] = kbdev->reg + 0x1d80; - kbdev->regmap.regs[JOB_CONTROL__JS11__TAIL] = kbdev->reg + 0x1d88; - kbdev->regmap.regs[JOB_CONTROL__JS11__AFFINITY] = kbdev->reg + 0x1d90; - kbdev->regmap.regs[JOB_CONTROL__JS11__CONFIG] = kbdev->reg + 0x1d98; - kbdev->regmap.regs[JOB_CONTROL__JS11__XAFFINITY] = kbdev->reg + 0x1d9c; - kbdev->regmap.regs[JOB_CONTROL__JS11__COMMAND] = kbdev->reg + 0x1da0; - kbdev->regmap.regs[JOB_CONTROL__JS11__STATUS] = kbdev->reg + 0x1da4; - kbdev->regmap.regs[JOB_CONTROL__JS11__FLUSH_ID] = kbdev->reg + 0x1db0; - kbdev->regmap.regs[JOB_CONTROL__JS11__EVENT_0] = kbdev->reg + 0x1db4; - kbdev->regmap.regs[JOB_CONTROL__JS11__EVENT_1] = kbdev->reg + 0x1db8; - kbdev->regmap.regs[JOB_CONTROL__JS11__EVENT_MASK] = kbdev->reg + 0x1dbc; - kbdev->regmap.regs[JOB_CONTROL__JS11__HEAD_NEXT] = kbdev->reg + 0x1dc0; - kbdev->regmap.regs[JOB_CONTROL__JS11__TAIL_NEXT] = kbdev->reg + 0x1dc8; - kbdev->regmap.regs[JOB_CONTROL__JS11__AFFINITY_NEXT] = kbdev->reg + 0x1dd0; - kbdev->regmap.regs[JOB_CONTROL__JS11__CONFIG_NEXT] = kbdev->reg + 0x1dd8; - kbdev->regmap.regs[JOB_CONTROL__JS11__XAFFINITY_NEXT] = kbdev->reg + 0x1ddc; - kbdev->regmap.regs[JOB_CONTROL__JS11__COMMAND_NEXT] = kbdev->reg + 0x1de0; - kbdev->regmap.regs[JOB_CONTROL__JS11__FLUSH_ID_NEXT] = kbdev->reg + 0x1df0; - kbdev->regmap.regs[JOB_CONTROL__JS11__EVENT_MASK_NEXT] = kbdev->reg + 0x1dfc; - kbdev->regmap.regs[JOB_CONTROL__JS12__HEAD] = kbdev->reg + 0x1e00; - kbdev->regmap.regs[JOB_CONTROL__JS12__TAIL] = kbdev->reg + 0x1e08; - kbdev->regmap.regs[JOB_CONTROL__JS12__AFFINITY] = kbdev->reg + 0x1e10; - kbdev->regmap.regs[JOB_CONTROL__JS12__CONFIG] = kbdev->reg + 0x1e18; - kbdev->regmap.regs[JOB_CONTROL__JS12__XAFFINITY] = kbdev->reg + 0x1e1c; - kbdev->regmap.regs[JOB_CONTROL__JS12__COMMAND] = kbdev->reg + 0x1e20; - kbdev->regmap.regs[JOB_CONTROL__JS12__STATUS] = kbdev->reg + 0x1e24; - kbdev->regmap.regs[JOB_CONTROL__JS12__FLUSH_ID] = kbdev->reg + 0x1e30; - kbdev->regmap.regs[JOB_CONTROL__JS12__EVENT_0] = kbdev->reg + 0x1e34; - kbdev->regmap.regs[JOB_CONTROL__JS12__EVENT_1] = kbdev->reg + 0x1e38; - kbdev->regmap.regs[JOB_CONTROL__JS12__EVENT_MASK] = kbdev->reg + 0x1e3c; - kbdev->regmap.regs[JOB_CONTROL__JS12__HEAD_NEXT] = kbdev->reg + 0x1e40; - kbdev->regmap.regs[JOB_CONTROL__JS12__TAIL_NEXT] = kbdev->reg + 0x1e48; - kbdev->regmap.regs[JOB_CONTROL__JS12__AFFINITY_NEXT] = kbdev->reg + 0x1e50; - kbdev->regmap.regs[JOB_CONTROL__JS12__CONFIG_NEXT] = kbdev->reg + 0x1e58; - kbdev->regmap.regs[JOB_CONTROL__JS12__XAFFINITY_NEXT] = kbdev->reg + 0x1e5c; - kbdev->regmap.regs[JOB_CONTROL__JS12__COMMAND_NEXT] = kbdev->reg + 0x1e60; - kbdev->regmap.regs[JOB_CONTROL__JS12__FLUSH_ID_NEXT] = kbdev->reg + 0x1e70; - kbdev->regmap.regs[JOB_CONTROL__JS12__EVENT_MASK_NEXT] = kbdev->reg + 0x1e7c; - kbdev->regmap.regs[JOB_CONTROL__JS13__HEAD] = kbdev->reg + 0x1e80; - kbdev->regmap.regs[JOB_CONTROL__JS13__TAIL] = kbdev->reg + 0x1e88; - kbdev->regmap.regs[JOB_CONTROL__JS13__AFFINITY] = kbdev->reg + 0x1e90; - kbdev->regmap.regs[JOB_CONTROL__JS13__CONFIG] = kbdev->reg + 0x1e98; - kbdev->regmap.regs[JOB_CONTROL__JS13__XAFFINITY] = kbdev->reg + 0x1e9c; - kbdev->regmap.regs[JOB_CONTROL__JS13__COMMAND] = kbdev->reg + 0x1ea0; - kbdev->regmap.regs[JOB_CONTROL__JS13__STATUS] = kbdev->reg + 0x1ea4; - kbdev->regmap.regs[JOB_CONTROL__JS13__FLUSH_ID] = kbdev->reg + 0x1eb0; - kbdev->regmap.regs[JOB_CONTROL__JS13__EVENT_0] = kbdev->reg + 0x1eb4; - kbdev->regmap.regs[JOB_CONTROL__JS13__EVENT_1] = kbdev->reg + 0x1eb8; - kbdev->regmap.regs[JOB_CONTROL__JS13__EVENT_MASK] = kbdev->reg + 0x1ebc; - kbdev->regmap.regs[JOB_CONTROL__JS13__HEAD_NEXT] = kbdev->reg + 0x1ec0; - kbdev->regmap.regs[JOB_CONTROL__JS13__TAIL_NEXT] = kbdev->reg + 0x1ec8; - kbdev->regmap.regs[JOB_CONTROL__JS13__AFFINITY_NEXT] = kbdev->reg + 0x1ed0; - kbdev->regmap.regs[JOB_CONTROL__JS13__CONFIG_NEXT] = kbdev->reg + 0x1ed8; - kbdev->regmap.regs[JOB_CONTROL__JS13__XAFFINITY_NEXT] = kbdev->reg + 0x1edc; - kbdev->regmap.regs[JOB_CONTROL__JS13__COMMAND_NEXT] = kbdev->reg + 0x1ee0; - kbdev->regmap.regs[JOB_CONTROL__JS13__FLUSH_ID_NEXT] = kbdev->reg + 0x1ef0; - kbdev->regmap.regs[JOB_CONTROL__JS13__EVENT_MASK_NEXT] = kbdev->reg + 0x1efc; - kbdev->regmap.regs[JOB_CONTROL__JS14__HEAD] = kbdev->reg + 0x1f00; - kbdev->regmap.regs[JOB_CONTROL__JS14__TAIL] = kbdev->reg + 0x1f08; - kbdev->regmap.regs[JOB_CONTROL__JS14__AFFINITY] = kbdev->reg + 0x1f10; - kbdev->regmap.regs[JOB_CONTROL__JS14__CONFIG] = kbdev->reg + 0x1f18; - kbdev->regmap.regs[JOB_CONTROL__JS14__XAFFINITY] = kbdev->reg + 0x1f1c; - kbdev->regmap.regs[JOB_CONTROL__JS14__COMMAND] = kbdev->reg + 0x1f20; - kbdev->regmap.regs[JOB_CONTROL__JS14__STATUS] = kbdev->reg + 0x1f24; - kbdev->regmap.regs[JOB_CONTROL__JS14__FLUSH_ID] = kbdev->reg + 0x1f30; - kbdev->regmap.regs[JOB_CONTROL__JS14__EVENT_0] = kbdev->reg + 0x1f34; - kbdev->regmap.regs[JOB_CONTROL__JS14__EVENT_1] = kbdev->reg + 0x1f38; - kbdev->regmap.regs[JOB_CONTROL__JS14__EVENT_MASK] = kbdev->reg + 0x1f3c; - kbdev->regmap.regs[JOB_CONTROL__JS14__HEAD_NEXT] = kbdev->reg + 0x1f40; - kbdev->regmap.regs[JOB_CONTROL__JS14__TAIL_NEXT] = kbdev->reg + 0x1f48; - kbdev->regmap.regs[JOB_CONTROL__JS14__AFFINITY_NEXT] = kbdev->reg + 0x1f50; - kbdev->regmap.regs[JOB_CONTROL__JS14__CONFIG_NEXT] = kbdev->reg + 0x1f58; - kbdev->regmap.regs[JOB_CONTROL__JS14__XAFFINITY_NEXT] = kbdev->reg + 0x1f5c; - kbdev->regmap.regs[JOB_CONTROL__JS14__COMMAND_NEXT] = kbdev->reg + 0x1f60; - kbdev->regmap.regs[JOB_CONTROL__JS14__FLUSH_ID_NEXT] = kbdev->reg + 0x1f70; - kbdev->regmap.regs[JOB_CONTROL__JS14__EVENT_MASK_NEXT] = kbdev->reg + 0x1f7c; - kbdev->regmap.regs[JOB_CONTROL__JS15__HEAD] = kbdev->reg + 0x1f80; - kbdev->regmap.regs[JOB_CONTROL__JS15__TAIL] = kbdev->reg + 0x1f88; - kbdev->regmap.regs[JOB_CONTROL__JS15__AFFINITY] = kbdev->reg + 0x1f90; - kbdev->regmap.regs[JOB_CONTROL__JS15__CONFIG] = kbdev->reg + 0x1f98; - kbdev->regmap.regs[JOB_CONTROL__JS15__XAFFINITY] = kbdev->reg + 0x1f9c; - kbdev->regmap.regs[JOB_CONTROL__JS15__COMMAND] = kbdev->reg + 0x1fa0; - kbdev->regmap.regs[JOB_CONTROL__JS15__STATUS] = kbdev->reg + 0x1fa4; - kbdev->regmap.regs[JOB_CONTROL__JS15__FLUSH_ID] = kbdev->reg + 0x1fb0; - kbdev->regmap.regs[JOB_CONTROL__JS15__EVENT_0] = kbdev->reg + 0x1fb4; - kbdev->regmap.regs[JOB_CONTROL__JS15__EVENT_1] = kbdev->reg + 0x1fb8; - kbdev->regmap.regs[JOB_CONTROL__JS15__EVENT_MASK] = kbdev->reg + 0x1fbc; - kbdev->regmap.regs[JOB_CONTROL__JS15__HEAD_NEXT] = kbdev->reg + 0x1fc0; - kbdev->regmap.regs[JOB_CONTROL__JS15__TAIL_NEXT] = kbdev->reg + 0x1fc8; - kbdev->regmap.regs[JOB_CONTROL__JS15__AFFINITY_NEXT] = kbdev->reg + 0x1fd0; - kbdev->regmap.regs[JOB_CONTROL__JS15__CONFIG_NEXT] = kbdev->reg + 0x1fd8; - kbdev->regmap.regs[JOB_CONTROL__JS15__XAFFINITY_NEXT] = kbdev->reg + 0x1fdc; - kbdev->regmap.regs[JOB_CONTROL__JS15__COMMAND_NEXT] = kbdev->reg + 0x1fe0; - kbdev->regmap.regs[JOB_CONTROL__JS15__FLUSH_ID_NEXT] = kbdev->reg + 0x1ff0; - kbdev->regmap.regs[JOB_CONTROL__JS15__EVENT_MASK_NEXT] = kbdev->reg + 0x1ffc; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__IRQ_RAWSTAT] = kbdev->reg + 0x2000; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__IRQ_CLEAR] = kbdev->reg + 0x2004; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__IRQ_MASK] = kbdev->reg + 0x2008; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__IRQ_STATUS] = kbdev->reg + 0x200c; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS0__TRANSTAB] = kbdev->reg + 0x2400; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS0__MEMATTR] = kbdev->reg + 0x2408; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS0__LOCKADDR] = kbdev->reg + 0x2410; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS0__COMMAND] = kbdev->reg + 0x2418; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS0__FAULTSTATUS] = kbdev->reg + 0x241c; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS0__FAULTADDRESS] = kbdev->reg + 0x2420; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS0__STATUS] = kbdev->reg + 0x2428; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS0__TRANSCFG] = kbdev->reg + 0x2430; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS0__FAULTEXTRA] = kbdev->reg + 0x2438; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS1__TRANSTAB] = kbdev->reg + 0x2440; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS1__MEMATTR] = kbdev->reg + 0x2448; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS1__LOCKADDR] = kbdev->reg + 0x2450; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS1__COMMAND] = kbdev->reg + 0x2458; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS1__FAULTSTATUS] = kbdev->reg + 0x245c; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS1__FAULTADDRESS] = kbdev->reg + 0x2460; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS1__STATUS] = kbdev->reg + 0x2468; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS1__TRANSCFG] = kbdev->reg + 0x2470; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS1__FAULTEXTRA] = kbdev->reg + 0x2478; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS2__TRANSTAB] = kbdev->reg + 0x2480; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS2__MEMATTR] = kbdev->reg + 0x2488; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS2__LOCKADDR] = kbdev->reg + 0x2490; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS2__COMMAND] = kbdev->reg + 0x2498; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS2__FAULTSTATUS] = kbdev->reg + 0x249c; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS2__FAULTADDRESS] = kbdev->reg + 0x24a0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS2__STATUS] = kbdev->reg + 0x24a8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS2__TRANSCFG] = kbdev->reg + 0x24b0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS2__FAULTEXTRA] = kbdev->reg + 0x24b8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS3__TRANSTAB] = kbdev->reg + 0x24c0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS3__MEMATTR] = kbdev->reg + 0x24c8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS3__LOCKADDR] = kbdev->reg + 0x24d0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS3__COMMAND] = kbdev->reg + 0x24d8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS3__FAULTSTATUS] = kbdev->reg + 0x24dc; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS3__FAULTADDRESS] = kbdev->reg + 0x24e0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS3__STATUS] = kbdev->reg + 0x24e8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS3__TRANSCFG] = kbdev->reg + 0x24f0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS3__FAULTEXTRA] = kbdev->reg + 0x24f8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS4__TRANSTAB] = kbdev->reg + 0x2500; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS4__MEMATTR] = kbdev->reg + 0x2508; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS4__LOCKADDR] = kbdev->reg + 0x2510; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS4__COMMAND] = kbdev->reg + 0x2518; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS4__FAULTSTATUS] = kbdev->reg + 0x251c; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS4__FAULTADDRESS] = kbdev->reg + 0x2520; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS4__STATUS] = kbdev->reg + 0x2528; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS4__TRANSCFG] = kbdev->reg + 0x2530; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS4__FAULTEXTRA] = kbdev->reg + 0x2538; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS5__TRANSTAB] = kbdev->reg + 0x2540; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS5__MEMATTR] = kbdev->reg + 0x2548; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS5__LOCKADDR] = kbdev->reg + 0x2550; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS5__COMMAND] = kbdev->reg + 0x2558; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS5__FAULTSTATUS] = kbdev->reg + 0x255c; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS5__FAULTADDRESS] = kbdev->reg + 0x2560; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS5__STATUS] = kbdev->reg + 0x2568; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS5__TRANSCFG] = kbdev->reg + 0x2570; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS5__FAULTEXTRA] = kbdev->reg + 0x2578; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS6__TRANSTAB] = kbdev->reg + 0x2580; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS6__MEMATTR] = kbdev->reg + 0x2588; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS6__LOCKADDR] = kbdev->reg + 0x2590; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS6__COMMAND] = kbdev->reg + 0x2598; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS6__FAULTSTATUS] = kbdev->reg + 0x259c; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS6__FAULTADDRESS] = kbdev->reg + 0x25a0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS6__STATUS] = kbdev->reg + 0x25a8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS6__TRANSCFG] = kbdev->reg + 0x25b0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS6__FAULTEXTRA] = kbdev->reg + 0x25b8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS7__TRANSTAB] = kbdev->reg + 0x25c0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS7__MEMATTR] = kbdev->reg + 0x25c8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS7__LOCKADDR] = kbdev->reg + 0x25d0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS7__COMMAND] = kbdev->reg + 0x25d8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS7__FAULTSTATUS] = kbdev->reg + 0x25dc; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS7__FAULTADDRESS] = kbdev->reg + 0x25e0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS7__STATUS] = kbdev->reg + 0x25e8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS7__TRANSCFG] = kbdev->reg + 0x25f0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS7__FAULTEXTRA] = kbdev->reg + 0x25f8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS8__TRANSTAB] = kbdev->reg + 0x2600; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS8__MEMATTR] = kbdev->reg + 0x2608; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS8__LOCKADDR] = kbdev->reg + 0x2610; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS8__COMMAND] = kbdev->reg + 0x2618; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS8__FAULTSTATUS] = kbdev->reg + 0x261c; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS8__FAULTADDRESS] = kbdev->reg + 0x2620; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS8__STATUS] = kbdev->reg + 0x2628; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS8__TRANSCFG] = kbdev->reg + 0x2630; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS8__FAULTEXTRA] = kbdev->reg + 0x2638; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS9__TRANSTAB] = kbdev->reg + 0x2640; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS9__MEMATTR] = kbdev->reg + 0x2648; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS9__LOCKADDR] = kbdev->reg + 0x2650; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS9__COMMAND] = kbdev->reg + 0x2658; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS9__FAULTSTATUS] = kbdev->reg + 0x265c; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS9__FAULTADDRESS] = kbdev->reg + 0x2660; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS9__STATUS] = kbdev->reg + 0x2668; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS9__TRANSCFG] = kbdev->reg + 0x2670; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS9__FAULTEXTRA] = kbdev->reg + 0x2678; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS10__TRANSTAB] = kbdev->reg + 0x2680; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS10__MEMATTR] = kbdev->reg + 0x2688; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS10__LOCKADDR] = kbdev->reg + 0x2690; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS10__COMMAND] = kbdev->reg + 0x2698; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS10__FAULTSTATUS] = kbdev->reg + 0x269c; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS10__FAULTADDRESS] = kbdev->reg + 0x26a0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS10__STATUS] = kbdev->reg + 0x26a8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS10__TRANSCFG] = kbdev->reg + 0x26b0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS10__FAULTEXTRA] = kbdev->reg + 0x26b8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS11__TRANSTAB] = kbdev->reg + 0x26c0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS11__MEMATTR] = kbdev->reg + 0x26c8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS11__LOCKADDR] = kbdev->reg + 0x26d0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS11__COMMAND] = kbdev->reg + 0x26d8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS11__FAULTSTATUS] = kbdev->reg + 0x26dc; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS11__FAULTADDRESS] = kbdev->reg + 0x26e0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS11__STATUS] = kbdev->reg + 0x26e8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS11__TRANSCFG] = kbdev->reg + 0x26f0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS11__FAULTEXTRA] = kbdev->reg + 0x26f8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS12__TRANSTAB] = kbdev->reg + 0x2700; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS12__MEMATTR] = kbdev->reg + 0x2708; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS12__LOCKADDR] = kbdev->reg + 0x2710; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS12__COMMAND] = kbdev->reg + 0x2718; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS12__FAULTSTATUS] = kbdev->reg + 0x271c; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS12__FAULTADDRESS] = kbdev->reg + 0x2720; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS12__STATUS] = kbdev->reg + 0x2728; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS12__TRANSCFG] = kbdev->reg + 0x2730; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS12__FAULTEXTRA] = kbdev->reg + 0x2738; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS13__TRANSTAB] = kbdev->reg + 0x2740; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS13__MEMATTR] = kbdev->reg + 0x2748; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS13__LOCKADDR] = kbdev->reg + 0x2750; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS13__COMMAND] = kbdev->reg + 0x2758; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS13__FAULTSTATUS] = kbdev->reg + 0x275c; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS13__FAULTADDRESS] = kbdev->reg + 0x2760; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS13__STATUS] = kbdev->reg + 0x2768; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS13__TRANSCFG] = kbdev->reg + 0x2770; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS13__FAULTEXTRA] = kbdev->reg + 0x2778; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS14__TRANSTAB] = kbdev->reg + 0x2780; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS14__MEMATTR] = kbdev->reg + 0x2788; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS14__LOCKADDR] = kbdev->reg + 0x2790; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS14__COMMAND] = kbdev->reg + 0x2798; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS14__FAULTSTATUS] = kbdev->reg + 0x279c; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS14__FAULTADDRESS] = kbdev->reg + 0x27a0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS14__STATUS] = kbdev->reg + 0x27a8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS14__TRANSCFG] = kbdev->reg + 0x27b0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS14__FAULTEXTRA] = kbdev->reg + 0x27b8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS15__TRANSTAB] = kbdev->reg + 0x27c0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS15__MEMATTR] = kbdev->reg + 0x27c8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS15__LOCKADDR] = kbdev->reg + 0x27d0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS15__COMMAND] = kbdev->reg + 0x27d8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS15__FAULTSTATUS] = kbdev->reg + 0x27dc; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS15__FAULTADDRESS] = kbdev->reg + 0x27e0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS15__STATUS] = kbdev->reg + 0x27e8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS15__TRANSCFG] = kbdev->reg + 0x27f0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS15__FAULTEXTRA] = kbdev->reg + 0x27f8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__IRQ_RAWSTAT] = kbdev->reg + 0x10000; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__IRQ_CLEAR] = kbdev->reg + 0x10004; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__IRQ_MASK] = kbdev->reg + 0x10008; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__IRQ_STATUS] = kbdev->reg + 0x1000c; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS0__TRANSTAB] = kbdev->reg + 0x10400; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS0__MEMATTR] = kbdev->reg + 0x10408; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS0__LOCKADDR] = kbdev->reg + 0x10410; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS0__COMMAND] = kbdev->reg + 0x10418; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS0__FAULTSTATUS] = kbdev->reg + 0x1041c; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS0__FAULTADDRESS] = kbdev->reg + 0x10420; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS0__STATUS] = kbdev->reg + 0x10428; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS0__TRANSCFG] = kbdev->reg + 0x10430; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS0__FAULTEXTRA] = kbdev->reg + 0x10438; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS1__TRANSTAB] = kbdev->reg + 0x10440; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS1__MEMATTR] = kbdev->reg + 0x10448; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS1__LOCKADDR] = kbdev->reg + 0x10450; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS1__COMMAND] = kbdev->reg + 0x10458; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS1__FAULTSTATUS] = kbdev->reg + 0x1045c; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS1__FAULTADDRESS] = kbdev->reg + 0x10460; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS1__STATUS] = kbdev->reg + 0x10468; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS1__TRANSCFG] = kbdev->reg + 0x10470; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS1__FAULTEXTRA] = kbdev->reg + 0x10478; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS2__TRANSTAB] = kbdev->reg + 0x10480; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS2__MEMATTR] = kbdev->reg + 0x10488; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS2__LOCKADDR] = kbdev->reg + 0x10490; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS2__COMMAND] = kbdev->reg + 0x10498; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS2__FAULTSTATUS] = kbdev->reg + 0x1049c; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS2__FAULTADDRESS] = kbdev->reg + 0x104a0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS2__STATUS] = kbdev->reg + 0x104a8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS2__TRANSCFG] = kbdev->reg + 0x104b0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS2__FAULTEXTRA] = kbdev->reg + 0x104b8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS3__TRANSTAB] = kbdev->reg + 0x104c0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS3__MEMATTR] = kbdev->reg + 0x104c8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS3__LOCKADDR] = kbdev->reg + 0x104d0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS3__COMMAND] = kbdev->reg + 0x104d8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS3__FAULTSTATUS] = kbdev->reg + 0x104dc; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS3__FAULTADDRESS] = kbdev->reg + 0x104e0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS3__STATUS] = kbdev->reg + 0x104e8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS3__TRANSCFG] = kbdev->reg + 0x104f0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS3__FAULTEXTRA] = kbdev->reg + 0x104f8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS4__TRANSTAB] = kbdev->reg + 0x10500; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS4__MEMATTR] = kbdev->reg + 0x10508; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS4__LOCKADDR] = kbdev->reg + 0x10510; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS4__COMMAND] = kbdev->reg + 0x10518; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS4__FAULTSTATUS] = kbdev->reg + 0x1051c; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS4__FAULTADDRESS] = kbdev->reg + 0x10520; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS4__STATUS] = kbdev->reg + 0x10528; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS4__TRANSCFG] = kbdev->reg + 0x10530; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS4__FAULTEXTRA] = kbdev->reg + 0x10538; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS5__TRANSTAB] = kbdev->reg + 0x10540; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS5__MEMATTR] = kbdev->reg + 0x10548; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS5__LOCKADDR] = kbdev->reg + 0x10550; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS5__COMMAND] = kbdev->reg + 0x10558; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS5__FAULTSTATUS] = kbdev->reg + 0x1055c; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS5__FAULTADDRESS] = kbdev->reg + 0x10560; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS5__STATUS] = kbdev->reg + 0x10568; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS5__TRANSCFG] = kbdev->reg + 0x10570; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS5__FAULTEXTRA] = kbdev->reg + 0x10578; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS6__TRANSTAB] = kbdev->reg + 0x10580; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS6__MEMATTR] = kbdev->reg + 0x10588; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS6__LOCKADDR] = kbdev->reg + 0x10590; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS6__COMMAND] = kbdev->reg + 0x10598; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS6__FAULTSTATUS] = kbdev->reg + 0x1059c; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS6__FAULTADDRESS] = kbdev->reg + 0x105a0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS6__STATUS] = kbdev->reg + 0x105a8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS6__TRANSCFG] = kbdev->reg + 0x105b0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS6__FAULTEXTRA] = kbdev->reg + 0x105b8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS7__TRANSTAB] = kbdev->reg + 0x105c0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS7__MEMATTR] = kbdev->reg + 0x105c8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS7__LOCKADDR] = kbdev->reg + 0x105d0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS7__COMMAND] = kbdev->reg + 0x105d8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS7__FAULTSTATUS] = kbdev->reg + 0x105dc; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS7__FAULTADDRESS] = kbdev->reg + 0x105e0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS7__STATUS] = kbdev->reg + 0x105e8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS7__TRANSCFG] = kbdev->reg + 0x105f0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS7__FAULTEXTRA] = kbdev->reg + 0x105f8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS8__TRANSTAB] = kbdev->reg + 0x10600; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS8__MEMATTR] = kbdev->reg + 0x10608; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS8__LOCKADDR] = kbdev->reg + 0x10610; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS8__COMMAND] = kbdev->reg + 0x10618; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS8__FAULTSTATUS] = kbdev->reg + 0x1061c; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS8__FAULTADDRESS] = kbdev->reg + 0x10620; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS8__STATUS] = kbdev->reg + 0x10628; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS8__TRANSCFG] = kbdev->reg + 0x10630; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS8__FAULTEXTRA] = kbdev->reg + 0x10638; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS9__TRANSTAB] = kbdev->reg + 0x10640; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS9__MEMATTR] = kbdev->reg + 0x10648; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS9__LOCKADDR] = kbdev->reg + 0x10650; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS9__COMMAND] = kbdev->reg + 0x10658; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS9__FAULTSTATUS] = kbdev->reg + 0x1065c; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS9__FAULTADDRESS] = kbdev->reg + 0x10660; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS9__STATUS] = kbdev->reg + 0x10668; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS9__TRANSCFG] = kbdev->reg + 0x10670; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS9__FAULTEXTRA] = kbdev->reg + 0x10678; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS10__TRANSTAB] = kbdev->reg + 0x10680; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS10__MEMATTR] = kbdev->reg + 0x10688; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS10__LOCKADDR] = kbdev->reg + 0x10690; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS10__COMMAND] = kbdev->reg + 0x10698; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS10__FAULTSTATUS] = kbdev->reg + 0x1069c; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS10__FAULTADDRESS] = kbdev->reg + 0x106a0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS10__STATUS] = kbdev->reg + 0x106a8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS10__TRANSCFG] = kbdev->reg + 0x106b0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS10__FAULTEXTRA] = kbdev->reg + 0x106b8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS11__TRANSTAB] = kbdev->reg + 0x106c0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS11__MEMATTR] = kbdev->reg + 0x106c8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS11__LOCKADDR] = kbdev->reg + 0x106d0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS11__COMMAND] = kbdev->reg + 0x106d8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS11__FAULTSTATUS] = kbdev->reg + 0x106dc; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS11__FAULTADDRESS] = kbdev->reg + 0x106e0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS11__STATUS] = kbdev->reg + 0x106e8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS11__TRANSCFG] = kbdev->reg + 0x106f0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS11__FAULTEXTRA] = kbdev->reg + 0x106f8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS12__TRANSTAB] = kbdev->reg + 0x10700; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS12__MEMATTR] = kbdev->reg + 0x10708; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS12__LOCKADDR] = kbdev->reg + 0x10710; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS12__COMMAND] = kbdev->reg + 0x10718; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS12__FAULTSTATUS] = kbdev->reg + 0x1071c; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS12__FAULTADDRESS] = kbdev->reg + 0x10720; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS12__STATUS] = kbdev->reg + 0x10728; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS12__TRANSCFG] = kbdev->reg + 0x10730; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS12__FAULTEXTRA] = kbdev->reg + 0x10738; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS13__TRANSTAB] = kbdev->reg + 0x10740; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS13__MEMATTR] = kbdev->reg + 0x10748; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS13__LOCKADDR] = kbdev->reg + 0x10750; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS13__COMMAND] = kbdev->reg + 0x10758; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS13__FAULTSTATUS] = kbdev->reg + 0x1075c; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS13__FAULTADDRESS] = kbdev->reg + 0x10760; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS13__STATUS] = kbdev->reg + 0x10768; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS13__TRANSCFG] = kbdev->reg + 0x10770; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS13__FAULTEXTRA] = kbdev->reg + 0x10778; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS14__TRANSTAB] = kbdev->reg + 0x10780; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS14__MEMATTR] = kbdev->reg + 0x10788; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS14__LOCKADDR] = kbdev->reg + 0x10790; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS14__COMMAND] = kbdev->reg + 0x10798; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS14__FAULTSTATUS] = kbdev->reg + 0x1079c; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS14__FAULTADDRESS] = kbdev->reg + 0x107a0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS14__STATUS] = kbdev->reg + 0x107a8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS14__TRANSCFG] = kbdev->reg + 0x107b0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS14__FAULTEXTRA] = kbdev->reg + 0x107b8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS15__TRANSTAB] = kbdev->reg + 0x107c0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS15__MEMATTR] = kbdev->reg + 0x107c8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS15__LOCKADDR] = kbdev->reg + 0x107d0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS15__COMMAND] = kbdev->reg + 0x107d8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS15__FAULTSTATUS] = kbdev->reg + 0x107dc; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS15__FAULTADDRESS] = kbdev->reg + 0x107e0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS15__STATUS] = kbdev->reg + 0x107e8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS15__TRANSCFG] = kbdev->reg + 0x107f0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS15__FAULTEXTRA] = kbdev->reg + 0x107f8; -} - -static void kbase_regmap_v6_2_init(struct kbase_device *kbdev) -{ - if (kbdev->regmap.regs == NULL && kbdev->regmap.flags == NULL) { - kbdev->regmap.size = NR_V6_2_REGS; - kbdev->regmap.regs = - kcalloc(kbdev->regmap.size, sizeof(void __iomem *), GFP_KERNEL); - kbdev->regmap.flags = kcalloc(kbdev->regmap.size, sizeof(u32), GFP_KERNEL); - } - - if (WARN_ON(kbdev->regmap.regs == NULL)) - return; - if (WARN_ON(kbdev->regmap.flags == NULL)) - return; - - kbase_regmap_v6_0_init(kbdev); - - kbdev->regmap.flags[GPU_CONTROL__REVIDR] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__STACK_PRESENT] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__STACK_PWROFF] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__STACK_PWRON] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__STACK_PWRTRANS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__STACK_READY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - - kbdev->regmap.regs[GPU_CONTROL__REVIDR] = kbdev->reg + 0x280; - kbdev->regmap.regs[GPU_CONTROL__STACK_PRESENT] = kbdev->reg + 0xe00; - kbdev->regmap.regs[GPU_CONTROL__STACK_PWROFF] = kbdev->reg + 0xe30; - kbdev->regmap.regs[GPU_CONTROL__STACK_PWRON] = kbdev->reg + 0xe20; - kbdev->regmap.regs[GPU_CONTROL__STACK_PWRTRANS] = kbdev->reg + 0xe40; - kbdev->regmap.regs[GPU_CONTROL__STACK_READY] = kbdev->reg + 0xe10; -} - -static void kbase_regmap_v7_0_init(struct kbase_device *kbdev) -{ - if (kbdev->regmap.regs == NULL && kbdev->regmap.flags == NULL) { - kbdev->regmap.size = NR_V7_0_REGS; - kbdev->regmap.regs = - kcalloc(kbdev->regmap.size, sizeof(void __iomem *), GFP_KERNEL); - kbdev->regmap.flags = kcalloc(kbdev->regmap.size, sizeof(u32), GFP_KERNEL); - } - - if (WARN_ON(kbdev->regmap.regs == NULL)) - return; - if (WARN_ON(kbdev->regmap.flags == NULL)) - return; - - kbase_regmap_v6_2_init(kbdev); - - kbdev->regmap.flags[GPU_CONTROL__TEXTURE_FEATURES_3] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - - kbdev->regmap.regs[GPU_CONTROL__TEXTURE_FEATURES_3] = kbdev->reg + 0xbc; -} - -static void kbase_regmap_v7_2_init(struct kbase_device *kbdev) -{ - if (kbdev->regmap.regs == NULL && kbdev->regmap.flags == NULL) { - kbdev->regmap.size = NR_V7_2_REGS; - kbdev->regmap.regs = - kcalloc(kbdev->regmap.size, sizeof(void __iomem *), GFP_KERNEL); - kbdev->regmap.flags = kcalloc(kbdev->regmap.size, sizeof(u32), GFP_KERNEL); - } - - if (WARN_ON(kbdev->regmap.regs == NULL)) - return; - if (WARN_ON(kbdev->regmap.flags == NULL)) - return; - - kbase_regmap_v7_0_init(kbdev); - - kbdev->regmap.flags[GPU_CONTROL__CORE_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__THREAD_TLS_ALLOC] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - - kbdev->regmap.regs[GPU_CONTROL__CORE_FEATURES] = kbdev->reg + 0x8; - kbdev->regmap.regs[GPU_CONTROL__THREAD_TLS_ALLOC] = kbdev->reg + 0x310; -} - -static void kbase_regmap_v9_0_init(struct kbase_device *kbdev) -{ - if (kbdev->regmap.regs == NULL && kbdev->regmap.flags == NULL) { - kbdev->regmap.size = NR_V9_0_REGS; - kbdev->regmap.regs = - kcalloc(kbdev->regmap.size, sizeof(void __iomem *), GFP_KERNEL); - kbdev->regmap.flags = kcalloc(kbdev->regmap.size, sizeof(u32), GFP_KERNEL); - } - - if (WARN_ON(kbdev->regmap.regs == NULL)) - return; - if (WARN_ON(kbdev->regmap.flags == NULL)) - return; - - kbase_regmap_v7_2_init(kbdev); - - kbdev->regmap.flags[GPU_CONTROL__CORE_FEATURES] = 0; - kbdev->regmap.flags[GPU_CONTROL__THREAD_TLS_ALLOC] = 0; - kbdev->regmap.flags[JOB_CONTROL__JOB_IRQ_THROTTLE] = 0; - - kbdev->regmap.regs[GPU_CONTROL__CORE_FEATURES] = NULL; - kbdev->regmap.regs[GPU_CONTROL__THREAD_TLS_ALLOC] = NULL; - kbdev->regmap.regs[JOB_CONTROL__JOB_IRQ_THROTTLE] = NULL; -} - -static void kbase_regmap_v9_2_init(struct kbase_device *kbdev) -{ - if (kbdev->regmap.regs == NULL && kbdev->regmap.flags == NULL) { - kbdev->regmap.size = NR_V9_2_REGS; - kbdev->regmap.regs = - kcalloc(kbdev->regmap.size, sizeof(void __iomem *), GFP_KERNEL); - kbdev->regmap.flags = kcalloc(kbdev->regmap.size, sizeof(u32), GFP_KERNEL); - } - - if (WARN_ON(kbdev->regmap.regs == NULL)) - return; - if (WARN_ON(kbdev->regmap.flags == NULL)) - return; - - kbase_regmap_v9_0_init(kbdev); - - kbdev->regmap.flags[GPU_CONTROL__L2_CONFIG] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - - kbdev->regmap.regs[GPU_CONTROL__L2_CONFIG] = kbdev->reg + 0x48; -} - -static void kbase_regmap_v9_14_init(struct kbase_device *kbdev) -{ - if (kbdev->regmap.regs == NULL && kbdev->regmap.flags == NULL) { - kbdev->regmap.size = NR_V9_14_REGS; - kbdev->regmap.regs = - kcalloc(kbdev->regmap.size, sizeof(void __iomem *), GFP_KERNEL); - kbdev->regmap.flags = kcalloc(kbdev->regmap.size, sizeof(u32), GFP_KERNEL); - } - - if (WARN_ON(kbdev->regmap.regs == NULL)) - return; - if (WARN_ON(kbdev->regmap.flags == NULL)) - return; - - kbase_regmap_v9_2_init(kbdev); - - kbdev->regmap.flags[PTM_AW_IRQ_CLEAR] = KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[PTM_AW_IRQ_INJECTION] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[PTM_AW_IRQ_MASK] = KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[PTM_AW_IRQ_RAWSTAT] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[PTM_AW_IRQ_STATUS] = KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE0] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE1] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE_STATUS] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[PTM_ID] = KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ; - - kbdev->regmap.regs[PTM_AW_IRQ_CLEAR] = kbdev->reg + 0x1ffc8; - kbdev->regmap.regs[PTM_AW_IRQ_INJECTION] = kbdev->reg + 0x1ffd4; - kbdev->regmap.regs[PTM_AW_IRQ_MASK] = kbdev->reg + 0x1ffcc; - kbdev->regmap.regs[PTM_AW_IRQ_RAWSTAT] = kbdev->reg + 0x1ffc4; - kbdev->regmap.regs[PTM_AW_IRQ_STATUS] = kbdev->reg + 0x1ffd0; - kbdev->regmap.regs[PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE0] = kbdev->reg + 0x1ffd8; - kbdev->regmap.regs[PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE1] = kbdev->reg + 0x1ffdc; - kbdev->regmap.regs[PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE0] = kbdev->reg + 0x1ffe4; - kbdev->regmap.regs[PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE1] = kbdev->reg + 0x1ffe8; - kbdev->regmap.regs[PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE_STATUS] = kbdev->reg + 0x1ffe0; - kbdev->regmap.regs[PTM_ID] = kbdev->reg + 0x1ffc0; -} - -u32 kbase_regmap_backend_init(struct kbase_device *kbdev) -{ - int i = 0; - - struct { - u32 arch_id; - void (*init)(struct kbase_device *kbdev); - } init_array[] = { - { GPU_ID_ARCH_MAKE(6, 0, 0), kbase_regmap_v6_0_init }, - { GPU_ID_ARCH_MAKE(6, 2, 0), kbase_regmap_v6_2_init }, - { GPU_ID_ARCH_MAKE(7, 0, 0), kbase_regmap_v7_0_init }, - { GPU_ID_ARCH_MAKE(7, 2, 0), kbase_regmap_v7_2_init }, - { GPU_ID_ARCH_MAKE(9, 0, 0), kbase_regmap_v9_0_init }, - { GPU_ID_ARCH_MAKE(9, 2, 0), kbase_regmap_v9_2_init }, - { GPU_ID_ARCH_MAKE(9, 14, 0), kbase_regmap_v9_14_init }, - }; - - for (i = 0; i < ARRAY_SIZE(init_array) - 1; i++) { - if (kbdev->gpu_props.gpu_id.arch_id < init_array[i + 1].arch_id) { - init_array[i].init(kbdev); - return init_array[i].arch_id; - } - } - - /* arch_id greater than last entry in init_array */ - init_array[i].init(kbdev); - return init_array[i].arch_id; -} - -#ifdef CONFIG_MALI_VALHALL_DEBUG -static char *enum_strings[] = { - [GPU_CONTROL__GPU_ID] = "GPU_CONTROL__GPU_ID", - [GPU_CONTROL__L2_FEATURES] = "GPU_CONTROL__L2_FEATURES", - [GPU_CONTROL__TILER_FEATURES] = "GPU_CONTROL__TILER_FEATURES", - [GPU_CONTROL__MEM_FEATURES] = "GPU_CONTROL__MEM_FEATURES", - [GPU_CONTROL__MMU_FEATURES] = "GPU_CONTROL__MMU_FEATURES", - [GPU_CONTROL__AS_PRESENT] = "GPU_CONTROL__AS_PRESENT", - [GPU_CONTROL__JS_PRESENT] = "GPU_CONTROL__JS_PRESENT", - [GPU_CONTROL__GPU_IRQ_RAWSTAT] = "GPU_CONTROL__GPU_IRQ_RAWSTAT", - [GPU_CONTROL__GPU_IRQ_CLEAR] = "GPU_CONTROL__GPU_IRQ_CLEAR", - [GPU_CONTROL__GPU_IRQ_MASK] = "GPU_CONTROL__GPU_IRQ_MASK", - [GPU_CONTROL__GPU_IRQ_STATUS] = "GPU_CONTROL__GPU_IRQ_STATUS", - [GPU_CONTROL__GPU_COMMAND] = "GPU_CONTROL__GPU_COMMAND", - [GPU_CONTROL__GPU_STATUS] = "GPU_CONTROL__GPU_STATUS", - [GPU_CONTROL__LATEST_FLUSH] = "GPU_CONTROL__LATEST_FLUSH", - [GPU_CONTROL__GPU_FAULTSTATUS] = "GPU_CONTROL__GPU_FAULTSTATUS", - [GPU_CONTROL__GPU_FAULTADDRESS] = "GPU_CONTROL__GPU_FAULTADDRESS", - [GPU_CONTROL__AFBC_FEATURES] = "GPU_CONTROL__AFBC_FEATURES", - [GPU_CONTROL__PWR_KEY] = "GPU_CONTROL__PWR_KEY", - [GPU_CONTROL__PWR_OVERRIDE0] = "GPU_CONTROL__PWR_OVERRIDE0", - [GPU_CONTROL__PWR_OVERRIDE1] = "GPU_CONTROL__PWR_OVERRIDE1", - [GPU_CONTROL__PRFCNT_BASE] = "GPU_CONTROL__PRFCNT_BASE", - [GPU_CONTROL__PRFCNT_CONFIG] = "GPU_CONTROL__PRFCNT_CONFIG", - [GPU_CONTROL__PRFCNT_JM_EN] = "GPU_CONTROL__PRFCNT_JM_EN", - [GPU_CONTROL__PRFCNT_SHADER_EN] = "GPU_CONTROL__PRFCNT_SHADER_EN", - [GPU_CONTROL__PRFCNT_TILER_EN] = "GPU_CONTROL__PRFCNT_TILER_EN", - [GPU_CONTROL__PRFCNT_MMU_L2_EN] = "GPU_CONTROL__PRFCNT_MMU_L2_EN", - [GPU_CONTROL__CYCLE_COUNT] = "GPU_CONTROL__CYCLE_COUNT", - [GPU_CONTROL__TIMESTAMP] = "GPU_CONTROL__TIMESTAMP", - [GPU_CONTROL__THREAD_MAX_THREADS] = "GPU_CONTROL__THREAD_MAX_THREADS", - [GPU_CONTROL__THREAD_MAX_WORKGROUP_SIZE] = "GPU_CONTROL__THREAD_MAX_WORKGROUP_SIZE", - [GPU_CONTROL__THREAD_MAX_BARRIER_SIZE] = "GPU_CONTROL__THREAD_MAX_BARRIER_SIZE", - [GPU_CONTROL__THREAD_FEATURES] = "GPU_CONTROL__THREAD_FEATURES", - [GPU_CONTROL__TEXTURE_FEATURES_0] = "GPU_CONTROL__TEXTURE_FEATURES_0", - [GPU_CONTROL__TEXTURE_FEATURES_1] = "GPU_CONTROL__TEXTURE_FEATURES_1", - [GPU_CONTROL__TEXTURE_FEATURES_2] = "GPU_CONTROL__TEXTURE_FEATURES_2", - [GPU_CONTROL__JS0_FEATURES] = "GPU_CONTROL__JS0_FEATURES", - [GPU_CONTROL__JS1_FEATURES] = "GPU_CONTROL__JS1_FEATURES", - [GPU_CONTROL__JS2_FEATURES] = "GPU_CONTROL__JS2_FEATURES", - [GPU_CONTROL__JS3_FEATURES] = "GPU_CONTROL__JS3_FEATURES", - [GPU_CONTROL__JS4_FEATURES] = "GPU_CONTROL__JS4_FEATURES", - [GPU_CONTROL__JS5_FEATURES] = "GPU_CONTROL__JS5_FEATURES", - [GPU_CONTROL__JS6_FEATURES] = "GPU_CONTROL__JS6_FEATURES", - [GPU_CONTROL__JS7_FEATURES] = "GPU_CONTROL__JS7_FEATURES", - [GPU_CONTROL__JS8_FEATURES] = "GPU_CONTROL__JS8_FEATURES", - [GPU_CONTROL__JS9_FEATURES] = "GPU_CONTROL__JS9_FEATURES", - [GPU_CONTROL__JS10_FEATURES] = "GPU_CONTROL__JS10_FEATURES", - [GPU_CONTROL__JS11_FEATURES] = "GPU_CONTROL__JS11_FEATURES", - [GPU_CONTROL__JS12_FEATURES] = "GPU_CONTROL__JS12_FEATURES", - [GPU_CONTROL__JS13_FEATURES] = "GPU_CONTROL__JS13_FEATURES", - [GPU_CONTROL__JS14_FEATURES] = "GPU_CONTROL__JS14_FEATURES", - [GPU_CONTROL__JS15_FEATURES] = "GPU_CONTROL__JS15_FEATURES", - [GPU_CONTROL__SHADER_PRESENT] = "GPU_CONTROL__SHADER_PRESENT", - [GPU_CONTROL__TILER_PRESENT] = "GPU_CONTROL__TILER_PRESENT", - [GPU_CONTROL__L2_PRESENT] = "GPU_CONTROL__L2_PRESENT", - [GPU_CONTROL__SHADER_READY] = "GPU_CONTROL__SHADER_READY", - [GPU_CONTROL__TILER_READY] = "GPU_CONTROL__TILER_READY", - [GPU_CONTROL__L2_READY] = "GPU_CONTROL__L2_READY", - [GPU_CONTROL__SHADER_PWRON] = "GPU_CONTROL__SHADER_PWRON", - [GPU_CONTROL__TILER_PWRON] = "GPU_CONTROL__TILER_PWRON", - [GPU_CONTROL__L2_PWRON] = "GPU_CONTROL__L2_PWRON", - [GPU_CONTROL__SHADER_PWROFF] = "GPU_CONTROL__SHADER_PWROFF", - [GPU_CONTROL__TILER_PWROFF] = "GPU_CONTROL__TILER_PWROFF", - [GPU_CONTROL__L2_PWROFF] = "GPU_CONTROL__L2_PWROFF", - [GPU_CONTROL__SHADER_PWRTRANS] = "GPU_CONTROL__SHADER_PWRTRANS", - [GPU_CONTROL__TILER_PWRTRANS] = "GPU_CONTROL__TILER_PWRTRANS", - [GPU_CONTROL__L2_PWRTRANS] = "GPU_CONTROL__L2_PWRTRANS", - [GPU_CONTROL__SHADER_PWRACTIVE] = "GPU_CONTROL__SHADER_PWRACTIVE", - [GPU_CONTROL__TILER_PWRACTIVE] = "GPU_CONTROL__TILER_PWRACTIVE", - [GPU_CONTROL__L2_PWRACTIVE] = "GPU_CONTROL__L2_PWRACTIVE", - [GPU_CONTROL__COHERENCY_FEATURES] = "GPU_CONTROL__COHERENCY_FEATURES", - [GPU_CONTROL__COHERENCY_ENABLE] = "GPU_CONTROL__COHERENCY_ENABLE", - [GPU_CONTROL__GPU_USER_INn] = "GPU_CONTROL__GPU_USER_INn", - [GPU_CONTROL__GPU_USER_OUTn] = "GPU_CONTROL__GPU_USER_OUTn", - [GPU_CONTROL__JM_CONFIG] = "GPU_CONTROL__JM_CONFIG", - [GPU_CONTROL__SHADER_CONFIG] = "GPU_CONTROL__SHADER_CONFIG", - [GPU_CONTROL__TILER_CONFIG] = "GPU_CONTROL__TILER_CONFIG", - [GPU_CONTROL__L2_MMU_CONFIG] = "GPU_CONTROL__L2_MMU_CONFIG", - [JOB_CONTROL__JOB_IRQ_RAWSTAT] = "JOB_CONTROL__JOB_IRQ_RAWSTAT", - [JOB_CONTROL__JOB_IRQ_CLEAR] = "JOB_CONTROL__JOB_IRQ_CLEAR", - [JOB_CONTROL__JOB_IRQ_MASK] = "JOB_CONTROL__JOB_IRQ_MASK", - [JOB_CONTROL__JOB_IRQ_STATUS] = "JOB_CONTROL__JOB_IRQ_STATUS", - [JOB_CONTROL__JOB_IRQ_JS_STATE] = "JOB_CONTROL__JOB_IRQ_JS_STATE", - [JOB_CONTROL__JOB_IRQ_THROTTLE] = "JOB_CONTROL__JOB_IRQ_THROTTLE", - [JOB_CONTROL__EVENT_IRQ_RAWSTAT] = "JOB_CONTROL__EVENT_IRQ_RAWSTAT", - [JOB_CONTROL__EVENT_IRQ_CLEAR] = "JOB_CONTROL__EVENT_IRQ_CLEAR", - [JOB_CONTROL__EVENT_IRQ_MASK] = "JOB_CONTROL__EVENT_IRQ_MASK", - [JOB_CONTROL__EVENT_IRQ_STATUS] = "JOB_CONTROL__EVENT_IRQ_STATUS", - [JOB_CONTROL__JS0__HEAD] = "JOB_CONTROL__JS0__HEAD", - [JOB_CONTROL__JS0__TAIL] = "JOB_CONTROL__JS0__TAIL", - [JOB_CONTROL__JS0__AFFINITY] = "JOB_CONTROL__JS0__AFFINITY", - [JOB_CONTROL__JS0__CONFIG] = "JOB_CONTROL__JS0__CONFIG", - [JOB_CONTROL__JS0__XAFFINITY] = "JOB_CONTROL__JS0__XAFFINITY", - [JOB_CONTROL__JS0__COMMAND] = "JOB_CONTROL__JS0__COMMAND", - [JOB_CONTROL__JS0__STATUS] = "JOB_CONTROL__JS0__STATUS", - [JOB_CONTROL__JS0__FLUSH_ID] = "JOB_CONTROL__JS0__FLUSH_ID", - [JOB_CONTROL__JS0__EVENT_0] = "JOB_CONTROL__JS0__EVENT_0", - [JOB_CONTROL__JS0__EVENT_1] = "JOB_CONTROL__JS0__EVENT_1", - [JOB_CONTROL__JS0__EVENT_MASK] = "JOB_CONTROL__JS0__EVENT_MASK", - [JOB_CONTROL__JS0__HEAD_NEXT] = "JOB_CONTROL__JS0__HEAD_NEXT", - [JOB_CONTROL__JS0__TAIL_NEXT] = "JOB_CONTROL__JS0__TAIL_NEXT", - [JOB_CONTROL__JS0__AFFINITY_NEXT] = "JOB_CONTROL__JS0__AFFINITY_NEXT", - [JOB_CONTROL__JS0__CONFIG_NEXT] = "JOB_CONTROL__JS0__CONFIG_NEXT", - [JOB_CONTROL__JS0__XAFFINITY_NEXT] = "JOB_CONTROL__JS0__XAFFINITY_NEXT", - [JOB_CONTROL__JS0__COMMAND_NEXT] = "JOB_CONTROL__JS0__COMMAND_NEXT", - [JOB_CONTROL__JS0__FLUSH_ID_NEXT] = "JOB_CONTROL__JS0__FLUSH_ID_NEXT", - [JOB_CONTROL__JS0__EVENT_MASK_NEXT] = "JOB_CONTROL__JS0__EVENT_MASK_NEXT", - [JOB_CONTROL__JS1__HEAD] = "JOB_CONTROL__JS1__HEAD", - [JOB_CONTROL__JS1__TAIL] = "JOB_CONTROL__JS1__TAIL", - [JOB_CONTROL__JS1__AFFINITY] = "JOB_CONTROL__JS1__AFFINITY", - [JOB_CONTROL__JS1__CONFIG] = "JOB_CONTROL__JS1__CONFIG", - [JOB_CONTROL__JS1__XAFFINITY] = "JOB_CONTROL__JS1__XAFFINITY", - [JOB_CONTROL__JS1__COMMAND] = "JOB_CONTROL__JS1__COMMAND", - [JOB_CONTROL__JS1__STATUS] = "JOB_CONTROL__JS1__STATUS", - [JOB_CONTROL__JS1__FLUSH_ID] = "JOB_CONTROL__JS1__FLUSH_ID", - [JOB_CONTROL__JS1__EVENT_0] = "JOB_CONTROL__JS1__EVENT_0", - [JOB_CONTROL__JS1__EVENT_1] = "JOB_CONTROL__JS1__EVENT_1", - [JOB_CONTROL__JS1__EVENT_MASK] = "JOB_CONTROL__JS1__EVENT_MASK", - [JOB_CONTROL__JS1__HEAD_NEXT] = "JOB_CONTROL__JS1__HEAD_NEXT", - [JOB_CONTROL__JS1__TAIL_NEXT] = "JOB_CONTROL__JS1__TAIL_NEXT", - [JOB_CONTROL__JS1__AFFINITY_NEXT] = "JOB_CONTROL__JS1__AFFINITY_NEXT", - [JOB_CONTROL__JS1__CONFIG_NEXT] = "JOB_CONTROL__JS1__CONFIG_NEXT", - [JOB_CONTROL__JS1__XAFFINITY_NEXT] = "JOB_CONTROL__JS1__XAFFINITY_NEXT", - [JOB_CONTROL__JS1__COMMAND_NEXT] = "JOB_CONTROL__JS1__COMMAND_NEXT", - [JOB_CONTROL__JS1__FLUSH_ID_NEXT] = "JOB_CONTROL__JS1__FLUSH_ID_NEXT", - [JOB_CONTROL__JS1__EVENT_MASK_NEXT] = "JOB_CONTROL__JS1__EVENT_MASK_NEXT", - [JOB_CONTROL__JS2__HEAD] = "JOB_CONTROL__JS2__HEAD", - [JOB_CONTROL__JS2__TAIL] = "JOB_CONTROL__JS2__TAIL", - [JOB_CONTROL__JS2__AFFINITY] = "JOB_CONTROL__JS2__AFFINITY", - [JOB_CONTROL__JS2__CONFIG] = "JOB_CONTROL__JS2__CONFIG", - [JOB_CONTROL__JS2__XAFFINITY] = "JOB_CONTROL__JS2__XAFFINITY", - [JOB_CONTROL__JS2__COMMAND] = "JOB_CONTROL__JS2__COMMAND", - [JOB_CONTROL__JS2__STATUS] = "JOB_CONTROL__JS2__STATUS", - [JOB_CONTROL__JS2__FLUSH_ID] = "JOB_CONTROL__JS2__FLUSH_ID", - [JOB_CONTROL__JS2__EVENT_0] = "JOB_CONTROL__JS2__EVENT_0", - [JOB_CONTROL__JS2__EVENT_1] = "JOB_CONTROL__JS2__EVENT_1", - [JOB_CONTROL__JS2__EVENT_MASK] = "JOB_CONTROL__JS2__EVENT_MASK", - [JOB_CONTROL__JS2__HEAD_NEXT] = "JOB_CONTROL__JS2__HEAD_NEXT", - [JOB_CONTROL__JS2__TAIL_NEXT] = "JOB_CONTROL__JS2__TAIL_NEXT", - [JOB_CONTROL__JS2__AFFINITY_NEXT] = "JOB_CONTROL__JS2__AFFINITY_NEXT", - [JOB_CONTROL__JS2__CONFIG_NEXT] = "JOB_CONTROL__JS2__CONFIG_NEXT", - [JOB_CONTROL__JS2__XAFFINITY_NEXT] = "JOB_CONTROL__JS2__XAFFINITY_NEXT", - [JOB_CONTROL__JS2__COMMAND_NEXT] = "JOB_CONTROL__JS2__COMMAND_NEXT", - [JOB_CONTROL__JS2__FLUSH_ID_NEXT] = "JOB_CONTROL__JS2__FLUSH_ID_NEXT", - [JOB_CONTROL__JS2__EVENT_MASK_NEXT] = "JOB_CONTROL__JS2__EVENT_MASK_NEXT", - [JOB_CONTROL__JS3__HEAD] = "JOB_CONTROL__JS3__HEAD", - [JOB_CONTROL__JS3__TAIL] = "JOB_CONTROL__JS3__TAIL", - [JOB_CONTROL__JS3__AFFINITY] = "JOB_CONTROL__JS3__AFFINITY", - [JOB_CONTROL__JS3__CONFIG] = "JOB_CONTROL__JS3__CONFIG", - [JOB_CONTROL__JS3__XAFFINITY] = "JOB_CONTROL__JS3__XAFFINITY", - [JOB_CONTROL__JS3__COMMAND] = "JOB_CONTROL__JS3__COMMAND", - [JOB_CONTROL__JS3__STATUS] = "JOB_CONTROL__JS3__STATUS", - [JOB_CONTROL__JS3__FLUSH_ID] = "JOB_CONTROL__JS3__FLUSH_ID", - [JOB_CONTROL__JS3__EVENT_0] = "JOB_CONTROL__JS3__EVENT_0", - [JOB_CONTROL__JS3__EVENT_1] = "JOB_CONTROL__JS3__EVENT_1", - [JOB_CONTROL__JS3__EVENT_MASK] = "JOB_CONTROL__JS3__EVENT_MASK", - [JOB_CONTROL__JS3__HEAD_NEXT] = "JOB_CONTROL__JS3__HEAD_NEXT", - [JOB_CONTROL__JS3__TAIL_NEXT] = "JOB_CONTROL__JS3__TAIL_NEXT", - [JOB_CONTROL__JS3__AFFINITY_NEXT] = "JOB_CONTROL__JS3__AFFINITY_NEXT", - [JOB_CONTROL__JS3__CONFIG_NEXT] = "JOB_CONTROL__JS3__CONFIG_NEXT", - [JOB_CONTROL__JS3__XAFFINITY_NEXT] = "JOB_CONTROL__JS3__XAFFINITY_NEXT", - [JOB_CONTROL__JS3__COMMAND_NEXT] = "JOB_CONTROL__JS3__COMMAND_NEXT", - [JOB_CONTROL__JS3__FLUSH_ID_NEXT] = "JOB_CONTROL__JS3__FLUSH_ID_NEXT", - [JOB_CONTROL__JS3__EVENT_MASK_NEXT] = "JOB_CONTROL__JS3__EVENT_MASK_NEXT", - [JOB_CONTROL__JS4__HEAD] = "JOB_CONTROL__JS4__HEAD", - [JOB_CONTROL__JS4__TAIL] = "JOB_CONTROL__JS4__TAIL", - [JOB_CONTROL__JS4__AFFINITY] = "JOB_CONTROL__JS4__AFFINITY", - [JOB_CONTROL__JS4__CONFIG] = "JOB_CONTROL__JS4__CONFIG", - [JOB_CONTROL__JS4__XAFFINITY] = "JOB_CONTROL__JS4__XAFFINITY", - [JOB_CONTROL__JS4__COMMAND] = "JOB_CONTROL__JS4__COMMAND", - [JOB_CONTROL__JS4__STATUS] = "JOB_CONTROL__JS4__STATUS", - [JOB_CONTROL__JS4__FLUSH_ID] = "JOB_CONTROL__JS4__FLUSH_ID", - [JOB_CONTROL__JS4__EVENT_0] = "JOB_CONTROL__JS4__EVENT_0", - [JOB_CONTROL__JS4__EVENT_1] = "JOB_CONTROL__JS4__EVENT_1", - [JOB_CONTROL__JS4__EVENT_MASK] = "JOB_CONTROL__JS4__EVENT_MASK", - [JOB_CONTROL__JS4__HEAD_NEXT] = "JOB_CONTROL__JS4__HEAD_NEXT", - [JOB_CONTROL__JS4__TAIL_NEXT] = "JOB_CONTROL__JS4__TAIL_NEXT", - [JOB_CONTROL__JS4__AFFINITY_NEXT] = "JOB_CONTROL__JS4__AFFINITY_NEXT", - [JOB_CONTROL__JS4__CONFIG_NEXT] = "JOB_CONTROL__JS4__CONFIG_NEXT", - [JOB_CONTROL__JS4__XAFFINITY_NEXT] = "JOB_CONTROL__JS4__XAFFINITY_NEXT", - [JOB_CONTROL__JS4__COMMAND_NEXT] = "JOB_CONTROL__JS4__COMMAND_NEXT", - [JOB_CONTROL__JS4__FLUSH_ID_NEXT] = "JOB_CONTROL__JS4__FLUSH_ID_NEXT", - [JOB_CONTROL__JS4__EVENT_MASK_NEXT] = "JOB_CONTROL__JS4__EVENT_MASK_NEXT", - [JOB_CONTROL__JS5__HEAD] = "JOB_CONTROL__JS5__HEAD", - [JOB_CONTROL__JS5__TAIL] = "JOB_CONTROL__JS5__TAIL", - [JOB_CONTROL__JS5__AFFINITY] = "JOB_CONTROL__JS5__AFFINITY", - [JOB_CONTROL__JS5__CONFIG] = "JOB_CONTROL__JS5__CONFIG", - [JOB_CONTROL__JS5__XAFFINITY] = "JOB_CONTROL__JS5__XAFFINITY", - [JOB_CONTROL__JS5__COMMAND] = "JOB_CONTROL__JS5__COMMAND", - [JOB_CONTROL__JS5__STATUS] = "JOB_CONTROL__JS5__STATUS", - [JOB_CONTROL__JS5__FLUSH_ID] = "JOB_CONTROL__JS5__FLUSH_ID", - [JOB_CONTROL__JS5__EVENT_0] = "JOB_CONTROL__JS5__EVENT_0", - [JOB_CONTROL__JS5__EVENT_1] = "JOB_CONTROL__JS5__EVENT_1", - [JOB_CONTROL__JS5__EVENT_MASK] = "JOB_CONTROL__JS5__EVENT_MASK", - [JOB_CONTROL__JS5__HEAD_NEXT] = "JOB_CONTROL__JS5__HEAD_NEXT", - [JOB_CONTROL__JS5__TAIL_NEXT] = "JOB_CONTROL__JS5__TAIL_NEXT", - [JOB_CONTROL__JS5__AFFINITY_NEXT] = "JOB_CONTROL__JS5__AFFINITY_NEXT", - [JOB_CONTROL__JS5__CONFIG_NEXT] = "JOB_CONTROL__JS5__CONFIG_NEXT", - [JOB_CONTROL__JS5__XAFFINITY_NEXT] = "JOB_CONTROL__JS5__XAFFINITY_NEXT", - [JOB_CONTROL__JS5__COMMAND_NEXT] = "JOB_CONTROL__JS5__COMMAND_NEXT", - [JOB_CONTROL__JS5__FLUSH_ID_NEXT] = "JOB_CONTROL__JS5__FLUSH_ID_NEXT", - [JOB_CONTROL__JS5__EVENT_MASK_NEXT] = "JOB_CONTROL__JS5__EVENT_MASK_NEXT", - [JOB_CONTROL__JS6__HEAD] = "JOB_CONTROL__JS6__HEAD", - [JOB_CONTROL__JS6__TAIL] = "JOB_CONTROL__JS6__TAIL", - [JOB_CONTROL__JS6__AFFINITY] = "JOB_CONTROL__JS6__AFFINITY", - [JOB_CONTROL__JS6__CONFIG] = "JOB_CONTROL__JS6__CONFIG", - [JOB_CONTROL__JS6__XAFFINITY] = "JOB_CONTROL__JS6__XAFFINITY", - [JOB_CONTROL__JS6__COMMAND] = "JOB_CONTROL__JS6__COMMAND", - [JOB_CONTROL__JS6__STATUS] = "JOB_CONTROL__JS6__STATUS", - [JOB_CONTROL__JS6__FLUSH_ID] = "JOB_CONTROL__JS6__FLUSH_ID", - [JOB_CONTROL__JS6__EVENT_0] = "JOB_CONTROL__JS6__EVENT_0", - [JOB_CONTROL__JS6__EVENT_1] = "JOB_CONTROL__JS6__EVENT_1", - [JOB_CONTROL__JS6__EVENT_MASK] = "JOB_CONTROL__JS6__EVENT_MASK", - [JOB_CONTROL__JS6__HEAD_NEXT] = "JOB_CONTROL__JS6__HEAD_NEXT", - [JOB_CONTROL__JS6__TAIL_NEXT] = "JOB_CONTROL__JS6__TAIL_NEXT", - [JOB_CONTROL__JS6__AFFINITY_NEXT] = "JOB_CONTROL__JS6__AFFINITY_NEXT", - [JOB_CONTROL__JS6__CONFIG_NEXT] = "JOB_CONTROL__JS6__CONFIG_NEXT", - [JOB_CONTROL__JS6__XAFFINITY_NEXT] = "JOB_CONTROL__JS6__XAFFINITY_NEXT", - [JOB_CONTROL__JS6__COMMAND_NEXT] = "JOB_CONTROL__JS6__COMMAND_NEXT", - [JOB_CONTROL__JS6__FLUSH_ID_NEXT] = "JOB_CONTROL__JS6__FLUSH_ID_NEXT", - [JOB_CONTROL__JS6__EVENT_MASK_NEXT] = "JOB_CONTROL__JS6__EVENT_MASK_NEXT", - [JOB_CONTROL__JS7__HEAD] = "JOB_CONTROL__JS7__HEAD", - [JOB_CONTROL__JS7__TAIL] = "JOB_CONTROL__JS7__TAIL", - [JOB_CONTROL__JS7__AFFINITY] = "JOB_CONTROL__JS7__AFFINITY", - [JOB_CONTROL__JS7__CONFIG] = "JOB_CONTROL__JS7__CONFIG", - [JOB_CONTROL__JS7__XAFFINITY] = "JOB_CONTROL__JS7__XAFFINITY", - [JOB_CONTROL__JS7__COMMAND] = "JOB_CONTROL__JS7__COMMAND", - [JOB_CONTROL__JS7__STATUS] = "JOB_CONTROL__JS7__STATUS", - [JOB_CONTROL__JS7__FLUSH_ID] = "JOB_CONTROL__JS7__FLUSH_ID", - [JOB_CONTROL__JS7__EVENT_0] = "JOB_CONTROL__JS7__EVENT_0", - [JOB_CONTROL__JS7__EVENT_1] = "JOB_CONTROL__JS7__EVENT_1", - [JOB_CONTROL__JS7__EVENT_MASK] = "JOB_CONTROL__JS7__EVENT_MASK", - [JOB_CONTROL__JS7__HEAD_NEXT] = "JOB_CONTROL__JS7__HEAD_NEXT", - [JOB_CONTROL__JS7__TAIL_NEXT] = "JOB_CONTROL__JS7__TAIL_NEXT", - [JOB_CONTROL__JS7__AFFINITY_NEXT] = "JOB_CONTROL__JS7__AFFINITY_NEXT", - [JOB_CONTROL__JS7__CONFIG_NEXT] = "JOB_CONTROL__JS7__CONFIG_NEXT", - [JOB_CONTROL__JS7__XAFFINITY_NEXT] = "JOB_CONTROL__JS7__XAFFINITY_NEXT", - [JOB_CONTROL__JS7__COMMAND_NEXT] = "JOB_CONTROL__JS7__COMMAND_NEXT", - [JOB_CONTROL__JS7__FLUSH_ID_NEXT] = "JOB_CONTROL__JS7__FLUSH_ID_NEXT", - [JOB_CONTROL__JS7__EVENT_MASK_NEXT] = "JOB_CONTROL__JS7__EVENT_MASK_NEXT", - [JOB_CONTROL__JS8__HEAD] = "JOB_CONTROL__JS8__HEAD", - [JOB_CONTROL__JS8__TAIL] = "JOB_CONTROL__JS8__TAIL", - [JOB_CONTROL__JS8__AFFINITY] = "JOB_CONTROL__JS8__AFFINITY", - [JOB_CONTROL__JS8__CONFIG] = "JOB_CONTROL__JS8__CONFIG", - [JOB_CONTROL__JS8__XAFFINITY] = "JOB_CONTROL__JS8__XAFFINITY", - [JOB_CONTROL__JS8__COMMAND] = "JOB_CONTROL__JS8__COMMAND", - [JOB_CONTROL__JS8__STATUS] = "JOB_CONTROL__JS8__STATUS", - [JOB_CONTROL__JS8__FLUSH_ID] = "JOB_CONTROL__JS8__FLUSH_ID", - [JOB_CONTROL__JS8__EVENT_0] = "JOB_CONTROL__JS8__EVENT_0", - [JOB_CONTROL__JS8__EVENT_1] = "JOB_CONTROL__JS8__EVENT_1", - [JOB_CONTROL__JS8__EVENT_MASK] = "JOB_CONTROL__JS8__EVENT_MASK", - [JOB_CONTROL__JS8__HEAD_NEXT] = "JOB_CONTROL__JS8__HEAD_NEXT", - [JOB_CONTROL__JS8__TAIL_NEXT] = "JOB_CONTROL__JS8__TAIL_NEXT", - [JOB_CONTROL__JS8__AFFINITY_NEXT] = "JOB_CONTROL__JS8__AFFINITY_NEXT", - [JOB_CONTROL__JS8__CONFIG_NEXT] = "JOB_CONTROL__JS8__CONFIG_NEXT", - [JOB_CONTROL__JS8__XAFFINITY_NEXT] = "JOB_CONTROL__JS8__XAFFINITY_NEXT", - [JOB_CONTROL__JS8__COMMAND_NEXT] = "JOB_CONTROL__JS8__COMMAND_NEXT", - [JOB_CONTROL__JS8__FLUSH_ID_NEXT] = "JOB_CONTROL__JS8__FLUSH_ID_NEXT", - [JOB_CONTROL__JS8__EVENT_MASK_NEXT] = "JOB_CONTROL__JS8__EVENT_MASK_NEXT", - [JOB_CONTROL__JS9__HEAD] = "JOB_CONTROL__JS9__HEAD", - [JOB_CONTROL__JS9__TAIL] = "JOB_CONTROL__JS9__TAIL", - [JOB_CONTROL__JS9__AFFINITY] = "JOB_CONTROL__JS9__AFFINITY", - [JOB_CONTROL__JS9__CONFIG] = "JOB_CONTROL__JS9__CONFIG", - [JOB_CONTROL__JS9__XAFFINITY] = "JOB_CONTROL__JS9__XAFFINITY", - [JOB_CONTROL__JS9__COMMAND] = "JOB_CONTROL__JS9__COMMAND", - [JOB_CONTROL__JS9__STATUS] = "JOB_CONTROL__JS9__STATUS", - [JOB_CONTROL__JS9__FLUSH_ID] = "JOB_CONTROL__JS9__FLUSH_ID", - [JOB_CONTROL__JS9__EVENT_0] = "JOB_CONTROL__JS9__EVENT_0", - [JOB_CONTROL__JS9__EVENT_1] = "JOB_CONTROL__JS9__EVENT_1", - [JOB_CONTROL__JS9__EVENT_MASK] = "JOB_CONTROL__JS9__EVENT_MASK", - [JOB_CONTROL__JS9__HEAD_NEXT] = "JOB_CONTROL__JS9__HEAD_NEXT", - [JOB_CONTROL__JS9__TAIL_NEXT] = "JOB_CONTROL__JS9__TAIL_NEXT", - [JOB_CONTROL__JS9__AFFINITY_NEXT] = "JOB_CONTROL__JS9__AFFINITY_NEXT", - [JOB_CONTROL__JS9__CONFIG_NEXT] = "JOB_CONTROL__JS9__CONFIG_NEXT", - [JOB_CONTROL__JS9__XAFFINITY_NEXT] = "JOB_CONTROL__JS9__XAFFINITY_NEXT", - [JOB_CONTROL__JS9__COMMAND_NEXT] = "JOB_CONTROL__JS9__COMMAND_NEXT", - [JOB_CONTROL__JS9__FLUSH_ID_NEXT] = "JOB_CONTROL__JS9__FLUSH_ID_NEXT", - [JOB_CONTROL__JS9__EVENT_MASK_NEXT] = "JOB_CONTROL__JS9__EVENT_MASK_NEXT", - [JOB_CONTROL__JS10__HEAD] = "JOB_CONTROL__JS10__HEAD", - [JOB_CONTROL__JS10__TAIL] = "JOB_CONTROL__JS10__TAIL", - [JOB_CONTROL__JS10__AFFINITY] = "JOB_CONTROL__JS10__AFFINITY", - [JOB_CONTROL__JS10__CONFIG] = "JOB_CONTROL__JS10__CONFIG", - [JOB_CONTROL__JS10__XAFFINITY] = "JOB_CONTROL__JS10__XAFFINITY", - [JOB_CONTROL__JS10__COMMAND] = "JOB_CONTROL__JS10__COMMAND", - [JOB_CONTROL__JS10__STATUS] = "JOB_CONTROL__JS10__STATUS", - [JOB_CONTROL__JS10__FLUSH_ID] = "JOB_CONTROL__JS10__FLUSH_ID", - [JOB_CONTROL__JS10__EVENT_0] = "JOB_CONTROL__JS10__EVENT_0", - [JOB_CONTROL__JS10__EVENT_1] = "JOB_CONTROL__JS10__EVENT_1", - [JOB_CONTROL__JS10__EVENT_MASK] = "JOB_CONTROL__JS10__EVENT_MASK", - [JOB_CONTROL__JS10__HEAD_NEXT] = "JOB_CONTROL__JS10__HEAD_NEXT", - [JOB_CONTROL__JS10__TAIL_NEXT] = "JOB_CONTROL__JS10__TAIL_NEXT", - [JOB_CONTROL__JS10__AFFINITY_NEXT] = "JOB_CONTROL__JS10__AFFINITY_NEXT", - [JOB_CONTROL__JS10__CONFIG_NEXT] = "JOB_CONTROL__JS10__CONFIG_NEXT", - [JOB_CONTROL__JS10__XAFFINITY_NEXT] = "JOB_CONTROL__JS10__XAFFINITY_NEXT", - [JOB_CONTROL__JS10__COMMAND_NEXT] = "JOB_CONTROL__JS10__COMMAND_NEXT", - [JOB_CONTROL__JS10__FLUSH_ID_NEXT] = "JOB_CONTROL__JS10__FLUSH_ID_NEXT", - [JOB_CONTROL__JS10__EVENT_MASK_NEXT] = "JOB_CONTROL__JS10__EVENT_MASK_NEXT", - [JOB_CONTROL__JS11__HEAD] = "JOB_CONTROL__JS11__HEAD", - [JOB_CONTROL__JS11__TAIL] = "JOB_CONTROL__JS11__TAIL", - [JOB_CONTROL__JS11__AFFINITY] = "JOB_CONTROL__JS11__AFFINITY", - [JOB_CONTROL__JS11__CONFIG] = "JOB_CONTROL__JS11__CONFIG", - [JOB_CONTROL__JS11__XAFFINITY] = "JOB_CONTROL__JS11__XAFFINITY", - [JOB_CONTROL__JS11__COMMAND] = "JOB_CONTROL__JS11__COMMAND", - [JOB_CONTROL__JS11__STATUS] = "JOB_CONTROL__JS11__STATUS", - [JOB_CONTROL__JS11__FLUSH_ID] = "JOB_CONTROL__JS11__FLUSH_ID", - [JOB_CONTROL__JS11__EVENT_0] = "JOB_CONTROL__JS11__EVENT_0", - [JOB_CONTROL__JS11__EVENT_1] = "JOB_CONTROL__JS11__EVENT_1", - [JOB_CONTROL__JS11__EVENT_MASK] = "JOB_CONTROL__JS11__EVENT_MASK", - [JOB_CONTROL__JS11__HEAD_NEXT] = "JOB_CONTROL__JS11__HEAD_NEXT", - [JOB_CONTROL__JS11__TAIL_NEXT] = "JOB_CONTROL__JS11__TAIL_NEXT", - [JOB_CONTROL__JS11__AFFINITY_NEXT] = "JOB_CONTROL__JS11__AFFINITY_NEXT", - [JOB_CONTROL__JS11__CONFIG_NEXT] = "JOB_CONTROL__JS11__CONFIG_NEXT", - [JOB_CONTROL__JS11__XAFFINITY_NEXT] = "JOB_CONTROL__JS11__XAFFINITY_NEXT", - [JOB_CONTROL__JS11__COMMAND_NEXT] = "JOB_CONTROL__JS11__COMMAND_NEXT", - [JOB_CONTROL__JS11__FLUSH_ID_NEXT] = "JOB_CONTROL__JS11__FLUSH_ID_NEXT", - [JOB_CONTROL__JS11__EVENT_MASK_NEXT] = "JOB_CONTROL__JS11__EVENT_MASK_NEXT", - [JOB_CONTROL__JS12__HEAD] = "JOB_CONTROL__JS12__HEAD", - [JOB_CONTROL__JS12__TAIL] = "JOB_CONTROL__JS12__TAIL", - [JOB_CONTROL__JS12__AFFINITY] = "JOB_CONTROL__JS12__AFFINITY", - [JOB_CONTROL__JS12__CONFIG] = "JOB_CONTROL__JS12__CONFIG", - [JOB_CONTROL__JS12__XAFFINITY] = "JOB_CONTROL__JS12__XAFFINITY", - [JOB_CONTROL__JS12__COMMAND] = "JOB_CONTROL__JS12__COMMAND", - [JOB_CONTROL__JS12__STATUS] = "JOB_CONTROL__JS12__STATUS", - [JOB_CONTROL__JS12__FLUSH_ID] = "JOB_CONTROL__JS12__FLUSH_ID", - [JOB_CONTROL__JS12__EVENT_0] = "JOB_CONTROL__JS12__EVENT_0", - [JOB_CONTROL__JS12__EVENT_1] = "JOB_CONTROL__JS12__EVENT_1", - [JOB_CONTROL__JS12__EVENT_MASK] = "JOB_CONTROL__JS12__EVENT_MASK", - [JOB_CONTROL__JS12__HEAD_NEXT] = "JOB_CONTROL__JS12__HEAD_NEXT", - [JOB_CONTROL__JS12__TAIL_NEXT] = "JOB_CONTROL__JS12__TAIL_NEXT", - [JOB_CONTROL__JS12__AFFINITY_NEXT] = "JOB_CONTROL__JS12__AFFINITY_NEXT", - [JOB_CONTROL__JS12__CONFIG_NEXT] = "JOB_CONTROL__JS12__CONFIG_NEXT", - [JOB_CONTROL__JS12__XAFFINITY_NEXT] = "JOB_CONTROL__JS12__XAFFINITY_NEXT", - [JOB_CONTROL__JS12__COMMAND_NEXT] = "JOB_CONTROL__JS12__COMMAND_NEXT", - [JOB_CONTROL__JS12__FLUSH_ID_NEXT] = "JOB_CONTROL__JS12__FLUSH_ID_NEXT", - [JOB_CONTROL__JS12__EVENT_MASK_NEXT] = "JOB_CONTROL__JS12__EVENT_MASK_NEXT", - [JOB_CONTROL__JS13__HEAD] = "JOB_CONTROL__JS13__HEAD", - [JOB_CONTROL__JS13__TAIL] = "JOB_CONTROL__JS13__TAIL", - [JOB_CONTROL__JS13__AFFINITY] = "JOB_CONTROL__JS13__AFFINITY", - [JOB_CONTROL__JS13__CONFIG] = "JOB_CONTROL__JS13__CONFIG", - [JOB_CONTROL__JS13__XAFFINITY] = "JOB_CONTROL__JS13__XAFFINITY", - [JOB_CONTROL__JS13__COMMAND] = "JOB_CONTROL__JS13__COMMAND", - [JOB_CONTROL__JS13__STATUS] = "JOB_CONTROL__JS13__STATUS", - [JOB_CONTROL__JS13__FLUSH_ID] = "JOB_CONTROL__JS13__FLUSH_ID", - [JOB_CONTROL__JS13__EVENT_0] = "JOB_CONTROL__JS13__EVENT_0", - [JOB_CONTROL__JS13__EVENT_1] = "JOB_CONTROL__JS13__EVENT_1", - [JOB_CONTROL__JS13__EVENT_MASK] = "JOB_CONTROL__JS13__EVENT_MASK", - [JOB_CONTROL__JS13__HEAD_NEXT] = "JOB_CONTROL__JS13__HEAD_NEXT", - [JOB_CONTROL__JS13__TAIL_NEXT] = "JOB_CONTROL__JS13__TAIL_NEXT", - [JOB_CONTROL__JS13__AFFINITY_NEXT] = "JOB_CONTROL__JS13__AFFINITY_NEXT", - [JOB_CONTROL__JS13__CONFIG_NEXT] = "JOB_CONTROL__JS13__CONFIG_NEXT", - [JOB_CONTROL__JS13__XAFFINITY_NEXT] = "JOB_CONTROL__JS13__XAFFINITY_NEXT", - [JOB_CONTROL__JS13__COMMAND_NEXT] = "JOB_CONTROL__JS13__COMMAND_NEXT", - [JOB_CONTROL__JS13__FLUSH_ID_NEXT] = "JOB_CONTROL__JS13__FLUSH_ID_NEXT", - [JOB_CONTROL__JS13__EVENT_MASK_NEXT] = "JOB_CONTROL__JS13__EVENT_MASK_NEXT", - [JOB_CONTROL__JS14__HEAD] = "JOB_CONTROL__JS14__HEAD", - [JOB_CONTROL__JS14__TAIL] = "JOB_CONTROL__JS14__TAIL", - [JOB_CONTROL__JS14__AFFINITY] = "JOB_CONTROL__JS14__AFFINITY", - [JOB_CONTROL__JS14__CONFIG] = "JOB_CONTROL__JS14__CONFIG", - [JOB_CONTROL__JS14__XAFFINITY] = "JOB_CONTROL__JS14__XAFFINITY", - [JOB_CONTROL__JS14__COMMAND] = "JOB_CONTROL__JS14__COMMAND", - [JOB_CONTROL__JS14__STATUS] = "JOB_CONTROL__JS14__STATUS", - [JOB_CONTROL__JS14__FLUSH_ID] = "JOB_CONTROL__JS14__FLUSH_ID", - [JOB_CONTROL__JS14__EVENT_0] = "JOB_CONTROL__JS14__EVENT_0", - [JOB_CONTROL__JS14__EVENT_1] = "JOB_CONTROL__JS14__EVENT_1", - [JOB_CONTROL__JS14__EVENT_MASK] = "JOB_CONTROL__JS14__EVENT_MASK", - [JOB_CONTROL__JS14__HEAD_NEXT] = "JOB_CONTROL__JS14__HEAD_NEXT", - [JOB_CONTROL__JS14__TAIL_NEXT] = "JOB_CONTROL__JS14__TAIL_NEXT", - [JOB_CONTROL__JS14__AFFINITY_NEXT] = "JOB_CONTROL__JS14__AFFINITY_NEXT", - [JOB_CONTROL__JS14__CONFIG_NEXT] = "JOB_CONTROL__JS14__CONFIG_NEXT", - [JOB_CONTROL__JS14__XAFFINITY_NEXT] = "JOB_CONTROL__JS14__XAFFINITY_NEXT", - [JOB_CONTROL__JS14__COMMAND_NEXT] = "JOB_CONTROL__JS14__COMMAND_NEXT", - [JOB_CONTROL__JS14__FLUSH_ID_NEXT] = "JOB_CONTROL__JS14__FLUSH_ID_NEXT", - [JOB_CONTROL__JS14__EVENT_MASK_NEXT] = "JOB_CONTROL__JS14__EVENT_MASK_NEXT", - [JOB_CONTROL__JS15__HEAD] = "JOB_CONTROL__JS15__HEAD", - [JOB_CONTROL__JS15__TAIL] = "JOB_CONTROL__JS15__TAIL", - [JOB_CONTROL__JS15__AFFINITY] = "JOB_CONTROL__JS15__AFFINITY", - [JOB_CONTROL__JS15__CONFIG] = "JOB_CONTROL__JS15__CONFIG", - [JOB_CONTROL__JS15__XAFFINITY] = "JOB_CONTROL__JS15__XAFFINITY", - [JOB_CONTROL__JS15__COMMAND] = "JOB_CONTROL__JS15__COMMAND", - [JOB_CONTROL__JS15__STATUS] = "JOB_CONTROL__JS15__STATUS", - [JOB_CONTROL__JS15__FLUSH_ID] = "JOB_CONTROL__JS15__FLUSH_ID", - [JOB_CONTROL__JS15__EVENT_0] = "JOB_CONTROL__JS15__EVENT_0", - [JOB_CONTROL__JS15__EVENT_1] = "JOB_CONTROL__JS15__EVENT_1", - [JOB_CONTROL__JS15__EVENT_MASK] = "JOB_CONTROL__JS15__EVENT_MASK", - [JOB_CONTROL__JS15__HEAD_NEXT] = "JOB_CONTROL__JS15__HEAD_NEXT", - [JOB_CONTROL__JS15__TAIL_NEXT] = "JOB_CONTROL__JS15__TAIL_NEXT", - [JOB_CONTROL__JS15__AFFINITY_NEXT] = "JOB_CONTROL__JS15__AFFINITY_NEXT", - [JOB_CONTROL__JS15__CONFIG_NEXT] = "JOB_CONTROL__JS15__CONFIG_NEXT", - [JOB_CONTROL__JS15__XAFFINITY_NEXT] = "JOB_CONTROL__JS15__XAFFINITY_NEXT", - [JOB_CONTROL__JS15__COMMAND_NEXT] = "JOB_CONTROL__JS15__COMMAND_NEXT", - [JOB_CONTROL__JS15__FLUSH_ID_NEXT] = "JOB_CONTROL__JS15__FLUSH_ID_NEXT", - [JOB_CONTROL__JS15__EVENT_MASK_NEXT] = "JOB_CONTROL__JS15__EVENT_MASK_NEXT", - [MMU_STAGE1__ST1MMU__IRQ_RAWSTAT] = "MMU_STAGE1__ST1MMU__IRQ_RAWSTAT", - [MMU_STAGE1__ST1MMU__IRQ_CLEAR] = "MMU_STAGE1__ST1MMU__IRQ_CLEAR", - [MMU_STAGE1__ST1MMU__IRQ_MASK] = "MMU_STAGE1__ST1MMU__IRQ_MASK", - [MMU_STAGE1__ST1MMU__IRQ_STATUS] = "MMU_STAGE1__ST1MMU__IRQ_STATUS", - [MMU_STAGE1__ST1MMU__AS0__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS0__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS0__MEMATTR] = "MMU_STAGE1__ST1MMU__AS0__MEMATTR", - [MMU_STAGE1__ST1MMU__AS0__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS0__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS0__COMMAND] = "MMU_STAGE1__ST1MMU__AS0__COMMAND", - [MMU_STAGE1__ST1MMU__AS0__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS0__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS0__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS0__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS0__STATUS] = "MMU_STAGE1__ST1MMU__AS0__STATUS", - [MMU_STAGE1__ST1MMU__AS0__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS0__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS0__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS0__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS1__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS1__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS1__MEMATTR] = "MMU_STAGE1__ST1MMU__AS1__MEMATTR", - [MMU_STAGE1__ST1MMU__AS1__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS1__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS1__COMMAND] = "MMU_STAGE1__ST1MMU__AS1__COMMAND", - [MMU_STAGE1__ST1MMU__AS1__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS1__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS1__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS1__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS1__STATUS] = "MMU_STAGE1__ST1MMU__AS1__STATUS", - [MMU_STAGE1__ST1MMU__AS1__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS1__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS1__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS1__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS2__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS2__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS2__MEMATTR] = "MMU_STAGE1__ST1MMU__AS2__MEMATTR", - [MMU_STAGE1__ST1MMU__AS2__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS2__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS2__COMMAND] = "MMU_STAGE1__ST1MMU__AS2__COMMAND", - [MMU_STAGE1__ST1MMU__AS2__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS2__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS2__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS2__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS2__STATUS] = "MMU_STAGE1__ST1MMU__AS2__STATUS", - [MMU_STAGE1__ST1MMU__AS2__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS2__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS2__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS2__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS3__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS3__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS3__MEMATTR] = "MMU_STAGE1__ST1MMU__AS3__MEMATTR", - [MMU_STAGE1__ST1MMU__AS3__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS3__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS3__COMMAND] = "MMU_STAGE1__ST1MMU__AS3__COMMAND", - [MMU_STAGE1__ST1MMU__AS3__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS3__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS3__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS3__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS3__STATUS] = "MMU_STAGE1__ST1MMU__AS3__STATUS", - [MMU_STAGE1__ST1MMU__AS3__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS3__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS3__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS3__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS4__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS4__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS4__MEMATTR] = "MMU_STAGE1__ST1MMU__AS4__MEMATTR", - [MMU_STAGE1__ST1MMU__AS4__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS4__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS4__COMMAND] = "MMU_STAGE1__ST1MMU__AS4__COMMAND", - [MMU_STAGE1__ST1MMU__AS4__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS4__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS4__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS4__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS4__STATUS] = "MMU_STAGE1__ST1MMU__AS4__STATUS", - [MMU_STAGE1__ST1MMU__AS4__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS4__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS4__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS4__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS5__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS5__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS5__MEMATTR] = "MMU_STAGE1__ST1MMU__AS5__MEMATTR", - [MMU_STAGE1__ST1MMU__AS5__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS5__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS5__COMMAND] = "MMU_STAGE1__ST1MMU__AS5__COMMAND", - [MMU_STAGE1__ST1MMU__AS5__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS5__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS5__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS5__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS5__STATUS] = "MMU_STAGE1__ST1MMU__AS5__STATUS", - [MMU_STAGE1__ST1MMU__AS5__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS5__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS5__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS5__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS6__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS6__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS6__MEMATTR] = "MMU_STAGE1__ST1MMU__AS6__MEMATTR", - [MMU_STAGE1__ST1MMU__AS6__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS6__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS6__COMMAND] = "MMU_STAGE1__ST1MMU__AS6__COMMAND", - [MMU_STAGE1__ST1MMU__AS6__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS6__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS6__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS6__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS6__STATUS] = "MMU_STAGE1__ST1MMU__AS6__STATUS", - [MMU_STAGE1__ST1MMU__AS6__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS6__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS6__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS6__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS7__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS7__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS7__MEMATTR] = "MMU_STAGE1__ST1MMU__AS7__MEMATTR", - [MMU_STAGE1__ST1MMU__AS7__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS7__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS7__COMMAND] = "MMU_STAGE1__ST1MMU__AS7__COMMAND", - [MMU_STAGE1__ST1MMU__AS7__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS7__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS7__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS7__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS7__STATUS] = "MMU_STAGE1__ST1MMU__AS7__STATUS", - [MMU_STAGE1__ST1MMU__AS7__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS7__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS7__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS7__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS8__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS8__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS8__MEMATTR] = "MMU_STAGE1__ST1MMU__AS8__MEMATTR", - [MMU_STAGE1__ST1MMU__AS8__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS8__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS8__COMMAND] = "MMU_STAGE1__ST1MMU__AS8__COMMAND", - [MMU_STAGE1__ST1MMU__AS8__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS8__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS8__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS8__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS8__STATUS] = "MMU_STAGE1__ST1MMU__AS8__STATUS", - [MMU_STAGE1__ST1MMU__AS8__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS8__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS8__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS8__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS9__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS9__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS9__MEMATTR] = "MMU_STAGE1__ST1MMU__AS9__MEMATTR", - [MMU_STAGE1__ST1MMU__AS9__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS9__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS9__COMMAND] = "MMU_STAGE1__ST1MMU__AS9__COMMAND", - [MMU_STAGE1__ST1MMU__AS9__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS9__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS9__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS9__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS9__STATUS] = "MMU_STAGE1__ST1MMU__AS9__STATUS", - [MMU_STAGE1__ST1MMU__AS9__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS9__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS9__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS9__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS10__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS10__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS10__MEMATTR] = "MMU_STAGE1__ST1MMU__AS10__MEMATTR", - [MMU_STAGE1__ST1MMU__AS10__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS10__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS10__COMMAND] = "MMU_STAGE1__ST1MMU__AS10__COMMAND", - [MMU_STAGE1__ST1MMU__AS10__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS10__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS10__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS10__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS10__STATUS] = "MMU_STAGE1__ST1MMU__AS10__STATUS", - [MMU_STAGE1__ST1MMU__AS10__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS10__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS10__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS10__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS11__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS11__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS11__MEMATTR] = "MMU_STAGE1__ST1MMU__AS11__MEMATTR", - [MMU_STAGE1__ST1MMU__AS11__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS11__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS11__COMMAND] = "MMU_STAGE1__ST1MMU__AS11__COMMAND", - [MMU_STAGE1__ST1MMU__AS11__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS11__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS11__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS11__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS11__STATUS] = "MMU_STAGE1__ST1MMU__AS11__STATUS", - [MMU_STAGE1__ST1MMU__AS11__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS11__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS11__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS11__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS12__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS12__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS12__MEMATTR] = "MMU_STAGE1__ST1MMU__AS12__MEMATTR", - [MMU_STAGE1__ST1MMU__AS12__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS12__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS12__COMMAND] = "MMU_STAGE1__ST1MMU__AS12__COMMAND", - [MMU_STAGE1__ST1MMU__AS12__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS12__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS12__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS12__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS12__STATUS] = "MMU_STAGE1__ST1MMU__AS12__STATUS", - [MMU_STAGE1__ST1MMU__AS12__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS12__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS12__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS12__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS13__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS13__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS13__MEMATTR] = "MMU_STAGE1__ST1MMU__AS13__MEMATTR", - [MMU_STAGE1__ST1MMU__AS13__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS13__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS13__COMMAND] = "MMU_STAGE1__ST1MMU__AS13__COMMAND", - [MMU_STAGE1__ST1MMU__AS13__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS13__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS13__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS13__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS13__STATUS] = "MMU_STAGE1__ST1MMU__AS13__STATUS", - [MMU_STAGE1__ST1MMU__AS13__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS13__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS13__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS13__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS14__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS14__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS14__MEMATTR] = "MMU_STAGE1__ST1MMU__AS14__MEMATTR", - [MMU_STAGE1__ST1MMU__AS14__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS14__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS14__COMMAND] = "MMU_STAGE1__ST1MMU__AS14__COMMAND", - [MMU_STAGE1__ST1MMU__AS14__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS14__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS14__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS14__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS14__STATUS] = "MMU_STAGE1__ST1MMU__AS14__STATUS", - [MMU_STAGE1__ST1MMU__AS14__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS14__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS14__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS14__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS15__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS15__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS15__MEMATTR] = "MMU_STAGE1__ST1MMU__AS15__MEMATTR", - [MMU_STAGE1__ST1MMU__AS15__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS15__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS15__COMMAND] = "MMU_STAGE1__ST1MMU__AS15__COMMAND", - [MMU_STAGE1__ST1MMU__AS15__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS15__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS15__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS15__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS15__STATUS] = "MMU_STAGE1__ST1MMU__AS15__STATUS", - [MMU_STAGE1__ST1MMU__AS15__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS15__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS15__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS15__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__IRQ_RAWSTAT] = "MMU_STAGE2__ST2MMU__IRQ_RAWSTAT", - [MMU_STAGE2__ST2MMU__IRQ_CLEAR] = "MMU_STAGE2__ST2MMU__IRQ_CLEAR", - [MMU_STAGE2__ST2MMU__IRQ_MASK] = "MMU_STAGE2__ST2MMU__IRQ_MASK", - [MMU_STAGE2__ST2MMU__IRQ_STATUS] = "MMU_STAGE2__ST2MMU__IRQ_STATUS", - [MMU_STAGE2__ST2MMU__AS0__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS0__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS0__MEMATTR] = "MMU_STAGE2__ST2MMU__AS0__MEMATTR", - [MMU_STAGE2__ST2MMU__AS0__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS0__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS0__COMMAND] = "MMU_STAGE2__ST2MMU__AS0__COMMAND", - [MMU_STAGE2__ST2MMU__AS0__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS0__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS0__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS0__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS0__STATUS] = "MMU_STAGE2__ST2MMU__AS0__STATUS", - [MMU_STAGE2__ST2MMU__AS0__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS0__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS0__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS0__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS1__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS1__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS1__MEMATTR] = "MMU_STAGE2__ST2MMU__AS1__MEMATTR", - [MMU_STAGE2__ST2MMU__AS1__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS1__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS1__COMMAND] = "MMU_STAGE2__ST2MMU__AS1__COMMAND", - [MMU_STAGE2__ST2MMU__AS1__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS1__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS1__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS1__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS1__STATUS] = "MMU_STAGE2__ST2MMU__AS1__STATUS", - [MMU_STAGE2__ST2MMU__AS1__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS1__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS1__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS1__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS2__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS2__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS2__MEMATTR] = "MMU_STAGE2__ST2MMU__AS2__MEMATTR", - [MMU_STAGE2__ST2MMU__AS2__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS2__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS2__COMMAND] = "MMU_STAGE2__ST2MMU__AS2__COMMAND", - [MMU_STAGE2__ST2MMU__AS2__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS2__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS2__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS2__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS2__STATUS] = "MMU_STAGE2__ST2MMU__AS2__STATUS", - [MMU_STAGE2__ST2MMU__AS2__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS2__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS2__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS2__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS3__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS3__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS3__MEMATTR] = "MMU_STAGE2__ST2MMU__AS3__MEMATTR", - [MMU_STAGE2__ST2MMU__AS3__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS3__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS3__COMMAND] = "MMU_STAGE2__ST2MMU__AS3__COMMAND", - [MMU_STAGE2__ST2MMU__AS3__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS3__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS3__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS3__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS3__STATUS] = "MMU_STAGE2__ST2MMU__AS3__STATUS", - [MMU_STAGE2__ST2MMU__AS3__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS3__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS3__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS3__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS4__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS4__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS4__MEMATTR] = "MMU_STAGE2__ST2MMU__AS4__MEMATTR", - [MMU_STAGE2__ST2MMU__AS4__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS4__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS4__COMMAND] = "MMU_STAGE2__ST2MMU__AS4__COMMAND", - [MMU_STAGE2__ST2MMU__AS4__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS4__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS4__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS4__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS4__STATUS] = "MMU_STAGE2__ST2MMU__AS4__STATUS", - [MMU_STAGE2__ST2MMU__AS4__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS4__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS4__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS4__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS5__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS5__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS5__MEMATTR] = "MMU_STAGE2__ST2MMU__AS5__MEMATTR", - [MMU_STAGE2__ST2MMU__AS5__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS5__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS5__COMMAND] = "MMU_STAGE2__ST2MMU__AS5__COMMAND", - [MMU_STAGE2__ST2MMU__AS5__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS5__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS5__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS5__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS5__STATUS] = "MMU_STAGE2__ST2MMU__AS5__STATUS", - [MMU_STAGE2__ST2MMU__AS5__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS5__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS5__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS5__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS6__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS6__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS6__MEMATTR] = "MMU_STAGE2__ST2MMU__AS6__MEMATTR", - [MMU_STAGE2__ST2MMU__AS6__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS6__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS6__COMMAND] = "MMU_STAGE2__ST2MMU__AS6__COMMAND", - [MMU_STAGE2__ST2MMU__AS6__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS6__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS6__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS6__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS6__STATUS] = "MMU_STAGE2__ST2MMU__AS6__STATUS", - [MMU_STAGE2__ST2MMU__AS6__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS6__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS6__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS6__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS7__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS7__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS7__MEMATTR] = "MMU_STAGE2__ST2MMU__AS7__MEMATTR", - [MMU_STAGE2__ST2MMU__AS7__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS7__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS7__COMMAND] = "MMU_STAGE2__ST2MMU__AS7__COMMAND", - [MMU_STAGE2__ST2MMU__AS7__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS7__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS7__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS7__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS7__STATUS] = "MMU_STAGE2__ST2MMU__AS7__STATUS", - [MMU_STAGE2__ST2MMU__AS7__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS7__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS7__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS7__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS8__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS8__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS8__MEMATTR] = "MMU_STAGE2__ST2MMU__AS8__MEMATTR", - [MMU_STAGE2__ST2MMU__AS8__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS8__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS8__COMMAND] = "MMU_STAGE2__ST2MMU__AS8__COMMAND", - [MMU_STAGE2__ST2MMU__AS8__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS8__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS8__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS8__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS8__STATUS] = "MMU_STAGE2__ST2MMU__AS8__STATUS", - [MMU_STAGE2__ST2MMU__AS8__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS8__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS8__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS8__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS9__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS9__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS9__MEMATTR] = "MMU_STAGE2__ST2MMU__AS9__MEMATTR", - [MMU_STAGE2__ST2MMU__AS9__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS9__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS9__COMMAND] = "MMU_STAGE2__ST2MMU__AS9__COMMAND", - [MMU_STAGE2__ST2MMU__AS9__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS9__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS9__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS9__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS9__STATUS] = "MMU_STAGE2__ST2MMU__AS9__STATUS", - [MMU_STAGE2__ST2MMU__AS9__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS9__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS9__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS9__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS10__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS10__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS10__MEMATTR] = "MMU_STAGE2__ST2MMU__AS10__MEMATTR", - [MMU_STAGE2__ST2MMU__AS10__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS10__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS10__COMMAND] = "MMU_STAGE2__ST2MMU__AS10__COMMAND", - [MMU_STAGE2__ST2MMU__AS10__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS10__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS10__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS10__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS10__STATUS] = "MMU_STAGE2__ST2MMU__AS10__STATUS", - [MMU_STAGE2__ST2MMU__AS10__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS10__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS10__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS10__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS11__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS11__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS11__MEMATTR] = "MMU_STAGE2__ST2MMU__AS11__MEMATTR", - [MMU_STAGE2__ST2MMU__AS11__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS11__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS11__COMMAND] = "MMU_STAGE2__ST2MMU__AS11__COMMAND", - [MMU_STAGE2__ST2MMU__AS11__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS11__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS11__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS11__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS11__STATUS] = "MMU_STAGE2__ST2MMU__AS11__STATUS", - [MMU_STAGE2__ST2MMU__AS11__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS11__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS11__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS11__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS12__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS12__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS12__MEMATTR] = "MMU_STAGE2__ST2MMU__AS12__MEMATTR", - [MMU_STAGE2__ST2MMU__AS12__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS12__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS12__COMMAND] = "MMU_STAGE2__ST2MMU__AS12__COMMAND", - [MMU_STAGE2__ST2MMU__AS12__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS12__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS12__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS12__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS12__STATUS] = "MMU_STAGE2__ST2MMU__AS12__STATUS", - [MMU_STAGE2__ST2MMU__AS12__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS12__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS12__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS12__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS13__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS13__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS13__MEMATTR] = "MMU_STAGE2__ST2MMU__AS13__MEMATTR", - [MMU_STAGE2__ST2MMU__AS13__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS13__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS13__COMMAND] = "MMU_STAGE2__ST2MMU__AS13__COMMAND", - [MMU_STAGE2__ST2MMU__AS13__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS13__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS13__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS13__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS13__STATUS] = "MMU_STAGE2__ST2MMU__AS13__STATUS", - [MMU_STAGE2__ST2MMU__AS13__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS13__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS13__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS13__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS14__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS14__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS14__MEMATTR] = "MMU_STAGE2__ST2MMU__AS14__MEMATTR", - [MMU_STAGE2__ST2MMU__AS14__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS14__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS14__COMMAND] = "MMU_STAGE2__ST2MMU__AS14__COMMAND", - [MMU_STAGE2__ST2MMU__AS14__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS14__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS14__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS14__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS14__STATUS] = "MMU_STAGE2__ST2MMU__AS14__STATUS", - [MMU_STAGE2__ST2MMU__AS14__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS14__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS14__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS14__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS15__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS15__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS15__MEMATTR] = "MMU_STAGE2__ST2MMU__AS15__MEMATTR", - [MMU_STAGE2__ST2MMU__AS15__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS15__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS15__COMMAND] = "MMU_STAGE2__ST2MMU__AS15__COMMAND", - [MMU_STAGE2__ST2MMU__AS15__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS15__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS15__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS15__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS15__STATUS] = "MMU_STAGE2__ST2MMU__AS15__STATUS", - [MMU_STAGE2__ST2MMU__AS15__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS15__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS15__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS15__FAULTEXTRA", - [GPU_CONTROL__REVIDR] = "GPU_CONTROL__REVIDR", - [GPU_CONTROL__STACK_PRESENT] = "GPU_CONTROL__STACK_PRESENT", - [GPU_CONTROL__STACK_PWROFF] = "GPU_CONTROL__STACK_PWROFF", - [GPU_CONTROL__STACK_PWRON] = "GPU_CONTROL__STACK_PWRON", - [GPU_CONTROL__STACK_PWRTRANS] = "GPU_CONTROL__STACK_PWRTRANS", - [GPU_CONTROL__STACK_READY] = "GPU_CONTROL__STACK_READY", - [GPU_CONTROL__TEXTURE_FEATURES_3] = "GPU_CONTROL__TEXTURE_FEATURES_3", - [GPU_CONTROL__CORE_FEATURES] = "GPU_CONTROL__CORE_FEATURES", - [GPU_CONTROL__THREAD_TLS_ALLOC] = "GPU_CONTROL__THREAD_TLS_ALLOC", - [GPU_CONTROL__L2_CONFIG] = "GPU_CONTROL__L2_CONFIG", - [PTM_AW_IRQ_CLEAR] = "PTM_AW_IRQ_CLEAR", - [PTM_AW_IRQ_INJECTION] = "PTM_AW_IRQ_INJECTION", - [PTM_AW_IRQ_MASK] = "PTM_AW_IRQ_MASK", - [PTM_AW_IRQ_RAWSTAT] = "PTM_AW_IRQ_RAWSTAT", - [PTM_AW_IRQ_STATUS] = "PTM_AW_IRQ_STATUS", - [PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE0] = "PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE0", - [PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE1] = "PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE1", - [PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE0] = "PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE0", - [PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE1] = "PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE1", - [PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE_STATUS] = - "PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE_STATUS", - [PTM_ID] = "PTM_ID", -}; - -const char *kbase_reg_get_enum_string(u32 reg_enum) -{ - if (reg_enum >= ARRAY_SIZE(enum_strings)) - return "INVALID_REG"; - return enum_strings[reg_enum]; -} -#endif /* CONFIG_MALI_VALHALL_DEBUG */ diff --git a/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_jm_enums.h b/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_jm_enums.h deleted file mode 100644 index 59d8745eaf4a..000000000000 --- a/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_jm_enums.h +++ /dev/null @@ -1,777 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * This header is autogenerated. Avoid modifying this file unless absolutely - * necessary. - */ - -#ifndef _MALI_KBASE_REGMAP_JM_ENUMS_H_ -#define _MALI_KBASE_REGMAP_JM_ENUMS_H_ - -#if MALI_USE_CSF -#error "Cannot be compiled with CSF" -#endif - -enum kbase_regmap_enum_v6_0 { - GPU_CONTROL__GPU_ID = 0, /* (RO) 32-bit 0x0 */ - GPU_CONTROL__L2_FEATURES, /* (RO) 32-bit 0x4 */ - GPU_CONTROL__TILER_FEATURES, /* (RO) 32-bit 0xC */ - GPU_CONTROL__MEM_FEATURES, /* (RO) 32-bit 0x10 */ - GPU_CONTROL__MMU_FEATURES, /* (RO) 32-bit 0x14 */ - GPU_CONTROL__AS_PRESENT, /* (RO) 32-bit 0x18 */ - GPU_CONTROL__JS_PRESENT, /* (RO) 32-bit 0x1C */ - GPU_CONTROL__GPU_IRQ_RAWSTAT, /* (RW) 32-bit 0x20 */ - GPU_CONTROL__GPU_IRQ_CLEAR, /* (WO) 32-bit 0x24 */ - GPU_CONTROL__GPU_IRQ_MASK, /* (RW) 32-bit 0x28 */ - GPU_CONTROL__GPU_IRQ_STATUS, /* (RO) 32-bit 0x2C */ - GPU_CONTROL__GPU_COMMAND, /* (WO) 32-bit 0x30 */ - GPU_CONTROL__GPU_STATUS, /* (RO) 32-bit 0x34 */ - GPU_CONTROL__LATEST_FLUSH, /* (RO) 32-bit 0x38 */ - GPU_CONTROL__GPU_FAULTSTATUS, /* (RO) 32-bit 0x3C */ - GPU_CONTROL__GPU_FAULTADDRESS, /* (RO) 64-bit 0x40 */ - GPU_CONTROL__AFBC_FEATURES, /* (RO) 32-bit 0x4C */ - GPU_CONTROL__PWR_KEY, /* (WO) 32-bit 0x50 */ - GPU_CONTROL__PWR_OVERRIDE0, /* (RW) 32-bit 0x54 */ - GPU_CONTROL__PWR_OVERRIDE1, /* (RW) 32-bit 0x58 */ - GPU_CONTROL__PRFCNT_BASE, /* (RW) 64-bit 0x60 */ - GPU_CONTROL__PRFCNT_CONFIG, /* (RW) 32-bit 0x68 */ - GPU_CONTROL__PRFCNT_JM_EN, /* (RW) 32-bit 0x6C */ - GPU_CONTROL__PRFCNT_SHADER_EN, /* (RW) 32-bit 0x70 */ - GPU_CONTROL__PRFCNT_TILER_EN, /* (RW) 32-bit 0x74 */ - GPU_CONTROL__PRFCNT_MMU_L2_EN, /* (RW) 32-bit 0x7C */ - GPU_CONTROL__CYCLE_COUNT, /* (RO) 64-bit 0x90 */ - GPU_CONTROL__TIMESTAMP, /* (RO) 64-bit 0x98 */ - GPU_CONTROL__THREAD_MAX_THREADS, /* (RO) 32-bit 0xA0 */ - GPU_CONTROL__THREAD_MAX_WORKGROUP_SIZE, /* (RO) 32-bit 0xA4 */ - GPU_CONTROL__THREAD_MAX_BARRIER_SIZE, /* (RO) 32-bit 0xA8 */ - GPU_CONTROL__THREAD_FEATURES, /* (RO) 32-bit 0xAC */ - GPU_CONTROL__TEXTURE_FEATURES_0, /* (RO) 32-bit 0xB0 */ - GPU_CONTROL__TEXTURE_FEATURES_1, /* (RO) 32-bit 0xB4 */ - GPU_CONTROL__TEXTURE_FEATURES_2, /* (RO) 32-bit 0xB8 */ - GPU_CONTROL__JS0_FEATURES, /* (RO) 32-bit 0xC0 */ - GPU_CONTROL__JS1_FEATURES, /* (RO) 32-bit 0xC4 */ - GPU_CONTROL__JS2_FEATURES, /* (RO) 32-bit 0xC8 */ - GPU_CONTROL__JS3_FEATURES, /* (RO) 32-bit 0xCC */ - GPU_CONTROL__JS4_FEATURES, /* (RO) 32-bit 0xD0 */ - GPU_CONTROL__JS5_FEATURES, /* (RO) 32-bit 0xD4 */ - GPU_CONTROL__JS6_FEATURES, /* (RO) 32-bit 0xD8 */ - GPU_CONTROL__JS7_FEATURES, /* (RO) 32-bit 0xDC */ - GPU_CONTROL__JS8_FEATURES, /* (RO) 32-bit 0xE0 */ - GPU_CONTROL__JS9_FEATURES, /* (RO) 32-bit 0xE4 */ - GPU_CONTROL__JS10_FEATURES, /* (RO) 32-bit 0xE8 */ - GPU_CONTROL__JS11_FEATURES, /* (RO) 32-bit 0xEC */ - GPU_CONTROL__JS12_FEATURES, /* (RO) 32-bit 0xF0 */ - GPU_CONTROL__JS13_FEATURES, /* (RO) 32-bit 0xF4 */ - GPU_CONTROL__JS14_FEATURES, /* (RO) 32-bit 0xF8 */ - GPU_CONTROL__JS15_FEATURES, /* (RO) 32-bit 0xFC */ - GPU_CONTROL__SHADER_PRESENT, /* (RO) 64-bit 0x100 */ - GPU_CONTROL__TILER_PRESENT, /* (RO) 64-bit 0x110 */ - GPU_CONTROL__L2_PRESENT, /* (RO) 64-bit 0x120 */ - GPU_CONTROL__SHADER_READY, /* (RO) 64-bit 0x140 */ - GPU_CONTROL__TILER_READY, /* (RO) 64-bit 0x150 */ - GPU_CONTROL__L2_READY, /* (RO) 64-bit 0x160 */ - GPU_CONTROL__SHADER_PWRON, /* (WO) 64-bit 0x180 */ - GPU_CONTROL__TILER_PWRON, /* (WO) 64-bit 0x190 */ - GPU_CONTROL__L2_PWRON, /* (WO) 64-bit 0x1A0 */ - GPU_CONTROL__SHADER_PWROFF, /* (WO) 64-bit 0x1C0 */ - GPU_CONTROL__TILER_PWROFF, /* (WO) 64-bit 0x1D0 */ - GPU_CONTROL__L2_PWROFF, /* (WO) 64-bit 0x1E0 */ - GPU_CONTROL__SHADER_PWRTRANS, /* (RO) 64-bit 0x200 */ - GPU_CONTROL__TILER_PWRTRANS, /* (RO) 64-bit 0x210 */ - GPU_CONTROL__L2_PWRTRANS, /* (RO) 64-bit 0x220 */ - GPU_CONTROL__SHADER_PWRACTIVE, /* (RO) 64-bit 0x240 */ - GPU_CONTROL__TILER_PWRACTIVE, /* (RO) 64-bit 0x250 */ - GPU_CONTROL__L2_PWRACTIVE, /* (RO) 64-bit 0x260 */ - GPU_CONTROL__COHERENCY_FEATURES, /* (RO) 32-bit 0x300 */ - GPU_CONTROL__COHERENCY_ENABLE, /* (RW) 32-bit 0x304 */ - GPU_CONTROL__GPU_USER_INn, /* (RO) 32-bit 0x400 */ - GPU_CONTROL__GPU_USER_OUTn, /* (RW) 32-bit 0x500 */ - GPU_CONTROL__JM_CONFIG, /* (RW) 32-bit 0xF00 */ - GPU_CONTROL__SHADER_CONFIG, /* (RW) 32-bit 0xF04 */ - GPU_CONTROL__TILER_CONFIG, /* (RW) 32-bit 0xF08 */ - GPU_CONTROL__L2_MMU_CONFIG, /* (RW) 32-bit 0xF0C */ - JOB_CONTROL__JOB_IRQ_RAWSTAT, /* (RW) 32-bit 0x1000 */ - JOB_CONTROL__JOB_IRQ_CLEAR, /* (WO) 32-bit 0x1004 */ - JOB_CONTROL__JOB_IRQ_MASK, /* (RW) 32-bit 0x1008 */ - JOB_CONTROL__JOB_IRQ_STATUS, /* (RO) 32-bit 0x100C */ - JOB_CONTROL__JOB_IRQ_JS_STATE, /* (RO) 32-bit 0x1010 */ - JOB_CONTROL__JOB_IRQ_THROTTLE, /* (RW) 32-bit 0x1014 */ - JOB_CONTROL__EVENT_IRQ_RAWSTAT, /* (RW) 32-bit 0x1020 */ - JOB_CONTROL__EVENT_IRQ_CLEAR, /* (WO) 32-bit 0x1024 */ - JOB_CONTROL__EVENT_IRQ_MASK, /* (RW) 32-bit 0x1028 */ - JOB_CONTROL__EVENT_IRQ_STATUS, /* (RO) 32-bit 0x102C */ - JOB_CONTROL__JS0__HEAD, /* (RO) 64-bit 0x1800 */ - JOB_CONTROL__JS0__TAIL, /* (RO) 64-bit 0x1808 */ - JOB_CONTROL__JS0__AFFINITY, /* (RO) 64-bit 0x1810 */ - JOB_CONTROL__JS0__CONFIG, /* (RO) 32-bit 0x1818 */ - JOB_CONTROL__JS0__XAFFINITY, /* (RO) 32-bit 0x181C */ - JOB_CONTROL__JS0__COMMAND, /* (RW) 32-bit 0x1820 */ - JOB_CONTROL__JS0__STATUS, /* (RO) 32-bit 0x1824 */ - JOB_CONTROL__JS0__FLUSH_ID, /* (RO) 32-bit 0x1830 */ - JOB_CONTROL__JS0__EVENT_0, /* (RW) 32-bit 0x1834 */ - JOB_CONTROL__JS0__EVENT_1, /* (RW) 32-bit 0x1838 */ - JOB_CONTROL__JS0__EVENT_MASK, /* (RW) 32-bit 0x183C */ - JOB_CONTROL__JS0__HEAD_NEXT, /* (RW) 64-bit 0x1840 */ - JOB_CONTROL__JS0__TAIL_NEXT, /* (RW) 64-bit 0x1848 */ - JOB_CONTROL__JS0__AFFINITY_NEXT, /* (RW) 64-bit 0x1850 */ - JOB_CONTROL__JS0__CONFIG_NEXT, /* (RW) 32-bit 0x1858 */ - JOB_CONTROL__JS0__XAFFINITY_NEXT, /* (RW) 32-bit 0x185C */ - JOB_CONTROL__JS0__COMMAND_NEXT, /* (RW) 32-bit 0x1860 */ - JOB_CONTROL__JS0__FLUSH_ID_NEXT, /* (RW) 32-bit 0x1870 */ - JOB_CONTROL__JS0__EVENT_MASK_NEXT, /* (RW) 32-bit 0x187C */ - JOB_CONTROL__JS1__HEAD, /* (RO) 64-bit 0x1880 */ - JOB_CONTROL__JS1__TAIL, /* (RO) 64-bit 0x1888 */ - JOB_CONTROL__JS1__AFFINITY, /* (RO) 64-bit 0x1890 */ - JOB_CONTROL__JS1__CONFIG, /* (RO) 32-bit 0x1898 */ - JOB_CONTROL__JS1__XAFFINITY, /* (RO) 32-bit 0x189C */ - JOB_CONTROL__JS1__COMMAND, /* (RW) 32-bit 0x18A0 */ - JOB_CONTROL__JS1__STATUS, /* (RO) 32-bit 0x18A4 */ - JOB_CONTROL__JS1__FLUSH_ID, /* (RO) 32-bit 0x18B0 */ - JOB_CONTROL__JS1__EVENT_0, /* (RW) 32-bit 0x18B4 */ - JOB_CONTROL__JS1__EVENT_1, /* (RW) 32-bit 0x18B8 */ - JOB_CONTROL__JS1__EVENT_MASK, /* (RW) 32-bit 0x18BC */ - JOB_CONTROL__JS1__HEAD_NEXT, /* (RW) 64-bit 0x18C0 */ - JOB_CONTROL__JS1__TAIL_NEXT, /* (RW) 64-bit 0x18C8 */ - JOB_CONTROL__JS1__AFFINITY_NEXT, /* (RW) 64-bit 0x18D0 */ - JOB_CONTROL__JS1__CONFIG_NEXT, /* (RW) 32-bit 0x18D8 */ - JOB_CONTROL__JS1__XAFFINITY_NEXT, /* (RW) 32-bit 0x18DC */ - JOB_CONTROL__JS1__COMMAND_NEXT, /* (RW) 32-bit 0x18E0 */ - JOB_CONTROL__JS1__FLUSH_ID_NEXT, /* (RW) 32-bit 0x18F0 */ - JOB_CONTROL__JS1__EVENT_MASK_NEXT, /* (RW) 32-bit 0x18FC */ - JOB_CONTROL__JS2__HEAD, /* (RO) 64-bit 0x1900 */ - JOB_CONTROL__JS2__TAIL, /* (RO) 64-bit 0x1908 */ - JOB_CONTROL__JS2__AFFINITY, /* (RO) 64-bit 0x1910 */ - JOB_CONTROL__JS2__CONFIG, /* (RO) 32-bit 0x1918 */ - JOB_CONTROL__JS2__XAFFINITY, /* (RO) 32-bit 0x191C */ - JOB_CONTROL__JS2__COMMAND, /* (RW) 32-bit 0x1920 */ - JOB_CONTROL__JS2__STATUS, /* (RO) 32-bit 0x1924 */ - JOB_CONTROL__JS2__FLUSH_ID, /* (RO) 32-bit 0x1930 */ - JOB_CONTROL__JS2__EVENT_0, /* (RW) 32-bit 0x1934 */ - JOB_CONTROL__JS2__EVENT_1, /* (RW) 32-bit 0x1938 */ - JOB_CONTROL__JS2__EVENT_MASK, /* (RW) 32-bit 0x193C */ - JOB_CONTROL__JS2__HEAD_NEXT, /* (RW) 64-bit 0x1940 */ - JOB_CONTROL__JS2__TAIL_NEXT, /* (RW) 64-bit 0x1948 */ - JOB_CONTROL__JS2__AFFINITY_NEXT, /* (RW) 64-bit 0x1950 */ - JOB_CONTROL__JS2__CONFIG_NEXT, /* (RW) 32-bit 0x1958 */ - JOB_CONTROL__JS2__XAFFINITY_NEXT, /* (RW) 32-bit 0x195C */ - JOB_CONTROL__JS2__COMMAND_NEXT, /* (RW) 32-bit 0x1960 */ - JOB_CONTROL__JS2__FLUSH_ID_NEXT, /* (RW) 32-bit 0x1970 */ - JOB_CONTROL__JS2__EVENT_MASK_NEXT, /* (RW) 32-bit 0x197C */ - JOB_CONTROL__JS3__HEAD, /* (RO) 64-bit 0x1980 */ - JOB_CONTROL__JS3__TAIL, /* (RO) 64-bit 0x1988 */ - JOB_CONTROL__JS3__AFFINITY, /* (RO) 64-bit 0x1990 */ - JOB_CONTROL__JS3__CONFIG, /* (RO) 32-bit 0x1998 */ - JOB_CONTROL__JS3__XAFFINITY, /* (RO) 32-bit 0x199C */ - JOB_CONTROL__JS3__COMMAND, /* (RW) 32-bit 0x19A0 */ - JOB_CONTROL__JS3__STATUS, /* (RO) 32-bit 0x19A4 */ - JOB_CONTROL__JS3__FLUSH_ID, /* (RO) 32-bit 0x19B0 */ - JOB_CONTROL__JS3__EVENT_0, /* (RW) 32-bit 0x19B4 */ - JOB_CONTROL__JS3__EVENT_1, /* (RW) 32-bit 0x19B8 */ - JOB_CONTROL__JS3__EVENT_MASK, /* (RW) 32-bit 0x19BC */ - JOB_CONTROL__JS3__HEAD_NEXT, /* (RW) 64-bit 0x19C0 */ - JOB_CONTROL__JS3__TAIL_NEXT, /* (RW) 64-bit 0x19C8 */ - JOB_CONTROL__JS3__AFFINITY_NEXT, /* (RW) 64-bit 0x19D0 */ - JOB_CONTROL__JS3__CONFIG_NEXT, /* (RW) 32-bit 0x19D8 */ - JOB_CONTROL__JS3__XAFFINITY_NEXT, /* (RW) 32-bit 0x19DC */ - JOB_CONTROL__JS3__COMMAND_NEXT, /* (RW) 32-bit 0x19E0 */ - JOB_CONTROL__JS3__FLUSH_ID_NEXT, /* (RW) 32-bit 0x19F0 */ - JOB_CONTROL__JS3__EVENT_MASK_NEXT, /* (RW) 32-bit 0x19FC */ - JOB_CONTROL__JS4__HEAD, /* (RO) 64-bit 0x1A00 */ - JOB_CONTROL__JS4__TAIL, /* (RO) 64-bit 0x1A08 */ - JOB_CONTROL__JS4__AFFINITY, /* (RO) 64-bit 0x1A10 */ - JOB_CONTROL__JS4__CONFIG, /* (RO) 32-bit 0x1A18 */ - JOB_CONTROL__JS4__XAFFINITY, /* (RO) 32-bit 0x1A1C */ - JOB_CONTROL__JS4__COMMAND, /* (RW) 32-bit 0x1A20 */ - JOB_CONTROL__JS4__STATUS, /* (RO) 32-bit 0x1A24 */ - JOB_CONTROL__JS4__FLUSH_ID, /* (RO) 32-bit 0x1A30 */ - JOB_CONTROL__JS4__EVENT_0, /* (RW) 32-bit 0x1A34 */ - JOB_CONTROL__JS4__EVENT_1, /* (RW) 32-bit 0x1A38 */ - JOB_CONTROL__JS4__EVENT_MASK, /* (RW) 32-bit 0x1A3C */ - JOB_CONTROL__JS4__HEAD_NEXT, /* (RW) 64-bit 0x1A40 */ - JOB_CONTROL__JS4__TAIL_NEXT, /* (RW) 64-bit 0x1A48 */ - JOB_CONTROL__JS4__AFFINITY_NEXT, /* (RW) 64-bit 0x1A50 */ - JOB_CONTROL__JS4__CONFIG_NEXT, /* (RW) 32-bit 0x1A58 */ - JOB_CONTROL__JS4__XAFFINITY_NEXT, /* (RW) 32-bit 0x1A5C */ - JOB_CONTROL__JS4__COMMAND_NEXT, /* (RW) 32-bit 0x1A60 */ - JOB_CONTROL__JS4__FLUSH_ID_NEXT, /* (RW) 32-bit 0x1A70 */ - JOB_CONTROL__JS4__EVENT_MASK_NEXT, /* (RW) 32-bit 0x1A7C */ - JOB_CONTROL__JS5__HEAD, /* (RO) 64-bit 0x1A80 */ - JOB_CONTROL__JS5__TAIL, /* (RO) 64-bit 0x1A88 */ - JOB_CONTROL__JS5__AFFINITY, /* (RO) 64-bit 0x1A90 */ - JOB_CONTROL__JS5__CONFIG, /* (RO) 32-bit 0x1A98 */ - JOB_CONTROL__JS5__XAFFINITY, /* (RO) 32-bit 0x1A9C */ - JOB_CONTROL__JS5__COMMAND, /* (RW) 32-bit 0x1AA0 */ - JOB_CONTROL__JS5__STATUS, /* (RO) 32-bit 0x1AA4 */ - JOB_CONTROL__JS5__FLUSH_ID, /* (RO) 32-bit 0x1AB0 */ - JOB_CONTROL__JS5__EVENT_0, /* (RW) 32-bit 0x1AB4 */ - JOB_CONTROL__JS5__EVENT_1, /* (RW) 32-bit 0x1AB8 */ - JOB_CONTROL__JS5__EVENT_MASK, /* (RW) 32-bit 0x1ABC */ - JOB_CONTROL__JS5__HEAD_NEXT, /* (RW) 64-bit 0x1AC0 */ - JOB_CONTROL__JS5__TAIL_NEXT, /* (RW) 64-bit 0x1AC8 */ - JOB_CONTROL__JS5__AFFINITY_NEXT, /* (RW) 64-bit 0x1AD0 */ - JOB_CONTROL__JS5__CONFIG_NEXT, /* (RW) 32-bit 0x1AD8 */ - JOB_CONTROL__JS5__XAFFINITY_NEXT, /* (RW) 32-bit 0x1ADC */ - JOB_CONTROL__JS5__COMMAND_NEXT, /* (RW) 32-bit 0x1AE0 */ - JOB_CONTROL__JS5__FLUSH_ID_NEXT, /* (RW) 32-bit 0x1AF0 */ - JOB_CONTROL__JS5__EVENT_MASK_NEXT, /* (RW) 32-bit 0x1AFC */ - JOB_CONTROL__JS6__HEAD, /* (RO) 64-bit 0x1B00 */ - JOB_CONTROL__JS6__TAIL, /* (RO) 64-bit 0x1B08 */ - JOB_CONTROL__JS6__AFFINITY, /* (RO) 64-bit 0x1B10 */ - JOB_CONTROL__JS6__CONFIG, /* (RO) 32-bit 0x1B18 */ - JOB_CONTROL__JS6__XAFFINITY, /* (RO) 32-bit 0x1B1C */ - JOB_CONTROL__JS6__COMMAND, /* (RW) 32-bit 0x1B20 */ - JOB_CONTROL__JS6__STATUS, /* (RO) 32-bit 0x1B24 */ - JOB_CONTROL__JS6__FLUSH_ID, /* (RO) 32-bit 0x1B30 */ - JOB_CONTROL__JS6__EVENT_0, /* (RW) 32-bit 0x1B34 */ - JOB_CONTROL__JS6__EVENT_1, /* (RW) 32-bit 0x1B38 */ - JOB_CONTROL__JS6__EVENT_MASK, /* (RW) 32-bit 0x1B3C */ - JOB_CONTROL__JS6__HEAD_NEXT, /* (RW) 64-bit 0x1B40 */ - JOB_CONTROL__JS6__TAIL_NEXT, /* (RW) 64-bit 0x1B48 */ - JOB_CONTROL__JS6__AFFINITY_NEXT, /* (RW) 64-bit 0x1B50 */ - JOB_CONTROL__JS6__CONFIG_NEXT, /* (RW) 32-bit 0x1B58 */ - JOB_CONTROL__JS6__XAFFINITY_NEXT, /* (RW) 32-bit 0x1B5C */ - JOB_CONTROL__JS6__COMMAND_NEXT, /* (RW) 32-bit 0x1B60 */ - JOB_CONTROL__JS6__FLUSH_ID_NEXT, /* (RW) 32-bit 0x1B70 */ - JOB_CONTROL__JS6__EVENT_MASK_NEXT, /* (RW) 32-bit 0x1B7C */ - JOB_CONTROL__JS7__HEAD, /* (RO) 64-bit 0x1B80 */ - JOB_CONTROL__JS7__TAIL, /* (RO) 64-bit 0x1B88 */ - JOB_CONTROL__JS7__AFFINITY, /* (RO) 64-bit 0x1B90 */ - JOB_CONTROL__JS7__CONFIG, /* (RO) 32-bit 0x1B98 */ - JOB_CONTROL__JS7__XAFFINITY, /* (RO) 32-bit 0x1B9C */ - JOB_CONTROL__JS7__COMMAND, /* (RW) 32-bit 0x1BA0 */ - JOB_CONTROL__JS7__STATUS, /* (RO) 32-bit 0x1BA4 */ - JOB_CONTROL__JS7__FLUSH_ID, /* (RO) 32-bit 0x1BB0 */ - JOB_CONTROL__JS7__EVENT_0, /* (RW) 32-bit 0x1BB4 */ - JOB_CONTROL__JS7__EVENT_1, /* (RW) 32-bit 0x1BB8 */ - JOB_CONTROL__JS7__EVENT_MASK, /* (RW) 32-bit 0x1BBC */ - JOB_CONTROL__JS7__HEAD_NEXT, /* (RW) 64-bit 0x1BC0 */ - JOB_CONTROL__JS7__TAIL_NEXT, /* (RW) 64-bit 0x1BC8 */ - JOB_CONTROL__JS7__AFFINITY_NEXT, /* (RW) 64-bit 0x1BD0 */ - JOB_CONTROL__JS7__CONFIG_NEXT, /* (RW) 32-bit 0x1BD8 */ - JOB_CONTROL__JS7__XAFFINITY_NEXT, /* (RW) 32-bit 0x1BDC */ - JOB_CONTROL__JS7__COMMAND_NEXT, /* (RW) 32-bit 0x1BE0 */ - JOB_CONTROL__JS7__FLUSH_ID_NEXT, /* (RW) 32-bit 0x1BF0 */ - JOB_CONTROL__JS7__EVENT_MASK_NEXT, /* (RW) 32-bit 0x1BFC */ - JOB_CONTROL__JS8__HEAD, /* (RO) 64-bit 0x1C00 */ - JOB_CONTROL__JS8__TAIL, /* (RO) 64-bit 0x1C08 */ - JOB_CONTROL__JS8__AFFINITY, /* (RO) 64-bit 0x1C10 */ - JOB_CONTROL__JS8__CONFIG, /* (RO) 32-bit 0x1C18 */ - JOB_CONTROL__JS8__XAFFINITY, /* (RO) 32-bit 0x1C1C */ - JOB_CONTROL__JS8__COMMAND, /* (RW) 32-bit 0x1C20 */ - JOB_CONTROL__JS8__STATUS, /* (RO) 32-bit 0x1C24 */ - JOB_CONTROL__JS8__FLUSH_ID, /* (RO) 32-bit 0x1C30 */ - JOB_CONTROL__JS8__EVENT_0, /* (RW) 32-bit 0x1C34 */ - JOB_CONTROL__JS8__EVENT_1, /* (RW) 32-bit 0x1C38 */ - JOB_CONTROL__JS8__EVENT_MASK, /* (RW) 32-bit 0x1C3C */ - JOB_CONTROL__JS8__HEAD_NEXT, /* (RW) 64-bit 0x1C40 */ - JOB_CONTROL__JS8__TAIL_NEXT, /* (RW) 64-bit 0x1C48 */ - JOB_CONTROL__JS8__AFFINITY_NEXT, /* (RW) 64-bit 0x1C50 */ - JOB_CONTROL__JS8__CONFIG_NEXT, /* (RW) 32-bit 0x1C58 */ - JOB_CONTROL__JS8__XAFFINITY_NEXT, /* (RW) 32-bit 0x1C5C */ - JOB_CONTROL__JS8__COMMAND_NEXT, /* (RW) 32-bit 0x1C60 */ - JOB_CONTROL__JS8__FLUSH_ID_NEXT, /* (RW) 32-bit 0x1C70 */ - JOB_CONTROL__JS8__EVENT_MASK_NEXT, /* (RW) 32-bit 0x1C7C */ - JOB_CONTROL__JS9__HEAD, /* (RO) 64-bit 0x1C80 */ - JOB_CONTROL__JS9__TAIL, /* (RO) 64-bit 0x1C88 */ - JOB_CONTROL__JS9__AFFINITY, /* (RO) 64-bit 0x1C90 */ - JOB_CONTROL__JS9__CONFIG, /* (RO) 32-bit 0x1C98 */ - JOB_CONTROL__JS9__XAFFINITY, /* (RO) 32-bit 0x1C9C */ - JOB_CONTROL__JS9__COMMAND, /* (RW) 32-bit 0x1CA0 */ - JOB_CONTROL__JS9__STATUS, /* (RO) 32-bit 0x1CA4 */ - JOB_CONTROL__JS9__FLUSH_ID, /* (RO) 32-bit 0x1CB0 */ - JOB_CONTROL__JS9__EVENT_0, /* (RW) 32-bit 0x1CB4 */ - JOB_CONTROL__JS9__EVENT_1, /* (RW) 32-bit 0x1CB8 */ - JOB_CONTROL__JS9__EVENT_MASK, /* (RW) 32-bit 0x1CBC */ - JOB_CONTROL__JS9__HEAD_NEXT, /* (RW) 64-bit 0x1CC0 */ - JOB_CONTROL__JS9__TAIL_NEXT, /* (RW) 64-bit 0x1CC8 */ - JOB_CONTROL__JS9__AFFINITY_NEXT, /* (RW) 64-bit 0x1CD0 */ - JOB_CONTROL__JS9__CONFIG_NEXT, /* (RW) 32-bit 0x1CD8 */ - JOB_CONTROL__JS9__XAFFINITY_NEXT, /* (RW) 32-bit 0x1CDC */ - JOB_CONTROL__JS9__COMMAND_NEXT, /* (RW) 32-bit 0x1CE0 */ - JOB_CONTROL__JS9__FLUSH_ID_NEXT, /* (RW) 32-bit 0x1CF0 */ - JOB_CONTROL__JS9__EVENT_MASK_NEXT, /* (RW) 32-bit 0x1CFC */ - JOB_CONTROL__JS10__HEAD, /* (RO) 64-bit 0x1D00 */ - JOB_CONTROL__JS10__TAIL, /* (RO) 64-bit 0x1D08 */ - JOB_CONTROL__JS10__AFFINITY, /* (RO) 64-bit 0x1D10 */ - JOB_CONTROL__JS10__CONFIG, /* (RO) 32-bit 0x1D18 */ - JOB_CONTROL__JS10__XAFFINITY, /* (RO) 32-bit 0x1D1C */ - JOB_CONTROL__JS10__COMMAND, /* (RW) 32-bit 0x1D20 */ - JOB_CONTROL__JS10__STATUS, /* (RO) 32-bit 0x1D24 */ - JOB_CONTROL__JS10__FLUSH_ID, /* (RO) 32-bit 0x1D30 */ - JOB_CONTROL__JS10__EVENT_0, /* (RW) 32-bit 0x1D34 */ - JOB_CONTROL__JS10__EVENT_1, /* (RW) 32-bit 0x1D38 */ - JOB_CONTROL__JS10__EVENT_MASK, /* (RW) 32-bit 0x1D3C */ - JOB_CONTROL__JS10__HEAD_NEXT, /* (RW) 64-bit 0x1D40 */ - JOB_CONTROL__JS10__TAIL_NEXT, /* (RW) 64-bit 0x1D48 */ - JOB_CONTROL__JS10__AFFINITY_NEXT, /* (RW) 64-bit 0x1D50 */ - JOB_CONTROL__JS10__CONFIG_NEXT, /* (RW) 32-bit 0x1D58 */ - JOB_CONTROL__JS10__XAFFINITY_NEXT, /* (RW) 32-bit 0x1D5C */ - JOB_CONTROL__JS10__COMMAND_NEXT, /* (RW) 32-bit 0x1D60 */ - JOB_CONTROL__JS10__FLUSH_ID_NEXT, /* (RW) 32-bit 0x1D70 */ - JOB_CONTROL__JS10__EVENT_MASK_NEXT, /* (RW) 32-bit 0x1D7C */ - JOB_CONTROL__JS11__HEAD, /* (RO) 64-bit 0x1D80 */ - JOB_CONTROL__JS11__TAIL, /* (RO) 64-bit 0x1D88 */ - JOB_CONTROL__JS11__AFFINITY, /* (RO) 64-bit 0x1D90 */ - JOB_CONTROL__JS11__CONFIG, /* (RO) 32-bit 0x1D98 */ - JOB_CONTROL__JS11__XAFFINITY, /* (RO) 32-bit 0x1D9C */ - JOB_CONTROL__JS11__COMMAND, /* (RW) 32-bit 0x1DA0 */ - JOB_CONTROL__JS11__STATUS, /* (RO) 32-bit 0x1DA4 */ - JOB_CONTROL__JS11__FLUSH_ID, /* (RO) 32-bit 0x1DB0 */ - JOB_CONTROL__JS11__EVENT_0, /* (RW) 32-bit 0x1DB4 */ - JOB_CONTROL__JS11__EVENT_1, /* (RW) 32-bit 0x1DB8 */ - JOB_CONTROL__JS11__EVENT_MASK, /* (RW) 32-bit 0x1DBC */ - JOB_CONTROL__JS11__HEAD_NEXT, /* (RW) 64-bit 0x1DC0 */ - JOB_CONTROL__JS11__TAIL_NEXT, /* (RW) 64-bit 0x1DC8 */ - JOB_CONTROL__JS11__AFFINITY_NEXT, /* (RW) 64-bit 0x1DD0 */ - JOB_CONTROL__JS11__CONFIG_NEXT, /* (RW) 32-bit 0x1DD8 */ - JOB_CONTROL__JS11__XAFFINITY_NEXT, /* (RW) 32-bit 0x1DDC */ - JOB_CONTROL__JS11__COMMAND_NEXT, /* (RW) 32-bit 0x1DE0 */ - JOB_CONTROL__JS11__FLUSH_ID_NEXT, /* (RW) 32-bit 0x1DF0 */ - JOB_CONTROL__JS11__EVENT_MASK_NEXT, /* (RW) 32-bit 0x1DFC */ - JOB_CONTROL__JS12__HEAD, /* (RO) 64-bit 0x1E00 */ - JOB_CONTROL__JS12__TAIL, /* (RO) 64-bit 0x1E08 */ - JOB_CONTROL__JS12__AFFINITY, /* (RO) 64-bit 0x1E10 */ - JOB_CONTROL__JS12__CONFIG, /* (RO) 32-bit 0x1E18 */ - JOB_CONTROL__JS12__XAFFINITY, /* (RO) 32-bit 0x1E1C */ - JOB_CONTROL__JS12__COMMAND, /* (RW) 32-bit 0x1E20 */ - JOB_CONTROL__JS12__STATUS, /* (RO) 32-bit 0x1E24 */ - JOB_CONTROL__JS12__FLUSH_ID, /* (RO) 32-bit 0x1E30 */ - JOB_CONTROL__JS12__EVENT_0, /* (RW) 32-bit 0x1E34 */ - JOB_CONTROL__JS12__EVENT_1, /* (RW) 32-bit 0x1E38 */ - JOB_CONTROL__JS12__EVENT_MASK, /* (RW) 32-bit 0x1E3C */ - JOB_CONTROL__JS12__HEAD_NEXT, /* (RW) 64-bit 0x1E40 */ - JOB_CONTROL__JS12__TAIL_NEXT, /* (RW) 64-bit 0x1E48 */ - JOB_CONTROL__JS12__AFFINITY_NEXT, /* (RW) 64-bit 0x1E50 */ - JOB_CONTROL__JS12__CONFIG_NEXT, /* (RW) 32-bit 0x1E58 */ - JOB_CONTROL__JS12__XAFFINITY_NEXT, /* (RW) 32-bit 0x1E5C */ - JOB_CONTROL__JS12__COMMAND_NEXT, /* (RW) 32-bit 0x1E60 */ - JOB_CONTROL__JS12__FLUSH_ID_NEXT, /* (RW) 32-bit 0x1E70 */ - JOB_CONTROL__JS12__EVENT_MASK_NEXT, /* (RW) 32-bit 0x1E7C */ - JOB_CONTROL__JS13__HEAD, /* (RO) 64-bit 0x1E80 */ - JOB_CONTROL__JS13__TAIL, /* (RO) 64-bit 0x1E88 */ - JOB_CONTROL__JS13__AFFINITY, /* (RO) 64-bit 0x1E90 */ - JOB_CONTROL__JS13__CONFIG, /* (RO) 32-bit 0x1E98 */ - JOB_CONTROL__JS13__XAFFINITY, /* (RO) 32-bit 0x1E9C */ - JOB_CONTROL__JS13__COMMAND, /* (RW) 32-bit 0x1EA0 */ - JOB_CONTROL__JS13__STATUS, /* (RO) 32-bit 0x1EA4 */ - JOB_CONTROL__JS13__FLUSH_ID, /* (RO) 32-bit 0x1EB0 */ - JOB_CONTROL__JS13__EVENT_0, /* (RW) 32-bit 0x1EB4 */ - JOB_CONTROL__JS13__EVENT_1, /* (RW) 32-bit 0x1EB8 */ - JOB_CONTROL__JS13__EVENT_MASK, /* (RW) 32-bit 0x1EBC */ - JOB_CONTROL__JS13__HEAD_NEXT, /* (RW) 64-bit 0x1EC0 */ - JOB_CONTROL__JS13__TAIL_NEXT, /* (RW) 64-bit 0x1EC8 */ - JOB_CONTROL__JS13__AFFINITY_NEXT, /* (RW) 64-bit 0x1ED0 */ - JOB_CONTROL__JS13__CONFIG_NEXT, /* (RW) 32-bit 0x1ED8 */ - JOB_CONTROL__JS13__XAFFINITY_NEXT, /* (RW) 32-bit 0x1EDC */ - JOB_CONTROL__JS13__COMMAND_NEXT, /* (RW) 32-bit 0x1EE0 */ - JOB_CONTROL__JS13__FLUSH_ID_NEXT, /* (RW) 32-bit 0x1EF0 */ - JOB_CONTROL__JS13__EVENT_MASK_NEXT, /* (RW) 32-bit 0x1EFC */ - JOB_CONTROL__JS14__HEAD, /* (RO) 64-bit 0x1F00 */ - JOB_CONTROL__JS14__TAIL, /* (RO) 64-bit 0x1F08 */ - JOB_CONTROL__JS14__AFFINITY, /* (RO) 64-bit 0x1F10 */ - JOB_CONTROL__JS14__CONFIG, /* (RO) 32-bit 0x1F18 */ - JOB_CONTROL__JS14__XAFFINITY, /* (RO) 32-bit 0x1F1C */ - JOB_CONTROL__JS14__COMMAND, /* (RW) 32-bit 0x1F20 */ - JOB_CONTROL__JS14__STATUS, /* (RO) 32-bit 0x1F24 */ - JOB_CONTROL__JS14__FLUSH_ID, /* (RO) 32-bit 0x1F30 */ - JOB_CONTROL__JS14__EVENT_0, /* (RW) 32-bit 0x1F34 */ - JOB_CONTROL__JS14__EVENT_1, /* (RW) 32-bit 0x1F38 */ - JOB_CONTROL__JS14__EVENT_MASK, /* (RW) 32-bit 0x1F3C */ - JOB_CONTROL__JS14__HEAD_NEXT, /* (RW) 64-bit 0x1F40 */ - JOB_CONTROL__JS14__TAIL_NEXT, /* (RW) 64-bit 0x1F48 */ - JOB_CONTROL__JS14__AFFINITY_NEXT, /* (RW) 64-bit 0x1F50 */ - JOB_CONTROL__JS14__CONFIG_NEXT, /* (RW) 32-bit 0x1F58 */ - JOB_CONTROL__JS14__XAFFINITY_NEXT, /* (RW) 32-bit 0x1F5C */ - JOB_CONTROL__JS14__COMMAND_NEXT, /* (RW) 32-bit 0x1F60 */ - JOB_CONTROL__JS14__FLUSH_ID_NEXT, /* (RW) 32-bit 0x1F70 */ - JOB_CONTROL__JS14__EVENT_MASK_NEXT, /* (RW) 32-bit 0x1F7C */ - JOB_CONTROL__JS15__HEAD, /* (RO) 64-bit 0x1F80 */ - JOB_CONTROL__JS15__TAIL, /* (RO) 64-bit 0x1F88 */ - JOB_CONTROL__JS15__AFFINITY, /* (RO) 64-bit 0x1F90 */ - JOB_CONTROL__JS15__CONFIG, /* (RO) 32-bit 0x1F98 */ - JOB_CONTROL__JS15__XAFFINITY, /* (RO) 32-bit 0x1F9C */ - JOB_CONTROL__JS15__COMMAND, /* (RW) 32-bit 0x1FA0 */ - JOB_CONTROL__JS15__STATUS, /* (RO) 32-bit 0x1FA4 */ - JOB_CONTROL__JS15__FLUSH_ID, /* (RO) 32-bit 0x1FB0 */ - JOB_CONTROL__JS15__EVENT_0, /* (RW) 32-bit 0x1FB4 */ - JOB_CONTROL__JS15__EVENT_1, /* (RW) 32-bit 0x1FB8 */ - JOB_CONTROL__JS15__EVENT_MASK, /* (RW) 32-bit 0x1FBC */ - JOB_CONTROL__JS15__HEAD_NEXT, /* (RW) 64-bit 0x1FC0 */ - JOB_CONTROL__JS15__TAIL_NEXT, /* (RW) 64-bit 0x1FC8 */ - JOB_CONTROL__JS15__AFFINITY_NEXT, /* (RW) 64-bit 0x1FD0 */ - JOB_CONTROL__JS15__CONFIG_NEXT, /* (RW) 32-bit 0x1FD8 */ - JOB_CONTROL__JS15__XAFFINITY_NEXT, /* (RW) 32-bit 0x1FDC */ - JOB_CONTROL__JS15__COMMAND_NEXT, /* (RW) 32-bit 0x1FE0 */ - JOB_CONTROL__JS15__FLUSH_ID_NEXT, /* (RW) 32-bit 0x1FF0 */ - JOB_CONTROL__JS15__EVENT_MASK_NEXT, /* (RW) 32-bit 0x1FFC */ - MMU_STAGE1__ST1MMU__IRQ_RAWSTAT, /* (RW) 32-bit 0x2000 */ - MMU_STAGE1__ST1MMU__IRQ_CLEAR, /* (WO) 32-bit 0x2004 */ - MMU_STAGE1__ST1MMU__IRQ_MASK, /* (RW) 32-bit 0x2008 */ - MMU_STAGE1__ST1MMU__IRQ_STATUS, /* (RO) 32-bit 0x200C */ - MMU_STAGE1__ST1MMU__AS0__TRANSTAB, /* (RW) 64-bit 0x2400 */ - MMU_STAGE1__ST1MMU__AS0__MEMATTR, /* (RW) 64-bit 0x2408 */ - MMU_STAGE1__ST1MMU__AS0__LOCKADDR, /* (RW) 64-bit 0x2410 */ - MMU_STAGE1__ST1MMU__AS0__COMMAND, /* (WO) 32-bit 0x2418 */ - MMU_STAGE1__ST1MMU__AS0__FAULTSTATUS, /* (RO) 32-bit 0x241C */ - MMU_STAGE1__ST1MMU__AS0__FAULTADDRESS, /* (RO) 64-bit 0x2420 */ - MMU_STAGE1__ST1MMU__AS0__STATUS, /* (RO) 32-bit 0x2428 */ - MMU_STAGE1__ST1MMU__AS0__TRANSCFG, /* (RW) 64-bit 0x2430 */ - MMU_STAGE1__ST1MMU__AS0__FAULTEXTRA, /* (RO) 64-bit 0x2438 */ - MMU_STAGE1__ST1MMU__AS1__TRANSTAB, /* (RW) 64-bit 0x2440 */ - MMU_STAGE1__ST1MMU__AS1__MEMATTR, /* (RW) 64-bit 0x2448 */ - MMU_STAGE1__ST1MMU__AS1__LOCKADDR, /* (RW) 64-bit 0x2450 */ - MMU_STAGE1__ST1MMU__AS1__COMMAND, /* (WO) 32-bit 0x2458 */ - MMU_STAGE1__ST1MMU__AS1__FAULTSTATUS, /* (RO) 32-bit 0x245C */ - MMU_STAGE1__ST1MMU__AS1__FAULTADDRESS, /* (RO) 64-bit 0x2460 */ - MMU_STAGE1__ST1MMU__AS1__STATUS, /* (RO) 32-bit 0x2468 */ - MMU_STAGE1__ST1MMU__AS1__TRANSCFG, /* (RW) 64-bit 0x2470 */ - MMU_STAGE1__ST1MMU__AS1__FAULTEXTRA, /* (RO) 64-bit 0x2478 */ - MMU_STAGE1__ST1MMU__AS2__TRANSTAB, /* (RW) 64-bit 0x2480 */ - MMU_STAGE1__ST1MMU__AS2__MEMATTR, /* (RW) 64-bit 0x2488 */ - MMU_STAGE1__ST1MMU__AS2__LOCKADDR, /* (RW) 64-bit 0x2490 */ - MMU_STAGE1__ST1MMU__AS2__COMMAND, /* (WO) 32-bit 0x2498 */ - MMU_STAGE1__ST1MMU__AS2__FAULTSTATUS, /* (RO) 32-bit 0x249C */ - MMU_STAGE1__ST1MMU__AS2__FAULTADDRESS, /* (RO) 64-bit 0x24A0 */ - MMU_STAGE1__ST1MMU__AS2__STATUS, /* (RO) 32-bit 0x24A8 */ - MMU_STAGE1__ST1MMU__AS2__TRANSCFG, /* (RW) 64-bit 0x24B0 */ - MMU_STAGE1__ST1MMU__AS2__FAULTEXTRA, /* (RO) 64-bit 0x24B8 */ - MMU_STAGE1__ST1MMU__AS3__TRANSTAB, /* (RW) 64-bit 0x24C0 */ - MMU_STAGE1__ST1MMU__AS3__MEMATTR, /* (RW) 64-bit 0x24C8 */ - MMU_STAGE1__ST1MMU__AS3__LOCKADDR, /* (RW) 64-bit 0x24D0 */ - MMU_STAGE1__ST1MMU__AS3__COMMAND, /* (WO) 32-bit 0x24D8 */ - MMU_STAGE1__ST1MMU__AS3__FAULTSTATUS, /* (RO) 32-bit 0x24DC */ - MMU_STAGE1__ST1MMU__AS3__FAULTADDRESS, /* (RO) 64-bit 0x24E0 */ - MMU_STAGE1__ST1MMU__AS3__STATUS, /* (RO) 32-bit 0x24E8 */ - MMU_STAGE1__ST1MMU__AS3__TRANSCFG, /* (RW) 64-bit 0x24F0 */ - MMU_STAGE1__ST1MMU__AS3__FAULTEXTRA, /* (RO) 64-bit 0x24F8 */ - MMU_STAGE1__ST1MMU__AS4__TRANSTAB, /* (RW) 64-bit 0x2500 */ - MMU_STAGE1__ST1MMU__AS4__MEMATTR, /* (RW) 64-bit 0x2508 */ - MMU_STAGE1__ST1MMU__AS4__LOCKADDR, /* (RW) 64-bit 0x2510 */ - MMU_STAGE1__ST1MMU__AS4__COMMAND, /* (WO) 32-bit 0x2518 */ - MMU_STAGE1__ST1MMU__AS4__FAULTSTATUS, /* (RO) 32-bit 0x251C */ - MMU_STAGE1__ST1MMU__AS4__FAULTADDRESS, /* (RO) 64-bit 0x2520 */ - MMU_STAGE1__ST1MMU__AS4__STATUS, /* (RO) 32-bit 0x2528 */ - MMU_STAGE1__ST1MMU__AS4__TRANSCFG, /* (RW) 64-bit 0x2530 */ - MMU_STAGE1__ST1MMU__AS4__FAULTEXTRA, /* (RO) 64-bit 0x2538 */ - MMU_STAGE1__ST1MMU__AS5__TRANSTAB, /* (RW) 64-bit 0x2540 */ - MMU_STAGE1__ST1MMU__AS5__MEMATTR, /* (RW) 64-bit 0x2548 */ - MMU_STAGE1__ST1MMU__AS5__LOCKADDR, /* (RW) 64-bit 0x2550 */ - MMU_STAGE1__ST1MMU__AS5__COMMAND, /* (WO) 32-bit 0x2558 */ - MMU_STAGE1__ST1MMU__AS5__FAULTSTATUS, /* (RO) 32-bit 0x255C */ - MMU_STAGE1__ST1MMU__AS5__FAULTADDRESS, /* (RO) 64-bit 0x2560 */ - MMU_STAGE1__ST1MMU__AS5__STATUS, /* (RO) 32-bit 0x2568 */ - MMU_STAGE1__ST1MMU__AS5__TRANSCFG, /* (RW) 64-bit 0x2570 */ - MMU_STAGE1__ST1MMU__AS5__FAULTEXTRA, /* (RO) 64-bit 0x2578 */ - MMU_STAGE1__ST1MMU__AS6__TRANSTAB, /* (RW) 64-bit 0x2580 */ - MMU_STAGE1__ST1MMU__AS6__MEMATTR, /* (RW) 64-bit 0x2588 */ - MMU_STAGE1__ST1MMU__AS6__LOCKADDR, /* (RW) 64-bit 0x2590 */ - MMU_STAGE1__ST1MMU__AS6__COMMAND, /* (WO) 32-bit 0x2598 */ - MMU_STAGE1__ST1MMU__AS6__FAULTSTATUS, /* (RO) 32-bit 0x259C */ - MMU_STAGE1__ST1MMU__AS6__FAULTADDRESS, /* (RO) 64-bit 0x25A0 */ - MMU_STAGE1__ST1MMU__AS6__STATUS, /* (RO) 32-bit 0x25A8 */ - MMU_STAGE1__ST1MMU__AS6__TRANSCFG, /* (RW) 64-bit 0x25B0 */ - MMU_STAGE1__ST1MMU__AS6__FAULTEXTRA, /* (RO) 64-bit 0x25B8 */ - MMU_STAGE1__ST1MMU__AS7__TRANSTAB, /* (RW) 64-bit 0x25C0 */ - MMU_STAGE1__ST1MMU__AS7__MEMATTR, /* (RW) 64-bit 0x25C8 */ - MMU_STAGE1__ST1MMU__AS7__LOCKADDR, /* (RW) 64-bit 0x25D0 */ - MMU_STAGE1__ST1MMU__AS7__COMMAND, /* (WO) 32-bit 0x25D8 */ - MMU_STAGE1__ST1MMU__AS7__FAULTSTATUS, /* (RO) 32-bit 0x25DC */ - MMU_STAGE1__ST1MMU__AS7__FAULTADDRESS, /* (RO) 64-bit 0x25E0 */ - MMU_STAGE1__ST1MMU__AS7__STATUS, /* (RO) 32-bit 0x25E8 */ - MMU_STAGE1__ST1MMU__AS7__TRANSCFG, /* (RW) 64-bit 0x25F0 */ - MMU_STAGE1__ST1MMU__AS7__FAULTEXTRA, /* (RO) 64-bit 0x25F8 */ - MMU_STAGE1__ST1MMU__AS8__TRANSTAB, /* (RW) 64-bit 0x2600 */ - MMU_STAGE1__ST1MMU__AS8__MEMATTR, /* (RW) 64-bit 0x2608 */ - MMU_STAGE1__ST1MMU__AS8__LOCKADDR, /* (RW) 64-bit 0x2610 */ - MMU_STAGE1__ST1MMU__AS8__COMMAND, /* (WO) 32-bit 0x2618 */ - MMU_STAGE1__ST1MMU__AS8__FAULTSTATUS, /* (RO) 32-bit 0x261C */ - MMU_STAGE1__ST1MMU__AS8__FAULTADDRESS, /* (RO) 64-bit 0x2620 */ - MMU_STAGE1__ST1MMU__AS8__STATUS, /* (RO) 32-bit 0x2628 */ - MMU_STAGE1__ST1MMU__AS8__TRANSCFG, /* (RW) 64-bit 0x2630 */ - MMU_STAGE1__ST1MMU__AS8__FAULTEXTRA, /* (RO) 64-bit 0x2638 */ - MMU_STAGE1__ST1MMU__AS9__TRANSTAB, /* (RW) 64-bit 0x2640 */ - MMU_STAGE1__ST1MMU__AS9__MEMATTR, /* (RW) 64-bit 0x2648 */ - MMU_STAGE1__ST1MMU__AS9__LOCKADDR, /* (RW) 64-bit 0x2650 */ - MMU_STAGE1__ST1MMU__AS9__COMMAND, /* (WO) 32-bit 0x2658 */ - MMU_STAGE1__ST1MMU__AS9__FAULTSTATUS, /* (RO) 32-bit 0x265C */ - MMU_STAGE1__ST1MMU__AS9__FAULTADDRESS, /* (RO) 64-bit 0x2660 */ - MMU_STAGE1__ST1MMU__AS9__STATUS, /* (RO) 32-bit 0x2668 */ - MMU_STAGE1__ST1MMU__AS9__TRANSCFG, /* (RW) 64-bit 0x2670 */ - MMU_STAGE1__ST1MMU__AS9__FAULTEXTRA, /* (RO) 64-bit 0x2678 */ - MMU_STAGE1__ST1MMU__AS10__TRANSTAB, /* (RW) 64-bit 0x2680 */ - MMU_STAGE1__ST1MMU__AS10__MEMATTR, /* (RW) 64-bit 0x2688 */ - MMU_STAGE1__ST1MMU__AS10__LOCKADDR, /* (RW) 64-bit 0x2690 */ - MMU_STAGE1__ST1MMU__AS10__COMMAND, /* (WO) 32-bit 0x2698 */ - MMU_STAGE1__ST1MMU__AS10__FAULTSTATUS, /* (RO) 32-bit 0x269C */ - MMU_STAGE1__ST1MMU__AS10__FAULTADDRESS, /* (RO) 64-bit 0x26A0 */ - MMU_STAGE1__ST1MMU__AS10__STATUS, /* (RO) 32-bit 0x26A8 */ - MMU_STAGE1__ST1MMU__AS10__TRANSCFG, /* (RW) 64-bit 0x26B0 */ - MMU_STAGE1__ST1MMU__AS10__FAULTEXTRA, /* (RO) 64-bit 0x26B8 */ - MMU_STAGE1__ST1MMU__AS11__TRANSTAB, /* (RW) 64-bit 0x26C0 */ - MMU_STAGE1__ST1MMU__AS11__MEMATTR, /* (RW) 64-bit 0x26C8 */ - MMU_STAGE1__ST1MMU__AS11__LOCKADDR, /* (RW) 64-bit 0x26D0 */ - MMU_STAGE1__ST1MMU__AS11__COMMAND, /* (WO) 32-bit 0x26D8 */ - MMU_STAGE1__ST1MMU__AS11__FAULTSTATUS, /* (RO) 32-bit 0x26DC */ - MMU_STAGE1__ST1MMU__AS11__FAULTADDRESS, /* (RO) 64-bit 0x26E0 */ - MMU_STAGE1__ST1MMU__AS11__STATUS, /* (RO) 32-bit 0x26E8 */ - MMU_STAGE1__ST1MMU__AS11__TRANSCFG, /* (RW) 64-bit 0x26F0 */ - MMU_STAGE1__ST1MMU__AS11__FAULTEXTRA, /* (RO) 64-bit 0x26F8 */ - MMU_STAGE1__ST1MMU__AS12__TRANSTAB, /* (RW) 64-bit 0x2700 */ - MMU_STAGE1__ST1MMU__AS12__MEMATTR, /* (RW) 64-bit 0x2708 */ - MMU_STAGE1__ST1MMU__AS12__LOCKADDR, /* (RW) 64-bit 0x2710 */ - MMU_STAGE1__ST1MMU__AS12__COMMAND, /* (WO) 32-bit 0x2718 */ - MMU_STAGE1__ST1MMU__AS12__FAULTSTATUS, /* (RO) 32-bit 0x271C */ - MMU_STAGE1__ST1MMU__AS12__FAULTADDRESS, /* (RO) 64-bit 0x2720 */ - MMU_STAGE1__ST1MMU__AS12__STATUS, /* (RO) 32-bit 0x2728 */ - MMU_STAGE1__ST1MMU__AS12__TRANSCFG, /* (RW) 64-bit 0x2730 */ - MMU_STAGE1__ST1MMU__AS12__FAULTEXTRA, /* (RO) 64-bit 0x2738 */ - MMU_STAGE1__ST1MMU__AS13__TRANSTAB, /* (RW) 64-bit 0x2740 */ - MMU_STAGE1__ST1MMU__AS13__MEMATTR, /* (RW) 64-bit 0x2748 */ - MMU_STAGE1__ST1MMU__AS13__LOCKADDR, /* (RW) 64-bit 0x2750 */ - MMU_STAGE1__ST1MMU__AS13__COMMAND, /* (WO) 32-bit 0x2758 */ - MMU_STAGE1__ST1MMU__AS13__FAULTSTATUS, /* (RO) 32-bit 0x275C */ - MMU_STAGE1__ST1MMU__AS13__FAULTADDRESS, /* (RO) 64-bit 0x2760 */ - MMU_STAGE1__ST1MMU__AS13__STATUS, /* (RO) 32-bit 0x2768 */ - MMU_STAGE1__ST1MMU__AS13__TRANSCFG, /* (RW) 64-bit 0x2770 */ - MMU_STAGE1__ST1MMU__AS13__FAULTEXTRA, /* (RO) 64-bit 0x2778 */ - MMU_STAGE1__ST1MMU__AS14__TRANSTAB, /* (RW) 64-bit 0x2780 */ - MMU_STAGE1__ST1MMU__AS14__MEMATTR, /* (RW) 64-bit 0x2788 */ - MMU_STAGE1__ST1MMU__AS14__LOCKADDR, /* (RW) 64-bit 0x2790 */ - MMU_STAGE1__ST1MMU__AS14__COMMAND, /* (WO) 32-bit 0x2798 */ - MMU_STAGE1__ST1MMU__AS14__FAULTSTATUS, /* (RO) 32-bit 0x279C */ - MMU_STAGE1__ST1MMU__AS14__FAULTADDRESS, /* (RO) 64-bit 0x27A0 */ - MMU_STAGE1__ST1MMU__AS14__STATUS, /* (RO) 32-bit 0x27A8 */ - MMU_STAGE1__ST1MMU__AS14__TRANSCFG, /* (RW) 64-bit 0x27B0 */ - MMU_STAGE1__ST1MMU__AS14__FAULTEXTRA, /* (RO) 64-bit 0x27B8 */ - MMU_STAGE1__ST1MMU__AS15__TRANSTAB, /* (RW) 64-bit 0x27C0 */ - MMU_STAGE1__ST1MMU__AS15__MEMATTR, /* (RW) 64-bit 0x27C8 */ - MMU_STAGE1__ST1MMU__AS15__LOCKADDR, /* (RW) 64-bit 0x27D0 */ - MMU_STAGE1__ST1MMU__AS15__COMMAND, /* (WO) 32-bit 0x27D8 */ - MMU_STAGE1__ST1MMU__AS15__FAULTSTATUS, /* (RO) 32-bit 0x27DC */ - MMU_STAGE1__ST1MMU__AS15__FAULTADDRESS, /* (RO) 64-bit 0x27E0 */ - MMU_STAGE1__ST1MMU__AS15__STATUS, /* (RO) 32-bit 0x27E8 */ - MMU_STAGE1__ST1MMU__AS15__TRANSCFG, /* (RW) 64-bit 0x27F0 */ - MMU_STAGE1__ST1MMU__AS15__FAULTEXTRA, /* (RO) 64-bit 0x27F8 */ - MMU_STAGE2__ST2MMU__IRQ_RAWSTAT, /* (RW) 32-bit 0x10000 */ - MMU_STAGE2__ST2MMU__IRQ_CLEAR, /* (WO) 32-bit 0x10004 */ - MMU_STAGE2__ST2MMU__IRQ_MASK, /* (RW) 32-bit 0x10008 */ - MMU_STAGE2__ST2MMU__IRQ_STATUS, /* (RO) 32-bit 0x1000C */ - MMU_STAGE2__ST2MMU__AS0__TRANSTAB, /* (RW) 64-bit 0x10400 */ - MMU_STAGE2__ST2MMU__AS0__MEMATTR, /* (RW) 64-bit 0x10408 */ - MMU_STAGE2__ST2MMU__AS0__LOCKADDR, /* (RW) 64-bit 0x10410 */ - MMU_STAGE2__ST2MMU__AS0__COMMAND, /* (WO) 32-bit 0x10418 */ - MMU_STAGE2__ST2MMU__AS0__FAULTSTATUS, /* (RO) 32-bit 0x1041C */ - MMU_STAGE2__ST2MMU__AS0__FAULTADDRESS, /* (RO) 64-bit 0x10420 */ - MMU_STAGE2__ST2MMU__AS0__STATUS, /* (RO) 32-bit 0x10428 */ - MMU_STAGE2__ST2MMU__AS0__TRANSCFG, /* (RW) 64-bit 0x10430 */ - MMU_STAGE2__ST2MMU__AS0__FAULTEXTRA, /* (RO) 64-bit 0x10438 */ - MMU_STAGE2__ST2MMU__AS1__TRANSTAB, /* (RW) 64-bit 0x10440 */ - MMU_STAGE2__ST2MMU__AS1__MEMATTR, /* (RW) 64-bit 0x10448 */ - MMU_STAGE2__ST2MMU__AS1__LOCKADDR, /* (RW) 64-bit 0x10450 */ - MMU_STAGE2__ST2MMU__AS1__COMMAND, /* (WO) 32-bit 0x10458 */ - MMU_STAGE2__ST2MMU__AS1__FAULTSTATUS, /* (RO) 32-bit 0x1045C */ - MMU_STAGE2__ST2MMU__AS1__FAULTADDRESS, /* (RO) 64-bit 0x10460 */ - MMU_STAGE2__ST2MMU__AS1__STATUS, /* (RO) 32-bit 0x10468 */ - MMU_STAGE2__ST2MMU__AS1__TRANSCFG, /* (RW) 64-bit 0x10470 */ - MMU_STAGE2__ST2MMU__AS1__FAULTEXTRA, /* (RO) 64-bit 0x10478 */ - MMU_STAGE2__ST2MMU__AS2__TRANSTAB, /* (RW) 64-bit 0x10480 */ - MMU_STAGE2__ST2MMU__AS2__MEMATTR, /* (RW) 64-bit 0x10488 */ - MMU_STAGE2__ST2MMU__AS2__LOCKADDR, /* (RW) 64-bit 0x10490 */ - MMU_STAGE2__ST2MMU__AS2__COMMAND, /* (WO) 32-bit 0x10498 */ - MMU_STAGE2__ST2MMU__AS2__FAULTSTATUS, /* (RO) 32-bit 0x1049C */ - MMU_STAGE2__ST2MMU__AS2__FAULTADDRESS, /* (RO) 64-bit 0x104A0 */ - MMU_STAGE2__ST2MMU__AS2__STATUS, /* (RO) 32-bit 0x104A8 */ - MMU_STAGE2__ST2MMU__AS2__TRANSCFG, /* (RW) 64-bit 0x104B0 */ - MMU_STAGE2__ST2MMU__AS2__FAULTEXTRA, /* (RO) 64-bit 0x104B8 */ - MMU_STAGE2__ST2MMU__AS3__TRANSTAB, /* (RW) 64-bit 0x104C0 */ - MMU_STAGE2__ST2MMU__AS3__MEMATTR, /* (RW) 64-bit 0x104C8 */ - MMU_STAGE2__ST2MMU__AS3__LOCKADDR, /* (RW) 64-bit 0x104D0 */ - MMU_STAGE2__ST2MMU__AS3__COMMAND, /* (WO) 32-bit 0x104D8 */ - MMU_STAGE2__ST2MMU__AS3__FAULTSTATUS, /* (RO) 32-bit 0x104DC */ - MMU_STAGE2__ST2MMU__AS3__FAULTADDRESS, /* (RO) 64-bit 0x104E0 */ - MMU_STAGE2__ST2MMU__AS3__STATUS, /* (RO) 32-bit 0x104E8 */ - MMU_STAGE2__ST2MMU__AS3__TRANSCFG, /* (RW) 64-bit 0x104F0 */ - MMU_STAGE2__ST2MMU__AS3__FAULTEXTRA, /* (RO) 64-bit 0x104F8 */ - MMU_STAGE2__ST2MMU__AS4__TRANSTAB, /* (RW) 64-bit 0x10500 */ - MMU_STAGE2__ST2MMU__AS4__MEMATTR, /* (RW) 64-bit 0x10508 */ - MMU_STAGE2__ST2MMU__AS4__LOCKADDR, /* (RW) 64-bit 0x10510 */ - MMU_STAGE2__ST2MMU__AS4__COMMAND, /* (WO) 32-bit 0x10518 */ - MMU_STAGE2__ST2MMU__AS4__FAULTSTATUS, /* (RO) 32-bit 0x1051C */ - MMU_STAGE2__ST2MMU__AS4__FAULTADDRESS, /* (RO) 64-bit 0x10520 */ - MMU_STAGE2__ST2MMU__AS4__STATUS, /* (RO) 32-bit 0x10528 */ - MMU_STAGE2__ST2MMU__AS4__TRANSCFG, /* (RW) 64-bit 0x10530 */ - MMU_STAGE2__ST2MMU__AS4__FAULTEXTRA, /* (RO) 64-bit 0x10538 */ - MMU_STAGE2__ST2MMU__AS5__TRANSTAB, /* (RW) 64-bit 0x10540 */ - MMU_STAGE2__ST2MMU__AS5__MEMATTR, /* (RW) 64-bit 0x10548 */ - MMU_STAGE2__ST2MMU__AS5__LOCKADDR, /* (RW) 64-bit 0x10550 */ - MMU_STAGE2__ST2MMU__AS5__COMMAND, /* (WO) 32-bit 0x10558 */ - MMU_STAGE2__ST2MMU__AS5__FAULTSTATUS, /* (RO) 32-bit 0x1055C */ - MMU_STAGE2__ST2MMU__AS5__FAULTADDRESS, /* (RO) 64-bit 0x10560 */ - MMU_STAGE2__ST2MMU__AS5__STATUS, /* (RO) 32-bit 0x10568 */ - MMU_STAGE2__ST2MMU__AS5__TRANSCFG, /* (RW) 64-bit 0x10570 */ - MMU_STAGE2__ST2MMU__AS5__FAULTEXTRA, /* (RO) 64-bit 0x10578 */ - MMU_STAGE2__ST2MMU__AS6__TRANSTAB, /* (RW) 64-bit 0x10580 */ - MMU_STAGE2__ST2MMU__AS6__MEMATTR, /* (RW) 64-bit 0x10588 */ - MMU_STAGE2__ST2MMU__AS6__LOCKADDR, /* (RW) 64-bit 0x10590 */ - MMU_STAGE2__ST2MMU__AS6__COMMAND, /* (WO) 32-bit 0x10598 */ - MMU_STAGE2__ST2MMU__AS6__FAULTSTATUS, /* (RO) 32-bit 0x1059C */ - MMU_STAGE2__ST2MMU__AS6__FAULTADDRESS, /* (RO) 64-bit 0x105A0 */ - MMU_STAGE2__ST2MMU__AS6__STATUS, /* (RO) 32-bit 0x105A8 */ - MMU_STAGE2__ST2MMU__AS6__TRANSCFG, /* (RW) 64-bit 0x105B0 */ - MMU_STAGE2__ST2MMU__AS6__FAULTEXTRA, /* (RO) 64-bit 0x105B8 */ - MMU_STAGE2__ST2MMU__AS7__TRANSTAB, /* (RW) 64-bit 0x105C0 */ - MMU_STAGE2__ST2MMU__AS7__MEMATTR, /* (RW) 64-bit 0x105C8 */ - MMU_STAGE2__ST2MMU__AS7__LOCKADDR, /* (RW) 64-bit 0x105D0 */ - MMU_STAGE2__ST2MMU__AS7__COMMAND, /* (WO) 32-bit 0x105D8 */ - MMU_STAGE2__ST2MMU__AS7__FAULTSTATUS, /* (RO) 32-bit 0x105DC */ - MMU_STAGE2__ST2MMU__AS7__FAULTADDRESS, /* (RO) 64-bit 0x105E0 */ - MMU_STAGE2__ST2MMU__AS7__STATUS, /* (RO) 32-bit 0x105E8 */ - MMU_STAGE2__ST2MMU__AS7__TRANSCFG, /* (RW) 64-bit 0x105F0 */ - MMU_STAGE2__ST2MMU__AS7__FAULTEXTRA, /* (RO) 64-bit 0x105F8 */ - MMU_STAGE2__ST2MMU__AS8__TRANSTAB, /* (RW) 64-bit 0x10600 */ - MMU_STAGE2__ST2MMU__AS8__MEMATTR, /* (RW) 64-bit 0x10608 */ - MMU_STAGE2__ST2MMU__AS8__LOCKADDR, /* (RW) 64-bit 0x10610 */ - MMU_STAGE2__ST2MMU__AS8__COMMAND, /* (WO) 32-bit 0x10618 */ - MMU_STAGE2__ST2MMU__AS8__FAULTSTATUS, /* (RO) 32-bit 0x1061C */ - MMU_STAGE2__ST2MMU__AS8__FAULTADDRESS, /* (RO) 64-bit 0x10620 */ - MMU_STAGE2__ST2MMU__AS8__STATUS, /* (RO) 32-bit 0x10628 */ - MMU_STAGE2__ST2MMU__AS8__TRANSCFG, /* (RW) 64-bit 0x10630 */ - MMU_STAGE2__ST2MMU__AS8__FAULTEXTRA, /* (RO) 64-bit 0x10638 */ - MMU_STAGE2__ST2MMU__AS9__TRANSTAB, /* (RW) 64-bit 0x10640 */ - MMU_STAGE2__ST2MMU__AS9__MEMATTR, /* (RW) 64-bit 0x10648 */ - MMU_STAGE2__ST2MMU__AS9__LOCKADDR, /* (RW) 64-bit 0x10650 */ - MMU_STAGE2__ST2MMU__AS9__COMMAND, /* (WO) 32-bit 0x10658 */ - MMU_STAGE2__ST2MMU__AS9__FAULTSTATUS, /* (RO) 32-bit 0x1065C */ - MMU_STAGE2__ST2MMU__AS9__FAULTADDRESS, /* (RO) 64-bit 0x10660 */ - MMU_STAGE2__ST2MMU__AS9__STATUS, /* (RO) 32-bit 0x10668 */ - MMU_STAGE2__ST2MMU__AS9__TRANSCFG, /* (RW) 64-bit 0x10670 */ - MMU_STAGE2__ST2MMU__AS9__FAULTEXTRA, /* (RO) 64-bit 0x10678 */ - MMU_STAGE2__ST2MMU__AS10__TRANSTAB, /* (RW) 64-bit 0x10680 */ - MMU_STAGE2__ST2MMU__AS10__MEMATTR, /* (RW) 64-bit 0x10688 */ - MMU_STAGE2__ST2MMU__AS10__LOCKADDR, /* (RW) 64-bit 0x10690 */ - MMU_STAGE2__ST2MMU__AS10__COMMAND, /* (WO) 32-bit 0x10698 */ - MMU_STAGE2__ST2MMU__AS10__FAULTSTATUS, /* (RO) 32-bit 0x1069C */ - MMU_STAGE2__ST2MMU__AS10__FAULTADDRESS, /* (RO) 64-bit 0x106A0 */ - MMU_STAGE2__ST2MMU__AS10__STATUS, /* (RO) 32-bit 0x106A8 */ - MMU_STAGE2__ST2MMU__AS10__TRANSCFG, /* (RW) 64-bit 0x106B0 */ - MMU_STAGE2__ST2MMU__AS10__FAULTEXTRA, /* (RO) 64-bit 0x106B8 */ - MMU_STAGE2__ST2MMU__AS11__TRANSTAB, /* (RW) 64-bit 0x106C0 */ - MMU_STAGE2__ST2MMU__AS11__MEMATTR, /* (RW) 64-bit 0x106C8 */ - MMU_STAGE2__ST2MMU__AS11__LOCKADDR, /* (RW) 64-bit 0x106D0 */ - MMU_STAGE2__ST2MMU__AS11__COMMAND, /* (WO) 32-bit 0x106D8 */ - MMU_STAGE2__ST2MMU__AS11__FAULTSTATUS, /* (RO) 32-bit 0x106DC */ - MMU_STAGE2__ST2MMU__AS11__FAULTADDRESS, /* (RO) 64-bit 0x106E0 */ - MMU_STAGE2__ST2MMU__AS11__STATUS, /* (RO) 32-bit 0x106E8 */ - MMU_STAGE2__ST2MMU__AS11__TRANSCFG, /* (RW) 64-bit 0x106F0 */ - MMU_STAGE2__ST2MMU__AS11__FAULTEXTRA, /* (RO) 64-bit 0x106F8 */ - MMU_STAGE2__ST2MMU__AS12__TRANSTAB, /* (RW) 64-bit 0x10700 */ - MMU_STAGE2__ST2MMU__AS12__MEMATTR, /* (RW) 64-bit 0x10708 */ - MMU_STAGE2__ST2MMU__AS12__LOCKADDR, /* (RW) 64-bit 0x10710 */ - MMU_STAGE2__ST2MMU__AS12__COMMAND, /* (WO) 32-bit 0x10718 */ - MMU_STAGE2__ST2MMU__AS12__FAULTSTATUS, /* (RO) 32-bit 0x1071C */ - MMU_STAGE2__ST2MMU__AS12__FAULTADDRESS, /* (RO) 64-bit 0x10720 */ - MMU_STAGE2__ST2MMU__AS12__STATUS, /* (RO) 32-bit 0x10728 */ - MMU_STAGE2__ST2MMU__AS12__TRANSCFG, /* (RW) 64-bit 0x10730 */ - MMU_STAGE2__ST2MMU__AS12__FAULTEXTRA, /* (RO) 64-bit 0x10738 */ - MMU_STAGE2__ST2MMU__AS13__TRANSTAB, /* (RW) 64-bit 0x10740 */ - MMU_STAGE2__ST2MMU__AS13__MEMATTR, /* (RW) 64-bit 0x10748 */ - MMU_STAGE2__ST2MMU__AS13__LOCKADDR, /* (RW) 64-bit 0x10750 */ - MMU_STAGE2__ST2MMU__AS13__COMMAND, /* (WO) 32-bit 0x10758 */ - MMU_STAGE2__ST2MMU__AS13__FAULTSTATUS, /* (RO) 32-bit 0x1075C */ - MMU_STAGE2__ST2MMU__AS13__FAULTADDRESS, /* (RO) 64-bit 0x10760 */ - MMU_STAGE2__ST2MMU__AS13__STATUS, /* (RO) 32-bit 0x10768 */ - MMU_STAGE2__ST2MMU__AS13__TRANSCFG, /* (RW) 64-bit 0x10770 */ - MMU_STAGE2__ST2MMU__AS13__FAULTEXTRA, /* (RO) 64-bit 0x10778 */ - MMU_STAGE2__ST2MMU__AS14__TRANSTAB, /* (RW) 64-bit 0x10780 */ - MMU_STAGE2__ST2MMU__AS14__MEMATTR, /* (RW) 64-bit 0x10788 */ - MMU_STAGE2__ST2MMU__AS14__LOCKADDR, /* (RW) 64-bit 0x10790 */ - MMU_STAGE2__ST2MMU__AS14__COMMAND, /* (WO) 32-bit 0x10798 */ - MMU_STAGE2__ST2MMU__AS14__FAULTSTATUS, /* (RO) 32-bit 0x1079C */ - MMU_STAGE2__ST2MMU__AS14__FAULTADDRESS, /* (RO) 64-bit 0x107A0 */ - MMU_STAGE2__ST2MMU__AS14__STATUS, /* (RO) 32-bit 0x107A8 */ - MMU_STAGE2__ST2MMU__AS14__TRANSCFG, /* (RW) 64-bit 0x107B0 */ - MMU_STAGE2__ST2MMU__AS14__FAULTEXTRA, /* (RO) 64-bit 0x107B8 */ - MMU_STAGE2__ST2MMU__AS15__TRANSTAB, /* (RW) 64-bit 0x107C0 */ - MMU_STAGE2__ST2MMU__AS15__MEMATTR, /* (RW) 64-bit 0x107C8 */ - MMU_STAGE2__ST2MMU__AS15__LOCKADDR, /* (RW) 64-bit 0x107D0 */ - MMU_STAGE2__ST2MMU__AS15__COMMAND, /* (WO) 32-bit 0x107D8 */ - MMU_STAGE2__ST2MMU__AS15__FAULTSTATUS, /* (RO) 32-bit 0x107DC */ - MMU_STAGE2__ST2MMU__AS15__FAULTADDRESS, /* (RO) 64-bit 0x107E0 */ - MMU_STAGE2__ST2MMU__AS15__STATUS, /* (RO) 32-bit 0x107E8 */ - MMU_STAGE2__ST2MMU__AS15__TRANSCFG, /* (RW) 64-bit 0x107F0 */ - MMU_STAGE2__ST2MMU__AS15__FAULTEXTRA, /* (RO) 64-bit 0x107F8 */ - NR_V6_0_REGS, -}; - -enum kbase_regmap_enum_v6_2 { - GPU_CONTROL__REVIDR = NR_V6_0_REGS, /* (RO) 32-bit 0x280 */ - GPU_CONTROL__STACK_PRESENT, /* (RO) 64-bit 0xE00 */ - GPU_CONTROL__STACK_PWROFF, /* (WO) 64-bit 0xE30 */ - GPU_CONTROL__STACK_PWRON, /* (WO) 64-bit 0xE20 */ - GPU_CONTROL__STACK_PWRTRANS, /* (RO) 64-bit 0xE40 */ - GPU_CONTROL__STACK_READY, /* (RO) 64-bit 0xE10 */ - NR_V6_2_REGS, -}; - -enum kbase_regmap_enum_v7_0 { - GPU_CONTROL__TEXTURE_FEATURES_3 = NR_V6_2_REGS, /* (RO) 32-bit 0xBC */ - NR_V7_0_REGS, -}; - -enum kbase_regmap_enum_v7_2 { - GPU_CONTROL__CORE_FEATURES = NR_V7_0_REGS, /* (RO) 32-bit 0x8 */ - GPU_CONTROL__THREAD_TLS_ALLOC, /* (RO) 32-bit 0x310 */ - NR_V7_2_REGS, -}; - -enum kbase_regmap_enum_v9_0 { - NR_V9_0_REGS = NR_V7_2_REGS, -}; - -/* - * V9_0_REMOVED_REGS: - * GPU_CONTROL__CORE_FEATURES - * GPU_CONTROL__THREAD_TLS_ALLOC - * JOB_CONTROL__JOB_IRQ_THROTTLE - */ - -enum kbase_regmap_enum_v9_2 { - GPU_CONTROL__L2_CONFIG = NR_V9_0_REGS, /* (RW) 32-bit 0x48 */ - NR_V9_2_REGS, -}; - -enum kbase_regmap_enum_v9_14 { - PTM_AW_IRQ_CLEAR = NR_V9_2_REGS, /* (RW) 32-bit 0x1FFC8 */ - PTM_AW_IRQ_INJECTION, /* (RW) 32-bit 0x1FFD4 */ - PTM_AW_IRQ_MASK, /* (RW) 32-bit 0x1FFCC */ - PTM_AW_IRQ_RAWSTAT, /* (RO) 32-bit 0x1FFC4 */ - PTM_AW_IRQ_STATUS, /* (RO) 32-bit 0x1FFD0 */ - PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE0, /* (RO) 32-bit 0x1FFD8 */ - PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE1, /* (RO) 32-bit 0x1FFDC */ - PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE0, /* (RW) 32-bit 0x1FFE4 */ - PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE1, /* (RW) 32-bit 0x1FFE8 */ - PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE_STATUS, /* (RO) 32-bit 0x1FFE0 */ - PTM_ID, /* (RO) 32-bit 0x1FFC0 */ - NR_V9_14_REGS, -}; - -#endif /* _MALI_KBASE_REGMAP_JM_ENUMS_H_ */ diff --git a/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_jm_macros.h b/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_jm_macros.h deleted file mode 100644 index 27d2702a06ce..000000000000 --- a/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_jm_macros.h +++ /dev/null @@ -1,308 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#ifndef _MALI_KBASE_REGMAP_JM_MACROS_H_ -#define _MALI_KBASE_REGMAP_JM_MACROS_H_ - -#if MALI_USE_CSF -#error "Cannot be compiled with CSF" -#endif - -#define ENUM_OFFSET(_index, _base, _next) (_base + _index * (_next - _base)) - -#define GPU_CONTROL_ENUM(regname) GPU_CONTROL__##regname -#define GPU_TEXTURE_FEATURES_ENUM(n) GPU_CONTROL_ENUM(TEXTURE_FEATURES_##n) -#define GPU_TEXTURE_FEATURES_OFFSET(n) (GPU_TEXTURE_FEATURES_ENUM(0) + n) -#define GPU_JS_FEATURES_ENUM(n) GPU_CONTROL_ENUM(JS##n##_FEATURES) -#define GPU_JS_FEATURES_OFFSET(n) (GPU_JS_FEATURES_ENUM(0) + n) - -#define JOB_CONTROL_ENUM(regname) JOB_CONTROL__##regname -#define JOB_SLOT_ENUM(n, regname) JOB_CONTROL_ENUM(JS##n##__##regname) -#define JOB_SLOT_BASE_ENUM(n) JOB_SLOT_ENUM(n, HEAD) -#define JOB_SLOT_OFFSET(n, regname) \ - ENUM_OFFSET(n, JOB_SLOT_ENUM(0, regname), JOB_SLOT_ENUM(1, regname)) -#define JOB_SLOT_BASE_OFFSET(n) JOB_SLOT_OFFSET(n, HEAD) - -#define MMU_CONTROL_ENUM(regname) MMU_STAGE1__ST1MMU__##regname -#define MMU_AS_ENUM(n, regname) MMU_CONTROL_ENUM(AS##n##__##regname) -#define MMU_AS_BASE_ENUM(n) MMU_AS_ENUM(n, TRANSTAB) -#define MMU_AS_OFFSET(n, regname) ENUM_OFFSET(n, MMU_AS_ENUM(0, regname), MMU_AS_ENUM(1, regname)) -#define MMU_AS_BASE_OFFSET(n) MMU_AS_OFFSET(n, TRANSTAB) - -#define PTM_AW_ENUM(regname) PTM_AW__##regname -#define PTM_AW_MESSAGE_ENUM(regname) PTM_AW_MESSAGE__##regname - -/* register value macros */ -/* GPU_STATUS values */ -#define GPU_STATUS_PRFCNT_ACTIVE (1 << 2) /* Set if the performance counters are active. */ -#define GPU_STATUS_CYCLE_COUNT_ACTIVE (1 << 6) /* Set if the cycle counter is active. */ - -/* PRFCNT_CONFIG register values */ -#define PRFCNT_CONFIG_MODE_SHIFT 0 /* Counter mode position. */ -#define PRFCNT_CONFIG_AS_SHIFT 4 /* Address space bitmap position. */ -#define PRFCNT_CONFIG_SETSELECT_SHIFT 8 /* Set select position. */ - -/* The performance counters are disabled. */ -#define PRFCNT_CONFIG_MODE_OFF 0 -/* The performance counters are enabled, but are only written out when a - * PRFCNT_SAMPLE command is issued using the GPU_COMMAND register. - */ -#define PRFCNT_CONFIG_MODE_MANUAL 1 -/* The performance counters are enabled, and are written out each time a tile - * finishes rendering. - */ -#define PRFCNT_CONFIG_MODE_TILE 2 - -/* - * Begin AARCH64 MMU TRANSTAB register values - */ -#define AS_TRANSTAB_BASE_SHIFT GPU_U(4) -#define AS_TRANSTAB_BASE_MASK (GPU_ULL(0xFFFFFFFFFFFFFFF) << AS_TRANSTAB_BASE_SHIFT) -#define AS_TRANSTAB_BASE_GET(reg_val) (((reg_val)&AS_TRANSTAB_BASE_MASK) >> AS_TRANSTAB_BASE_SHIFT) -#define AS_TRANSTAB_BASE_SET(reg_val, value) \ - (~(~(reg_val) | AS_TRANSTAB_BASE_MASK) | \ - (((uint64_t)(value) << AS_TRANSTAB_BASE_SHIFT) & AS_TRANSTAB_BASE_MASK)) - -#define AS_FAULTSTATUS_EXCEPTION_TYPE_OK 0x0 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_DONE 0x1 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_STOPPED 0x3 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_TERMINATED 0x4 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_KABOOM 0x5 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_EUREKA 0x6 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_ACTIVE 0x8 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_JOB_CONFIG_FAULT 0x40 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_JOB_POWER_FAULT 0x41 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_JOB_READ_FAULT 0x42 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_JOB_WRITE_FAULT 0x43 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_JOB_AFFINITY_FAULT 0x44 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_JOB_BUS_FAULT 0x48 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_INSTR_INVALID_PC 0x50 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_INSTR_INVALID_ENC 0x51 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_INSTR_BARRIER_FAULT 0x55 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_DATA_INVALID_FAULT 0x58 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_TILE_RANGE_FAULT 0x59 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_ADDR_RANGE_FAULT 0x5A -#define AS_FAULTSTATUS_EXCEPTION_TYPE_IMPRECISE_FAULT 0x5B -#define AS_FAULTSTATUS_EXCEPTION_TYPE_OUT_OF_MEMORY 0x60 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_UNKNOWN 0x7F -#define AS_FAULTSTATUS_EXCEPTION_TYPE_DELAYED_BUS_FAULT 0x80 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_GPU_SHAREABILITY_FAULT 0x88 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_SYSTEM_SHAREABILITY_FAULT 0x89 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_GPU_CACHEABILITY_FAULT 0x8A -#define AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSLATION_FAULT_0 0xC0 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSLATION_FAULT_1 0xC1 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSLATION_FAULT_2 0xC2 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSLATION_FAULT_3 0xC3 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSLATION_FAULT_4 0xC4 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSLATION_FAULT_IDENTITY 0xC7 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_PERMISSION_FAULT_0 0xC8 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_PERMISSION_FAULT_1 0xC9 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_PERMISSION_FAULT_2 0xCA -#define AS_FAULTSTATUS_EXCEPTION_TYPE_PERMISSION_FAULT_3 0xCB -#define AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSTAB_BUS_FAULT_0 0xD0 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSTAB_BUS_FAULT_1 0xD1 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSTAB_BUS_FAULT_2 0xD2 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSTAB_BUS_FAULT_3 0xD3 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_ACCESS_FLAG_0 0xD8 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_ACCESS_FLAG_1 0xD9 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_ACCESS_FLAG_2 0xDA -#define AS_FAULTSTATUS_EXCEPTION_TYPE_ACCESS_FLAG_3 0xDB -#define AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_IN0 0xE0 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_IN1 0xE1 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_IN2 0xE2 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_IN3 0xE3 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_OUT0 0xE4 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_OUT1 0xE5 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_OUT2 0xE6 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_OUT3 0xE7 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_FAULT_0 0xE8 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_FAULT_1 0xE9 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_FAULT_2 0xEA -#define AS_FAULTSTATUS_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_FAULT_3 0xEB -#define AS_FAULTSTATUS_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_NONCACHEABLE_0 0xEC -#define AS_FAULTSTATUS_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_NONCACHEABLE_1 0xED -#define AS_FAULTSTATUS_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_NONCACHEABLE_2 0xEE -#define AS_FAULTSTATUS_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_NONCACHEABLE_3 0xEF - -/* No JM-specific MMU control registers */ -/* No JM-specific MMU address space control registers */ - -/* JS_COMMAND register commands */ -#define JS_COMMAND_NOP 0x00 /* NOP Operation. Writing this value is ignored */ -#define JS_COMMAND_START 0x01 /* Start processing a job chain. Writing this value is ignored */ -#define JS_COMMAND_SOFT_STOP 0x02 /* Gently stop processing a job chain */ -#define JS_COMMAND_HARD_STOP 0x03 /* Rudely stop processing a job chain */ -#define JS_COMMAND_SOFT_STOP_0 0x04 /* Execute SOFT_STOP if JOB_CHAIN_FLAG is 0 */ -#define JS_COMMAND_HARD_STOP_0 0x05 /* Execute HARD_STOP if JOB_CHAIN_FLAG is 0 */ -#define JS_COMMAND_SOFT_STOP_1 0x06 /* Execute SOFT_STOP if JOB_CHAIN_FLAG is 1 */ -#define JS_COMMAND_HARD_STOP_1 0x07 /* Execute HARD_STOP if JOB_CHAIN_FLAG is 1 */ - -#define JS_COMMAND_MASK 0x07 /* Mask of bits currently in use by the HW */ - -/* Possible values of JS_CONFIG and JS_CONFIG_NEXT registers */ -#define JS_CONFIG_START_FLUSH_NO_ACTION (0u << 0) -#define JS_CONFIG_START_FLUSH_CLEAN (1u << 8) -#define JS_CONFIG_START_FLUSH_INV_SHADER_OTHER (2u << 8) -#define JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE (3u << 8) -#define JS_CONFIG_START_MMU (1u << 10) -#define JS_CONFIG_JOB_CHAIN_FLAG (1u << 11) -#define JS_CONFIG_END_FLUSH_NO_ACTION JS_CONFIG_START_FLUSH_NO_ACTION -#define JS_CONFIG_END_FLUSH_CLEAN (1u << 12) -#define JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE (3u << 12) -#define JS_CONFIG_ENABLE_FLUSH_REDUCTION (1u << 14) -#define JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK (1u << 15) -#define JS_CONFIG_THREAD_PRI(n) ((n) << 16) - -/* JS_XAFFINITY register values */ -#define JS_XAFFINITY_XAFFINITY_ENABLE (1u << 0) -#define JS_XAFFINITY_TILER_ENABLE (1u << 8) -#define JS_XAFFINITY_CACHE_ENABLE (1u << 16) - -/* JS_STATUS register values */ - -/* NOTE: Please keep this values in sync with enum base_jd_event_code in mali_base_kernel.h. - * The values are separated to avoid dependency of userspace and kernel code. - */ - -/* Group of values representing the job status instead of a particular fault */ -#define JS_STATUS_NO_EXCEPTION_BASE 0x00 -#define JS_STATUS_INTERRUPTED (JS_STATUS_NO_EXCEPTION_BASE + 0x02) /* 0x02 means INTERRUPTED */ -#define JS_STATUS_STOPPED (JS_STATUS_NO_EXCEPTION_BASE + 0x03) /* 0x03 means STOPPED */ -#define JS_STATUS_TERMINATED (JS_STATUS_NO_EXCEPTION_BASE + 0x04) /* 0x04 means TERMINATED */ - -/* General fault values */ -#define JS_STATUS_FAULT_BASE 0x40 -#define JS_STATUS_CONFIG_FAULT (JS_STATUS_FAULT_BASE) /* 0x40 means CONFIG FAULT */ -#define JS_STATUS_POWER_FAULT (JS_STATUS_FAULT_BASE + 0x01) /* 0x41 means POWER FAULT */ -#define JS_STATUS_READ_FAULT (JS_STATUS_FAULT_BASE + 0x02) /* 0x42 means READ FAULT */ -#define JS_STATUS_WRITE_FAULT (JS_STATUS_FAULT_BASE + 0x03) /* 0x43 means WRITE FAULT */ -#define JS_STATUS_AFFINITY_FAULT (JS_STATUS_FAULT_BASE + 0x04) /* 0x44 means AFFINITY FAULT */ -#define JS_STATUS_BUS_FAULT (JS_STATUS_FAULT_BASE + 0x08) /* 0x48 means BUS FAULT */ - -/* Instruction or data faults */ -#define JS_STATUS_INSTRUCTION_FAULT_BASE 0x50 -#define JS_STATUS_INSTR_INVALID_PC \ - (JS_STATUS_INSTRUCTION_FAULT_BASE) /* 0x50 means INSTR INVALID PC */ -#define JS_STATUS_INSTR_INVALID_ENC \ - (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x01) /* 0x51 means INSTR INVALID ENC */ -#define JS_STATUS_INSTR_TYPE_MISMATCH \ - (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x02) /* 0x52 means INSTR TYPE MISMATCH */ -#define JS_STATUS_INSTR_OPERAND_FAULT \ - (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x03) /* 0x53 means INSTR OPERAND FAULT */ -#define JS_STATUS_INSTR_TLS_FAULT \ - (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x04) /* 0x54 means INSTR TLS FAULT */ -#define JS_STATUS_INSTR_BARRIER_FAULT \ - (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x05) /* 0x55 means INSTR BARRIER FAULT */ -#define JS_STATUS_INSTR_ALIGN_FAULT \ - (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x06) /* 0x56 means INSTR ALIGN FAULT */ -/* NOTE: No fault with 0x57 code defined in spec. */ -#define JS_STATUS_DATA_INVALID_FAULT \ - (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x08) /* 0x58 means DATA INVALID FAULT */ -#define JS_STATUS_TILE_RANGE_FAULT \ - (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x09) /* 0x59 means TILE RANGE FAULT */ -#define JS_STATUS_ADDRESS_RANGE_FAULT \ - (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x0A) /* 0x5A means ADDRESS RANGE FAULT */ - -/* Other faults */ -#define JS_STATUS_MEMORY_FAULT_BASE 0x60 -#define JS_STATUS_OUT_OF_MEMORY (JS_STATUS_MEMORY_FAULT_BASE) /* 0x60 means OUT OF MEMORY */ -#define JS_STATUS_UNKNOWN 0x7F /* 0x7F means UNKNOWN */ - -/* JS_FEATURES register */ -#define JS_FEATURE_NULL_JOB (1u << 1) -#define JS_FEATURE_SET_VALUE_JOB (1u << 2) -#define JS_FEATURE_CACHE_FLUSH_JOB (1u << 3) -#define JS_FEATURE_COMPUTE_JOB (1u << 4) -#define JS_FEATURE_VERTEX_JOB (1u << 5) -#define JS_FEATURE_GEOMETRY_JOB (1u << 6) -#define JS_FEATURE_TILER_JOB (1u << 7) -#define JS_FEATURE_FUSED_JOB (1u << 8) -#define JS_FEATURE_FRAGMENT_JOB (1u << 9) - -/* JM_CONFIG register */ -#define JM_TIMESTAMP_OVERRIDE (1ul << 0) -#define JM_CLOCK_GATE_OVERRIDE (1ul << 1) -#define JM_JOB_THROTTLE_ENABLE (1ul << 2) -#define JM_JOB_THROTTLE_LIMIT_SHIFT (3) -#define JM_MAX_JOB_THROTTLE_LIMIT (0x3F) -#define JM_FORCE_COHERENCY_FEATURES_SHIFT (2) - -/* GPU_COMMAND values */ -#define GPU_COMMAND_NOP 0x00 /* No operation, nothing happens */ -#define GPU_COMMAND_SOFT_RESET \ - 0x01 /* Stop all external bus interfaces, and then reset the entire GPU. */ -#define GPU_COMMAND_HARD_RESET 0x02 /* Immediately reset the entire GPU. */ -#define GPU_COMMAND_PRFCNT_CLEAR \ - 0x03 /* Clear all performance counters, setting them all to zero. */ -#define GPU_COMMAND_PRFCNT_SAMPLE \ - 0x04 /* Sample all performance counters, writing them out to memory */ -#define GPU_COMMAND_CYCLE_COUNT_START \ - 0x05 /* Starts the cycle counter, and system timestamp propagation */ -#define GPU_COMMAND_CYCLE_COUNT_STOP \ - 0x06 /* Stops the cycle counter, and system timestamp propagation */ -#define GPU_COMMAND_CLEAN_CACHES 0x07 /* Clean all caches */ -#define GPU_COMMAND_CLEAN_INV_CACHES 0x08 /* Clean and invalidate all caches */ -#define GPU_COMMAND_SET_PROTECTED_MODE 0x09 /* Places the GPU in protected mode */ - -/* GPU_COMMAND cache flush alias to CSF command payload */ -#define GPU_COMMAND_CACHE_CLN_INV_L2 GPU_COMMAND_CLEAN_INV_CACHES -#define GPU_COMMAND_CACHE_CLN_INV_L2_LSC GPU_COMMAND_CLEAN_INV_CACHES -#define GPU_COMMAND_CACHE_CLN_INV_FULL GPU_COMMAND_CLEAN_INV_CACHES -#define GPU_COMMAND_CACHE_CLN_INV_LSC GPU_COMMAND_CLEAN_INV_CACHES - -/* Merge cache flush commands */ -#define GPU_COMMAND_FLUSH_CACHE_MERGE(cmd1, cmd2) ((cmd1) > (cmd2) ? (cmd1) : (cmd2)) - -/* IRQ flags */ -#define GPU_FAULT (1U << 0) /* A GPU Fault has occurred */ -#define MULTIPLE_GPU_FAULTS (1U << 7) /* More than one GPU Fault occurred. */ -#define RESET_COMPLETED (1U << 8) /* Set when a reset has completed. */ -#define POWER_CHANGED_SINGLE \ - (1U << 9) /* Set when a single core has finished powering up or down. */ -#define POWER_CHANGED_ALL (1U << 10) /* Set when all cores have finished powering up or down. */ -#define PRFCNT_SAMPLE_COMPLETED (1U << 16) /* Set when a performance count sample has completed. */ -#define CLEAN_CACHES_COMPLETED (1U << 17) /* Set when a cache clean operation has completed. */ -#define FLUSH_PA_RANGE_COMPLETED \ - (1 << 20) /* Set when a physical range cache clean operation has completed. */ - -/* - * In Debug build, - * GPU_IRQ_REG_COMMON | POWER_CHANGED_SINGLE is used to clear and enable - * interrupts sources of GPU_IRQ by writing it onto GPU_IRQ_CLEAR/MASK registers. - * - * In Release build, - * GPU_IRQ_REG_COMMON is used. - * - * Note: - * CLEAN_CACHES_COMPLETED - Used separately for cache operation. - */ -#define GPU_IRQ_REG_COMMON \ - (GPU_FAULT | MULTIPLE_GPU_FAULTS | RESET_COMPLETED | POWER_CHANGED_ALL | \ - PRFCNT_SAMPLE_COMPLETED) - -#define WINDOW_IRQ_MESSAGE (1U << 0) -#define WINDOW_IRQ_INVALID_ACCESS (1U << 1) -#define WINDOW_IRQ_GPU (1U << 2) -#define WINDOW_IRQ_JOB (1U << 3) -#define WINDOW_IRQ_MMU (1U << 4) -#define WINDOW_IRQ_EVENT (1U << 5) - -#endif /* _MALI_KBASE_REGMAP_JM_MACROS_H_ */ diff --git a/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_legacy_csf.h b/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_legacy_csf.h index 0e3a56e9d664..0678cf76fc0a 100644 --- a/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_legacy_csf.h +++ b/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_legacy_csf.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -22,9 +22,6 @@ #ifndef _MALI_KBASE_REGMAP_LEGACY_CSF_H_ #define _MALI_KBASE_REGMAP_LEGACY_CSF_H_ -#if !MALI_USE_CSF && defined(__KERNEL__) -#error "Cannot be compiled with JM" -#endif /* GPU control registers */ #define MCU_CONTROL 0x700 diff --git a/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_legacy_jm.h b/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_legacy_jm.h deleted file mode 100644 index c1a54991f4c6..000000000000 --- a/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_legacy_jm.h +++ /dev/null @@ -1,108 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#ifndef _MALI_KBASE_REGMAP_LEGACY_JM_H_ -#define _MALI_KBASE_REGMAP_LEGACY_JM_H_ - -#if MALI_USE_CSF && defined(__KERNEL__) -#error "Cannot be compiled with CSF" -#endif - -/* GPU control registers */ -#define CORE_FEATURES 0x008 /* (RO) Shader Core Features */ -#define JS_PRESENT 0x01C /* (RO) Job slots present */ -#define LATEST_FLUSH 0x038 /* (RO) Flush ID of latest clean-and-invalidate operation */ -#define PRFCNT_BASE_LO 0x060 /* (RW) Performance counter memory region base address, low word */ -#define PRFCNT_BASE_HI 0x064 /* (RW) Performance counter memory region base address, high word */ -#define PRFCNT_CONFIG 0x068 /* (RW) Performance counter configuration */ -#define PRFCNT_JM_EN 0x06C /* (RW) Performance counter enable flags for Job Manager */ -#define PRFCNT_SHADER_EN 0x070 /* (RW) Performance counter enable flags for shader cores */ -#define PRFCNT_TILER_EN 0x074 /* (RW) Performance counter enable flags for tiler */ -#define PRFCNT_MMU_L2_EN 0x07C /* (RW) Performance counter enable flags for MMU/L2 cache */ - -#define JS0_FEATURES 0x0C0 /* (RO) Features of job slot 0 */ -#define JS1_FEATURES 0x0C4 /* (RO) Features of job slot 1 */ -#define JS2_FEATURES 0x0C8 /* (RO) Features of job slot 2 */ -#define JS3_FEATURES 0x0CC /* (RO) Features of job slot 3 */ -#define JS4_FEATURES 0x0D0 /* (RO) Features of job slot 4 */ -#define JS5_FEATURES 0x0D4 /* (RO) Features of job slot 5 */ -#define JS6_FEATURES 0x0D8 /* (RO) Features of job slot 6 */ -#define JS7_FEATURES 0x0DC /* (RO) Features of job slot 7 */ -#define JS8_FEATURES 0x0E0 /* (RO) Features of job slot 8 */ -#define JS9_FEATURES 0x0E4 /* (RO) Features of job slot 9 */ -#define JS10_FEATURES 0x0E8 /* (RO) Features of job slot 10 */ -#define JS11_FEATURES 0x0EC /* (RO) Features of job slot 11 */ -#define JS12_FEATURES 0x0F0 /* (RO) Features of job slot 12 */ -#define JS13_FEATURES 0x0F4 /* (RO) Features of job slot 13 */ -#define JS14_FEATURES 0x0F8 /* (RO) Features of job slot 14 */ -#define JS15_FEATURES 0x0FC /* (RO) Features of job slot 15 */ - -#define JS_FEATURES_REG(n) GPU_CONTROL_REG(JS0_FEATURES + ((n) << 2)) - -#define JM_CONFIG 0xF00 /* (RW) Job manager configuration (implementation-specific) */ - -/* Job control registers */ -/* status==active and _next == busy snapshot from last JOB_IRQ_CLEAR */ -#define JOB_IRQ_JS_STATE 0x010 -/* cycles to delay delivering an interrupt externally. The JOB_IRQ_STATUS - * is NOT affected by this, just the delivery of the interrupt. - */ -#define JOB_IRQ_THROTTLE 0x014 - -#define JOB_SLOT0 0x800 /* Configuration registers for job slot 0 */ -#define JOB_SLOT_REG(n, r) (JOB_CONTROL_REG(JOB_SLOT0 + ((n) << 7)) + (r)) -#define JOB_SLOT1 0x880 /* Configuration registers for job slot 1 */ -#define JOB_SLOT2 0x900 /* Configuration registers for job slot 2 */ -#define JOB_SLOT3 0x980 /* Configuration registers for job slot 3 */ -#define JOB_SLOT4 0xA00 /* Configuration registers for job slot 4 */ -#define JOB_SLOT5 0xA80 /* Configuration registers for job slot 5 */ -#define JOB_SLOT6 0xB00 /* Configuration registers for job slot 6 */ -#define JOB_SLOT7 0xB80 /* Configuration registers for job slot 7 */ -#define JOB_SLOT8 0xC00 /* Configuration registers for job slot 8 */ -#define JOB_SLOT9 0xC80 /* Configuration registers for job slot 9 */ -#define JOB_SLOT10 0xD00 /* Configuration registers for job slot 10 */ -#define JOB_SLOT11 0xD80 /* Configuration registers for job slot 11 */ -#define JOB_SLOT12 0xE00 /* Configuration registers for job slot 12 */ -#define JOB_SLOT13 0xE80 /* Configuration registers for job slot 13 */ -#define JOB_SLOT14 0xF00 /* Configuration registers for job slot 14 */ -#define JOB_SLOT15 0xF80 /* Configuration registers for job slot 15 */ - -/* JM Job control register definitions for mali_kbase_debug_job_fault */ -#define JS_HEAD_LO 0x00 /* (RO) Job queue head pointer for job slot n, low word */ -#define JS_HEAD_HI 0x04 /* (RO) Job queue head pointer for job slot n, high word */ -#define JS_TAIL_LO 0x08 /* (RO) Job queue tail pointer for job slot n, low word */ -#define JS_TAIL_HI 0x0C /* (RO) Job queue tail pointer for job slot n, high word */ -#define JS_AFFINITY_LO 0x10 /* (RO) Core affinity mask for job slot n, low word */ -#define JS_AFFINITY_HI 0x14 /* (RO) Core affinity mask for job slot n, high word */ -#define JS_CONFIG 0x18 /* (RO) Configuration settings for job slot n */ -#define JS_XAFFINITY 0x1C /* (RO) Extended affinity mask for job slot n*/ -#define JS_COMMAND 0x20 /* (WO) Command register for job slot n */ -#define JS_STATUS 0x24 /* (RO) Status register for job slot n */ -#define JS_HEAD_NEXT_LO 0x40 /* (RW) Next job queue head pointer for job slot n, low word */ -#define JS_HEAD_NEXT_HI 0x44 /* (RW) Next job queue head pointer for job slot n, high word */ -#define JS_AFFINITY_NEXT_LO 0x50 /* (RW) Next core affinity mask for job slot n, low word */ -#define JS_AFFINITY_NEXT_HI 0x54 /* (RW) Next core affinity mask for job slot n, high word */ -#define JS_CONFIG_NEXT 0x58 /* (RW) Next configuration settings for job slot n */ -#define JS_XAFFINITY_NEXT 0x5C /* (RW) Next extended affinity mask for job slot n */ -#define JS_COMMAND_NEXT 0x60 /* (RW) Next command register for job slot n */ -#define JS_FLUSH_ID_NEXT 0x70 /* (RW) Next job slot n cache flush ID */ - -#endif /* _MALI_KBASE_REGMAP_LEGACY_JM_H_ */ diff --git a/drivers/gpu/arm/valhall/hwcnt/Kbuild b/drivers/gpu/arm/valhall/hwcnt/Kbuild index f26717e92e0f..2bd5848984f3 100644 --- a/drivers/gpu/arm/valhall/hwcnt/Kbuild +++ b/drivers/gpu/arm/valhall/hwcnt/Kbuild @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2022 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2022, 2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -23,14 +23,6 @@ valhall_kbase-y += \ hwcnt/mali_kbase_hwcnt_gpu.o \ hwcnt/mali_kbase_hwcnt_types.o \ hwcnt/mali_kbase_hwcnt_virtualizer.o \ - hwcnt/mali_kbase_hwcnt_watchdog_if_timer.o - -ifeq ($(CONFIG_MALI_VALHALL_CSF_SUPPORT),y) - valhall_kbase-y += \ - hwcnt/backend/mali_kbase_hwcnt_backend_csf.o \ - hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.o -else - valhall_kbase-y += \ - hwcnt/backend/mali_kbase_hwcnt_backend_jm.o \ - hwcnt/backend/mali_kbase_hwcnt_backend_jm_watchdog.o -endif + hwcnt/mali_kbase_hwcnt_watchdog_if_timer.o \ + hwcnt/backend/mali_kbase_hwcnt_backend_csf.o \ + hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.o diff --git a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf.c b/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf.c index 723ae207070e..7f33aa6e3942 100644 --- a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf.c +++ b/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf.c @@ -39,6 +39,9 @@ /* Used to check for a sample in which all counters in the block are disabled */ #define HWCNT_BLOCK_EMPTY_SAMPLE (2) +#define RING_BUF_REFCNT_MAX 2 +#define RING_BUF_REFCNT_MIN 0 + /** * enum kbase_hwcnt_backend_csf_dump_state - HWC CSF backend dumping states. * @@ -99,17 +102,26 @@ enum kbase_hwcnt_backend_csf_dump_state { * @KBASE_HWCNT_BACKEND_CSF_DISABLED: Initial state, and the state when backend * is disabled. * - * @KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED: Enable request is in - * progress, waiting for firmware acknowledgment. + * @KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_REQ: Backend is + * being enabled, pending to send enable REQ to firmware. + * + * @KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_ACK: Enable REQ + * has been sent to FW, waiting for firmware acknowledgment. * * @KBASE_HWCNT_BACKEND_CSF_ENABLED: Enable request has been acknowledged, * enable is done. * - * @KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED: Disable request is in - * progress, waiting for firmware acknowledgment. + * @KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_REQ: Backend is + * being disabled, pending to send disable REQ to firmware. + * + * @KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK: Disable REQ + * has been sent to FW, waiting for firmware acknowledgment. * * @KBASE_HWCNT_BACKEND_CSF_DISABLED_WAIT_FOR_WORKER: Disable request has been * acknowledged, waiting for dump workers to be finished. + * Note: This is mainly used when recovering from a recoverable error. During a + * clean disable procedure, backend will transition directly to DISABLED state + * after being ACKed. * * @KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR_WAIT_FOR_WORKER: An * unrecoverable error happened, waiting for dump workers to be finished. @@ -118,26 +130,52 @@ enum kbase_hwcnt_backend_csf_dump_state { * happened, and dump workers have finished, waiting for reset. * * Valid state transitions: - * DISABLED -> TRANSITIONING_TO_ENABLED (on enable) - * TRANSITIONING_TO_ENABLED -> ENABLED (on enable ack) - * ENABLED -> TRANSITIONING_TO_DISABLED (on disable) - * TRANSITIONING_TO_DISABLED -> DISABLED_WAIT_FOR_WORKER (on disable ack) - * DISABLED_WAIT_FOR_WORKER -> DISABLED (after workers are flushed) - * DISABLED -> UNRECOVERABLE_ERROR (on unrecoverable error) - * ANY but DISABLED -> UNRECOVERABLE_ERROR_WAIT_FOR_WORKER (on unrecoverable - * error) - * UNRECOVERABLE_ERROR -> DISABLED (on before reset) + * - DISABLED -> TRANSITIONING_TO_ENABLED_PENDING_REQ (on enable before sending REQ) + * - TRANSITIONING_TO_ENABLED_PENDING_REQ -> + * TRANSITIONING_TO_ENABLED_PENDING_ACK (on sending enable REQ) + * - TRANSITIONING_TO_ENABLED_PENDING_ACK -> ENABLED (on enable ack) + * - ENABLED -> TRANSITIONING_TO_DISABLED_PENDING_REQ (on disable before sending REQ) + * - TRANSITIONING_TO_DISABLED_PENDING_REQ -> + * TRANSITIONING_TO_DISABLED_PENDING_ACK (on sending disable REQ) + * - TRANSITIONING_TO_DISABLED_PENDING_ACK -> DISABLED (on disable ack) + * - TRANSITIONING_TO_DISABLED_PENDING_ACK -> DISABLED_WAIT_FOR_WORKER (on disable ack + * after a recoverable error) + * - DISABLED_WAIT_FOR_WORKER -> DISABLED (after workers are flushed) + * - UNRECOVERABLE_ERROR_WAIT_FOR_WORKER -> UNRECOVERABLE_ERROR (after workers are flushed) + * - DISABLED -> UNRECOVERABLE_ERROR (on unrecoverable error) + * - ANY but DISABLED -> UNRECOVERABLE_ERROR_WAIT_FOR_WORKER (on unrecoverable error) + * - UNRECOVERABLE_ERROR -> DISABLED (on before reset) + * + * More valid state transitions due to MCU being powered off: + * - TRANSITIONING_TO_ENABLED_PENDING_* -> + * TRANSITIONING_TO_DISABLED_PENDING_* (backend needs to be disabled + * but enable REQ hasn't been sent or ACK not received + * due to MCU being OFF) + * - TRANSITIONING_TO_DISABLED_PENDING_* -> + * TRANSITIONING_TO_ENABLED_PENDING_* (opposite case to the previous one) */ enum kbase_hwcnt_backend_csf_enable_state { KBASE_HWCNT_BACKEND_CSF_DISABLED, - KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED, + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_REQ, + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_ACK, KBASE_HWCNT_BACKEND_CSF_ENABLED, - KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED, + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_REQ, + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK, KBASE_HWCNT_BACKEND_CSF_DISABLED_WAIT_FOR_WORKER, KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR_WAIT_FOR_WORKER, KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR, }; +enum kbase_hwcnt_backend_block_type { + KBASE_HWCNT_BLOCK_METADATA, + KBASE_HWCNT_BLOCK_FW, + KBASE_HWCNT_BLOCK_CSHW, + KBASE_HWCNT_BLOCK_TILER, + KBASE_HWCNT_BLOCK_MEMSYS, + KBASE_HWCNT_BLOCK_SHADER, + KBASE_HWCNT_BLOCK_NEURAL +}; + /** * struct kbase_hwcnt_backend_csf_info - Information used to create an instance * of a CSF hardware counter backend. @@ -153,6 +191,19 @@ enum kbase_hwcnt_backend_csf_enable_state { * @prfcnt_info: Performance counter information. * @watchdog_if: Watchdog interface object pointer. * @watchdog_timer_interval_ms: Watchdog timer interval + * @last_transition: Last backend state transition obtained + * from the metadata block. + * @mcu_on: True if MCU is powered on. + * @ring_buf: Opaque pointer for ring buffer object. + * @ring_buf_cpu_base: CPU base address of the allocated ring buffer. + * @ring_buf_refcnt: Reference counter for ring buffer, used to control + * its allocation across MCU power cycles. + * @hwc_deferred_op_workq: Workqueue for deferred operations on the backend, + * e.g. due to MCU power cycle. + * @hwc_dump_reenable_work: Worker for re-enabling the backend after MCU ON notification. + * @hwc_ring_buf_free_work: Worker for freeing the ring buffer. + * @ring_buf_cleanup: Pointer to the ring buffer that needs to be freed. + * @pending_disable_req: True if disable REQ is pending to be sent when MCU is powered on. */ struct kbase_hwcnt_backend_csf_info { struct kbase_hwcnt_backend_csf *backend; @@ -165,6 +216,19 @@ struct kbase_hwcnt_backend_csf_info { struct kbase_hwcnt_backend_csf_if_prfcnt_info prfcnt_info; struct kbase_hwcnt_watchdog_interface *watchdog_if; u32 watchdog_timer_interval_ms; + enum kbase_hwcnt_backend_sample_reason last_transition; + bool mcu_on; + struct kbase_hwcnt_backend_csf_if_ring_buf *ring_buf; + void *ring_buf_cpu_base; + struct kbasep_hwcnt_ring_buf_refcnt { + int val; + bool incremented_on_mcu_off; + } ring_buf_refcnt; + struct workqueue_struct *hwc_deferred_op_workq; + struct work_struct hwc_dump_reenable_work; + struct work_struct hwc_ring_buf_free_work; + struct kbase_hwcnt_backend_csf_if_ring_buf *ring_buf_cleanup; + bool pending_disable_req; }; /** @@ -179,7 +243,10 @@ struct kbase_hwcnt_backend_csf_info { * @hw_block_cnt: Total number of hardware counter blocks. The hw counters blocks are * sub-categorized into 4 classes: front-end, tiler, memory system, and shader. * hw_block_cnt = fe_cnt + tiler_cnt + mmu_l2_cnt + shader_cnt. - * @block_cnt: Total block count (sum of all counter blocks: hw_block_cnt + fw_block_cnt). + * @md_block_cnt: Total number of metadata counter blocks. + * @ringbuf_block_cnt: Total block count (sum of all counter blocks: hw_block_cnt + fw_block_cnt + * + md_block_cnt). + * @payload_block_cnt: Total number of payload counter blocks: hw_block_cnt + fw_block_cnt. * @shader_avail_mask: Bitmap of all shader cores in the system. * @enable_mask_offset: Offset in array elements of enable mask in each block * starting from the beginning of block. @@ -195,7 +262,9 @@ struct kbase_hwcnt_csf_physical_layout { u8 shader_cnt; u8 fw_block_cnt; u8 hw_block_cnt; - u8 block_cnt; + u8 md_block_cnt; + u8 ringbuf_block_cnt; + u8 payload_block_cnt; u64 shader_avail_mask; size_t enable_mask_offset; size_t headers_per_block; @@ -229,8 +298,6 @@ struct kbase_hwcnt_csf_physical_layout { * seen, to check any new firmware automatic * samples generated during the watchdog * period. - * @ring_buf: Opaque pointer for ring buffer object. - * @ring_buf_cpu_base: CPU base address of the allocated ring buffer. * @clk_enable_map: The enable map specifying enabled clock domains. * @cycle_count_elapsed: Cycle count elapsed for a given sample period. * @prev_cycle_count: Previous cycle count to calculate the cycle @@ -258,6 +325,13 @@ struct kbase_hwcnt_csf_physical_layout { * is converted from the TIMESTAMP of a dump to CPU MONOTONIC * time. Then it's returned to the caller via 'dump_time_ns' * parameter of dump_get function. + * @last_enable_map: Record of the last requested enable map. + * Used to reapply dump enable request on MCU ON notification. + * @dump_reenable_in_progress: True if backend reenable procedure is in progress. + * @flush_on_disable_in_progress: True when the dump workers are being flushed during dump + * disable procedure. This is needed to prevent the backend + * from being enabled. + * @flush_workers_needed: True when the dump workers need to be flushed. */ struct kbase_hwcnt_backend_csf { struct kbase_hwcnt_backend_csf_info *info; @@ -272,8 +346,6 @@ struct kbase_hwcnt_backend_csf { blk_stt_t *block_states; blk_stt_t *to_user_block_states; u32 watchdog_last_seen_insert_idx; - struct kbase_hwcnt_backend_csf_if_ring_buf *ring_buf; - void *ring_buf_cpu_base; u64 clk_enable_map; u64 cycle_count_elapsed[BASE_MAX_NR_CLOCKS_REGULATORS]; u64 prev_cycle_count[BASE_MAX_NR_CLOCKS_REGULATORS]; @@ -286,8 +358,36 @@ struct kbase_hwcnt_backend_csf { size_t num_l2_slices; u64 powered_shader_core_mask; u64 dump_time_ns; + struct kbase_hwcnt_enable_map last_enable_map; + bool dump_reenable_in_progress; + bool flush_on_disable_in_progress; + bool flush_workers_needed; }; +/* Forward declarations. */ +static void +kbasep_hwcnt_backend_csf_handle_unrecoverable_error(struct kbase_hwcnt_backend_csf *backend_csf); + +static inline bool +kbasep_hwcnt_backend_csf_is_state_trans_to_enabled(enum kbase_hwcnt_backend_csf_enable_state state) +{ + return (state == KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_REQ) || + (state == KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_ACK); +} + +static inline bool +kbasep_hwcnt_backend_csf_is_state_trans_to_disabled(enum kbase_hwcnt_backend_csf_enable_state state) +{ + return (state == KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_REQ) || + (state == KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK); +} + +static inline bool +kbasep_hwcnt_backend_csf_is_error_state(enum kbase_hwcnt_backend_csf_enable_state state) +{ + return (state == KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR_WAIT_FOR_WORKER) || + (state == KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR); +} static bool kbasep_hwcnt_backend_csf_backend_exists(struct kbase_hwcnt_backend_csf_info *csf_info) { if (WARN_ON(!csf_info)) @@ -338,7 +438,8 @@ void kbase_hwcnt_backend_csf_set_hw_availability(struct kbase_hwcnt_backend_inte norm_shader_present = virtual_core_mask; } - if (WARN_ON(csf_info->backend->enable_state != KBASE_HWCNT_BACKEND_CSF_DISABLED)) + /* MCU needs to be powered off. */ + if (WARN_ON(csf_info->mcu_on)) return; if (WARN_ON(num_l2_slices > csf_info->backend->phys_layout.mmu_l2_cnt) || @@ -448,6 +549,7 @@ static void kbasep_hwcnt_backend_csf_init_layout( size_t values_per_block; size_t fw_block_cnt; size_t hw_block_cnt; + size_t md_block_cnt; size_t core_cnt; size_t ne_core_cnt; @@ -458,6 +560,7 @@ static void kbasep_hwcnt_backend_csf_init_layout( values_per_block = prfcnt_info->prfcnt_block_size / KBASE_HWCNT_VALUE_HW_BYTES; fw_block_cnt = div_u64(prfcnt_info->prfcnt_fw_size, prfcnt_info->prfcnt_block_size); hw_block_cnt = div_u64(prfcnt_info->prfcnt_hw_size, prfcnt_info->prfcnt_block_size); + md_block_cnt = div_u64(prfcnt_info->metadata_size, prfcnt_info->prfcnt_block_size); core_cnt = shader_core_cnt; /* In the presence of heterogeneous NE, the SCs that don't have dedicated @@ -479,7 +582,9 @@ static void kbasep_hwcnt_backend_csf_init_layout( .shader_cnt = shader_core_cnt, .fw_block_cnt = fw_block_cnt, .hw_block_cnt = hw_block_cnt, - .block_cnt = fw_block_cnt + hw_block_cnt, + .md_block_cnt = md_block_cnt, + .ringbuf_block_cnt = fw_block_cnt + hw_block_cnt + md_block_cnt, + .payload_block_cnt = fw_block_cnt + hw_block_cnt, .shader_avail_mask = prfcnt_info->sc_core_mask, .headers_per_block = KBASE_HWCNT_V5_HEADERS_PER_BLOCK, .values_per_block = values_per_block, @@ -493,7 +598,7 @@ static void kbasep_hwcnt_backend_csf_reset_internal_buffers(struct kbase_hwcnt_backend_csf *backend_csf) { size_t user_buf_bytes = backend_csf->info->metadata->dump_buf_bytes; - size_t block_state_bytes = backend_csf->phys_layout.block_cnt * + size_t block_state_bytes = backend_csf->phys_layout.payload_block_cnt * KBASE_HWCNT_BLOCK_STATE_BYTES * KBASE_HWCNT_BLOCK_STATE_STRIDE; memset(backend_csf->accum_buf, 0, user_buf_bytes); @@ -506,7 +611,7 @@ static void kbasep_hwcnt_backend_csf_reset_consumed_buffers(struct kbase_hwcnt_backend_csf *backend_csf) { size_t user_buf_bytes = backend_csf->info->metadata->dump_buf_bytes; - size_t block_state_bytes = backend_csf->phys_layout.block_cnt * + size_t block_state_bytes = backend_csf->phys_layout.payload_block_cnt * KBASE_HWCNT_BLOCK_STATE_BYTES * KBASE_HWCNT_BLOCK_STATE_STRIDE; memset(backend_csf->to_user_buf, 0, user_buf_bytes); @@ -523,7 +628,7 @@ kbasep_hwcnt_backend_csf_zero_sample_prfcnt_en_header(struct kbase_hwcnt_backend phys_layout = &backend_csf->phys_layout; - for (block_idx = 0; block_idx < phys_layout->block_cnt; block_idx++) { + for (block_idx = 0; block_idx < phys_layout->ringbuf_block_cnt; block_idx++) { block_buf = sample + block_idx * phys_layout->values_per_block; block_buf[phys_layout->enable_mask_offset] = 0; } @@ -537,7 +642,9 @@ kbasep_hwcnt_backend_csf_zero_all_prfcnt_en_header(struct kbase_hwcnt_backend_cs char *cpu_dump_base; size_t dump_bytes = backend_csf->info->prfcnt_info.dump_bytes; - cpu_dump_base = (char *)backend_csf->ring_buf_cpu_base; + cpu_dump_base = (char *)backend_csf->info->ring_buf_cpu_base; + if (WARN_ON(!cpu_dump_base)) + return; for (idx = 0; idx < backend_csf->info->ring_buf_cnt; idx++) { sample = (u32 *)&cpu_dump_base[idx * dump_bytes]; @@ -548,18 +655,18 @@ kbasep_hwcnt_backend_csf_zero_all_prfcnt_en_header(struct kbase_hwcnt_backend_cs static void kbasep_hwcnt_backend_csf_update_user_sample(struct kbase_hwcnt_backend_csf *backend_csf) { const size_t user_buf_bytes = backend_csf->info->metadata->dump_buf_bytes; - const size_t block_cnt = backend_csf->phys_layout.block_cnt; + const size_t payload_block_cnt = backend_csf->phys_layout.payload_block_cnt; const size_t block_state_bytes = - block_cnt * KBASE_HWCNT_BLOCK_STATE_BYTES * KBASE_HWCNT_BLOCK_STATE_STRIDE; + payload_block_cnt * KBASE_HWCNT_BLOCK_STATE_BYTES * KBASE_HWCNT_BLOCK_STATE_STRIDE; size_t i; /* Copy the data into the sample and wait for the user to get it. */ memcpy(backend_csf->to_user_buf, backend_csf->accum_buf, user_buf_bytes); - for (i = 0; i < block_cnt; i++) + for (i = 0; i < payload_block_cnt; i++) kbase_hwcnt_block_state_append(&backend_csf->to_user_block_states[i], backend_csf->block_states[i]); - /* After copied data into user sample, clear the accumulator values to + /* After copying data into user sample, clear the accumulator values to * prepare for the next accumulator, such as the next request or * threshold. */ @@ -568,45 +675,99 @@ static void kbasep_hwcnt_backend_csf_update_user_sample(struct kbase_hwcnt_backe memset(backend_csf->block_states, 0, block_state_bytes); } -void kbasep_hwcnt_backend_csf_update_block_state(struct kbase_hwcnt_backend_csf *backend, - const u32 enable_mask, bool exiting_protm, - size_t block_idx, blk_stt_t *const block_state, - bool fw_in_protected_mode) +static enum kbase_hwcnt_backend_block_type kbase_hwcnt_backend_csf_identify_block_type( + const struct kbase_hwcnt_csf_physical_layout *phys_layout, size_t block_idx, + size_t *const block_in_type_idx) { - const struct kbase_hwcnt_csf_physical_layout *phys_layout = &backend->phys_layout; - /* Offset of shader core blocks from the start of the HW blocks in the sample */ - size_t shader_core_block_offset = - (size_t)(phys_layout->block_cnt - phys_layout->shader_cnt); - bool is_shader_core_block; + const size_t fw_block_offset = phys_layout->md_block_cnt; + const size_t cshw_block_offset = fw_block_offset + phys_layout->fw_block_cnt; + const size_t tiler_block_offset = cshw_block_offset + phys_layout->fe_cnt; + const size_t memsys_block_offset = tiler_block_offset + phys_layout->tiler_cnt; + const size_t shader_core_block_offset = memsys_block_offset + phys_layout->mmu_l2_cnt; + const size_t neural_core_block_offset = shader_core_block_offset + phys_layout->shader_cnt; - size_t neural_core_block_offset = phys_layout->block_cnt - phys_layout->ne_cnt; - bool is_neural_core_block = (block_idx >= neural_core_block_offset); - shader_core_block_offset -= phys_layout->ne_cnt; - is_shader_core_block = - ((block_idx >= shader_core_block_offset) && (block_idx < neural_core_block_offset)); + if (block_idx < fw_block_offset) { + *block_in_type_idx = block_idx; + + return KBASE_HWCNT_BLOCK_METADATA; + } + + if ((fw_block_offset <= block_idx) && (block_idx < cshw_block_offset)) { + *block_in_type_idx = block_idx - fw_block_offset; + + return KBASE_HWCNT_BLOCK_FW; + } + + if ((cshw_block_offset <= block_idx) && (block_idx < tiler_block_offset)) { + *block_in_type_idx = block_idx - cshw_block_offset; + + return KBASE_HWCNT_BLOCK_CSHW; + } + + if ((tiler_block_offset <= block_idx) && (block_idx < memsys_block_offset)) { + *block_in_type_idx = block_idx - tiler_block_offset; + + return KBASE_HWCNT_BLOCK_TILER; + } + + if ((memsys_block_offset <= block_idx) && (block_idx < shader_core_block_offset)) { + *block_in_type_idx = block_idx - memsys_block_offset; + + return KBASE_HWCNT_BLOCK_MEMSYS; + } + if ((shader_core_block_offset <= block_idx) && (block_idx < neural_core_block_offset)) { + *block_in_type_idx = block_idx - shader_core_block_offset; + + return KBASE_HWCNT_BLOCK_SHADER; + } + + if ((neural_core_block_offset <= block_idx) && + (block_idx < phys_layout->ringbuf_block_cnt)) { + *block_in_type_idx = block_idx - neural_core_block_offset; + + return KBASE_HWCNT_BLOCK_NEURAL; + } + + WARN(1, "Unknown block index %zu", block_idx); + *block_in_type_idx = 0; + return KBASE_HWCNT_BLOCK_CSHW; +} + +void kbasep_hwcnt_backend_csf_update_block_state( + struct kbase_hwcnt_backend_csf *backend, const u32 enable_mask, size_t block_idx, + blk_stt_t *const block_state, enum kbase_hwcnt_backend_sample_reason prev_sample_reason, + enum kbase_hwcnt_backend_sample_reason curr_sample_reason) +{ + size_t idx_in_block_type = 0; + enum kbase_hwcnt_backend_block_type type = kbase_hwcnt_backend_csf_identify_block_type( + &backend->phys_layout, block_idx, &idx_in_block_type); + + bool sample_in_protected_mode = (curr_sample_reason == SAMPLE_REASON_BEFORE_PROTM); + bool sample_exiting_protm = (prev_sample_reason == SAMPLE_REASON_BEFORE_PROTM); + bool is_block_on = false; /* Set power bits for the block state for the block, for the sample */ switch (backend->enable_state) { /* Disabled states */ case KBASE_HWCNT_BACKEND_CSF_DISABLED: - case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED: + case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_REQ: + case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_ACK: case KBASE_HWCNT_BACKEND_CSF_DISABLED_WAIT_FOR_WORKER: - kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_OFF); + /* Nothing to do */ break; /* Enabled states */ case KBASE_HWCNT_BACKEND_CSF_ENABLED: - case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED: - if (!is_shader_core_block) - kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_ON); - else if (!exiting_protm) { + case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_REQ: + case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK: + if (type != KBASE_HWCNT_BLOCK_SHADER) + is_block_on = true; + else if (!sample_exiting_protm) { /* When not exiting protected mode, a zero enable mask on a shader core * counter block indicates the block was powered off for the sample, and * a non-zero counter enable mask indicates the block was powered on for * the sample. */ - kbase_hwcnt_block_state_append(block_state, - (enable_mask ? KBASE_HWCNT_STATE_ON : - KBASE_HWCNT_STATE_OFF)); + is_block_on = !!enable_mask; } break; /* Error states */ @@ -617,6 +778,12 @@ void kbasep_hwcnt_backend_csf_update_block_state(struct kbase_hwcnt_backend_csf break; } + if (curr_sample_reason == SAMPLE_REASON_BEFORE_SLEEP) + is_block_on = false; + + kbase_hwcnt_block_state_append(block_state, + is_block_on ? KBASE_HWCNT_STATE_ON : KBASE_HWCNT_STATE_OFF); + /* The following four cases apply to a block state in either normal mode or protected mode: * 1. GPU executing in normal mode: Only set normal mode bit. * 2. First sample request after GPU enters protected mode: Set both normal mode and @@ -629,7 +796,7 @@ void kbasep_hwcnt_backend_csf_update_block_state(struct kbase_hwcnt_backend_csf * 4. Samples requested after GPU exits protected mode: Set both protected mode and normal * mode bits. */ - if (exiting_protm || fw_in_protected_mode) + if (sample_exiting_protm || sample_in_protected_mode) kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_PROTECTED | KBASE_HWCNT_STATE_NORMAL); else @@ -639,30 +806,54 @@ void kbasep_hwcnt_backend_csf_update_block_state(struct kbase_hwcnt_backend_csf * the shader present and the debug core mask, so explicit checking of the * core mask is not required here. */ - if (is_shader_core_block) { - u64 current_shader_core = 1ULL << (block_idx - shader_core_block_offset); + switch (type) { + case KBASE_HWCNT_BLOCK_METADATA: + case KBASE_HWCNT_BLOCK_FW: + case KBASE_HWCNT_BLOCK_CSHW: + case KBASE_HWCNT_BLOCK_TILER: + case KBASE_HWCNT_BLOCK_MEMSYS: + if (curr_sample_reason == SAMPLE_REASON_BEFORE_YIELD) + kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_UNAVAILABLE); + else + kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_AVAILABLE); + break; + case KBASE_HWCNT_BLOCK_SHADER: { + u64 current_shader_core = 1ULL << idx_in_block_type; - WARN_ON_ONCE(backend->phys_layout.shader_cnt > 64); + WARN_ONCE(backend->phys_layout.shader_cnt > 64, + "More than 64 SCs will cause an overflow!"); - if (current_shader_core & backend->info->backend->powered_shader_core_mask) + if (curr_sample_reason == SAMPLE_REASON_BEFORE_YIELD) + kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_UNAVAILABLE); + else if (current_shader_core & backend->info->backend->powered_shader_core_mask) kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_AVAILABLE); else if (current_shader_core & ~backend->info->backend->powered_shader_core_mask) kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_UNAVAILABLE); else - WARN_ON_ONCE(true); - } else if (is_neural_core_block) { - u64 current_neural_core = 1ULL << (block_idx - neural_core_block_offset); + WARN_ONCE(true, "Unknown SC availability state!"); + break; + } + case KBASE_HWCNT_BLOCK_NEURAL: { + u64 current_neural_core = 1ULL << idx_in_block_type; - WARN_ON_ONCE(backend->phys_layout.ne_cnt > 64); + WARN_ONCE(backend->phys_layout.ne_cnt > 64, + "More than 64 NE cores may cause an overflow!"); - if (current_neural_core & backend->info->prfcnt_info.ne_core_mask) + if (curr_sample_reason == SAMPLE_REASON_BEFORE_YIELD) + kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_UNAVAILABLE); + else if (current_neural_core & backend->info->prfcnt_info.ne_core_mask) kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_AVAILABLE); else if (current_neural_core & ~backend->info->prfcnt_info.ne_core_mask) kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_UNAVAILABLE); else - WARN_ON_ONCE(true); - } else - kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_AVAILABLE); + WARN_ONCE(true, "Unknown NE core availability state!"); + break; + } + default: + WARN_ONCE(true, "Unknown block type returned (type = %u)", + type); /* This shouldn't happen */ + break; + } } static void kbasep_hwcnt_backend_csf_accumulate_sample(struct kbase_hwcnt_backend_csf *backend, @@ -670,35 +861,40 @@ static void kbasep_hwcnt_backend_csf_accumulate_sample(struct kbase_hwcnt_backen const u32 *new_sample_buf) { const struct kbase_hwcnt_csf_physical_layout *phys_layout = &backend->phys_layout; - const size_t dump_bytes = backend->info->prfcnt_info.dump_bytes; + struct kbase_hwcnt_backend_csf_if_prfcnt_info *prfcnt_info = &backend->info->prfcnt_info; + const size_t dump_bytes = prfcnt_info->dump_bytes; const size_t values_per_block = phys_layout->values_per_block; blk_stt_t *const block_states = backend->block_states; - const bool fw_in_protected_mode = backend->info->fw_in_protected_mode; - const bool clearing_samples = backend->info->prfcnt_info.clearing_samples; + enum kbase_hwcnt_backend_sample_reason prev_sample_reason = backend->info->last_transition; + enum kbase_hwcnt_backend_sample_reason curr_sample_reason = SAMPLE_REASON_NONE; + const bool clearing_samples = prfcnt_info->clearing_samples; u64 *accum_buf = backend->accum_buf; size_t block_idx; + const size_t metadata_size = prfcnt_info->metadata_size; const u32 *old_block = old_sample_buf; const u32 *new_block = new_sample_buf; u64 *acc_block = accum_buf; - /* Flag to indicate whether current sample is exiting protected mode. */ - bool exiting_protm = false; - /* The block pointers now point to the first HW block, which is always a CSHW/front-end - * block. The counter enable mask for this block can be checked to determine whether this - * sample is taken after leaving protected mode - this is the only scenario where the CSHW - * block counter enable mask has only the first bit set, and no others. In this case, - * the values in this sample would not be meaningful, so they don't need to be accumulated. - */ - exiting_protm = (new_block[phys_layout->enable_mask_offset] == 1); + size_t num_meta_blocks = prfcnt_info->metadata_size / prfcnt_info->prfcnt_block_size; + size_t num_meta_counters = num_meta_blocks * values_per_block; - for (block_idx = 0; block_idx < phys_layout->block_cnt; block_idx++) { + const unsigned int METADATA_SAMPLE_REASON_OFFSET = 3; + + if (backend->phys_layout.md_block_cnt) + curr_sample_reason = new_block[METADATA_SAMPLE_REASON_OFFSET]; + + old_block += num_meta_counters; + new_block += num_meta_counters; + + for (block_idx = 0; block_idx < phys_layout->payload_block_cnt; block_idx++) { const u32 old_enable_mask = old_block[phys_layout->enable_mask_offset]; const u32 new_enable_mask = new_block[phys_layout->enable_mask_offset]; /* Update block state with information of the current sample */ - kbasep_hwcnt_backend_csf_update_block_state(backend, new_enable_mask, exiting_protm, - block_idx, &block_states[block_idx], - fw_in_protected_mode); + kbasep_hwcnt_backend_csf_update_block_state(backend, new_enable_mask, + block_idx + phys_layout->md_block_cnt, + &block_states[block_idx], + prev_sample_reason, curr_sample_reason); if (!(new_enable_mask & HWCNT_BLOCK_EMPTY_SAMPLE)) { /* Hardware block was unavailable or we didn't turn on @@ -728,8 +924,8 @@ static void kbasep_hwcnt_backend_csf_accumulate_sample(struct kbase_hwcnt_backen * request samples without interfering with each other. * * However some early HW may not support wrapping - * counters, for these GPUs counters must be cleared on - * sample to avoid loss of data due to counters + * counters. For these GPUs, counters must be cleared + * on sample to avoid loss of data due to counters * saturating at their maximum value. */ if (!clearing_samples) { @@ -769,8 +965,12 @@ static void kbasep_hwcnt_backend_csf_accumulate_sample(struct kbase_hwcnt_backen } WARN_ON(old_block != old_sample_buf + (dump_bytes / KBASE_HWCNT_VALUE_HW_BYTES)); WARN_ON(new_block != new_sample_buf + (dump_bytes / KBASE_HWCNT_VALUE_HW_BYTES)); - WARN_ON(acc_block != accum_buf + (dump_bytes / KBASE_HWCNT_VALUE_HW_BYTES)); + WARN_ON(acc_block != + accum_buf + ((dump_bytes - metadata_size) / KBASE_HWCNT_VALUE_HW_BYTES)); (void)dump_bytes; + + if (backend->phys_layout.md_block_cnt) + backend->info->last_transition = curr_sample_reason; } static void kbasep_hwcnt_backend_csf_accumulate_samples(struct kbase_hwcnt_backend_csf *backend_csf, @@ -779,33 +979,39 @@ static void kbasep_hwcnt_backend_csf_accumulate_samples(struct kbase_hwcnt_backe { u32 raw_idx; unsigned long flags = 0UL; - u8 *cpu_dump_base = (u8 *)backend_csf->ring_buf_cpu_base; + u8 *cpu_dump_base = (u8 *)backend_csf->info->ring_buf_cpu_base; const size_t ring_buf_cnt = backend_csf->info->ring_buf_cnt; const size_t buf_dump_bytes = backend_csf->info->prfcnt_info.dump_bytes; u32 *old_sample_buf = backend_csf->old_sample_buf; u32 *new_sample_buf = old_sample_buf; - const struct kbase_hwcnt_csf_physical_layout *phys_layout = &backend_csf->phys_layout; if (extract_index_to_start == insert_index_to_stop) { /* No samples to accumulate but block states need to be updated for dump. */ size_t block_idx; - for (block_idx = 0; block_idx < phys_layout->block_cnt; block_idx++) { + /* We cannot use the last sample reason here, because the sequencing with the + * previous dump is not entirely clear and the required flag may not be set + * at that time. + */ + blk_stt_t stt = backend_csf->info->fw_in_protected_mode ? + KBASE_HWCNT_STATE_PROTECTED : + KBASE_HWCNT_STATE_NORMAL; + + for (block_idx = 0; block_idx < backend_csf->phys_layout.payload_block_cnt; + block_idx++) { /* Set protected mode bit for block state if GPU is in protected mode, * otherwise set the normal mode bit. */ - kbase_hwcnt_block_state_append(&backend_csf->block_states[block_idx], - backend_csf->info->fw_in_protected_mode ? - KBASE_HWCNT_STATE_PROTECTED : - KBASE_HWCNT_STATE_NORMAL); + kbase_hwcnt_block_state_append(&backend_csf->block_states[block_idx], stt); } return; } /* Sync all the buffers to CPU side before read the data. */ backend_csf->info->csf_if->ring_buf_sync(backend_csf->info->csf_if->ctx, - backend_csf->ring_buf, extract_index_to_start, - insert_index_to_stop, true); + backend_csf->info->ring_buf, + extract_index_to_start, insert_index_to_stop, + true); /* Consider u32 wrap case, '!=' is used here instead of '<' operator */ for (raw_idx = extract_index_to_start; raw_idx != insert_index_to_stop; raw_idx++) { @@ -834,8 +1040,9 @@ static void kbasep_hwcnt_backend_csf_accumulate_samples(struct kbase_hwcnt_backe /* Sync zeroed buffers to avoid coherency issues on future use. */ backend_csf->info->csf_if->ring_buf_sync(backend_csf->info->csf_if->ctx, - backend_csf->ring_buf, extract_index_to_start, - insert_index_to_stop, false); + backend_csf->info->ring_buf, + extract_index_to_start, insert_index_to_stop, + false); /* After consuming all samples between extract_idx and insert_idx, * set the raw extract index to insert_idx so that the sample buffers @@ -858,6 +1065,7 @@ static void kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( backend_csf->info->csf_if->assert_lock_held(backend_csf->info->csf_if->ctx); if (backend_csf->enable_state != new_state) { + pr_debug("%s %d -> %d", __func__, backend_csf->enable_state, new_state); backend_csf->enable_state = new_state; wake_up(&backend_csf->enable_state_waitq); @@ -872,7 +1080,8 @@ static void kbasep_hwcnt_backend_watchdog_timer_cb(void *info) csf_info->csf_if->lock(csf_info->csf_if->ctx, &flags); - if (WARN_ON(!kbasep_hwcnt_backend_csf_backend_exists(csf_info))) { + /* Early-exit if backend doesn't exist. */ + if (!kbasep_hwcnt_backend_csf_backend_exists(csf_info)) { csf_info->csf_if->unlock(csf_info->csf_if->ctx, flags); return; } @@ -882,9 +1091,11 @@ static void kbasep_hwcnt_backend_watchdog_timer_cb(void *info) /* Only do watchdog request when all conditions are met: */ if (/* 1. Backend is enabled. */ (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_ENABLED) && - /* 2. FW is not in protected mode. */ + /* 2. MCU is ON. */ + csf_info->mcu_on && + /* 3. FW is not in protected mode. */ (!csf_info->fw_in_protected_mode) && - /* 3. dump state indicates no other dumping is in progress. */ + /* 4. dump state indicates no other dumping is in progress. */ ((backend_csf->dump_state == KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE) || (backend_csf->dump_state == KBASE_HWCNT_BACKEND_CSF_DUMP_COMPLETED))) { u32 extract_index = 0U; @@ -905,13 +1116,19 @@ static void kbasep_hwcnt_backend_watchdog_timer_cb(void *info) } } - /* Must schedule another callback when in the transitional state because - * this function can be called for the first time before the performance - * counter enabled interrupt. + /* Reschedule the next callback when watchdog is enabled (i.e. backend states ENABLED + * and KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_ACK). + * + * Keep in mind that this function can be called for the first time + * before the performance counter enabled interrupt. + * + * Do not reschedule if MCU is off. */ - if ((backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_ENABLED) || - (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED)) { - /* Reschedule the timer for next watchdog callback. */ + if (((backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_ENABLED) || + (backend_csf->enable_state == + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_ACK)) && + csf_info->mcu_on) { + /* Reschedule the timer for next watchdog callback.*/ csf_info->watchdog_if->modify(csf_info->watchdog_if->timer, csf_info->watchdog_timer_interval_ms); } @@ -935,7 +1152,9 @@ static void kbasep_hwcnt_backend_csf_dump_worker(struct work_struct *work) u32 extract_index = 0U; u32 insert_index = 0U; u64 ts_gpu, ts_dump_raw, ts_dump, ts_dump_request, ts_now; + u32 *ts_gpu_ptr; bool sample_accumulated; + size_t ts_offset_in_sample_buf; WARN_ON(!work); backend_csf = container_of(work, struct kbase_hwcnt_backend_csf, hwc_dump_work); @@ -982,7 +1201,11 @@ static void kbasep_hwcnt_backend_csf_dump_worker(struct work_struct *work) WARN_ON(backend_csf != backend_csf->info->backend); /* TIMESTAMP_LO/HI of the newest dump */ - ts_gpu = *(u64 *)backend_csf->old_sample_buf; + ts_offset_in_sample_buf = (backend_csf->phys_layout.md_block_cnt * + backend_csf->info->prfcnt_info.prfcnt_block_size) / + KBASE_HWCNT_VALUE_HW_BYTES; + ts_gpu_ptr = backend_csf->old_sample_buf + ts_offset_in_sample_buf; + ts_gpu = *(u64 *)ts_gpu_ptr; /* Convert TIMESTAMP_LO/HI of a newest dump into a CPU timestamp. */ ts_dump_raw = backend_csf->info->csf_if->time_convert_gpu_to_cpu( @@ -1170,17 +1393,53 @@ kbasep_hwcnt_backend_csf_append_block_states(struct kbase_hwcnt_backend_csf *bac { size_t i; - for (i = 0; i < backend_csf->phys_layout.block_cnt; i++) + for (i = 0; i < backend_csf->phys_layout.payload_block_cnt; i++) kbase_hwcnt_block_state_append(&backend_csf->to_user_block_states[i], block_state); } +static int kbasep_hwcnt_backend_csf_do_dump_enable(struct kbase_hwcnt_backend_csf *backend_csf, + const struct kbase_hwcnt_enable_map *enable_map) +{ + struct kbase_hwcnt_backend_csf_if_enable enable; + int err; + + if (!backend_csf || !enable_map || (enable_map->metadata != backend_csf->info->metadata) || + WARN_ON(!backend_csf->info->ring_buf)) + return -EINVAL; + + backend_csf->info->csf_if->assert_lock_held(backend_csf->info->csf_if->ctx); + WARN_ON(backend_csf->enable_state != + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_REQ); + + if (!backend_csf->info->mcu_on) + return 0; + + err = backend_csf->info->watchdog_if->enable(backend_csf->info->watchdog_if->timer, + backend_csf->info->watchdog_timer_interval_ms, + kbasep_hwcnt_backend_watchdog_timer_cb, + backend_csf->info); + if (err) + return err; + + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( + backend_csf, KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_ACK); + + kbasep_hwcnt_backend_csf_get_physical_enable(backend_csf, enable_map, &enable); + + backend_csf->info->csf_if->dump_enable(backend_csf->info->csf_if->ctx, + backend_csf->info->ring_buf, &enable); + kbasep_hwcnt_backend_csf_cc_initial_sample(backend_csf, enable_map); + + return 0; +} + /* CSF backend implementation of kbase_hwcnt_backend_dump_enable_nolock_fn */ static int kbasep_hwcnt_backend_csf_dump_enable_nolock(struct kbase_hwcnt_backend *backend, const struct kbase_hwcnt_enable_map *enable_map) { struct kbase_hwcnt_backend_csf *backend_csf = (struct kbase_hwcnt_backend_csf *)backend; - struct kbase_hwcnt_backend_csf_if_enable enable; + int err; if (!backend_csf || !enable_map || (enable_map->metadata != backend_csf->info->metadata)) @@ -1200,28 +1459,35 @@ kbasep_hwcnt_backend_csf_dump_enable_nolock(struct kbase_hwcnt_backend *backend, */ kbasep_hwcnt_backend_csf_append_block_states(backend_csf, KBASE_HWCNT_STATE_OFF); - kbasep_hwcnt_backend_csf_get_physical_enable(backend_csf, enable_map, &enable); - - /* enable_state should be DISABLED before we transfer it to enabled */ - if (backend_csf->enable_state != KBASE_HWCNT_BACKEND_CSF_DISABLED) + /* enable_state should be (or be transitioning to) DISABLED before we transfer it + * to enabled. In addition to that, there shouldn't be a concurrent dump worker flush during + * dump_disable() procedure (the lock is released then). + */ + if (backend_csf->flush_on_disable_in_progress || + (backend_csf->enable_state != KBASE_HWCNT_BACKEND_CSF_DISABLED && + !kbasep_hwcnt_backend_csf_is_state_trans_to_disabled(backend_csf->enable_state))) return -EIO; - err = backend_csf->info->watchdog_if->enable(backend_csf->info->watchdog_if->timer, - backend_csf->info->watchdog_timer_interval_ms, - kbasep_hwcnt_backend_watchdog_timer_cb, - backend_csf->info); - if (err) - return err; - backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE; WARN_ON(!completion_done(&backend_csf->dump_completed)); + + /* Save the enable_map. It will be used e.g. when handling the deferred enable REQ + * or during cold_boot procedure. + */ + kbase_hwcnt_enable_map_copy(&backend_csf->last_enable_map, enable_map); + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( - backend_csf, KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED); + backend_csf, KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_REQ); - backend_csf->info->csf_if->dump_enable(backend_csf->info->csf_if->ctx, - backend_csf->ring_buf, &enable); - - kbasep_hwcnt_backend_csf_cc_initial_sample(backend_csf, enable_map); + /* If MCU is powered off, backend will stay in TRANSITIONING_TO_ENABLED_PENDING_REQ. + * Enable REQ will be sent during MCU ON notification. + */ + if (backend_csf->info->mcu_on) { + err = kbasep_hwcnt_backend_csf_do_dump_enable(backend_csf, + &backend_csf->last_enable_map); + if (err) + return err; + } return 0; } @@ -1243,25 +1509,60 @@ static int kbasep_hwcnt_backend_csf_dump_enable(struct kbase_hwcnt_backend *back return errcode; } -static void kbasep_hwcnt_backend_csf_wait_enable_transition_complete( - struct kbase_hwcnt_backend_csf *backend_csf, unsigned long *lock_flags) +static void +kbasep_hwcnt_backend_csf_wait_state_transition_complete(struct kbase_hwcnt_backend_csf *backend_csf, + unsigned long *lock_flags) { backend_csf->info->csf_if->assert_lock_held(backend_csf->info->csf_if->ctx); - while ((backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED) || - (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED)) { + /* The wait loop ends when backend gets out of a transitioning state or MCU + * is powered off. + */ + while ((kbasep_hwcnt_backend_csf_is_state_trans_to_enabled(backend_csf->enable_state) || + kbasep_hwcnt_backend_csf_is_state_trans_to_disabled(backend_csf->enable_state)) && + backend_csf->info->mcu_on) { backend_csf->info->csf_if->unlock(backend_csf->info->csf_if->ctx, *lock_flags); wait_event(backend_csf->enable_state_waitq, - (backend_csf->enable_state != - KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED) && - (backend_csf->enable_state != - KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED)); + (!kbasep_hwcnt_backend_csf_is_state_trans_to_enabled( + backend_csf->enable_state) && + !kbasep_hwcnt_backend_csf_is_state_trans_to_disabled( + backend_csf->enable_state)) || + !backend_csf->info->mcu_on); backend_csf->info->csf_if->lock(backend_csf->info->csf_if->ctx, lock_flags); } } +static void kbasep_hwcnt_backend_csf_send_disable_req(struct kbase_hwcnt_backend_csf_info *csf_info) +{ + csf_info->csf_if->assert_lock_held(csf_info->csf_if->ctx); + + /* Send the disable REQ and mark that we don't have any pending disable REQ. */ + csf_info->csf_if->dump_disable(csf_info->csf_if->ctx); + csf_info->pending_disable_req = false; +} + +static void +kbasep_hwcnt_backend_csf_resolve_wait_for_worker_state(struct kbase_hwcnt_backend_csf *backend_csf) +{ + backend_csf->info->csf_if->assert_lock_held(backend_csf->info->csf_if->ctx); + + switch (backend_csf->enable_state) { + case KBASE_HWCNT_BACKEND_CSF_DISABLED_WAIT_FOR_WORKER: + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( + backend_csf, KBASE_HWCNT_BACKEND_CSF_DISABLED); + break; + case KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR_WAIT_FOR_WORKER: + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( + backend_csf, KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR); + break; + default: + /* Do nothing */ + break; + } +} + /* CSF backend implementation of kbase_hwcnt_backend_dump_disable_fn */ static void kbasep_hwcnt_backend_csf_dump_disable(struct kbase_hwcnt_backend *backend, struct kbase_hwcnt_dump_buffer *dump_buffer, @@ -1277,12 +1578,19 @@ static void kbasep_hwcnt_backend_csf_dump_disable(struct kbase_hwcnt_backend *ba (dump_buffer && !enable_map))) return; + /* Cancel any pending reenable worker so we can disable the backend cleanly. */ + cancel_work_sync(&backend_csf->info->hwc_dump_reenable_work); + backend_csf->info->csf_if->lock(backend_csf->info->csf_if->ctx, &flags); /* Make sure we wait until any previous enable or disable have completed * before doing anything. + * + * Note: Backend might still be in a transitioning state after + * kbasep_hwcnt_backend_csf_wait_state_transition_complete() if MCU was + * powered off. */ - kbasep_hwcnt_backend_csf_wait_enable_transition_complete(backend_csf, &flags); + kbasep_hwcnt_backend_csf_wait_state_transition_complete(backend_csf, &flags); if (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_DISABLED || backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR) { @@ -1293,27 +1601,36 @@ static void kbasep_hwcnt_backend_csf_dump_disable(struct kbase_hwcnt_backend *ba return; } - if (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_ENABLED) { + /* Send disable REQ if the backend was previously either enabled or was transitioning + * to enabled state but it was never REQed/ACKed due to MCU being powered off. + * + * In any other state the disable REQ have already been sent. + */ + if (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_ENABLED || + kbasep_hwcnt_backend_csf_is_state_trans_to_enabled(backend_csf->enable_state)) { kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( - backend_csf, KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED); + backend_csf, KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_REQ); backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE; complete_all(&backend_csf->dump_completed); - /* Only disable if we were previously enabled - in all other - * cases the call to disable will have already been made. - */ do_disable = true; } WARN_ON(backend_csf->dump_state != KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE); WARN_ON(!completion_done(&backend_csf->dump_completed)); + /* Mark that we are going to flush the workers (hence we are releasing the lock). + * This is needed to prevent any potential concurrent attempts to enable the backend. + */ + backend_csf->flush_on_disable_in_progress = true; + backend_csf->info->csf_if->unlock(backend_csf->info->csf_if->ctx, flags); /* Deregister the timer and block until any timer callback has completed. * We've transitioned out of the ENABLED state so we can guarantee it * won't reschedule itself. */ - backend_csf->info->watchdog_if->disable(backend_csf->info->watchdog_if->timer); + backend_csf->info->watchdog_if->disable(backend_csf->info->watchdog_if->timer, + KBASE_HWCNT_WATCHDOG_DISABLE_SHOULD_BLOCK); /* Block until any async work has completed. We have transitioned out of * the ENABLED state so we can guarantee no new work will concurrently @@ -1322,26 +1639,49 @@ static void kbasep_hwcnt_backend_csf_dump_disable(struct kbase_hwcnt_backend *ba flush_workqueue(backend_csf->hwc_dump_workq); backend_csf->info->csf_if->lock(backend_csf->info->csf_if->ctx, &flags); + backend_csf->flush_on_disable_in_progress = false; - if (do_disable) - backend_csf->info->csf_if->dump_disable(backend_csf->info->csf_if->ctx); - - kbasep_hwcnt_backend_csf_wait_enable_transition_complete(backend_csf, &flags); - - switch (backend_csf->enable_state) { - case KBASE_HWCNT_BACKEND_CSF_DISABLED_WAIT_FOR_WORKER: - kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( - backend_csf, KBASE_HWCNT_BACKEND_CSF_DISABLED); - break; - case KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR_WAIT_FOR_WORKER: - kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( - backend_csf, KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR); - break; - default: - WARN_ON(true); - break; + if (!backend_csf->info->mcu_on) { + /* Stay in TRANSITIONING_TO_DISABLED_PENDING_REQ state. + * Record that we wanted to disable HWC but the disable REQ hasn't reached MCU. + * We need to save this in a backend instance-independent structure, + * to be able to send the disable REQ at MCU ON notification even if the backend + * has been destroyed/recreated. + */ + backend_csf->info->pending_disable_req = true; + goto skip_disable_req; } + if (do_disable) { + /* Check if we have entered an error state while we released the lock. + * If yes, don't leave the error state. + */ + if (!kbasep_hwcnt_backend_csf_is_error_state(backend_csf->enable_state)) + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( + backend_csf, + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK); + kbasep_hwcnt_backend_csf_send_disable_req(backend_csf->info); + } + + if (!kbasep_hwcnt_backend_csf_is_error_state(backend_csf->enable_state)) + kbasep_hwcnt_backend_csf_wait_state_transition_complete(backend_csf, &flags); + + kbasep_hwcnt_backend_csf_resolve_wait_for_worker_state(backend_csf); + + /* At this point the correct states are : + * - KBASE_HWCNT_BACKEND_CSF_DISABLED + * - KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR + * - KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK (only if MCU + * is powered off). + */ + + WARN_ON((backend_csf->enable_state != KBASE_HWCNT_BACKEND_CSF_DISABLED) && + (backend_csf->enable_state != KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR) && + ((backend_csf->enable_state != + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK) || + backend_csf->info->mcu_on)); + +skip_disable_req: backend_csf->user_requested = false; backend_csf->watchdog_last_seen_insert_idx = 0; @@ -1354,7 +1694,7 @@ static void kbasep_hwcnt_backend_csf_dump_disable(struct kbase_hwcnt_backend *ba /* Sync zeroed buffers to avoid coherency issues on future use. */ backend_csf->info->csf_if->ring_buf_sync(backend_csf->info->csf_if->ctx, - backend_csf->ring_buf, 0, + backend_csf->info->ring_buf, 0, backend_csf->info->ring_buf_cnt, false); /* Disabling HWCNT is an indication that blocks have been powered off. This is important to @@ -1408,12 +1748,19 @@ static int kbasep_hwcnt_backend_csf_dump_request(struct kbase_hwcnt_backend *bac return -EINVAL; backend_csf->info->csf_if->lock(backend_csf->info->csf_if->ctx, &flags); - - /* If we're transitioning to enabled there's nothing to accumulate, and + /* If we're transitioning to enabled, then there's nothing to accumulate, and * the user dump buffer is already zeroed. We can just short circuit to * the DUMP_COMPLETED state. + * + * We can also early exit in case of an ongoing reenable procedure + * (dump_reenable_in_progress). It means that we must have disabled the backend + * first so there is nothing to accumulate. */ - if (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED) { + if (kbasep_hwcnt_backend_csf_is_state_trans_to_enabled(backend_csf->enable_state) || + (backend_csf->dump_reenable_in_progress && + ((backend_csf->enable_state == + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK) || + (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_DISABLED)))) { backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_COMPLETED; backend_csf->dump_time_ns = kbasep_hwcnt_backend_csf_timestamp_ns(backend); kbasep_hwcnt_backend_csf_cc_update(backend_csf); @@ -1421,12 +1768,11 @@ static int kbasep_hwcnt_backend_csf_dump_request(struct kbase_hwcnt_backend *bac * during multiple dumps, hence append the OFF state. */ kbasep_hwcnt_backend_csf_append_block_states(backend_csf, KBASE_HWCNT_STATE_OFF); - backend_csf->user_requested = true; + WARN_ON(!completion_done(&backend_csf->dump_completed)); backend_csf->info->csf_if->unlock(backend_csf->info->csf_if->ctx, flags); return 0; } - /* Otherwise, make sure we're already enabled. */ if (backend_csf->enable_state != KBASE_HWCNT_BACKEND_CSF_ENABLED) { backend_csf->info->csf_if->unlock(backend_csf->info->csf_if->ctx, flags); @@ -1461,9 +1807,9 @@ static int kbasep_hwcnt_backend_csf_dump_request(struct kbase_hwcnt_backend *bac watchdog_dumping = true; if ((backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_ENABLED) && - !backend_csf->info->fw_in_protected_mode) { - /* Only do the request if we are fully enabled and not in - * protected mode. + !backend_csf->info->fw_in_protected_mode && backend_csf->info->mcu_on) { + /* Do the request only if the MCU is on, we are fully enabled + * and not in protected mode. */ backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_REQUESTED; do_request = true; @@ -1496,13 +1842,16 @@ static int kbasep_hwcnt_backend_csf_dump_request(struct kbase_hwcnt_backend *bac } else kbase_hwcnt_backend_csf_submit_dump_worker(backend_csf->info); - backend_csf->info->csf_if->unlock(backend_csf->info->csf_if->ctx, flags); - /* Modify watchdog timer to delay the regular check time since - * just requested. + * just requested (only if MCU is ON). */ - backend_csf->info->watchdog_if->modify(backend_csf->info->watchdog_if->timer, - backend_csf->info->watchdog_timer_interval_ms); + if (backend_csf->info->mcu_on) { + backend_csf->info->watchdog_if->modify( + backend_csf->info->watchdog_if->timer, + backend_csf->info->watchdog_timer_interval_ms); + } + + backend_csf->info->csf_if->unlock(backend_csf->info->csf_if->ctx, flags); return 0; } @@ -1613,8 +1962,7 @@ static void kbasep_hwcnt_backend_csf_destroy(struct kbase_hwcnt_backend_csf *bac destroy_workqueue(backend_csf->hwc_dump_workq); - backend_csf->info->csf_if->ring_buf_free(backend_csf->info->csf_if->ctx, - backend_csf->ring_buf); + kbase_hwcnt_enable_map_free(&backend_csf->last_enable_map); kfree(backend_csf->accum_buf); backend_csf->accum_buf = NULL; @@ -1650,8 +1998,8 @@ static int kbasep_hwcnt_backend_csf_create(struct kbase_hwcnt_backend_csf_info * int errcode = -ENOMEM; size_t block_state_bytes; - WARN_ON(!csf_info); - WARN_ON(!out_backend); + if (WARN_ON(!out_backend || !csf_info || !csf_info->ring_buf)) + return -EINVAL; backend_csf = kzalloc(sizeof(*backend_csf), GFP_KERNEL); if (!backend_csf) @@ -1674,8 +2022,8 @@ static int kbasep_hwcnt_backend_csf_create(struct kbase_hwcnt_backend_csf_info * goto err_alloc_user_sample_buf; /* Allocate space to store block state values for each block */ - block_state_bytes = backend_csf->phys_layout.block_cnt * KBASE_HWCNT_BLOCK_STATE_BYTES * - KBASE_HWCNT_BLOCK_STATE_STRIDE; + block_state_bytes = backend_csf->phys_layout.payload_block_cnt * + KBASE_HWCNT_BLOCK_STATE_BYTES * KBASE_HWCNT_BLOCK_STATE_STRIDE; backend_csf->block_states = kzalloc(block_state_bytes, GFP_KERNEL); if (!backend_csf->block_states) goto err_alloc_block_states_buf; @@ -1684,11 +2032,14 @@ static int kbasep_hwcnt_backend_csf_create(struct kbase_hwcnt_backend_csf_info * if (!backend_csf->to_user_block_states) goto err_alloc_user_block_state_buf; - errcode = csf_info->csf_if->ring_buf_alloc(csf_info->csf_if->ctx, csf_info->ring_buf_cnt, - &backend_csf->ring_buf_cpu_base, - &backend_csf->ring_buf); + /* Allocate a local version of the enable_map to avoid a situation where the backend's + * caller frees the map when it's still required for the backend (e.g. enable_map + * being reapplied after MCU power on). + */ + errcode = kbase_hwcnt_enable_map_alloc(csf_info->metadata, &backend_csf->last_enable_map); if (errcode) - goto err_ring_buf_alloc; + goto err_alloc_last_enable_map; + errcode = -ENOMEM; /* Zero all performance enable header to prepare for first enable. */ @@ -1696,7 +2047,7 @@ static int kbasep_hwcnt_backend_csf_create(struct kbase_hwcnt_backend_csf_info * /* Sync zeroed buffers to avoid coherency issues on use. */ backend_csf->info->csf_if->ring_buf_sync(backend_csf->info->csf_if->ctx, - backend_csf->ring_buf, 0, + backend_csf->info->ring_buf, 0, backend_csf->info->ring_buf_cnt, false); init_completion(&backend_csf->dump_completed); @@ -1724,9 +2075,8 @@ static int kbasep_hwcnt_backend_csf_create(struct kbase_hwcnt_backend_csf_info * return 0; err_alloc_workqueue: - backend_csf->info->csf_if->ring_buf_free(backend_csf->info->csf_if->ctx, - backend_csf->ring_buf); -err_ring_buf_alloc: + kbase_hwcnt_enable_map_free(&backend_csf->last_enable_map); +err_alloc_last_enable_map: kfree(backend_csf->to_user_block_states); backend_csf->to_user_block_states = NULL; err_alloc_user_block_state_buf: @@ -1747,6 +2097,85 @@ alloc_error: return errcode; } +enum kbasep_hwcnt_backend_ring_buf_free_type { + KBASEP_RING_BUF_FREE_NORMAL, + KBASEP_RING_BUF_FREE_DEFERRED, +}; + +static inline void +kbasep_hwcnt_backend_csf_ring_buf_free_deferred(struct kbase_hwcnt_backend_csf_info *csf_info) +{ + if (WARN_ON(!csf_info)) + return; + + csf_info->csf_if->assert_lock_held(csf_info->csf_if->ctx); + + csf_info->ring_buf_cleanup = csf_info->ring_buf; + queue_work(csf_info->hwc_deferred_op_workq, &csf_info->hwc_ring_buf_free_work); + csf_info->ring_buf = NULL; +} + +static inline void kbasep_hwcnt_backend_csf_ring_buf_refcnt_increment_nolock( + struct kbase_hwcnt_backend_csf_info *csf_info) +{ + if (WARN_ON(!csf_info)) + return; + + csf_info->csf_if->assert_lock_held(csf_info->csf_if->ctx); + + csf_info->ring_buf_refcnt.val++; + WARN_ON(csf_info->ring_buf_refcnt.val > RING_BUF_REFCNT_MAX); +} + +static void kbasep_hwcnt_backend_csf_ring_buf_refcnt_decrement_nolock( + struct kbase_hwcnt_backend_csf_info *csf_info, unsigned long *lock_flags, + enum kbasep_hwcnt_backend_ring_buf_free_type type) +{ + if (WARN_ON(!csf_info)) + return; + + csf_info->csf_if->assert_lock_held(csf_info->csf_if->ctx); + + csf_info->ring_buf_refcnt.val--; + WARN_ON(csf_info->ring_buf_refcnt.val < RING_BUF_REFCNT_MIN); + + if (!csf_info->ring_buf_refcnt.val) { + struct kbase_hwcnt_backend_csf_if_ring_buf *tmp_ring_buf = csf_info->ring_buf; + + WARN_ON(!csf_info->ring_buf); + + switch (type) { + case KBASEP_RING_BUF_FREE_NORMAL: + if (WARN_ON(!lock_flags)) + break; + csf_info->ring_buf = NULL; + csf_info->csf_if->unlock(csf_info->csf_if->ctx, *lock_flags); + csf_info->csf_if->ring_buf_free(csf_info->csf_if->ctx, tmp_ring_buf); + csf_info->csf_if->lock(csf_info->csf_if->ctx, lock_flags); + break; + case KBASEP_RING_BUF_FREE_DEFERRED: + kbasep_hwcnt_backend_csf_ring_buf_free_deferred(csf_info); + break; + default: + break; + } + } +} + +static void kbasep_hwcnt_backend_csf_ring_buf_refcnt_decrement( + struct kbase_hwcnt_backend_csf_info *csf_info, + enum kbasep_hwcnt_backend_ring_buf_free_type type) +{ + unsigned long flags = 0UL; + + if (WARN_ON(!csf_info)) + return; + + csf_info->csf_if->lock(csf_info->csf_if->ctx, &flags); + kbasep_hwcnt_backend_csf_ring_buf_refcnt_decrement_nolock(csf_info, &flags, type); + csf_info->csf_if->unlock(csf_info->csf_if->ctx, flags); +} + /* CSF backend implementation of kbase_hwcnt_backend_init_fn */ static int kbasep_hwcnt_backend_csf_init(const struct kbase_hwcnt_backend_info *info, struct kbase_hwcnt_backend **out_backend) @@ -1760,10 +2189,35 @@ static int kbasep_hwcnt_backend_csf_init(const struct kbase_hwcnt_backend_info * if (!info || !out_backend) return -EINVAL; + /* Ring buffer allocation. */ + csf_info->csf_if->lock(csf_info->csf_if->ctx, &flags); + kbasep_hwcnt_backend_csf_ring_buf_refcnt_increment_nolock(csf_info); + if (csf_info->ring_buf_refcnt.val == 1) { + WARN_ON(csf_info->ring_buf); + csf_info->csf_if->unlock(csf_info->csf_if->ctx, flags); + + /* Flush any pending ring_buf_free_worker before allocating a new ring buffer. */ + flush_workqueue(csf_info->hwc_deferred_op_workq); + + errcode = csf_info->csf_if->ring_buf_alloc(csf_info->csf_if->ctx, + csf_info->ring_buf_cnt, + &csf_info->ring_buf_cpu_base, + &csf_info->ring_buf); + if (errcode) { + kbasep_hwcnt_backend_csf_ring_buf_refcnt_decrement( + csf_info, KBASEP_RING_BUF_FREE_NORMAL); + return errcode; + } + } else + csf_info->csf_if->unlock(csf_info->csf_if->ctx, flags); + /* Create the backend. */ errcode = kbasep_hwcnt_backend_csf_create(csf_info, &backend_csf); - if (errcode) + if (errcode) { + kbasep_hwcnt_backend_csf_ring_buf_refcnt_decrement(csf_info, + KBASEP_RING_BUF_FREE_NORMAL); return errcode; + } /* If it was not created before, attach it to csf_info. * Use spin lock to avoid concurrent initialization. @@ -1783,6 +2237,8 @@ static int kbasep_hwcnt_backend_csf_init(const struct kbase_hwcnt_backend_info * * function properly. */ if (!success) { + kbasep_hwcnt_backend_csf_ring_buf_refcnt_decrement(backend_csf->info, + KBASEP_RING_BUF_FREE_NORMAL); kbasep_hwcnt_backend_csf_destroy(backend_csf); return -EBUSY; } @@ -1806,6 +2262,8 @@ static void kbasep_hwcnt_backend_csf_term(struct kbase_hwcnt_backend *backend) */ backend_csf->info->csf_if->lock(backend_csf->info->csf_if->ctx, &flags); backend_csf->info->backend = NULL; + kbasep_hwcnt_backend_csf_ring_buf_refcnt_decrement_nolock(backend_csf->info, &flags, + KBASEP_RING_BUF_FREE_NORMAL); backend_csf->info->csf_if->unlock(backend_csf->info->csf_if->ctx, flags); kbasep_hwcnt_backend_csf_destroy(backend_csf); @@ -1827,6 +2285,141 @@ static void kbasep_hwcnt_backend_csf_release(const struct kbase_hwcnt_backend *b csf_info->csf_if->release(csf_info->csf_if->ctx); } +static inline bool +kbasep_hwcnt_backend_csf_abort_dump_reenable(struct kbase_hwcnt_backend_csf_info *csf_info) +{ + if (WARN_ON(!csf_info)) + return true; + + csf_info->csf_if->assert_lock_held(csf_info->csf_if->ctx); + /* Check if the dump reenabling is still valid. + * + * dump_reenable_in_progress can be set to false by a cold_boot procedure, + * which is a signal to abort the reenable work. + */ + return (!kbasep_hwcnt_backend_csf_backend_exists(csf_info) || + !csf_info->backend->dump_reenable_in_progress || + (csf_info->backend->enable_state != + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK && + csf_info->backend->enable_state != + KBASE_HWCNT_BACKEND_CSF_DISABLED_WAIT_FOR_WORKER && + csf_info->backend->enable_state != KBASE_HWCNT_BACKEND_CSF_DISABLED)); +} + +/** + * kbasep_hwcnt_backend_csf_dump_reenable_worker() - HWC backend reenable worker. + * + * @work: Work structure. + * + * Called when a HWC backend needs a disable-enable cycle, e.g. when we want to enable the backend, + * but the previous disable hasn't been performed due to MCU being powered off. + * + * This procedure assumes that the disable REQ has been already sent in the atomic context, + * and this worker waits for the disable ACK, and later sends an enable REQ. + */ +static void kbasep_hwcnt_backend_csf_dump_reenable_worker(struct work_struct *work) +{ + struct kbase_hwcnt_backend_csf_info *csf_info; + unsigned long flags = 0ULL; + int err; + + WARN_ON(!work); + + csf_info = container_of(work, struct kbase_hwcnt_backend_csf_info, hwc_dump_reenable_work); + + csf_info->csf_if->lock(csf_info->csf_if->ctx, &flags); + /* Check if the re-enabling is still valid. + * + * The thread that enqueued this worker has already sent the disable REQ, so we either wait + * for the disable ACK, or it was already ACKed. Any other state (or a destroyed backend) + * means that the re-enabling should be aborted. + */ + if (kbasep_hwcnt_backend_csf_abort_dump_reenable(csf_info)) + goto exit; + + /* Wait for the disable ACK. */ + kbasep_hwcnt_backend_csf_wait_state_transition_complete(csf_info->backend, &flags); + + /* kbasep_hwcnt_backend_csf_wait_state_transition_complete releases the lock, so we need to + * recheck if reenabling is still valid. + */ + if (kbasep_hwcnt_backend_csf_abort_dump_reenable(csf_info)) + goto exit; + + /* We are re-enabling the backend, so we must have already waited for the workers + * to complete while disabling the backend. It is safe to transition out of + * *_WAIT_FOR_WORKER states. + */ + kbasep_hwcnt_backend_csf_resolve_wait_for_worker_state(csf_info->backend); + + if (csf_info->backend->enable_state == KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR) + goto exit; + + /* The disabling part of the worker has concluded. + * + * Note that it is fine to be in TRANSITIONING_TO_DISABLED_PENDING_ACK at this point. + * It means that we have already sent disable REQ while the MCU was on, but the MCU + * was powered off while kbasep_hwcnt_backend_csf_wait_state_transition_complete was called + * previously. + * + * We can proceed with the re-enabling procedure because: + * - the pending disable will be ACKed at the following MCU power on, + * - enable REQ will be sent at the next MCU power on, because this reenable worker + * will set the TRANSITIONING_TO_ENABLED_PENDING_REQ state. + */ + WARN_ON(!(csf_info->backend->enable_state == KBASE_HWCNT_BACKEND_CSF_DISABLED || + csf_info->backend->enable_state == + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK)); + + /* Proceed to enabling part of the worker. */ + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( + csf_info->backend, KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_REQ); + + /* If MCU is powered off, backend will stay in TRANSITIONING_TO_ENABLED_PENDING_REQ. + * Enable REQ will be sent during MCU ON notification. + */ + if (csf_info->mcu_on) { + err = kbasep_hwcnt_backend_csf_do_dump_enable(csf_info->backend, + &csf_info->backend->last_enable_map); + if (err) + kbasep_hwcnt_backend_csf_handle_unrecoverable_error(csf_info->backend); + } + +exit: + if (kbasep_hwcnt_backend_csf_backend_exists(csf_info)) + csf_info->backend->dump_reenable_in_progress = false; + csf_info->csf_if->unlock(csf_info->csf_if->ctx, flags); +} + +/** + * kbasep_hwcnt_backend_csf_ring_buf_free_worker() - Ring buffer freeing worker. + * + * @work: Work structure. + * + * Called when a ring buffer needs to be freed. + */ +static void kbasep_hwcnt_backend_csf_ring_buf_free_worker(struct work_struct *work) +{ + struct kbase_hwcnt_backend_csf_info *csf_info; + struct kbase_hwcnt_backend_csf_if_ring_buf *ring_buf; + unsigned long flags = 0UL; + + WARN_ON(!work); + + csf_info = container_of(work, struct kbase_hwcnt_backend_csf_info, hwc_ring_buf_free_work); + csf_info->csf_if->lock(csf_info->csf_if->ctx, &flags); + + if (WARN_ON(!csf_info->ring_buf_cleanup)) { + csf_info->csf_if->unlock(csf_info->csf_if->ctx, flags); + return; + } + ring_buf = csf_info->ring_buf_cleanup; + csf_info->ring_buf_cleanup = NULL; + csf_info->csf_if->unlock(csf_info->csf_if->ctx, flags); + + csf_info->csf_if->ring_buf_free(csf_info->csf_if->ctx, ring_buf); +} + /** * kbasep_hwcnt_backend_csf_info_destroy() - Destroy a CSF backend info. * @info: Pointer to info to destroy. @@ -1839,12 +2432,18 @@ static void kbasep_hwcnt_backend_csf_info_destroy(const struct kbase_hwcnt_backe if (!info) return; + destroy_workqueue(info->hwc_deferred_op_workq); + /* The backend should be destroyed before the info object destroy. */ WARN_ON(info->backend != NULL); /* The metadata should be destroyed before the info object destroy. */ WARN_ON(info->metadata != NULL); + /* Ring buffer should be freed either during backend destruction or device termination.*/ + WARN_ON(info->ring_buf != NULL); + WARN_ON(info->ring_buf_cleanup != NULL); + kfree(info); } @@ -1892,7 +2491,19 @@ kbasep_hwcnt_backend_csf_info_create(struct kbase_hwcnt_backend_csf_if *csf_if, .fw_in_protected_mode = false, .unrecoverable_error_happened = false, .watchdog_if = watchdog_if, .watchdog_timer_interval_ms = watchdog_timer_interval_ms, + .last_transition = SAMPLE_REASON_NONE, .mcu_on = false, }; + + info->hwc_deferred_op_workq = + alloc_workqueue("mali_hwc_deferred_op_wq", WQ_HIGHPRI | WQ_UNBOUND, 1); + if (!info->hwc_deferred_op_workq) { + kfree(info); + return -ENOMEM; + } + + INIT_WORK(&info->hwc_dump_reenable_work, kbasep_hwcnt_backend_csf_dump_reenable_worker); + INIT_WORK(&info->hwc_ring_buf_free_work, kbasep_hwcnt_backend_csf_ring_buf_free_worker); + *out_info = info; return 0; @@ -1913,10 +2524,13 @@ kbasep_hwcnt_backend_csf_metadata(const struct kbase_hwcnt_backend_info *info) static void kbasep_hwcnt_backend_csf_handle_unrecoverable_error(struct kbase_hwcnt_backend_csf *backend_csf) { - bool do_disable = false; + bool send_disable_req = false; backend_csf->info->csf_if->assert_lock_held(backend_csf->info->csf_if->ctx); + /* Cancel any pending backend reenabling procedure. */ + backend_csf->dump_reenable_in_progress = false; + /* We are already in or transitioning to the unrecoverable error state. * Early out. */ @@ -1934,12 +2548,12 @@ kbasep_hwcnt_backend_csf_handle_unrecoverable_error(struct kbase_hwcnt_backend_c return; } - /* Trigger a disable only if we are not already transitioning to - * disabled, we don't want to disable twice if an unrecoverable error + /* Send a disable REQ only if we are not already transitioning to + * disabled. We don't want to send disable REQ twice if an unrecoverable error * happens while we are disabling. */ - do_disable = - (backend_csf->enable_state != KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED); + send_disable_req = + !kbasep_hwcnt_backend_csf_is_state_trans_to_disabled(backend_csf->enable_state); kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( backend_csf, KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR_WAIT_FOR_WORKER); @@ -1950,12 +2564,8 @@ kbasep_hwcnt_backend_csf_handle_unrecoverable_error(struct kbase_hwcnt_backend_c backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE; complete_all(&backend_csf->dump_completed); - /* Trigger a disable only if we are not already transitioning to - * disabled, - we don't want to disable twice if an unrecoverable error - * happens while we are disabling. - */ - if (do_disable) - backend_csf->info->csf_if->dump_disable(backend_csf->info->csf_if->ctx); + if (send_disable_req) + kbasep_hwcnt_backend_csf_send_disable_req(backend_csf->info); } static void @@ -1966,14 +2576,16 @@ kbasep_hwcnt_backend_csf_handle_recoverable_error(struct kbase_hwcnt_backend_csf switch (backend_csf->enable_state) { case KBASE_HWCNT_BACKEND_CSF_DISABLED: case KBASE_HWCNT_BACKEND_CSF_DISABLED_WAIT_FOR_WORKER: - case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED: + case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_REQ: + case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK: case KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR: case KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR_WAIT_FOR_WORKER: /* Already disabled or disabling, or in an unrecoverable error. * Nothing to be done to handle the error. */ return; - case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED: + case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_REQ: + case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_ACK: /* A seemingly recoverable error that occurs while we are * transitioning to enabled is probably unrecoverable. */ @@ -1986,14 +2598,15 @@ kbasep_hwcnt_backend_csf_handle_recoverable_error(struct kbase_hwcnt_backend_csf * disable(). */ kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( - backend_csf, KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED); - /* Transition the dump to the IDLE state and unblock any - * waiters. The IDLE state signifies an error. + backend_csf, KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK); + /* Transition the dump to the IDLE state and unblock any waiters. The IDLE state + * signifies an error. Mark that dump workers need to be flushed. */ + backend_csf->flush_workers_needed = true; backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE; complete_all(&backend_csf->dump_completed); - backend_csf->info->csf_if->dump_disable(backend_csf->info->csf_if->ctx); + kbasep_hwcnt_backend_csf_send_disable_req(backend_csf->info); return; } } @@ -2015,7 +2628,6 @@ void kbase_hwcnt_backend_csf_protm_entered(struct kbase_hwcnt_backend_interface void kbase_hwcnt_backend_csf_protm_exited(struct kbase_hwcnt_backend_interface *iface) { struct kbase_hwcnt_backend_csf_info *csf_info; - csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; csf_info->csf_if->assert_lock_held(csf_info->csf_if->ctx); @@ -2057,6 +2669,7 @@ void kbase_hwcnt_backend_csf_on_before_reset(struct kbase_hwcnt_backend_interfac csf_info->csf_if->unlock(csf_info->csf_if->ctx, flags); return; } + backend_csf = csf_info->backend; if ((backend_csf->enable_state != KBASE_HWCNT_BACKEND_CSF_DISABLED) && @@ -2164,15 +2777,22 @@ void kbase_hwcnt_backend_csf_on_prfcnt_enable(struct kbase_hwcnt_backend_interfa /* Early out if the backend does not exist. */ if (!kbasep_hwcnt_backend_csf_backend_exists(csf_info)) return; + backend_csf = csf_info->backend; - if (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED) { + if (backend_csf->enable_state == + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_ACK) { kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( backend_csf, KBASE_HWCNT_BACKEND_CSF_ENABLED); } else if (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_ENABLED) { /* Unexpected, but we are already in the right state so just * ignore it. */ + } else if (kbasep_hwcnt_backend_csf_is_state_trans_to_disabled(backend_csf->enable_state)) { + /* We might already be in the TRANSITION_TO_DISABLED_* states and + * the MCU is handling outstanding ACKs due to MCU power cycle, + * so just ignore it. + */ } else { /* Unexpected state change, assume everything is broken until * we reset. @@ -2189,17 +2809,45 @@ void kbase_hwcnt_backend_csf_on_prfcnt_disable(struct kbase_hwcnt_backend_interf csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; csf_info->csf_if->assert_lock_held(csf_info->csf_if->ctx); + /* We received a disable ACK, so we can decrement the ring buffer refcounter. */ + if (csf_info->ring_buf_refcnt.incremented_on_mcu_off) { + csf_info->ring_buf_refcnt.incremented_on_mcu_off = false; + kbasep_hwcnt_backend_csf_ring_buf_refcnt_decrement_nolock( + csf_info, NULL, KBASEP_RING_BUF_FREE_DEFERRED); + } + /* Early out if the backend does not exist. */ if (!kbasep_hwcnt_backend_csf_backend_exists(csf_info)) return; + backend_csf = csf_info->backend; - if (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED) { - kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( - backend_csf, KBASE_HWCNT_BACKEND_CSF_DISABLED_WAIT_FOR_WORKER); + if (backend_csf->enable_state == + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK) { + /* If the disable REQ was deferred due to MCU being off, we have already waited + * for the workers to complete. Hence it's safe to transition directly to + * DISABLED state. + * + * In any other case + * (e.g. KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK + * was set due to an recoverable error), we need to wait for workers to be flushed + * at the next dump_disable() call. + */ + enum kbase_hwcnt_backend_csf_enable_state new_state = + backend_csf->flush_workers_needed ? + KBASE_HWCNT_BACKEND_CSF_DISABLED_WAIT_FOR_WORKER : + KBASE_HWCNT_BACKEND_CSF_DISABLED; + backend_csf->flush_workers_needed = false; + + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters(backend_csf, new_state); + } else if (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_DISABLED) { - /* Unexpected, but we are already in the right state so just - * ignore it. + /* This might be expected if we are aligning the backend state with MCU state. + * We are already in the right state so just ignore it. + */ + } else if (kbasep_hwcnt_backend_csf_is_state_trans_to_enabled(backend_csf->enable_state)) { + /* We might already be in the TRANSITION_TO_ENABLED_* states and the MCU is handling + * outstanding ACKs due to MCU power cycle, so just ignore it. */ } else { /* Unexpected state change, assume everything is broken until @@ -2209,6 +2857,272 @@ void kbase_hwcnt_backend_csf_on_prfcnt_disable(struct kbase_hwcnt_backend_interf } } +static void +kbasep_hwcnt_backend_csf_on_after_mcu_off_nolock(struct kbase_hwcnt_backend_interface *iface) +{ + struct kbase_hwcnt_backend_csf_info *csf_info; + struct kbase_hwcnt_backend_csf *backend_csf; + + csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; + + csf_info->csf_if->assert_lock_held(csf_info->csf_if->ctx); + + WARN_ON(!csf_info->mcu_on); + csf_info->mcu_on = false; + /* Early out if the backend does not exist. */ + if (!kbasep_hwcnt_backend_csf_backend_exists(csf_info)) + return; + + backend_csf = csf_info->backend; + + /* Notify about the MCU OFF event. */ + wake_up(&backend_csf->enable_state_waitq); + + /* Ring buffer refcount makes sure the ring buffer is only freed when FW has ACKed + * any outstanding HWC disable REQs. This means that the refcount should be incremented + * on MCU_OFF event when the backend is enabled (or about to be enabled) from + * FW perspective, i.e. states: + * - KBASE_HWCNT_BACKEND_CSF_ENABLED, + * - KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_*, because the disable + * is not yet ACKed, + * - KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_ACK, to prevent + * a race condition between enable ACK IRQ and the ring buffer freeing + * after MCU ON notification. + * + * It's safe to free the ring buffer (i.e. to not increase the refcount) while MCU is OFF + * if the backend disable has been ACKed by FW or the enabling REQ never reached FW + * (e.g. KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_REQ). + * + * Do not increase the refcnt again if FW still hasn't ACKed the backend disable since + * the last MCU power cycle. + */ + if (!backend_csf->info->ring_buf_refcnt.incremented_on_mcu_off && + (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_ENABLED || + kbasep_hwcnt_backend_csf_is_state_trans_to_disabled(backend_csf->enable_state) || + backend_csf->enable_state == + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_ACK)) { + kbasep_hwcnt_backend_csf_ring_buf_refcnt_increment_nolock(csf_info); + backend_csf->info->ring_buf_refcnt.incremented_on_mcu_off = true; + } + + kbasep_hwcnt_backend_csf_append_block_states(backend_csf, KBASE_HWCNT_STATE_OFF); + + /* ACK the sample */ + kbase_hwcnt_backend_csf_on_prfcnt_sample(iface); +} + +void kbase_hwcnt_backend_csf_on_after_mcu_off(struct kbase_hwcnt_backend_interface *iface) +{ + struct kbase_hwcnt_backend_csf_info *csf_info; + unsigned long flags = 0UL; + + csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; + + csf_info->csf_if->lock(csf_info->csf_if->ctx, &flags); + kbasep_hwcnt_backend_csf_on_after_mcu_off_nolock(iface); + csf_info->csf_if->unlock(csf_info->csf_if->ctx, flags); +} + +void kbase_hwcnt_backend_csf_on_after_mcu_off_reset(struct kbase_hwcnt_backend_interface *iface) +{ + struct kbase_hwcnt_backend_csf_info *csf_info; + unsigned long flags = 0UL; + + csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; + csf_info->csf_if->lock(csf_info->csf_if->ctx, &flags); + + /* It's uncertain at which MCU state reset was called. + * Hence, only notify the backend about MCU_OFF if it hasn't been done yet. + */ + if (csf_info->mcu_on) + kbasep_hwcnt_backend_csf_on_after_mcu_off_nolock(iface); + csf_info->csf_if->unlock(csf_info->csf_if->ctx, flags); +} + +void kbase_hwcnt_backend_csf_on_after_mcu_on(struct kbase_hwcnt_backend_interface *iface) +{ + struct kbase_hwcnt_backend_csf_info *csf_info; + struct kbase_hwcnt_backend_csf *backend_csf; + const enum kbase_hwcnt_backend_csf_enable_state trans_to_disabled_pending_ack = + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK; + + unsigned long flags = 0UL; + + csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; + csf_info->csf_if->lock(csf_info->csf_if->ctx, &flags); + + WARN_ON(csf_info->mcu_on); + csf_info->mcu_on = true; + + if (kbasep_hwcnt_backend_csf_backend_exists(csf_info)) { + backend_csf = (struct kbase_hwcnt_backend_csf *)csf_info->backend; + + kbasep_hwcnt_backend_csf_append_block_states(backend_csf, KBASE_HWCNT_STATE_ON); + + switch (backend_csf->enable_state) { + case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_REQ: + /* pending_disable_req means that prior to enabling, we need to disable + * the backend first. + */ + if (backend_csf->info->pending_disable_req) { + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( + backend_csf, trans_to_disabled_pending_ack); + /* Send the disable REQ now. */ + kbasep_hwcnt_backend_csf_send_disable_req(backend_csf->info); + + backend_csf->dump_reenable_in_progress = true; + /* Queue a worker to wait for disable ACK and to send enable REQ. */ + queue_work(backend_csf->info->hwc_deferred_op_workq, + &backend_csf->info->hwc_dump_reenable_work); + /* Exit the backend state handling. */ + break; + } + + /* At this point we are sure that the previous disable was ACKed by FW, + * so we can proceed directly to the enable REQ. + */ + if (kbasep_hwcnt_backend_csf_do_dump_enable(backend_csf, + &backend_csf->last_enable_map)) + kbasep_hwcnt_backend_csf_handle_unrecoverable_error(backend_csf); + break; + case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_ACK: + case KBASE_HWCNT_BACKEND_CSF_ENABLED: + /* Resume the timer. */ + csf_info->watchdog_if->modify(csf_info->watchdog_if->timer, + csf_info->watchdog_timer_interval_ms); + break; + case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_REQ: + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( + backend_csf, + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK); + kbasep_hwcnt_backend_csf_send_disable_req(backend_csf->info); + + break; + case KBASE_HWCNT_BACKEND_CSF_DISABLED: + /* Here we handle a case where the backend was recreated (but not yet + * desired to be enabled) while MCU was off. If the disable REQ + * never reached MCU, we need to apply it now to align the HWC state + * in MCU and in the kbase. + * The disable ACK should be ignored. + */ + if (backend_csf->info->pending_disable_req) + kbasep_hwcnt_backend_csf_send_disable_req(backend_csf->info); + + break; + default: + /* Do nothing. */ + break; + } + + /* Notify about the MCU ON event. */ + wake_up(&backend_csf->enable_state_waitq); + + } else { + /* Backend has been already destroyed. */ + if (csf_info->pending_disable_req) { + /* Finish the backend destruction by sending the disable REQ to FW. */ + kbasep_hwcnt_backend_csf_send_disable_req(csf_info); + } + } + + csf_info->csf_if->unlock(csf_info->csf_if->ctx, flags); +} + +void kbase_hwcnt_backend_csf_on_before_mcu_cold_boot(struct kbase_hwcnt_backend_interface *iface) +{ + struct kbase_hwcnt_backend_csf_info *csf_info; + struct kbase_hwcnt_backend_csf *backend_csf; + unsigned long flags = 0UL; + + csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; + csf_info->csf_if->lock(csf_info->csf_if->ctx, &flags); + + WARN_ON(csf_info->mcu_on); + + /* Reset the internal variables to match the cold-booted FW state.*/ + csf_info->pending_disable_req = false; + + if (kbasep_hwcnt_backend_csf_backend_exists(csf_info)) { + bool do_reenable = false; + + backend_csf = (struct kbase_hwcnt_backend_csf *)csf_info->backend; + + if (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_ENABLED || + kbasep_hwcnt_backend_csf_is_state_trans_to_enabled(backend_csf->enable_state)) { + /* If the backend was enabled (or was transitioning to) prior to the + * cold boot, we need to reenable it so it can continue to process + * dump requests after the boot. + */ + do_reenable = true; + + /* Disable watchdog in a non-blocking manner. Pending watchdog callback will + * safely early-exit when called. + * + * Exclude TRANSITIONING_TO_ENABLED_PENDING_REQ state because watchdog + * is disabled in this state. + */ + if (backend_csf->enable_state != + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_REQ) + backend_csf->info->watchdog_if->disable( + backend_csf->info->watchdog_if->timer, + KBASE_HWCNT_WATCHDOG_DISABLE_SHOULD_NOT_BLOCK); + } + + backend_csf->flush_workers_needed = false; + /* Cancel any reenable work in progress. */ + backend_csf->dump_reenable_in_progress = false; + /* Set backend state to DISABLED and cancel any pending dump work + * by resetting dump-related variables. + * + * Do not leave an error state. A dump_disable and a subsequent reset is the only + * way to leave the error state. + */ + if (!kbasep_hwcnt_backend_csf_is_error_state(backend_csf->enable_state)) { + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( + backend_csf, KBASE_HWCNT_BACKEND_CSF_DISABLED); + backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE; + complete_all(&backend_csf->dump_completed); + } + + if (WARN_ON(backend_csf->info->ring_buf == NULL)) + goto unlock; + + /* Keep using the allocated ring buffer. Reset its refcount to 1. */ + csf_info->ring_buf_refcnt = (struct kbasep_hwcnt_ring_buf_refcnt){ .val = 1 }; + /* Zero all performance enable header to prepare for first enable + * after a cold boot. + */ + kbasep_hwcnt_backend_csf_zero_all_prfcnt_en_header(backend_csf); + /* Sync zeroed buffers to avoid coherency issues on future use. */ + backend_csf->info->csf_if->ring_buf_sync(backend_csf->info->csf_if->ctx, + backend_csf->info->ring_buf, 0, + backend_csf->info->ring_buf_cnt, false); + + if (do_reenable) { + int errcode; + + errcode = kbasep_hwcnt_backend_csf_dump_enable_nolock( + (struct kbase_hwcnt_backend *)backend_csf, + &backend_csf->last_enable_map); + if (errcode) { + kbasep_hwcnt_backend_csf_handle_unrecoverable_error(backend_csf); + goto unlock; + } + } + } else { + csf_info->ring_buf_refcnt = (struct kbasep_hwcnt_ring_buf_refcnt){ 0 }; + if (csf_info->ring_buf) { + /* It's safe to ignore ring buffer's refcounter because HWC is disabled + * during the cold boot. + */ + kbasep_hwcnt_backend_csf_ring_buf_free_deferred(csf_info); + } + } + +unlock: + csf_info->csf_if->unlock(csf_info->csf_if->ctx, flags); +} + int kbase_hwcnt_backend_csf_metadata_init(struct kbase_hwcnt_backend_interface *iface) { struct kbase_hwcnt_backend_csf_info *csf_info; @@ -2268,6 +3182,29 @@ void kbase_hwcnt_backend_csf_metadata_term(struct kbase_hwcnt_backend_interface } } +void kbase_hwcnt_backend_csf_ring_buf_term(struct kbase_hwcnt_backend_interface *iface) +{ + struct kbase_hwcnt_backend_csf_info *csf_info; + + if (!iface) + return; + + csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; + + /* Wait for any ring_buf_free worker to finish. */ + flush_workqueue(csf_info->hwc_deferred_op_workq); + + /* Free ring buffer, if not yet freed. + * It's safe to ignore ring buffer's refcounter because firmware is being terminated, + * hence MCU will not be booted again. + * Needs to be done before firmware is terminated. + */ + if (csf_info->ring_buf != NULL) { + csf_info->csf_if->ring_buf_free(csf_info->csf_if->ctx, csf_info->ring_buf); + csf_info->ring_buf = NULL; + } +} + int kbase_hwcnt_backend_csf_create(struct kbase_hwcnt_backend_csf_if *csf_if, u32 ring_buf_cnt, struct kbase_hwcnt_watchdog_interface *watchdog_if, struct kbase_hwcnt_backend_interface *iface, diff --git a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf.h b/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf.h index 171b28deb6bc..5e3b96fa3d61 100644 --- a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf.h +++ b/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf.h @@ -31,10 +31,35 @@ #include "hwcnt/backend/mali_kbase_hwcnt_backend_csf_if.h" #include "hwcnt/mali_kbase_hwcnt_watchdog_if.h" #include "hwcnt/mali_kbase_hwcnt_types.h" +#include struct kbase_hwcnt_physical_enable_map; struct kbase_hwcnt_backend_csf; +/** + * enum kbase_hwcnt_backend_sample_reason - HWC CSF metadata block sample + * reasons. + * + * @SAMPLE_REASON_NONE: No metadata was enabled or available. + * @SAMPLE_REASON_BEFORE_PROTM: The sample was taken just before the GPU enters protected mode. + * @SAMPLE_REASON_BEFORE_HALT: The sample was taken just before the GPU enters the HALT state. + * @SAMPLE_REASON_BEFORE_SLEEP: The sample was taken just before the GPU enters the SLEEP state. + * @SAMPLE_REASON_BEFORE_SUSPEND: The sample was taken just before the GPU enters the SUSPEND + * state. + * @SAMPLE_REASON_BEFORE_YIELD: The sample was taken just before GPU subinstance yields the + * access window. + * @SAMPLE_REASON_AFTER_WARM_BOOT: The sample taken after GPU subinstance completed a warm boot. + */ +enum kbase_hwcnt_backend_sample_reason { + SAMPLE_REASON_NONE, + SAMPLE_REASON_BEFORE_PROTM, + SAMPLE_REASON_BEFORE_HALT, + SAMPLE_REASON_BEFORE_SLEEP, + SAMPLE_REASON_BEFORE_SUSPEND, + SAMPLE_REASON_BEFORE_YIELD, + SAMPLE_REASON_AFTER_WARM_BOOT +}; + /** * kbase_hwcnt_backend_csf_create() - Create a CSF hardware counter backend * interface, with set timer interval. @@ -72,6 +97,18 @@ int kbase_hwcnt_backend_csf_metadata_init(struct kbase_hwcnt_backend_interface * */ void kbase_hwcnt_backend_csf_metadata_term(struct kbase_hwcnt_backend_interface *iface); +/** + * kbase_hwcnt_backend_csf_ring_buf_term() - Terminate the ring buffers for a CSF + * hardware counter backend. Must be called while + * FW is still loaded. + * + * @iface: Non-NULL pointer to backend interface structure. + * + * If the ring buffer was not freed while HWC backend was present + * (e.g. due to MCU being powered off), this function will free it. + */ +void kbase_hwcnt_backend_csf_ring_buf_term(struct kbase_hwcnt_backend_interface *iface); + /** * kbase_hwcnt_backend_csf_destroy() - Destroy a CSF hardware counter backend * interface. @@ -123,8 +160,8 @@ void kbase_hwcnt_backend_csf_on_before_reset(struct kbase_hwcnt_backend_interfac /** * kbase_hwcnt_backend_csf_set_hw_availability() - CSF HWC backend function to * set current HW configuration. - * HWC must be disabled before - * this function is called. + * Can only be called when the MCU is off. + * * @iface: Non-NULL pointer to HWC backend interface. * @num_l2_slices: Current number of L2 slices allocated to the GPU. * @shader_present: Shader_present of the current configuration. @@ -179,21 +216,61 @@ void kbase_hwcnt_backend_csf_on_prfcnt_enable(struct kbase_hwcnt_backend_interfa */ void kbase_hwcnt_backend_csf_on_prfcnt_disable(struct kbase_hwcnt_backend_interface *iface); +/** + * kbase_hwcnt_backend_csf_on_after_mcu_off() - CSF HWC backend function to be called immediately + * after the MCU shut down process is completed, + * informing the backend that it is no longer valid + * to send commands to the MCU and that any + * outstanding commands will not be ACKed. + * @iface: Non-NULL pointer to HWC backend interface. + */ +void kbase_hwcnt_backend_csf_on_after_mcu_off(struct kbase_hwcnt_backend_interface *iface); + +/** + * kbase_hwcnt_backend_csf_on_after_mcu_off_reset() - Similar to + * kbase_hwcnt_backend_csf_on_after_mcu_off(). + * To be called only after a completed reset. + * This function will notify backend about MCU + * being powered off only if it hasn't been + * done yet. This is due to the fact that it's + * uncertain if the MCU_OFF notification was + * raised prior to the reset. + * @iface: Non-NULL pointer to HWC backend interface. + */ +void kbase_hwcnt_backend_csf_on_after_mcu_off_reset(struct kbase_hwcnt_backend_interface *iface); + +/** + * kbase_hwcnt_backend_csf_on_after_mcu_on() - CSF HWC backend function to be called immediately + * after the MCU has booted, informing the backend + * that it is now valid to send commands to the MCU. + * @iface: Non-NULL pointer to HWC backend interface. + */ +void kbase_hwcnt_backend_csf_on_after_mcu_on(struct kbase_hwcnt_backend_interface *iface); + +/** + * kbase_hwcnt_backend_csf_on_before_mcu_cold_boot() - CSF HWC backend function to be called + * immediately before a cold MCU boot, + * allowing the backend to reset its internal + * state machine to match the cold-booted + * FW state. + * @iface: Non-NULL pointer to HWC backend interface. + */ +void kbase_hwcnt_backend_csf_on_before_mcu_cold_boot(struct kbase_hwcnt_backend_interface *iface); + /** * kbasep_hwcnt_backend_csf_update_block_state - Update block state of a block instance with * information from a sample. * @backend: CSF hardware counter backend. * @enable_mask: Counter enable mask for the block whose state is being updated. - * @exiting_protm: Whether or not the sample is taken when the GPU is exiting - * protected mode. - * @block_idx: Index of block within the ringbuffer. + * @block_idx: Index of the block in the dump. * @block_state: Pointer to existing block state of the block whose state is being * updated. - * @fw_in_protected_mode: Whether or not GPU is in protected mode during sampling. + * @prev_sample_reason: The previous sample reason + * @curr_sample_reason: The current sample reason */ -void kbasep_hwcnt_backend_csf_update_block_state(struct kbase_hwcnt_backend_csf *backend, - const u32 enable_mask, bool exiting_protm, - size_t block_idx, blk_stt_t *const block_state, - bool fw_in_protected_mode); +void kbasep_hwcnt_backend_csf_update_block_state( + struct kbase_hwcnt_backend_csf *backend, const u32 enable_mask, size_t block_idx, + blk_stt_t *const block_state, enum kbase_hwcnt_backend_sample_reason prev_sample_reason, + enum kbase_hwcnt_backend_sample_reason curr_sample_reason); #endif /* _KBASE_HWCNT_BACKEND_CSF_H_ */ diff --git a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if.h b/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if.h index 220a63de779a..f8ba2acb4b67 100644 --- a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if.h +++ b/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if.h @@ -61,15 +61,16 @@ struct kbase_hwcnt_backend_csf_if_enable { /** * struct kbase_hwcnt_backend_csf_if_prfcnt_info - Performance counter * information. - * @prfcnt_hw_size: Total length in bytes of all the hardware counters data. The hardware - * counters are sub-divided into 4 classes: front-end, shader, tiler, and - * memory system (l2 cache + MMU). - * @prfcnt_fw_size: Total length in bytes of all the firmware counters data. - * @dump_bytes: Bytes of GPU memory required to perform a performance - * counter dump. dump_bytes = prfcnt_hw_size + prfcnt_fw_size. - * @prfcnt_block_size: Bytes of each performance counter block. - * @l2_count: The MMU L2 cache count. - * @csg_count: The total number of CSGs in the system + * @prfcnt_hw_size: Total length in bytes of all the hardware counters data. The hardware + * counters are sub-divided into 4 classes: front-end, shader, tiler, and + * memory system (l2 cache + MMU). + * @prfcnt_fw_size: Total length in bytes of all the firmware counters data. + * @metadata_size: Total length in bytes of the dedicated metadata block. + * @dump_bytes: Bytes of GPU memory required to perform a performance counter dump. + * dump_bytes = prfcnt_metadata_size + prfcnt_hw_size + prfcnt_fw_size. + * @prfcnt_block_size: Bytes of each performance counter block. + * @l2_count: The MMU L2 cache count. + * @csg_count: The total number of CSGs in the system * @sc_core_mask: Shader core mask. * @clk_cnt: Clock domain count in the system. * @clearing_samples: Indicates whether counters are cleared after each sample @@ -81,6 +82,7 @@ struct kbase_hwcnt_backend_csf_if_enable { struct kbase_hwcnt_backend_csf_if_prfcnt_info { size_t prfcnt_hw_size; size_t prfcnt_fw_size; + size_t metadata_size; size_t dump_bytes; size_t prfcnt_block_size; size_t l2_count; diff --git a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.c b/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.c index 15f5ff4987f0..93d6f0d656d4 100644 --- a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.c +++ b/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.c @@ -303,6 +303,7 @@ static void kbasep_hwcnt_backend_csf_if_fw_get_prfcnt_info( KBASE_DUMMY_MODEL_MAX_NUM_HARDWARE_BLOCKS * KBASE_DUMMY_MODEL_BLOCK_SIZE, .prfcnt_fw_size = KBASE_DUMMY_MODEL_MAX_FIRMWARE_BLOCKS * KBASE_DUMMY_MODEL_BLOCK_SIZE, + .metadata_size = 0, .dump_bytes = KBASE_DUMMY_MODEL_MAX_SAMPLE_SIZE, .prfcnt_block_size = KBASE_DUMMY_MODEL_BLOCK_SIZE, .clk_cnt = 1, @@ -317,6 +318,8 @@ static void kbasep_hwcnt_backend_csf_if_fw_get_prfcnt_info( u32 prfcnt_size; u32 prfcnt_hw_size; u32 prfcnt_fw_size; + u32 prfcnt_features; + u32 metadata_size; u32 csg_count; u32 fw_block_count = 0; u32 prfcnt_block_size = @@ -330,8 +333,10 @@ static void kbasep_hwcnt_backend_csf_if_fw_get_prfcnt_info( kbdev = fw_ctx->kbdev; csg_count = kbdev->csf.global_iface.group_num; prfcnt_size = kbdev->csf.global_iface.prfcnt_size; + prfcnt_features = kbdev->csf.global_iface.prfcnt_features; prfcnt_hw_size = GLB_PRFCNT_SIZE_HARDWARE_SIZE_GET(prfcnt_size); prfcnt_fw_size = GLB_PRFCNT_SIZE_FIRMWARE_SIZE_GET(prfcnt_size); + metadata_size = GLB_PRFCNT_FEATURES_METADATA_SIZE_GET(prfcnt_features); has_virtual_core_ids = kbdev->gpu_props.gpu_id.arch_id >= GPU_ID_ARCH_MAKE(14, 8, 4); /* Read the block size if the GPU has the register PRFCNT_FEATURES @@ -355,10 +360,11 @@ static void kbasep_hwcnt_backend_csf_if_fw_get_prfcnt_info( else WARN_ON_ONCE(true); - fw_ctx->buf_bytes = prfcnt_hw_size + prfcnt_fw_size; + fw_ctx->buf_bytes = prfcnt_hw_size + prfcnt_fw_size + metadata_size; *prfcnt_info = (struct kbase_hwcnt_backend_csf_if_prfcnt_info){ .prfcnt_hw_size = prfcnt_hw_size, .prfcnt_fw_size = prfcnt_fw_size, + .metadata_size = metadata_size, .dump_bytes = fw_ctx->buf_bytes, .prfcnt_block_size = prfcnt_block_size, .l2_count = kbdev->gpu_props.num_l2_slices, @@ -380,10 +386,17 @@ static void kbasep_hwcnt_backend_csf_if_fw_get_prfcnt_info( else WARN_ON(prfcnt_info->ne_core_mask != 0); + WARN((metadata_size % prfcnt_info->prfcnt_block_size) != 0, + "Metadata block size is not aligned to block size (metadata size %u, block size %zu)", + metadata_size, prfcnt_info->prfcnt_block_size); /* Block size must be multiple of counter size. */ - WARN_ON((prfcnt_info->prfcnt_block_size % KBASE_HWCNT_VALUE_HW_BYTES) != 0); + WARN((prfcnt_info->prfcnt_block_size % KBASE_HWCNT_VALUE_HW_BYTES) != 0, + "Block size is not a multiple of counter size (block size %zu, counter size %lu)", + prfcnt_info->prfcnt_block_size, (unsigned long)KBASE_HWCNT_VALUE_HW_BYTES); /* Total size must be multiple of block size. */ - WARN_ON((prfcnt_info->dump_bytes % prfcnt_info->prfcnt_block_size) != 0); + WARN((prfcnt_info->dump_bytes % prfcnt_info->prfcnt_block_size) != 0, + "Total size should be a multiple of block size (total size %zu, block size %zu)", + prfcnt_info->dump_bytes, prfcnt_info->prfcnt_block_size); #endif } @@ -440,8 +453,7 @@ static int kbasep_hwcnt_backend_csf_if_fw_ring_buf_alloc( goto page_list_alloc_error; /* Get physical page for the buffer */ - ret = kbase_mem_pool_alloc_pages(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], num_pages, - phys, false, NULL); + ret = kbase_mem_pool_alloc_pages(&kbdev->fw_mem_pools.small, num_pages, phys, false, NULL); if ((size_t)ret != num_pages) goto phys_mem_pool_alloc_error; @@ -484,8 +496,7 @@ static int kbasep_hwcnt_backend_csf_if_fw_ring_buf_alloc( mmu_insert_failed: vunmap(cpu_addr); vmap_error: - kbase_mem_pool_free_pages(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], num_pages, phys, - false, false); + kbase_mem_pool_free_pages(&kbdev->fw_mem_pools.small, num_pages, phys, false, false); phys_mem_pool_alloc_error: kfree(page_list); page_list_alloc_error: @@ -604,7 +615,7 @@ kbasep_hwcnt_backend_csf_if_fw_ring_buf_free(struct kbase_hwcnt_backend_csf_if_c /* After zeroing, the ring_buf pages are dirty so need to pass the 'dirty' flag * as true when freeing the pages to the Global pool. */ - kbase_mem_pool_free_pages(&fw_ctx->kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], + kbase_mem_pool_free_pages(&fw_ctx->kbdev->fw_mem_pools.small, fw_ring_buf->num_pages, fw_ring_buf->phys, true, false); kfree(fw_ring_buf->phys); @@ -640,6 +651,7 @@ kbasep_hwcnt_backend_csf_if_fw_dump_enable(struct kbase_hwcnt_backend_csf_if_ctx /* Configure */ prfcnt_config = GLB_PRFCNT_CONFIG_SIZE_SET(0, fw_ring_buf->buf_count); prfcnt_config = GLB_PRFCNT_CONFIG_SET_SELECT_SET(prfcnt_config, enable->counter_set); + prfcnt_config = GLB_PRFCNT_CONFIG_METADATA_ENABLE_SET(prfcnt_config, 1); /* Configure the ring buffer base address */ kbase_csf_fw_io_global_write(&kbdev->csf.fw_io, GLB_PRFCNT_JASID, fw_ring_buf->as_nr); @@ -808,15 +820,27 @@ kbasep_hwcnt_backend_csf_if_fw_get_gpu_cycle_count(struct kbase_hwcnt_backend_cs if (!(clk_enable_map & (1ull << clk))) continue; - if (clk == KBASE_CLOCK_DOMAIN_TOP) { + /* Collect cycle count from GPU register. + * If GPU is unavailable, fall back to a SW estimation. + */ + if (clk == KBASE_CLOCK_DOMAIN_TOP && kbase_io_has_gpu(fw_ctx->kbdev)) { /* Read cycle count for top clock domain. */ kbase_backend_get_gpu_time_norequest(fw_ctx->kbdev, &cycle_counts[clk], NULL, NULL); - } else { - /* Estimate cycle count for non-top clock domain. */ - cycle_counts[clk] = - kbase_ccswe_cycle_at(&fw_ctx->ccswe_shader_cores, timestamp_ns); + + /* Check again if the device is lost. + * If we still have GPU, then the value is correct and we can continue with + * the next loop iteration. + * If it's lost, we can't guarantee we have read good data out + * from CYCLE_COUNT, so fall back to the SW estimated cycle count. + */ + if (kbase_io_has_gpu(fw_ctx->kbdev)) + continue; } + /* Estimate cycle count for non-top clock domain or for the top clock domain while + * the device is lost. + */ + cycle_counts[clk] = kbase_ccswe_cycle_at(&fw_ctx->ccswe_shader_cores, timestamp_ns); } } diff --git a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_jm.c b/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_jm.c deleted file mode 100644 index 3467c5fd4d8b..000000000000 --- a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_jm.c +++ /dev/null @@ -1,981 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include "hwcnt/backend/mali_kbase_hwcnt_backend_jm.h" -#include "hwcnt/mali_kbase_hwcnt_gpu.h" -#include "hwcnt/mali_kbase_hwcnt_types.h" -#include "mali_kbase.h" -#include "mali_kbase_mem_flags.h" -#include "backend/gpu/mali_kbase_pm_ca.h" -#include "mali_kbase_hwaccess_instr.h" -#include "mali_kbase_hwaccess_time.h" -#include "mali_kbase_ccswe.h" -#include "backend/gpu/mali_kbase_model_linux.h" -#include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h" - -#include "backend/gpu/mali_kbase_pm_internal.h" - -/** - * struct kbase_hwcnt_backend_jm_info - Information used to create an instance - * of a JM hardware counter backend. - * @kbdev: KBase device. - * @counter_set: The performance counter set to use. - * @metadata: Hardware counter metadata. - * @dump_bytes: Bytes of GPU memory required to perform a - * hardware counter dump. - * @hwcnt_gpu_info: Hardware counter block information. - */ -struct kbase_hwcnt_backend_jm_info { - struct kbase_device *kbdev; - enum kbase_hwcnt_set counter_set; - const struct kbase_hwcnt_metadata *metadata; - size_t dump_bytes; - struct kbase_hwcnt_gpu_info hwcnt_gpu_info; -}; - -/** - * struct kbase_hwcnt_jm_physical_layout - HWC sample memory physical layout - * information. - * @fe_cnt: Front end block count. - * @tiler_cnt: Tiler block count. - * @mmu_l2_cnt: Memory system(MMU and L2 cache) block count. - * @shader_cnt: Shader Core block count. - * @block_cnt: Total block count (sum of all other block counts). - * @shader_avail_mask: Bitmap of all shader cores in the system. - * @enable_mask_offset: Offset in array elements of enable mask in each block - * starting from the beginning of block. - * @headers_per_block: Header size per block. - * @counters_per_block: Counters size per block. - * @values_per_block: Total size per block. - */ -struct kbase_hwcnt_jm_physical_layout { - u8 fe_cnt; - u8 tiler_cnt; - u8 mmu_l2_cnt; - u8 shader_cnt; - u8 block_cnt; - u64 shader_avail_mask; - size_t enable_mask_offset; - size_t headers_per_block; - size_t counters_per_block; - size_t values_per_block; -}; - -/** - * struct kbase_hwcnt_backend_jm - Instance of a JM hardware counter backend. - * @info: Info used to create the backend. - * @kctx: KBase context used for GPU memory allocation and - * counter dumping. - * @gpu_dump_va: GPU hardware counter dump buffer virtual address. - * @cpu_dump_va: CPU mapping of gpu_dump_va. - * @vmap: Dump buffer vmap. - * @to_user_buf: HWC sample buffer for client user, size - * metadata.dump_buf_bytes. - * @to_user_block_states: HWC sample block states for client user. - * @enabled: True if dumping has been enabled, else false. - * @debug_core_mask: User-set mask of shader cores that can be used. - * @pm_core_mask: PM state sync-ed shaders core mask for the enabled - * dumping. - * @curr_config: Current allocated hardware resources to correctly map the - * source raw dump buffer to the destination dump buffer. - * @max_core_mask: Core mask of all cores allocated to the GPU (non - * virtualized platforms) or resource group (virtualized - * platforms). - * @max_l2_slices: Maximum number of L2 slices allocated to the GPU (non - * virtualized platforms) or resource group (virtualized - * platforms). - * @clk_enable_map: The enable map specifying enabled clock domains. - * @cycle_count_elapsed: Cycle count elapsed for a given sample period. - * The top clock cycle, index 0, is read directly from - * hardware, but the other clock domains need to be - * calculated with software estimation. - * @prev_cycle_count: Previous cycle count to calculate the cycle count for - * sample period. - * @rate_listener: Clock rate listener callback state. - * @ccswe_shader_cores: Shader cores cycle count software estimator. - * @phys_layout: Physical memory layout information of HWC sample buffer. - * @dump_time_ns: Holds the CPU timestamp captured at the time of - * dump_request(). - */ -struct kbase_hwcnt_backend_jm { - const struct kbase_hwcnt_backend_jm_info *info; - struct kbase_context *kctx; - u64 gpu_dump_va; - void *cpu_dump_va; - struct kbase_vmap_struct *vmap; - u64 *to_user_buf; - blk_stt_t *to_user_block_states; - bool enabled; - u64 debug_core_mask; - u64 pm_core_mask; - struct kbase_hwcnt_curr_config curr_config; - u64 max_core_mask; - size_t max_l2_slices; - u64 clk_enable_map; - u64 cycle_count_elapsed[BASE_MAX_NR_CLOCKS_REGULATORS]; - u64 prev_cycle_count[BASE_MAX_NR_CLOCKS_REGULATORS]; - struct kbase_clk_rate_listener rate_listener; - struct kbase_ccswe ccswe_shader_cores; - struct kbase_hwcnt_jm_physical_layout phys_layout; - u64 dump_time_ns; -}; - -/** - * kbasep_hwcnt_backend_jm_gpu_info_init() - Initialise an info structure used - * to create the hwcnt metadata. - * @kbdev: Non-NULL pointer to kbase device. - * @info: Non-NULL pointer to data structure to be filled in. - * - * The initialised info struct will only be valid for use while kbdev is valid. - * - * Return: 0 on success, else error code. - */ -static int kbasep_hwcnt_backend_jm_gpu_info_init(struct kbase_device *kbdev, - struct kbase_hwcnt_gpu_info *info) -{ - size_t clk, l2_count, core_mask; - - if (!kbdev || !info) - return -EINVAL; - -#if IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI) - l2_count = KBASE_DUMMY_MODEL_MAX_MEMSYS_BLOCKS; - core_mask = (1ull << KBASE_DUMMY_MODEL_MAX_SHADER_CORES) - 1; -#else - l2_count = kbdev->gpu_props.num_l2_slices; - core_mask = kbdev->gpu_props.coherency_info.group.core_mask; -#endif - - info->l2_count = l2_count; - info->sc_core_mask = core_mask; - info->prfcnt_values_per_block = KBASE_HWCNT_V5_DEFAULT_VALUES_PER_BLOCK; - - /* Determine the number of available clock domains. */ - for (clk = 0; clk < BASE_MAX_NR_CLOCKS_REGULATORS; clk++) { - if (kbdev->pm.clk_rtm.clks[clk] == NULL) - break; - } - info->clk_cnt = clk; - - return 0; -} - -static void kbasep_hwcnt_backend_jm_init_layout(const struct kbase_hwcnt_gpu_info *gpu_info, - struct kbase_hwcnt_jm_physical_layout *phys_layout) -{ - u8 shader_core_cnt; - - WARN_ON(!gpu_info); - WARN_ON(!phys_layout); - - shader_core_cnt = fls64(gpu_info->sc_core_mask); - - *phys_layout = (struct kbase_hwcnt_jm_physical_layout){ - .fe_cnt = KBASE_HWCNT_V5_FE_BLOCK_COUNT, - .tiler_cnt = KBASE_HWCNT_V5_TILER_BLOCK_COUNT, - .mmu_l2_cnt = gpu_info->l2_count, - .shader_cnt = shader_core_cnt, - .block_cnt = KBASE_HWCNT_V5_FE_BLOCK_COUNT + KBASE_HWCNT_V5_TILER_BLOCK_COUNT + - gpu_info->l2_count + shader_core_cnt, - .shader_avail_mask = gpu_info->sc_core_mask, - .headers_per_block = KBASE_HWCNT_V5_HEADERS_PER_BLOCK, - .values_per_block = gpu_info->prfcnt_values_per_block, - .counters_per_block = - gpu_info->prfcnt_values_per_block - KBASE_HWCNT_V5_HEADERS_PER_BLOCK, - .enable_mask_offset = KBASE_HWCNT_V5_PRFCNT_EN_HEADER, - }; -} - -static void -kbasep_hwcnt_backend_jm_dump_sample(const struct kbase_hwcnt_backend_jm *const backend_jm) -{ - size_t block_idx; - const u32 *new_sample_buf = backend_jm->cpu_dump_va; - const u32 *new_block = new_sample_buf; - u64 *dst_buf = backend_jm->to_user_buf; - u64 *dst_block = dst_buf; - const size_t values_per_block = backend_jm->phys_layout.values_per_block; - const size_t dump_bytes = backend_jm->info->dump_bytes; - - for (block_idx = 0; block_idx < backend_jm->phys_layout.block_cnt; block_idx++) { - size_t ctr_idx; - - for (ctr_idx = 0; ctr_idx < values_per_block; ctr_idx++) - dst_block[ctr_idx] = new_block[ctr_idx]; - - new_block += values_per_block; - dst_block += values_per_block; - } - - WARN_ON(new_block != new_sample_buf + (dump_bytes / KBASE_HWCNT_VALUE_HW_BYTES)); - WARN_ON(dst_block != dst_buf + (dump_bytes / KBASE_HWCNT_VALUE_HW_BYTES)); -} - -/** - * kbasep_hwcnt_backend_jm_on_freq_change() - On freq change callback - * - * @rate_listener: Callback state - * @clk_index: Clock index - * @clk_rate_hz: Clock frequency(hz) - */ -static void kbasep_hwcnt_backend_jm_on_freq_change(struct kbase_clk_rate_listener *rate_listener, - u32 clk_index, u32 clk_rate_hz) -{ - struct kbase_hwcnt_backend_jm *backend_jm = - container_of(rate_listener, struct kbase_hwcnt_backend_jm, rate_listener); - u64 timestamp_ns; - - if (clk_index != KBASE_CLOCK_DOMAIN_SHADER_CORES) - return; - - timestamp_ns = ktime_get_raw_ns(); - kbase_ccswe_freq_change(&backend_jm->ccswe_shader_cores, timestamp_ns, clk_rate_hz); -} - -/** - * kbasep_hwcnt_backend_jm_cc_enable() - Enable cycle count tracking - * - * @backend_jm: Non-NULL pointer to backend. - * @enable_map: Non-NULL pointer to enable map specifying enabled counters. - * @timestamp_ns: Timestamp(ns) when HWCNT were enabled. - */ -static void kbasep_hwcnt_backend_jm_cc_enable(struct kbase_hwcnt_backend_jm *backend_jm, - const struct kbase_hwcnt_enable_map *enable_map, - u64 timestamp_ns) -{ - struct kbase_device *kbdev = backend_jm->kctx->kbdev; - u64 clk_enable_map = enable_map->clk_enable_map; - u64 cycle_count; - - if (kbase_hwcnt_clk_enable_map_enabled(clk_enable_map, KBASE_CLOCK_DOMAIN_TOP)) { - /* turn on the cycle counter */ - kbase_pm_request_gpu_cycle_counter_l2_is_on(kbdev); - /* Read cycle count for top clock domain. */ - kbase_backend_get_gpu_time_norequest(kbdev, &cycle_count, NULL, NULL); - - backend_jm->prev_cycle_count[KBASE_CLOCK_DOMAIN_TOP] = cycle_count; - } - - if (kbase_hwcnt_clk_enable_map_enabled(clk_enable_map, KBASE_CLOCK_DOMAIN_SHADER_CORES)) { - /* software estimation for non-top clock domains */ - struct kbase_clk_rate_trace_manager *rtm = &kbdev->pm.clk_rtm; - const struct kbase_clk_data *clk_data = rtm->clks[KBASE_CLOCK_DOMAIN_SHADER_CORES]; - u32 cur_freq; - unsigned long flags; - - spin_lock_irqsave(&rtm->lock, flags); - - cur_freq = (u32)clk_data->clock_val; - kbase_ccswe_reset(&backend_jm->ccswe_shader_cores); - kbase_ccswe_freq_change(&backend_jm->ccswe_shader_cores, timestamp_ns, cur_freq); - - kbase_clk_rate_trace_manager_subscribe_no_lock(rtm, &backend_jm->rate_listener); - - spin_unlock_irqrestore(&rtm->lock, flags); - - /* ccswe was reset. The estimated cycle is zero. */ - backend_jm->prev_cycle_count[KBASE_CLOCK_DOMAIN_SHADER_CORES] = 0; - } - - /* Keep clk_enable_map for dump_request. */ - backend_jm->clk_enable_map = clk_enable_map; -} - -/** - * kbasep_hwcnt_backend_jm_cc_disable() - Disable cycle count tracking - * - * @backend_jm: Non-NULL pointer to backend. - */ -static void kbasep_hwcnt_backend_jm_cc_disable(struct kbase_hwcnt_backend_jm *backend_jm) -{ - struct kbase_device *kbdev = backend_jm->kctx->kbdev; - struct kbase_clk_rate_trace_manager *rtm = &kbdev->pm.clk_rtm; - u64 clk_enable_map = backend_jm->clk_enable_map; - - if (kbase_hwcnt_clk_enable_map_enabled(clk_enable_map, KBASE_CLOCK_DOMAIN_TOP)) { - /* turn off the cycle counter */ - kbase_pm_release_gpu_cycle_counter(kbdev); - } - - if (kbase_hwcnt_clk_enable_map_enabled(clk_enable_map, KBASE_CLOCK_DOMAIN_SHADER_CORES)) { - kbase_clk_rate_trace_manager_unsubscribe(rtm, &backend_jm->rate_listener); - } -} - -/** - * kbasep_hwcnt_gpu_update_curr_config() - Update the destination buffer with - * current config information. - * @kbdev: Non-NULL pointer to kbase device. - * @curr_config: Non-NULL pointer to return the current configuration of - * hardware allocated to the GPU. - * - * The current configuration information is used for architectures where the - * max_config interface is available from the Arbiter. In this case the current - * allocated hardware is not always the same, so the current config information - * is used to correctly map the current allocated resources to the memory layout - * that is copied to the user space. - * - * Return: 0 on success, else error code. - */ -static int kbasep_hwcnt_gpu_update_curr_config(struct kbase_device *kbdev, - struct kbase_hwcnt_curr_config *curr_config) -{ - if (WARN_ON(!kbdev) || WARN_ON(!curr_config)) - return -EINVAL; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - curr_config->num_l2_slices = kbdev->gpu_props.curr_config.l2_slices; - curr_config->shader_present = kbdev->gpu_props.curr_config.shader_present; - return 0; -} - -/* JM backend implementation of kbase_hwcnt_backend_timestamp_ns_fn */ -static u64 kbasep_hwcnt_backend_jm_timestamp_ns(struct kbase_hwcnt_backend *backend) -{ - (void)backend; - return ktime_get_raw_ns(); -} - -static void kbasep_hwcnt_backend_jm_append_block_states(struct kbase_hwcnt_backend_jm *backend_jm, - blk_stt_t block_state) -{ - size_t i; - - for (i = 0; i < backend_jm->phys_layout.block_cnt; i++) - kbase_hwcnt_block_state_append(&backend_jm->to_user_block_states[i], block_state); -} - -static void -kbasep_hwcnt_backend_jm_get_physical_enable(struct kbase_hwcnt_backend_jm *backend_jm, - const struct kbase_hwcnt_enable_map *enable_map, - struct kbase_instr_hwcnt_enable *enable) -{ - enum kbase_hwcnt_physical_set phys_counter_set; - struct kbase_hwcnt_physical_enable_map phys_enable_map = { 0 }; - - kbase_hwcnt_gpu_enable_map_to_physical(&phys_enable_map, enable_map); - - kbase_hwcnt_gpu_set_to_physical(&phys_counter_set, backend_jm->info->counter_set); - - *enable = (struct kbase_instr_hwcnt_enable){ - .fe_bm = phys_enable_map.fe_bm, - .shader_bm = phys_enable_map.shader_bm, - .tiler_bm = phys_enable_map.tiler_bm, - .mmu_l2_bm = phys_enable_map.mmu_l2_bm, - .counter_set = phys_counter_set, - .dump_buffer = backend_jm->gpu_dump_va, - .dump_buffer_bytes = backend_jm->info->dump_bytes, - }; - - /* The dummy model needs the CPU mapping. */ - if (IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI)) - enable->dump_buffer = (uintptr_t)backend_jm->cpu_dump_va; -} - -/* JM backend implementation of kbase_hwcnt_backend_dump_enable_nolock_fn */ -static int -kbasep_hwcnt_backend_jm_dump_enable_nolock(struct kbase_hwcnt_backend *backend, - const struct kbase_hwcnt_enable_map *enable_map) -{ - int errcode; - struct kbase_hwcnt_backend_jm *backend_jm = (struct kbase_hwcnt_backend_jm *)backend; - struct kbase_context *kctx; - struct kbase_device *kbdev; - struct kbase_instr_hwcnt_enable enable = { 0 }; - u64 timestamp_ns; - - if (!backend_jm || !enable_map || backend_jm->enabled || - (enable_map->metadata != backend_jm->info->metadata)) - return -EINVAL; - - kctx = backend_jm->kctx; - kbdev = backend_jm->kctx->kbdev; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - kbasep_hwcnt_backend_jm_get_physical_enable(backend_jm, enable_map, &enable); - - timestamp_ns = kbasep_hwcnt_backend_jm_timestamp_ns(backend); - - /* Update the current configuration information. */ - errcode = kbasep_hwcnt_gpu_update_curr_config(kbdev, &backend_jm->curr_config); - if (errcode) - goto error; - - errcode = kbase_instr_hwcnt_enable_internal(kbdev, kctx, &enable); - if (errcode) - goto error; - - backend_jm->debug_core_mask = kbase_pm_ca_get_debug_core_mask(kbdev); - backend_jm->max_l2_slices = backend_jm->info->hwcnt_gpu_info.l2_count; - backend_jm->max_core_mask = backend_jm->info->hwcnt_gpu_info.sc_core_mask; - - backend_jm->pm_core_mask = kbase_pm_ca_get_instr_core_mask(kbdev); - - backend_jm->enabled = true; - - /* Enabling counters is an indication that the power may have previously been off for all - * blocks. - * - * In any case, the counters would not have been counting recently, so an 'off' block state - * is an approximation for this. - * - * This will be transferred to the dump only after a dump_wait(), or dump_disable() in - * cases where the caller requested such information. This is to handle when a - * dump_enable() happens in between dump_wait() and dump_get(). - */ - kbasep_hwcnt_backend_jm_append_block_states(backend_jm, KBASE_HWCNT_STATE_OFF); - - kbasep_hwcnt_backend_jm_cc_enable(backend_jm, enable_map, timestamp_ns); - - return 0; -error: - return errcode; -} - -static void -kbasep_hwcnt_backend_jm_reset_consumed_buffers(struct kbase_hwcnt_backend_jm *backend_jm) -{ - size_t block_state_bytes = backend_jm->phys_layout.block_cnt * - KBASE_HWCNT_BLOCK_STATE_BYTES * KBASE_HWCNT_BLOCK_STATE_STRIDE; - - memset(backend_jm->to_user_block_states, 0, block_state_bytes); -} - -/* JM backend implementation of kbase_hwcnt_backend_dump_enable_fn */ -static int kbasep_hwcnt_backend_jm_dump_enable(struct kbase_hwcnt_backend *backend, - const struct kbase_hwcnt_enable_map *enable_map) -{ - unsigned long flags; - int errcode; - struct kbase_hwcnt_backend_jm *backend_jm = (struct kbase_hwcnt_backend_jm *)backend; - struct kbase_device *kbdev; - - if (!backend_jm) - return -EINVAL; - - kbdev = backend_jm->kctx->kbdev; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - errcode = kbasep_hwcnt_backend_jm_dump_enable_nolock(backend, enable_map); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - return errcode; -} - -/* JM backend implementation of kbase_hwcnt_backend_dump_disable_fn */ -static void kbasep_hwcnt_backend_jm_dump_disable(struct kbase_hwcnt_backend *backend, - struct kbase_hwcnt_dump_buffer *dump_buffer, - const struct kbase_hwcnt_enable_map *enable_map) -{ - int errcode; - struct kbase_hwcnt_backend_jm *backend_jm = (struct kbase_hwcnt_backend_jm *)backend; - - if (WARN_ON(!backend_jm || - (dump_buffer && (backend_jm->info->metadata != dump_buffer->metadata)) || - (enable_map && (backend_jm->info->metadata != enable_map->metadata)) || - (dump_buffer && !enable_map))) - return; - - /* No WARN needed here, but still return early if backend is already disabled */ - if (!backend_jm->enabled) - return; - - kbasep_hwcnt_backend_jm_cc_disable(backend_jm); - - errcode = kbase_instr_hwcnt_disable_internal(backend_jm->kctx); - WARN_ON(errcode); - - kbasep_hwcnt_backend_jm_append_block_states(backend_jm, KBASE_HWCNT_STATE_OFF); - - if (dump_buffer) { - /* In some use-cases, the caller will need the information whilst the counters are - * disabled, but will not be able to call into the backend to dump them. Instead, - * they have an opportunity here to request them to be accumulated into their - * buffer immediately. - * - * This consists of taking a sample of the accumulated block state (as though a - * real dump_get() had happened), then transfer ownership of that to the caller - * (i.e. erasing our copy of it). - */ - kbase_hwcnt_dump_buffer_append_block_states(dump_buffer, enable_map, - backend_jm->to_user_block_states); - - /* Now the block state has been passed out into the caller's own accumulation - * buffer, clear our own accumulated and sampled block state - ownership has been - * transferred. - */ - kbasep_hwcnt_backend_jm_reset_consumed_buffers(backend_jm); - } - - backend_jm->enabled = false; -} - -/* JM backend implementation of kbase_hwcnt_backend_dump_clear_fn */ -static int kbasep_hwcnt_backend_jm_dump_clear(struct kbase_hwcnt_backend *backend) -{ - struct kbase_hwcnt_backend_jm *backend_jm = (struct kbase_hwcnt_backend_jm *)backend; - - if (!backend_jm || !backend_jm->enabled) - return -EINVAL; - - return kbase_instr_hwcnt_clear(backend_jm->kctx); -} - -/* JM backend implementation of kbase_hwcnt_backend_dump_request_fn */ -static int kbasep_hwcnt_backend_jm_dump_request(struct kbase_hwcnt_backend *backend) -{ - struct kbase_hwcnt_backend_jm *backend_jm = (struct kbase_hwcnt_backend_jm *)backend; - struct kbase_device *kbdev; - const struct kbase_hwcnt_metadata *metadata; - u64 current_cycle_count; - size_t clk; - int ret; - - if (!backend_jm || !backend_jm->enabled) - return -EINVAL; - - kbdev = backend_jm->kctx->kbdev; - metadata = backend_jm->info->metadata; - - /* Disable pre-emption, to make the timestamp as accurate as possible */ - preempt_disable(); - { - backend_jm->dump_time_ns = kbasep_hwcnt_backend_jm_timestamp_ns(backend); - ret = kbase_instr_hwcnt_request_dump(backend_jm->kctx); - - kbase_hwcnt_metadata_for_each_clock(metadata, clk) { - if (!kbase_hwcnt_clk_enable_map_enabled(backend_jm->clk_enable_map, clk)) - continue; - - if (clk == KBASE_CLOCK_DOMAIN_TOP) { - /* Read cycle count for top clock domain. */ - kbase_backend_get_gpu_time_norequest(kbdev, ¤t_cycle_count, - NULL, NULL); - } else { - /* - * Estimate cycle count for non-top clock - * domain. - */ - current_cycle_count = kbase_ccswe_cycle_at( - &backend_jm->ccswe_shader_cores, backend_jm->dump_time_ns); - } - backend_jm->cycle_count_elapsed[clk] = - current_cycle_count - backend_jm->prev_cycle_count[clk]; - - /* - * Keep the current cycle count for later calculation. - */ - backend_jm->prev_cycle_count[clk] = current_cycle_count; - } - } - preempt_enable(); - - return ret; -} - -/* JM backend implementation of kbase_hwcnt_backend_dump_wait_fn */ -static int kbasep_hwcnt_backend_jm_dump_wait(struct kbase_hwcnt_backend *backend) -{ - int errcode; - struct kbase_hwcnt_backend_jm *backend_jm = (struct kbase_hwcnt_backend_jm *)backend; - - if (!backend_jm || !backend_jm->enabled) - return -EINVAL; - - errcode = kbase_instr_hwcnt_wait_for_dump(backend_jm->kctx); - if (errcode) - return errcode; - - return errcode; -} - -/* JM backend implementation of kbase_hwcnt_backend_dump_get_fn */ -static int kbasep_hwcnt_backend_jm_dump_get(struct kbase_hwcnt_backend *backend, - struct kbase_hwcnt_dump_buffer *dst, - const struct kbase_hwcnt_enable_map *dst_enable_map, - bool accumulate, u64 *dump_time_ns) -{ - struct kbase_hwcnt_backend_jm *backend_jm = (struct kbase_hwcnt_backend_jm *)backend; - size_t clk; -#if IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI) - struct kbase_device *kbdev; - unsigned long flags; -#endif /* CONFIG_MALI_VALHALL_NO_MALI */ - int errcode; - - if (!backend_jm || !dst || !dst_enable_map || !dump_time_ns || - (backend_jm->info->metadata != dst->metadata) || - (dst_enable_map->metadata != dst->metadata)) - return -EINVAL; - - /* Invalidate the kernel buffer before reading from it. */ - kbase_sync_mem_regions(backend_jm->kctx, backend_jm->vmap, KBASE_SYNC_TO_CPU); - - /* Dump sample to the internal 64-bit user buffer. */ - kbasep_hwcnt_backend_jm_dump_sample(backend_jm); - - /* Extract elapsed cycle count for each clock domain if enabled. */ - kbase_hwcnt_metadata_for_each_clock(dst_enable_map->metadata, clk) { - if (!kbase_hwcnt_clk_enable_map_enabled(dst_enable_map->clk_enable_map, clk)) - continue; - - /* Reset the counter to zero if accumulation is off. */ - if (!accumulate) - dst->clk_cnt_buf[clk] = 0; - dst->clk_cnt_buf[clk] += backend_jm->cycle_count_elapsed[clk]; - } - -#if IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI) - kbdev = backend_jm->kctx->kbdev; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - /* Update the current configuration information. */ - errcode = kbasep_hwcnt_gpu_update_curr_config(kbdev, &backend_jm->curr_config); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - if (errcode) - return errcode; -#endif /* CONFIG_MALI_VALHALL_NO_MALI */ - errcode = kbase_hwcnt_jm_dump_get(dst, backend_jm->to_user_buf, dst_enable_map, - backend_jm->pm_core_mask, backend_jm->debug_core_mask, - backend_jm->max_l2_slices, &backend_jm->curr_config, - accumulate); - - if (errcode) - return errcode; - - *dump_time_ns = backend_jm->dump_time_ns; - kbase_hwcnt_dump_buffer_append_block_states(dst, dst_enable_map, - backend_jm->to_user_block_states); - kbasep_hwcnt_backend_jm_reset_consumed_buffers(backend_jm); - - return errcode; -} - -/** - * kbasep_hwcnt_backend_jm_dump_alloc() - Allocate a GPU dump buffer. - * @info: Non-NULL pointer to JM backend info. - * @kctx: Non-NULL pointer to kbase context. - * @gpu_dump_va: Non-NULL pointer to where GPU dump buffer virtual address - * is stored on success. - * - * Return: 0 on success, else error code. - */ -static int kbasep_hwcnt_backend_jm_dump_alloc(const struct kbase_hwcnt_backend_jm_info *info, - struct kbase_context *kctx, u64 *gpu_dump_va) -{ - struct kbase_va_region *reg; - base_mem_alloc_flags flags; - u64 nr_pages; - - /* Calls to this function are inherently asynchronous, with respect to - * MMU operations. - */ - const enum kbase_caller_mmu_sync_info mmu_sync_info = CALLER_MMU_ASYNC; - - WARN_ON(!info); - WARN_ON(!kctx); - WARN_ON(!gpu_dump_va); - - flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_WR | BASEP_MEM_PERMANENT_KERNEL_MAPPING | - BASE_MEM_CACHED_CPU | BASE_MEM_UNCACHED_GPU; - - nr_pages = PFN_UP(info->dump_bytes); - - reg = kbase_mem_alloc(kctx, nr_pages, nr_pages, 0, &flags, gpu_dump_va, mmu_sync_info); - - if (!reg) - return -ENOMEM; - - return 0; -} - -/** - * kbasep_hwcnt_backend_jm_dump_free() - Free an allocated GPU dump buffer. - * @kctx: Non-NULL pointer to kbase context. - * @gpu_dump_va: GPU dump buffer virtual address. - */ -static void kbasep_hwcnt_backend_jm_dump_free(struct kbase_context *kctx, u64 gpu_dump_va) -{ - WARN_ON(!kctx); - if (gpu_dump_va) - kbase_mem_free(kctx, gpu_dump_va); -} - -/** - * kbasep_hwcnt_backend_jm_destroy() - Destroy a JM backend. - * @backend: Pointer to JM backend to destroy. - * - * Can be safely called on a backend in any state of partial construction. - */ -static void kbasep_hwcnt_backend_jm_destroy(struct kbase_hwcnt_backend_jm *backend) -{ - if (!backend) - return; - - if (backend->kctx) { - struct kbase_context *kctx = backend->kctx; - struct kbase_device *kbdev = kctx->kbdev; - - if (backend->cpu_dump_va) - kbase_phy_alloc_mapping_put(kctx, backend->vmap); - - if (backend->gpu_dump_va) - kbasep_hwcnt_backend_jm_dump_free(kctx, backend->gpu_dump_va); - - kbasep_js_release_privileged_ctx(kbdev, kctx); - kbase_destroy_context(kctx); - } - - kfree(backend->to_user_buf); - - kfree(backend->to_user_block_states); - - kfree(backend); -} - -/** - * kbasep_hwcnt_backend_jm_create() - Create a JM backend. - * @info: Non-NULL pointer to backend info. - * @out_backend: Non-NULL pointer to where backend is stored on success. - * - * Return: 0 on success, else error code. - */ -static int kbasep_hwcnt_backend_jm_create(const struct kbase_hwcnt_backend_jm_info *info, - struct kbase_hwcnt_backend_jm **out_backend) -{ - int errcode; - struct kbase_device *kbdev; - struct kbase_hwcnt_backend_jm *backend = NULL; - size_t block_state_bytes; - - WARN_ON(!info); - WARN_ON(!out_backend); - - kbdev = info->kbdev; - - backend = kzalloc(sizeof(*backend), GFP_KERNEL); - if (!backend) - goto alloc_error; - - backend->info = info; - kbasep_hwcnt_backend_jm_init_layout(&info->hwcnt_gpu_info, &backend->phys_layout); - - backend->kctx = kbase_create_context(kbdev, true, - BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED, 0, NULL); - if (!backend->kctx) - goto alloc_error; - - kbasep_js_schedule_privileged_ctx(kbdev, backend->kctx); - - errcode = kbasep_hwcnt_backend_jm_dump_alloc(info, backend->kctx, &backend->gpu_dump_va); - if (errcode) - goto error; - - backend->cpu_dump_va = - kbase_phy_alloc_mapping_get(backend->kctx, backend->gpu_dump_va, &backend->vmap); - if (!backend->cpu_dump_va || !backend->vmap) - goto alloc_error; - - backend->to_user_buf = kzalloc(info->metadata->dump_buf_bytes, GFP_KERNEL); - if (!backend->to_user_buf) - goto alloc_error; - - block_state_bytes = backend->phys_layout.block_cnt * KBASE_HWCNT_BLOCK_STATE_BYTES * - KBASE_HWCNT_BLOCK_STATE_STRIDE; - - backend->to_user_block_states = kzalloc(block_state_bytes, GFP_KERNEL); - if (!backend->to_user_block_states) - goto alloc_error; - - memset(backend->to_user_block_states, 0, block_state_bytes); - - kbase_ccswe_init(&backend->ccswe_shader_cores); - backend->rate_listener.notify = kbasep_hwcnt_backend_jm_on_freq_change; - - *out_backend = backend; - return 0; - -alloc_error: - errcode = -ENOMEM; -error: - kbasep_hwcnt_backend_jm_destroy(backend); - return errcode; -} - -/* JM backend implementation of kbase_hwcnt_backend_metadata_fn */ -static const struct kbase_hwcnt_metadata * -kbasep_hwcnt_backend_jm_metadata(const struct kbase_hwcnt_backend_info *info) -{ - if (!info) - return NULL; - - return ((const struct kbase_hwcnt_backend_jm_info *)info)->metadata; -} - -/* JM backend implementation of kbase_hwcnt_backend_init_fn */ -static int kbasep_hwcnt_backend_jm_init(const struct kbase_hwcnt_backend_info *info, - struct kbase_hwcnt_backend **out_backend) -{ - int errcode; - struct kbase_hwcnt_backend_jm *backend = NULL; - - if (!info || !out_backend) - return -EINVAL; - - errcode = kbasep_hwcnt_backend_jm_create((const struct kbase_hwcnt_backend_jm_info *)info, - &backend); - if (errcode) - return errcode; - - *out_backend = (struct kbase_hwcnt_backend *)backend; - - return 0; -} - -/* JM backend implementation of kbase_hwcnt_backend_term_fn */ -static void kbasep_hwcnt_backend_jm_term(struct kbase_hwcnt_backend *backend) -{ - if (!backend) - return; - - kbasep_hwcnt_backend_jm_dump_disable(backend, NULL, NULL); - kbasep_hwcnt_backend_jm_destroy((struct kbase_hwcnt_backend_jm *)backend); -} - -static void kbasep_hwcnt_backend_jm_acquire(const struct kbase_hwcnt_backend *backend) -{ -} - -static void kbasep_hwcnt_backend_jm_release(const struct kbase_hwcnt_backend *backend) -{ -} - -/** - * kbasep_hwcnt_backend_jm_info_destroy() - Destroy a JM backend info. - * @info: Pointer to info to destroy. - * - * Can be safely called on a backend info in any state of partial construction. - */ -static void kbasep_hwcnt_backend_jm_info_destroy(const struct kbase_hwcnt_backend_jm_info *info) -{ - if (!info) - return; - - kbase_hwcnt_metadata_destroy(info->metadata); - kfree(info); -} - -/** - * kbasep_hwcnt_backend_jm_info_create() - Create a JM backend info. - * @kbdev: Non_NULL pointer to kbase device. - * @out_info: Non-NULL pointer to where info is stored on success. - * - * Return: 0 on success, else error code. - */ -static int kbasep_hwcnt_backend_jm_info_create(struct kbase_device *kbdev, - const struct kbase_hwcnt_backend_jm_info **out_info) -{ - int errcode = -ENOMEM; - struct kbase_hwcnt_backend_jm_info *info = NULL; - - WARN_ON(!kbdev); - WARN_ON(!out_info); - - info = kzalloc(sizeof(*info), GFP_KERNEL); - if (!info) - return errcode; - - info->kbdev = kbdev; - -#ifdef CONFIG_MALI_VALHALL_PRFCNT_SET_SECONDARY - info->counter_set = KBASE_HWCNT_SET_SECONDARY; -#elif defined(CONFIG_MALI_VALHALL_PRFCNT_SET_TERTIARY) - info->counter_set = KBASE_HWCNT_SET_TERTIARY; -#else - /* Default to primary */ - info->counter_set = KBASE_HWCNT_SET_PRIMARY; -#endif - - errcode = kbasep_hwcnt_backend_jm_gpu_info_init(kbdev, &info->hwcnt_gpu_info); - if (errcode) - goto error; - - errcode = kbase_hwcnt_jm_metadata_create(&info->hwcnt_gpu_info, info->counter_set, - &info->metadata, &info->dump_bytes); - if (errcode) - goto error; - - *out_info = info; - - return 0; -error: - kbasep_hwcnt_backend_jm_info_destroy(info); - return errcode; -} - -int kbase_hwcnt_backend_jm_create(struct kbase_device *kbdev, - struct kbase_hwcnt_backend_interface *iface) -{ - int errcode; - const struct kbase_hwcnt_backend_jm_info *info = NULL; - - if (!kbdev || !iface) - return -EINVAL; - - errcode = kbasep_hwcnt_backend_jm_info_create(kbdev, &info); - - if (errcode) - return errcode; - - iface->info = (struct kbase_hwcnt_backend_info *)info; - iface->metadata = kbasep_hwcnt_backend_jm_metadata; - iface->init = kbasep_hwcnt_backend_jm_init; - iface->term = kbasep_hwcnt_backend_jm_term; - iface->acquire = kbasep_hwcnt_backend_jm_acquire; - iface->release = kbasep_hwcnt_backend_jm_release; - iface->timestamp_ns = kbasep_hwcnt_backend_jm_timestamp_ns; - iface->dump_enable = kbasep_hwcnt_backend_jm_dump_enable; - iface->dump_enable_nolock = kbasep_hwcnt_backend_jm_dump_enable_nolock; - iface->dump_disable = kbasep_hwcnt_backend_jm_dump_disable; - iface->dump_clear = kbasep_hwcnt_backend_jm_dump_clear; - iface->dump_request = kbasep_hwcnt_backend_jm_dump_request; - iface->dump_wait = kbasep_hwcnt_backend_jm_dump_wait; - iface->dump_get = kbasep_hwcnt_backend_jm_dump_get; - - return 0; -} - -void kbase_hwcnt_backend_jm_destroy(struct kbase_hwcnt_backend_interface *iface) -{ - if (!iface) - return; - - kbasep_hwcnt_backend_jm_info_destroy( - (const struct kbase_hwcnt_backend_jm_info *)iface->info); - memset(iface, 0, sizeof(*iface)); -} diff --git a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_jm.h b/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_jm.h deleted file mode 100644 index 4a6293c25473..000000000000 --- a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_jm.h +++ /dev/null @@ -1,58 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2018, 2020-2022 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Concrete implementation of mali_kbase_hwcnt_backend interface for JM - * backend. - */ - -#ifndef _KBASE_HWCNT_BACKEND_JM_H_ -#define _KBASE_HWCNT_BACKEND_JM_H_ - -#include "hwcnt/backend/mali_kbase_hwcnt_backend.h" - -struct kbase_device; - -/** - * kbase_hwcnt_backend_jm_create() - Create a JM hardware counter backend - * interface. - * @kbdev: Non-NULL pointer to kbase device. - * @iface: Non-NULL pointer to backend interface structure that is filled in - * on creation success. - * - * Calls to iface->dump_enable_nolock() require kbdev->hwaccess_lock held. - * - * Return: 0 on success, else error code. - */ -int kbase_hwcnt_backend_jm_create(struct kbase_device *kbdev, - struct kbase_hwcnt_backend_interface *iface); - -/** - * kbase_hwcnt_backend_jm_destroy() - Destroy a JM hardware counter backend - * interface. - * @iface: Pointer to interface to destroy. - * - * Can be safely called on an all-zeroed interface, or on an already destroyed - * interface. - */ -void kbase_hwcnt_backend_jm_destroy(struct kbase_hwcnt_backend_interface *iface); - -#endif /* _KBASE_HWCNT_BACKEND_JM_H_ */ diff --git a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_jm_watchdog.c b/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_jm_watchdog.c deleted file mode 100644 index a68be8c64fd9..000000000000 --- a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_jm_watchdog.c +++ /dev/null @@ -1,841 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2021-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include - -#include -#include - -#include -#include -#include - -/* - * IDLE_BUFFER_EMPTY -> USER_DUMPING_BUFFER_EMPTY on dump_request. - * IDLE_BUFFER_EMPTY -> TIMER_DUMPING after - * hwcnt_backend_watchdog_timer_interval_ms - * milliseconds, if no dump_request has been - * called in the meantime. - * IDLE_BUFFER_FULL -> USER_DUMPING_BUFFER_FULL on dump_request. - * IDLE_BUFFER_FULL -> TIMER_DUMPING after - * hwcnt_backend_watchdog_timer_interval_ms - * milliseconds, if no dump_request has been - * called in the meantime. - * IDLE_BUFFER_FULL -> IDLE_BUFFER_EMPTY on dump_disable, upon discarding undumped - * counter values since the last dump_get. - * IDLE_BUFFER_EMPTY -> BUFFER_CLEARING on dump_clear, before calling job manager - * backend dump_clear. - * IDLE_BUFFER_FULL -> BUFFER_CLEARING on dump_clear, before calling job manager - * backend dump_clear. - * USER_DUMPING_BUFFER_EMPTY -> BUFFER_CLEARING on dump_clear, before calling job manager - * backend dump_clear. - * USER_DUMPING_BUFFER_FULL -> BUFFER_CLEARING on dump_clear, before calling job manager - * backend dump_clear. - * BUFFER_CLEARING -> IDLE_BUFFER_EMPTY on dump_clear, upon job manager backend - * dump_clear completion. - * TIMER_DUMPING -> IDLE_BUFFER_FULL on timer's callback completion. - * TIMER_DUMPING -> TIMER_DUMPING_USER_CLEAR on dump_clear, notifies the callback thread - * that there is no need for dumping the buffer - * anymore, and that the client will proceed - * clearing the buffer. - * TIMER_DUMPING_USER_CLEAR -> IDLE_BUFFER_EMPTY on timer's callback completion, when a user - * requested a dump_clear. - * TIMER_DUMPING -> TIMER_DUMPING_USER_REQUESTED on dump_request, when a client performs a - * dump request while the timer is dumping (the - * timer will perform the dump and (once - * completed) the client will retrieve the value - * from the buffer). - * TIMER_DUMPING_USER_REQUESTED -> IDLE_BUFFER_EMPTY on dump_get, when a timer completed and the - * user reads the periodic dump buffer. - * Any -> ERROR if the job manager backend returns an error - * (of any kind). - * USER_DUMPING_BUFFER_EMPTY -> IDLE_BUFFER_EMPTY on dump_get (performs get, ignores the - * periodic dump buffer and returns). - * USER_DUMPING_BUFFER_FULL -> IDLE_BUFFER_EMPTY on dump_get (performs get, accumulates with - * periodic dump buffer and returns). - */ - -/** enum backend_watchdog_state State used to synchronize timer callbacks with the main thread. - * @HWCNT_JM_WD_ERROR: Received an error from the job manager backend calls. - * @HWCNT_JM_WD_IDLE_BUFFER_EMPTY: Initial state. Watchdog timer enabled, periodic dump buffer is - * empty. - * @HWCNT_JM_WD_IDLE_BUFFER_FULL: Watchdog timer enabled, periodic dump buffer is full. - * @HWCNT_JM_WD_BUFFER_CLEARING: The client is performing a dump clear. A concurrent timer callback - * thread should just ignore and reschedule another callback in - * hwcnt_backend_watchdog_timer_interval_ms milliseconds. - * @HWCNT_JM_WD_TIMER_DUMPING: The timer ran out. The callback is performing a periodic dump. - * @HWCNT_JM_WD_TIMER_DUMPING_USER_REQUESTED: While the timer is performing a periodic dump, user - * requested a dump. - * @HWCNT_JM_WD_TIMER_DUMPING_USER_CLEAR: While the timer is performing a dump, user requested a - * dump_clear. The timer has to complete the periodic dump - * and clear buffer (internal and job manager backend). - * @HWCNT_JM_WD_USER_DUMPING_BUFFER_EMPTY: From IDLE state, user requested a dump. The periodic - * dump buffer is empty. - * @HWCNT_JM_WD_USER_DUMPING_BUFFER_FULL: From IDLE state, user requested a dump. The periodic dump - * buffer is full. - * - * While the state machine is in HWCNT_JM_WD_TIMER_DUMPING*, only the timer callback thread is - * allowed to call the job manager backend layer. - */ -enum backend_watchdog_state { - HWCNT_JM_WD_ERROR, - HWCNT_JM_WD_IDLE_BUFFER_EMPTY, - HWCNT_JM_WD_IDLE_BUFFER_FULL, - HWCNT_JM_WD_BUFFER_CLEARING, - HWCNT_JM_WD_TIMER_DUMPING, - HWCNT_JM_WD_TIMER_DUMPING_USER_REQUESTED, - HWCNT_JM_WD_TIMER_DUMPING_USER_CLEAR, - HWCNT_JM_WD_USER_DUMPING_BUFFER_EMPTY, - HWCNT_JM_WD_USER_DUMPING_BUFFER_FULL, -}; - -/** enum wd_init_state - State machine for initialization / termination of the backend resources - */ -enum wd_init_state { - HWCNT_JM_WD_INIT_START, - HWCNT_JM_WD_INIT_BACKEND = HWCNT_JM_WD_INIT_START, - HWCNT_JM_WD_INIT_ENABLE_MAP, - HWCNT_JM_WD_INIT_DUMP_BUFFER, - HWCNT_JM_WD_INIT_END -}; - -/** - * struct kbase_hwcnt_backend_jm_watchdog_info - Immutable information used to initialize an - * instance of the job manager watchdog backend. - * @jm_backend_iface: Hardware counter backend interface. This module extends - * this interface with a watchdog that performs regular - * dumps. The new interface this module provides complies - * with the old backend interface. - * @dump_watchdog_iface: Dump watchdog interface, used to periodically dump the - * hardware counter in case no reads are requested within - * a certain time, used to avoid hardware counter's buffer - * saturation. - * @watchdog_timer_interval_ms: Interval in milliseconds between hwcnt samples. - */ -struct kbase_hwcnt_backend_jm_watchdog_info { - struct kbase_hwcnt_backend_interface *jm_backend_iface; - struct kbase_hwcnt_watchdog_interface *dump_watchdog_iface; - u32 watchdog_timer_interval_ms; -}; - -/** - * struct kbase_hwcnt_backend_jm_watchdog - An instance of the job manager watchdog backend. - * @info: Immutable information used to create the job manager watchdog backend. - * @jm_backend: Job manager's backend internal state. To be passed as argument during parent calls. - * @timeout_ms: Time period in milliseconds for hardware counters dumping. - * @wd_dump_buffer: Used to store periodic dumps done by a timer callback function. Contents are - * valid in state %HWCNT_JM_WD_TIMER_DUMPING_USER_REQUESTED, - * %HWCNT_JM_WD_IDLE_BUFFER_FULL or %HWCNT_JM_WD_USER_DUMPING_BUFFER_FULL. - * @wd_enable_map: Watchdog backend internal buffer mask, initialized during dump_enable copying - * the enable_map passed as argument. - * @wd_dump_timestamp: Holds the dumping timestamp for potential future client dump_request, filled - * during watchdog timer dumps. - * @watchdog_complete: Used for synchronization between watchdog dumper thread and client calls. - * @locked: Members protected from concurrent access by different threads. - * @locked.watchdog_lock: Lock used to access fields within this struct (that require mutual - * exclusion). - * @locked.is_enabled: If true then the wrapped job manager hardware counter backend and the - * watchdog timer are both enabled. If false then both are disabled (or soon - * will be). Races between enable and disable have undefined behavior. - * @locked.state: State used to synchronize timer callbacks with the main thread. - */ -struct kbase_hwcnt_backend_jm_watchdog { - const struct kbase_hwcnt_backend_jm_watchdog_info *info; - struct kbase_hwcnt_backend *jm_backend; - u32 timeout_ms; - struct kbase_hwcnt_dump_buffer wd_dump_buffer; - struct kbase_hwcnt_enable_map wd_enable_map; - u64 wd_dump_timestamp; - struct completion watchdog_complete; - struct { - spinlock_t watchdog_lock; - bool is_enabled; - enum backend_watchdog_state state; - } locked; -}; - -/* timer's callback function */ -static void kbasep_hwcnt_backend_jm_watchdog_timer_callback(void *backend) -{ - struct kbase_hwcnt_backend_jm_watchdog *wd_backend = backend; - unsigned long flags; - bool wd_accumulate; - - spin_lock_irqsave(&wd_backend->locked.watchdog_lock, flags); - - if (!wd_backend->locked.is_enabled || wd_backend->locked.state == HWCNT_JM_WD_ERROR) { - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - return; - } - - if (!(wd_backend->locked.state == HWCNT_JM_WD_IDLE_BUFFER_EMPTY || - wd_backend->locked.state == HWCNT_JM_WD_IDLE_BUFFER_FULL)) { - /*resetting the timer. Calling modify on a disabled timer enables it.*/ - wd_backend->info->dump_watchdog_iface->modify( - wd_backend->info->dump_watchdog_iface->timer, wd_backend->timeout_ms); - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - return; - } - /*start performing the dump*/ - - /* if there has been a previous timeout use accumulating dump_get() - * otherwise use non-accumulating to overwrite buffer - */ - wd_accumulate = (wd_backend->locked.state == HWCNT_JM_WD_IDLE_BUFFER_FULL); - - wd_backend->locked.state = HWCNT_JM_WD_TIMER_DUMPING; - - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - - if (wd_backend->info->jm_backend_iface->dump_request(wd_backend->jm_backend) || - wd_backend->info->jm_backend_iface->dump_wait(wd_backend->jm_backend) || - wd_backend->info->jm_backend_iface->dump_get( - wd_backend->jm_backend, &wd_backend->wd_dump_buffer, &wd_backend->wd_enable_map, - wd_accumulate, &wd_backend->wd_dump_timestamp)) { - spin_lock_irqsave(&wd_backend->locked.watchdog_lock, flags); - WARN_ON(wd_backend->locked.state != HWCNT_JM_WD_TIMER_DUMPING && - wd_backend->locked.state != HWCNT_JM_WD_TIMER_DUMPING_USER_CLEAR && - wd_backend->locked.state != HWCNT_JM_WD_TIMER_DUMPING_USER_REQUESTED); - wd_backend->locked.state = HWCNT_JM_WD_ERROR; - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - /* Unblock user if it's waiting. */ - complete_all(&wd_backend->watchdog_complete); - return; - } - - spin_lock_irqsave(&wd_backend->locked.watchdog_lock, flags); - WARN_ON(wd_backend->locked.state != HWCNT_JM_WD_TIMER_DUMPING && - wd_backend->locked.state != HWCNT_JM_WD_TIMER_DUMPING_USER_CLEAR && - wd_backend->locked.state != HWCNT_JM_WD_TIMER_DUMPING_USER_REQUESTED); - - if (wd_backend->locked.state == HWCNT_JM_WD_TIMER_DUMPING) { - /* If there is no user request/clear, transit to HWCNT_JM_WD_IDLE_BUFFER_FULL - * to indicate timer dump is done and the buffer is full. If state changed to - * HWCNT_JM_WD_TIMER_DUMPING_USER_REQUESTED or - * HWCNT_JM_WD_TIMER_DUMPING_USER_CLEAR then user will transit the state - * machine to next state. - */ - wd_backend->locked.state = HWCNT_JM_WD_IDLE_BUFFER_FULL; - } - if (wd_backend->locked.state != HWCNT_JM_WD_ERROR && wd_backend->locked.is_enabled) { - /* reset the timer to schedule another callback. Calling modify on a - * disabled timer enables it. - */ - /*The spin lock needs to be held in case the client calls dump_enable*/ - wd_backend->info->dump_watchdog_iface->modify( - wd_backend->info->dump_watchdog_iface->timer, wd_backend->timeout_ms); - } - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - - /* Unblock user if it's waiting. */ - complete_all(&wd_backend->watchdog_complete); -} - -/* helper methods, info structure creation and destruction*/ - -static struct kbase_hwcnt_backend_jm_watchdog_info * -kbasep_hwcnt_backend_jm_watchdog_info_create(struct kbase_hwcnt_backend_interface *backend_iface, - struct kbase_hwcnt_watchdog_interface *watchdog_iface, - u32 watchdog_timer_interval_ms) -{ - struct kbase_hwcnt_backend_jm_watchdog_info *const info = - kmalloc(sizeof(*info), GFP_KERNEL); - - if (!info) - return NULL; - - *info = (struct kbase_hwcnt_backend_jm_watchdog_info){ - .jm_backend_iface = backend_iface, - .dump_watchdog_iface = watchdog_iface, - .watchdog_timer_interval_ms = watchdog_timer_interval_ms - }; - - return info; -} - -/****** kbase_hwcnt_backend_interface implementation *******/ - -/* Job manager watchdog backend, implementation of kbase_hwcnt_backend_metadata_fn */ -static const struct kbase_hwcnt_metadata * -kbasep_hwcnt_backend_jm_watchdog_metadata(const struct kbase_hwcnt_backend_info *info) -{ - const struct kbase_hwcnt_backend_jm_watchdog_info *wd_info = (void *)info; - - if (WARN_ON(!info)) - return NULL; - - return wd_info->jm_backend_iface->metadata(wd_info->jm_backend_iface->info); -} - -static void -kbasep_hwcnt_backend_jm_watchdog_term_partial(struct kbase_hwcnt_backend_jm_watchdog *wd_backend, - enum wd_init_state state) -{ - if (!wd_backend) - return; - - WARN_ON(state > HWCNT_JM_WD_INIT_END); - - while (state-- > HWCNT_JM_WD_INIT_START) { - switch (state) { - case HWCNT_JM_WD_INIT_BACKEND: - wd_backend->info->jm_backend_iface->term(wd_backend->jm_backend); - break; - case HWCNT_JM_WD_INIT_ENABLE_MAP: - kbase_hwcnt_enable_map_free(&wd_backend->wd_enable_map); - break; - case HWCNT_JM_WD_INIT_DUMP_BUFFER: - kbase_hwcnt_dump_buffer_free(&wd_backend->wd_dump_buffer); - break; - case HWCNT_JM_WD_INIT_END: - break; - } - } - - kfree(wd_backend); -} - -static void kbasep_hwcnt_backend_jm_watchdog_acquire(const struct kbase_hwcnt_backend *backend) -{ -} - -static void kbasep_hwcnt_backend_jm_watchdog_release(const struct kbase_hwcnt_backend *backend) -{ -} - -/* Job manager watchdog backend, implementation of kbase_hwcnt_backend_term_fn - * Calling term does *not* destroy the interface - */ -static void kbasep_hwcnt_backend_jm_watchdog_term(struct kbase_hwcnt_backend *backend) -{ - struct kbase_hwcnt_backend_jm_watchdog *wd_backend = - (struct kbase_hwcnt_backend_jm_watchdog *)backend; - - if (!backend) - return; - - /* disable timer thread to avoid concurrent access to shared resources */ - wd_backend->info->dump_watchdog_iface->disable( - wd_backend->info->dump_watchdog_iface->timer); - - kbasep_hwcnt_backend_jm_watchdog_term_partial(wd_backend, HWCNT_JM_WD_INIT_END); -} - -/* Job manager watchdog backend, implementation of kbase_hwcnt_backend_init_fn */ -static int kbasep_hwcnt_backend_jm_watchdog_init(const struct kbase_hwcnt_backend_info *info, - struct kbase_hwcnt_backend **out_backend) -{ - int errcode = 0; - struct kbase_hwcnt_backend_jm_watchdog *wd_backend = NULL; - struct kbase_hwcnt_backend_jm_watchdog_info *const wd_info = (void *)info; - const struct kbase_hwcnt_backend_info *jm_info; - const struct kbase_hwcnt_metadata *metadata; - enum wd_init_state state = HWCNT_JM_WD_INIT_START; - - if (WARN_ON(!info) || WARN_ON(!out_backend)) - return -EINVAL; - - jm_info = wd_info->jm_backend_iface->info; - metadata = wd_info->jm_backend_iface->metadata(wd_info->jm_backend_iface->info); - - wd_backend = kmalloc(sizeof(*wd_backend), GFP_KERNEL); - if (!wd_backend) { - *out_backend = NULL; - return -ENOMEM; - } - - *wd_backend = (struct kbase_hwcnt_backend_jm_watchdog){ - .info = wd_info, - .timeout_ms = wd_info->watchdog_timer_interval_ms, - .locked = { .state = HWCNT_JM_WD_IDLE_BUFFER_EMPTY, .is_enabled = false } - }; - - while (state < HWCNT_JM_WD_INIT_END && !errcode) { - switch (state) { - case HWCNT_JM_WD_INIT_BACKEND: - errcode = wd_info->jm_backend_iface->init(jm_info, &wd_backend->jm_backend); - break; - case HWCNT_JM_WD_INIT_ENABLE_MAP: - errcode = - kbase_hwcnt_enable_map_alloc(metadata, &wd_backend->wd_enable_map); - break; - case HWCNT_JM_WD_INIT_DUMP_BUFFER: - errcode = kbase_hwcnt_dump_buffer_alloc(metadata, - &wd_backend->wd_dump_buffer); - break; - case HWCNT_JM_WD_INIT_END: - break; - } - if (!errcode) - state++; - } - - if (errcode) { - kbasep_hwcnt_backend_jm_watchdog_term_partial(wd_backend, state); - *out_backend = NULL; - return errcode; - } - - WARN_ON(state != HWCNT_JM_WD_INIT_END); - - spin_lock_init(&wd_backend->locked.watchdog_lock); - init_completion(&wd_backend->watchdog_complete); - - *out_backend = (struct kbase_hwcnt_backend *)wd_backend; - return 0; -} - -/* Job manager watchdog backend, implementation of timestamp_ns */ -static u64 kbasep_hwcnt_backend_jm_watchdog_timestamp_ns(struct kbase_hwcnt_backend *backend) -{ - struct kbase_hwcnt_backend_jm_watchdog *const wd_backend = (void *)backend; - - return wd_backend->info->jm_backend_iface->timestamp_ns(wd_backend->jm_backend); -} - -static int kbasep_hwcnt_backend_jm_watchdog_dump_enable_common( - struct kbase_hwcnt_backend_jm_watchdog *wd_backend, - const struct kbase_hwcnt_enable_map *enable_map, kbase_hwcnt_backend_dump_enable_fn enabler) -{ - int errcode = -EPERM; - unsigned long flags; - - if (WARN_ON(!wd_backend) || WARN_ON(!enable_map)) - return -EINVAL; - - spin_lock_irqsave(&wd_backend->locked.watchdog_lock, flags); - - /* If the backend is already enabled return an error */ - if (wd_backend->locked.is_enabled) { - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - return -EPERM; - } - - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - - /*We copy the enable map into our watchdog backend copy, for future usage*/ - kbase_hwcnt_enable_map_copy(&wd_backend->wd_enable_map, enable_map); - - errcode = enabler(wd_backend->jm_backend, enable_map); - if (!errcode) { - /*Enable dump watchdog*/ - errcode = wd_backend->info->dump_watchdog_iface->enable( - wd_backend->info->dump_watchdog_iface->timer, wd_backend->timeout_ms, - kbasep_hwcnt_backend_jm_watchdog_timer_callback, wd_backend); - if (!errcode) { - spin_lock_irqsave(&wd_backend->locked.watchdog_lock, flags); - WARN_ON(wd_backend->locked.is_enabled); - wd_backend->locked.is_enabled = true; - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - } else - /*Reverting the job manager backend back to disabled*/ - wd_backend->info->jm_backend_iface->dump_disable(wd_backend->jm_backend, - NULL, NULL); - } - - return errcode; -} - -/* Job manager watchdog backend, implementation of dump_enable */ -static int -kbasep_hwcnt_backend_jm_watchdog_dump_enable(struct kbase_hwcnt_backend *backend, - const struct kbase_hwcnt_enable_map *enable_map) -{ - struct kbase_hwcnt_backend_jm_watchdog *const wd_backend = (void *)backend; - - return kbasep_hwcnt_backend_jm_watchdog_dump_enable_common( - wd_backend, enable_map, wd_backend->info->jm_backend_iface->dump_enable); -} - -/* Job manager watchdog backend, implementation of dump_enable_nolock */ -static int -kbasep_hwcnt_backend_jm_watchdog_dump_enable_nolock(struct kbase_hwcnt_backend *backend, - const struct kbase_hwcnt_enable_map *enable_map) -{ - struct kbase_hwcnt_backend_jm_watchdog *const wd_backend = (void *)backend; - - return kbasep_hwcnt_backend_jm_watchdog_dump_enable_common( - wd_backend, enable_map, wd_backend->info->jm_backend_iface->dump_enable_nolock); -} - -/* Job manager watchdog backend, implementation of dump_disable */ -static void -kbasep_hwcnt_backend_jm_watchdog_dump_disable(struct kbase_hwcnt_backend *backend, - struct kbase_hwcnt_dump_buffer *dump_buffer, - const struct kbase_hwcnt_enable_map *buf_enable_map) -{ - struct kbase_hwcnt_backend_jm_watchdog *const wd_backend = (void *)backend; - unsigned long flags; - - if (WARN_ON(!backend)) - return; - - spin_lock_irqsave(&wd_backend->locked.watchdog_lock, flags); - if (!wd_backend->locked.is_enabled) { - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - return; - } - - wd_backend->locked.is_enabled = false; - - /* Discard undumped counter values since the last dump_get. */ - if (wd_backend->locked.state == HWCNT_JM_WD_IDLE_BUFFER_FULL) - wd_backend->locked.state = HWCNT_JM_WD_IDLE_BUFFER_EMPTY; - - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - - wd_backend->info->dump_watchdog_iface->disable( - wd_backend->info->dump_watchdog_iface->timer); - - wd_backend->info->jm_backend_iface->dump_disable(wd_backend->jm_backend, dump_buffer, - buf_enable_map); -} - -/* Job manager watchdog backend, implementation of dump_clear */ -static int kbasep_hwcnt_backend_jm_watchdog_dump_clear(struct kbase_hwcnt_backend *backend) -{ - int errcode = -EPERM; - bool clear_wd_wait_completion = false; - unsigned long flags; - struct kbase_hwcnt_backend_jm_watchdog *const wd_backend = (void *)backend; - - if (WARN_ON(!backend)) - return -EINVAL; - - spin_lock_irqsave(&wd_backend->locked.watchdog_lock, flags); - if (!wd_backend->locked.is_enabled) { - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - return -EPERM; - } - - switch (wd_backend->locked.state) { - case HWCNT_JM_WD_IDLE_BUFFER_FULL: - case HWCNT_JM_WD_USER_DUMPING_BUFFER_FULL: - case HWCNT_JM_WD_IDLE_BUFFER_EMPTY: - case HWCNT_JM_WD_USER_DUMPING_BUFFER_EMPTY: - wd_backend->locked.state = HWCNT_JM_WD_BUFFER_CLEARING; - errcode = 0; - break; - case HWCNT_JM_WD_TIMER_DUMPING: - /* The timer asked for a dump request, when complete, the job manager backend - * buffer will be zero - */ - clear_wd_wait_completion = true; - /* This thread will have to wait for the callback to terminate and then call a - * dump_clear on the job manager backend. We change the state to - * HWCNT_JM_WD_TIMER_DUMPING_USER_CLEAR to notify the callback thread there is - * no more need to dump the buffer (since we will clear it right after anyway). - * We set up a wait queue to synchronize with the callback. - */ - reinit_completion(&wd_backend->watchdog_complete); - wd_backend->locked.state = HWCNT_JM_WD_TIMER_DUMPING_USER_CLEAR; - errcode = 0; - break; - default: - errcode = -EPERM; - break; - } - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - - if (!errcode) { - if (clear_wd_wait_completion) { - /* Waiting for the callback to finish */ - wait_for_completion(&wd_backend->watchdog_complete); - } - - /* Clearing job manager backend buffer */ - errcode = wd_backend->info->jm_backend_iface->dump_clear(wd_backend->jm_backend); - - spin_lock_irqsave(&wd_backend->locked.watchdog_lock, flags); - - WARN_ON(wd_backend->locked.state != HWCNT_JM_WD_TIMER_DUMPING_USER_CLEAR && - wd_backend->locked.state != HWCNT_JM_WD_BUFFER_CLEARING && - wd_backend->locked.state != HWCNT_JM_WD_ERROR); - - WARN_ON(!wd_backend->locked.is_enabled); - - if (!errcode && wd_backend->locked.state != HWCNT_JM_WD_ERROR) { - /* Setting the internal buffer state to EMPTY */ - wd_backend->locked.state = HWCNT_JM_WD_IDLE_BUFFER_EMPTY; - /* Resetting the timer. Calling modify on a disabled timer - * enables it. - */ - wd_backend->info->dump_watchdog_iface->modify( - wd_backend->info->dump_watchdog_iface->timer, - wd_backend->timeout_ms); - } else { - wd_backend->locked.state = HWCNT_JM_WD_ERROR; - errcode = -EPERM; - } - - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - } - - return errcode; -} - -/* Job manager watchdog backend, implementation of dump_request */ -static int kbasep_hwcnt_backend_jm_watchdog_dump_request(struct kbase_hwcnt_backend *backend) -{ - bool call_dump_request = false; - int errcode = 0; - unsigned long flags; - struct kbase_hwcnt_backend_jm_watchdog *const wd_backend = (void *)backend; - - if (WARN_ON(!backend)) - return -EINVAL; - - spin_lock_irqsave(&wd_backend->locked.watchdog_lock, flags); - - if (!wd_backend->locked.is_enabled) { - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - return -EPERM; - } - - switch (wd_backend->locked.state) { - case HWCNT_JM_WD_IDLE_BUFFER_EMPTY: - /* progressing the state to avoid callbacks running while calling the job manager - * backend - */ - wd_backend->locked.state = HWCNT_JM_WD_USER_DUMPING_BUFFER_EMPTY; - call_dump_request = true; - break; - case HWCNT_JM_WD_IDLE_BUFFER_FULL: - wd_backend->locked.state = HWCNT_JM_WD_USER_DUMPING_BUFFER_FULL; - call_dump_request = true; - break; - case HWCNT_JM_WD_TIMER_DUMPING: - /* On the next client call (dump_wait) the thread will have to wait for the - * callback to finish the dumping. - * We set up a wait queue to synchronize with the callback. - */ - reinit_completion(&wd_backend->watchdog_complete); - wd_backend->locked.state = HWCNT_JM_WD_TIMER_DUMPING_USER_REQUESTED; - break; - default: - errcode = -EPERM; - break; - } - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - - if (call_dump_request) { - errcode = wd_backend->info->jm_backend_iface->dump_request(wd_backend->jm_backend); - if (!errcode) { - /*resetting the timer. Calling modify on a disabled timer enables it*/ - wd_backend->info->dump_watchdog_iface->modify( - wd_backend->info->dump_watchdog_iface->timer, - wd_backend->timeout_ms); - } else { - spin_lock_irqsave(&wd_backend->locked.watchdog_lock, flags); - WARN_ON(!wd_backend->locked.is_enabled); - wd_backend->locked.state = HWCNT_JM_WD_ERROR; - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - } - } - - return errcode; -} - -/* Job manager watchdog backend, implementation of dump_wait */ -static int kbasep_hwcnt_backend_jm_watchdog_dump_wait(struct kbase_hwcnt_backend *backend) -{ - int errcode = -EPERM; - bool wait_for_auto_dump = false, wait_for_user_dump = false; - struct kbase_hwcnt_backend_jm_watchdog *const wd_backend = (void *)backend; - unsigned long flags; - - if (WARN_ON(!backend)) - return -EINVAL; - - spin_lock_irqsave(&wd_backend->locked.watchdog_lock, flags); - if (!wd_backend->locked.is_enabled) { - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - return -EPERM; - } - - switch (wd_backend->locked.state) { - case HWCNT_JM_WD_TIMER_DUMPING_USER_REQUESTED: - wait_for_auto_dump = true; - errcode = 0; - break; - case HWCNT_JM_WD_USER_DUMPING_BUFFER_EMPTY: - case HWCNT_JM_WD_USER_DUMPING_BUFFER_FULL: - wait_for_user_dump = true; - errcode = 0; - break; - default: - errcode = -EPERM; - break; - } - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - - if (wait_for_auto_dump) - wait_for_completion(&wd_backend->watchdog_complete); - else if (wait_for_user_dump) { - errcode = wd_backend->info->jm_backend_iface->dump_wait(wd_backend->jm_backend); - if (errcode) { - spin_lock_irqsave(&wd_backend->locked.watchdog_lock, flags); - WARN_ON(!wd_backend->locked.is_enabled); - wd_backend->locked.state = HWCNT_JM_WD_ERROR; - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - } - } - - return errcode; -} - -/* Job manager watchdog backend, implementation of dump_get */ -static int kbasep_hwcnt_backend_jm_watchdog_dump_get( - struct kbase_hwcnt_backend *backend, struct kbase_hwcnt_dump_buffer *dump_buffer, - const struct kbase_hwcnt_enable_map *enable_map, bool accumulate, u64 *dump_time_ns) -{ - bool call_dump_get = false; - struct kbase_hwcnt_backend_jm_watchdog *const wd_backend = (void *)backend; - unsigned long flags; - int errcode = 0; - - if (WARN_ON(!backend) || WARN_ON(!dump_buffer) || WARN_ON(!enable_map) || - WARN_ON(!dump_time_ns)) - return -EINVAL; - - /* The resultant contents of the dump buffer are only well defined if a prior - * call to dump_wait returned successfully, and a new dump has not yet been - * requested by a call to dump_request. - */ - - spin_lock_irqsave(&wd_backend->locked.watchdog_lock, flags); - - switch (wd_backend->locked.state) { - case HWCNT_JM_WD_TIMER_DUMPING_USER_REQUESTED: - /*we assume dump_wait has been called and completed successfully*/ - if (accumulate) - kbase_hwcnt_dump_buffer_accumulate(dump_buffer, &wd_backend->wd_dump_buffer, - enable_map); - else - kbase_hwcnt_dump_buffer_copy(dump_buffer, &wd_backend->wd_dump_buffer, - enable_map); - - /*use state to indicate the the buffer is now empty*/ - wd_backend->locked.state = HWCNT_JM_WD_IDLE_BUFFER_EMPTY; - /* Retrieve timing information from previous dump_request */ - *dump_time_ns = wd_backend->wd_dump_timestamp; - break; - case HWCNT_JM_WD_USER_DUMPING_BUFFER_FULL: - /*accumulate or copy watchdog data to user buffer first so that dump_get can set - * the header correctly - */ - if (accumulate) - kbase_hwcnt_dump_buffer_accumulate(dump_buffer, &wd_backend->wd_dump_buffer, - enable_map); - else - kbase_hwcnt_dump_buffer_copy(dump_buffer, &wd_backend->wd_dump_buffer, - enable_map); - - /*accumulate backend data into user buffer on top of watchdog data*/ - accumulate = true; - call_dump_get = true; - break; - case HWCNT_JM_WD_USER_DUMPING_BUFFER_EMPTY: - call_dump_get = true; - break; - default: - errcode = -EPERM; - break; - } - - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - - if (call_dump_get && !errcode) { - /*we just dump the job manager backend into the user buffer, following - *accumulate flag - */ - errcode = wd_backend->info->jm_backend_iface->dump_get( - wd_backend->jm_backend, dump_buffer, enable_map, accumulate, dump_time_ns); - - spin_lock_irqsave(&wd_backend->locked.watchdog_lock, flags); - - WARN_ON(wd_backend->locked.state != HWCNT_JM_WD_USER_DUMPING_BUFFER_EMPTY && - wd_backend->locked.state != HWCNT_JM_WD_USER_DUMPING_BUFFER_FULL && - wd_backend->locked.state != HWCNT_JM_WD_TIMER_DUMPING_USER_REQUESTED); - - if (!errcode) - wd_backend->locked.state = HWCNT_JM_WD_IDLE_BUFFER_EMPTY; - else - wd_backend->locked.state = HWCNT_JM_WD_ERROR; - - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - } - - return errcode; -} - -/* exposed methods */ - -int kbase_hwcnt_backend_jm_watchdog_create(struct kbase_hwcnt_backend_interface *backend_iface, - struct kbase_hwcnt_watchdog_interface *watchdog_iface, - struct kbase_hwcnt_backend_interface *out_iface, - u32 watchdog_timer_interval_ms) -{ - struct kbase_hwcnt_backend_jm_watchdog_info *info = NULL; - - if (WARN_ON(!backend_iface) || WARN_ON(!watchdog_iface) || WARN_ON(!out_iface)) - return -EINVAL; - - info = kbasep_hwcnt_backend_jm_watchdog_info_create(backend_iface, watchdog_iface, - watchdog_timer_interval_ms); - if (!info) - return -ENOMEM; - - /*linking the info table with the output iface, to allow the callbacks below to access the - *info object later on - */ - *out_iface = (struct kbase_hwcnt_backend_interface){ - .info = (void *)info, - .metadata = kbasep_hwcnt_backend_jm_watchdog_metadata, - .init = kbasep_hwcnt_backend_jm_watchdog_init, - .term = kbasep_hwcnt_backend_jm_watchdog_term, - .acquire = kbasep_hwcnt_backend_jm_watchdog_acquire, - .release = kbasep_hwcnt_backend_jm_watchdog_release, - .timestamp_ns = kbasep_hwcnt_backend_jm_watchdog_timestamp_ns, - .dump_enable = kbasep_hwcnt_backend_jm_watchdog_dump_enable, - .dump_enable_nolock = kbasep_hwcnt_backend_jm_watchdog_dump_enable_nolock, - .dump_disable = kbasep_hwcnt_backend_jm_watchdog_dump_disable, - .dump_clear = kbasep_hwcnt_backend_jm_watchdog_dump_clear, - .dump_request = kbasep_hwcnt_backend_jm_watchdog_dump_request, - .dump_wait = kbasep_hwcnt_backend_jm_watchdog_dump_wait, - .dump_get = kbasep_hwcnt_backend_jm_watchdog_dump_get - }; - - /*registering watchdog backend module methods on the output interface*/ - - return 0; -} - -void kbase_hwcnt_backend_jm_watchdog_destroy(struct kbase_hwcnt_backend_interface *iface) -{ - if (!iface || !iface->info) - return; - - kfree((struct kbase_hwcnt_backend_jm_watchdog_info *)iface->info); - - /*blanking the watchdog backend interface*/ - memset(iface, 0, sizeof(*iface)); -} diff --git a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_jm_watchdog.h b/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_jm_watchdog.h deleted file mode 100644 index d5ce27685186..000000000000 --- a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_jm_watchdog.h +++ /dev/null @@ -1,67 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2021-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Concrete implementation of mali_kbase_hwcnt_backend interface for job manager - * backend. This module functionally interleaves between the hardware counter - * (hwcnt_accumulator) module (the interface consumer) and the job manager - * backend module (hwcnt_backend_jm). This module provides buffering - * functionality for the dumping requests requested by the hwcnt_accumulator - * consumer. This module is NOT multi-thread safe. The programmer must - * ensure the exposed methods are called by at most one thread at any time. - */ - -#ifndef _KBASE_HWCNT_BACKEND_JM_WATCHDOG_H_ -#define _KBASE_HWCNT_BACKEND_JM_WATCHDOG_H_ - -#include -#include - -/** - * kbase_hwcnt_backend_jm_watchdog_create() - Create a job manager hardware counter watchdog - * backend interface. - * @backend_iface: Non-NULL pointer to the backend interface structure that this module will - * extend. - * @watchdog_iface: Non-NULL pointer to an hardware counter watchdog interface. - * @out_iface: Non-NULL pointer to backend interface structure that is filled in - * on creation success. - * @watchdog_timer_interval_ms: Interval in milliseconds between hwcnt samples. - * - * Calls to out_iface->dump_enable_nolock() require kbdev->hwaccess_lock held. - * - * Return: 0 on success, error otherwise. - */ -int kbase_hwcnt_backend_jm_watchdog_create(struct kbase_hwcnt_backend_interface *backend_iface, - struct kbase_hwcnt_watchdog_interface *watchdog_iface, - struct kbase_hwcnt_backend_interface *out_iface, - u32 watchdog_timer_interval_ms); - -/** - * kbase_hwcnt_backend_jm_watchdog_destroy() - Destroy a job manager hardware counter watchdog - * backend interface. - * @iface: Pointer to interface to destroy. - * - * Can be safely called on an all-zeroed interface, or on an already destroyed - * interface. - */ -void kbase_hwcnt_backend_jm_watchdog_destroy(struct kbase_hwcnt_backend_interface *iface); - -#endif /* _KBASE_HWCNT_BACKEND_JM_WATCHDOG_H_ */ diff --git a/drivers/gpu/arm/valhall/hwcnt/mali_kbase_hwcnt_watchdog_if.h b/drivers/gpu/arm/valhall/hwcnt/mali_kbase_hwcnt_watchdog_if.h index 501c0087b7e6..f8b12c12a4ef 100644 --- a/drivers/gpu/arm/valhall/hwcnt/mali_kbase_hwcnt_watchdog_if.h +++ b/drivers/gpu/arm/valhall/hwcnt/mali_kbase_hwcnt_watchdog_if.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2021-2022 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2021-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -33,6 +33,15 @@ */ struct kbase_hwcnt_watchdog_info; +/* + * Enum describing different types of watchdog disable calls. + */ +enum kbase_hwcnt_watchdog_disable_type { + KBASE_HWCNT_WATCHDOG_DISABLE_SHOULD_BLOCK, + KBASE_HWCNT_WATCHDOG_DISABLE_SHOULD_NOT_BLOCK, + KBASE_HWCNT_WATCHDOG_DISABLE_TYPE_CNT +}; + /** * typedef kbase_hwcnt_watchdog_callback_fn - Callback function when watchdog timer is done * @@ -59,8 +68,10 @@ typedef int kbase_hwcnt_watchdog_enable_fn(const struct kbase_hwcnt_watchdog_inf * typedef kbase_hwcnt_watchdog_disable_fn - Disable watchdog timer * * @timer: Non-NULL pointer to a watchdog timer interface context + * @type: Disable type to use */ -typedef void kbase_hwcnt_watchdog_disable_fn(const struct kbase_hwcnt_watchdog_info *timer); +typedef void kbase_hwcnt_watchdog_disable_fn(const struct kbase_hwcnt_watchdog_info *timer, + enum kbase_hwcnt_watchdog_disable_type type); /** * typedef kbase_hwcnt_watchdog_modify_fn - Modify watchdog timer's timeout diff --git a/drivers/gpu/arm/valhall/hwcnt/mali_kbase_hwcnt_watchdog_if_timer.c b/drivers/gpu/arm/valhall/hwcnt/mali_kbase_hwcnt_watchdog_if_timer.c index 4caa832cd587..eae1cf6c0aae 100644 --- a/drivers/gpu/arm/valhall/hwcnt/mali_kbase_hwcnt_watchdog_if_timer.c +++ b/drivers/gpu/arm/valhall/hwcnt/mali_kbase_hwcnt_watchdog_if_timer.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2021-2022 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2021-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -57,7 +57,8 @@ static void kbasep_hwcnt_watchdog_callback(struct work_struct *const work) struct kbase_hwcnt_watchdog_if_timer_info *const info = container_of(work, struct kbase_hwcnt_watchdog_if_timer_info, dwork.work); - if (info->callback) + /* Issue the callback only if timer is enabled. */ + if (info->callback && info->timer_enabled) info->callback(info->user_data); } @@ -80,7 +81,8 @@ static int kbasep_hwcnt_watchdog_if_timer_enable( } static void -kbasep_hwcnt_watchdog_if_timer_disable(const struct kbase_hwcnt_watchdog_info *const timer) +kbasep_hwcnt_watchdog_if_timer_disable(const struct kbase_hwcnt_watchdog_info *const timer, + enum kbase_hwcnt_watchdog_disable_type type) { struct kbase_hwcnt_watchdog_if_timer_info *const timer_info = (void *)timer; @@ -90,7 +92,16 @@ kbasep_hwcnt_watchdog_if_timer_disable(const struct kbase_hwcnt_watchdog_info *c if (!timer_info->timer_enabled) return; - cancel_delayed_work_sync(&timer_info->dwork); + /* Cancel any pending work. + * Use a blocking call if the caller can afford to wait. + * In an opposite case (KBASE_HWCNT_WATCHDOG_DISABLE_SHOULD_NOT_BLOCK), + * timer callback will be effectively canceled by setting timer_enabled to false. + * This will prevent the timer callback from being issued and from rescheduling + * itself. + */ + if (type == KBASE_HWCNT_WATCHDOG_DISABLE_SHOULD_BLOCK) + cancel_delayed_work_sync(&timer_info->dwork); + timer_info->timer_enabled = false; } @@ -100,7 +111,7 @@ kbasep_hwcnt_watchdog_if_timer_modify(const struct kbase_hwcnt_watchdog_info *co { struct kbase_hwcnt_watchdog_if_timer_info *const timer_info = (void *)timer; - if (WARN_ON(!timer) || WARN_ON(!timer_info->timer_enabled)) + if (WARN_ON(!timer) || !timer_info->timer_enabled) return; mod_delayed_work(timer_info->workq, &timer_info->dwork, msecs_to_jiffies(delay_ms)); @@ -118,6 +129,7 @@ void kbase_hwcnt_watchdog_if_timer_destroy(struct kbase_hwcnt_watchdog_interface if (WARN_ON(!timer_info)) return; + cancel_delayed_work_sync(&timer_info->dwork); destroy_workqueue(timer_info->workq); kfree(timer_info); diff --git a/drivers/gpu/arm/valhall/ipa/Kbuild b/drivers/gpu/arm/valhall/ipa/Kbuild index 9d8a79d8dc29..968157446956 100644 --- a/drivers/gpu/arm/valhall/ipa/Kbuild +++ b/drivers/gpu/arm/valhall/ipa/Kbuild @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2016-2018, 2020-2021 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2016-2018, 2020-2021, 2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -20,16 +20,8 @@ valhall_kbase-y += \ ipa/mali_kbase_ipa_simple.o \ - ipa/mali_kbase_ipa.o + ipa/mali_kbase_ipa.o \ + ipa/backend/mali_kbase_ipa_counter_csf.o \ + ipa/backend/mali_kbase_ipa_counter_common_csf.o valhall_kbase-$(CONFIG_DEBUG_FS) += ipa/mali_kbase_ipa_debugfs.o - -ifeq ($(MALI_USE_CSF),1) - valhall_kbase-y += \ - ipa/backend/mali_kbase_ipa_counter_csf.o \ - ipa/backend/mali_kbase_ipa_counter_common_csf.o -else - valhall_kbase-y += \ - ipa/backend/mali_kbase_ipa_counter_jm.o \ - ipa/backend/mali_kbase_ipa_counter_common_jm.o -endif diff --git a/drivers/gpu/arm/valhall/ipa/backend/mali_kbase_ipa_counter_common_jm.c b/drivers/gpu/arm/valhall/ipa/backend/mali_kbase_ipa_counter_common_jm.c deleted file mode 100644 index a716e15b3df3..000000000000 --- a/drivers/gpu/arm/valhall/ipa/backend/mali_kbase_ipa_counter_common_jm.c +++ /dev/null @@ -1,339 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2017-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include "mali_kbase_ipa_counter_common_jm.h" -#include "ipa/mali_kbase_ipa_debugfs.h" - -#define DEFAULT_SCALING_FACTOR 5 - -/* If the value of GPU_ACTIVE is below this, use the simple model - * instead, to avoid extrapolating small amounts of counter data across - * large sample periods. - */ -#define DEFAULT_MIN_SAMPLE_CYCLES 10000 - -/** - * kbase_ipa_read_hwcnt() - read a counter value - * @model_data: pointer to model data - * @offset: offset, in bytes, into vinstr buffer - * - * Return: A 32-bit counter value. Range: 0 < value < 2^27 (worst case would be - * incrementing every cycle over a ~100ms sample period at a high frequency, - * e.g. 1 GHz: 2^30 * 0.1seconds ~= 2^27. - */ -static inline u32 kbase_ipa_read_hwcnt(struct kbase_ipa_model_vinstr_data *model_data, u32 offset) -{ - u8 *p = (u8 *)model_data->dump_buf.dump_buf; - u64 val = *(u64 *)&p[offset]; - - return (val > U32_MAX) ? U32_MAX : (u32)val; -} - -static inline s64 kbase_ipa_add_saturate(s64 a, s64 b) -{ - s64 rtn; - - if (a > 0 && (S64_MAX - a) < b) - rtn = S64_MAX; - else if (a < 0 && (S64_MIN - a) > b) - rtn = S64_MIN; - else - rtn = a + b; - - return rtn; -} - -s64 kbase_ipa_sum_all_shader_cores(struct kbase_ipa_model_vinstr_data *model_data, s32 coeff, - u32 counter) -{ - struct kbase_device *kbdev = model_data->kbdev; - u64 core_mask; - u32 base = 0; - s64 ret = 0; - - core_mask = kbdev->gpu_props.coherency_info.group.core_mask; - while (core_mask != 0ull) { - if ((core_mask & 1ull) != 0ull) { - /* 0 < counter_value < 2^27 */ - u32 counter_value = kbase_ipa_read_hwcnt(model_data, base + counter); - - /* 0 < ret < 2^27 * max_num_cores = 2^32 */ - ret = kbase_ipa_add_saturate(ret, counter_value); - } - base += KBASE_IPA_NR_BYTES_PER_BLOCK; - core_mask >>= 1; - } - - /* Range: -2^54 < ret * coeff < 2^54 */ - return ret * coeff; -} - -s64 kbase_ipa_sum_all_memsys_blocks(struct kbase_ipa_model_vinstr_data *model_data, s32 coeff, - u32 counter) -{ - struct kbase_device *kbdev = model_data->kbdev; - const u32 num_blocks = kbdev->gpu_props.num_l2_slices; - u32 base = 0; - s64 ret = 0; - u32 i; - - for (i = 0; i < num_blocks; i++) { - /* 0 < counter_value < 2^27 */ - u32 counter_value = kbase_ipa_read_hwcnt(model_data, base + counter); - - /* 0 < ret < 2^27 * max_num_memsys_blocks = 2^29 */ - ret = kbase_ipa_add_saturate(ret, counter_value); - base += KBASE_IPA_NR_BYTES_PER_BLOCK; - } - - /* Range: -2^51 < ret * coeff < 2^51 */ - return ret * coeff; -} - -s64 kbase_ipa_single_counter(struct kbase_ipa_model_vinstr_data *model_data, s32 coeff, u32 counter) -{ - /* Range: 0 < counter_value < 2^27 */ - const u32 counter_value = kbase_ipa_read_hwcnt(model_data, counter); - - /* Range: -2^49 < ret < 2^49 */ - return counter_value * (s64)coeff; -} - -int kbase_ipa_attach_vinstr(struct kbase_ipa_model_vinstr_data *model_data) -{ - int errcode; - struct kbase_device *kbdev = model_data->kbdev; - struct kbase_hwcnt_virtualizer *hvirt = kbdev->hwcnt_gpu_virt; - struct kbase_hwcnt_enable_map enable_map; - const struct kbase_hwcnt_metadata *metadata = kbase_hwcnt_virtualizer_metadata(hvirt); - - if (!metadata) - return -1; - - errcode = kbase_hwcnt_enable_map_alloc(metadata, &enable_map); - if (errcode) { - dev_err(kbdev->dev, "Failed to allocate IPA enable map"); - return errcode; - } - - kbase_hwcnt_enable_map_enable_all(&enable_map); - - /* Disable cycle counter only. */ - enable_map.clk_enable_map = 0; - - errcode = kbase_hwcnt_virtualizer_client_create(hvirt, &enable_map, &model_data->hvirt_cli); - kbase_hwcnt_enable_map_free(&enable_map); - if (errcode) { - dev_err(kbdev->dev, "Failed to register IPA with virtualizer"); - model_data->hvirt_cli = NULL; - return errcode; - } - - errcode = kbase_hwcnt_dump_buffer_alloc(metadata, &model_data->dump_buf); - if (errcode) { - dev_err(kbdev->dev, "Failed to allocate IPA dump buffer"); - kbase_hwcnt_virtualizer_client_destroy(model_data->hvirt_cli); - model_data->hvirt_cli = NULL; - return errcode; - } - - return 0; -} - -void kbase_ipa_detach_vinstr(struct kbase_ipa_model_vinstr_data *model_data) -{ - if (model_data->hvirt_cli) { - kbase_hwcnt_virtualizer_client_destroy(model_data->hvirt_cli); - kbase_hwcnt_dump_buffer_free(&model_data->dump_buf); - model_data->hvirt_cli = NULL; - } -} - -int kbase_ipa_vinstr_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp) -{ - struct kbase_ipa_model_vinstr_data *model_data = - (struct kbase_ipa_model_vinstr_data *)model->model_data; - s64 energy = 0; - size_t i; - u64 coeff = 0, coeff_mul = 0; - u64 start_ts_ns, end_ts_ns; - u32 active_cycles; - int err = 0; - - err = kbase_hwcnt_virtualizer_client_dump(model_data->hvirt_cli, &start_ts_ns, &end_ts_ns, - &model_data->dump_buf); - if (err) - goto err0; - - /* Range: 0 (GPU not used at all), to the max sampling interval, say - * 1s, * max GPU frequency (GPU 100% utilized). - * 0 <= active_cycles <= 1 * ~2GHz - * 0 <= active_cycles < 2^31 - */ - active_cycles = model_data->get_active_cycles(model_data); - - if (active_cycles < (u32)max(model_data->min_sample_cycles, 0)) { - err = -ENODATA; - goto err0; - } - - /* Range: 1 <= active_cycles < 2^31 */ - active_cycles = max(1u, active_cycles); - - /* Range of 'energy' is +/- 2^54 * number of IPA groups (~8), so around - * -2^57 < energy < 2^57 - */ - for (i = 0; i < model_data->groups_def_num; i++) { - const struct kbase_ipa_group *group = &model_data->groups_def[i]; - s32 coeff = model_data->group_values[i]; - s64 group_energy = group->op(model_data, coeff, group->counter_block_offset); - - energy = kbase_ipa_add_saturate(energy, group_energy); - } - - /* Range: 0 <= coeff < 2^57 */ - if (energy > 0) - coeff = (u64)energy; - - /* Range: 0 <= coeff < 2^57 (because active_cycles >= 1). However, this - * can be constrained further: Counter values can only be increased by - * a theoretical maximum of about 64k per clock cycle. Beyond this, - * we'd have to sample every 1ms to avoid them overflowing at the - * lowest clock frequency (say 100MHz). Therefore, we can write the - * range of 'coeff' in terms of active_cycles: - * - * coeff = SUM(coeffN * counterN * num_cores_for_counterN) - * coeff <= SUM(coeffN * counterN) * max_num_cores - * coeff <= num_IPA_groups * max_coeff * max_counter * max_num_cores - * (substitute max_counter = 2^16 * active_cycles) - * coeff <= num_IPA_groups * max_coeff * 2^16 * active_cycles * max_num_cores - * coeff <= 2^3 * 2^22 * 2^16 * active_cycles * 2^5 - * coeff <= 2^46 * active_cycles - * - * So after the division: 0 <= coeff <= 2^46 - */ - coeff = div_u64(coeff, active_cycles); - - /* Not all models were derived at the same reference voltage. Voltage - * scaling is done by multiplying by V^2, so we need to *divide* by - * Vref^2 here. - * Range: 0 <= coeff <= 2^49 - */ - coeff = div_u64(coeff * 1000, max(model_data->reference_voltage, 1)); - /* Range: 0 <= coeff <= 2^52 */ - coeff = div_u64(coeff * 1000, max(model_data->reference_voltage, 1)); - - /* Scale by user-specified integer factor. - * Range: 0 <= coeff_mul < 2^57 - */ - coeff_mul = coeff * (u64)model_data->scaling_factor; - - /* The power models have results with units - * mW/(MHz V^2), i.e. nW/(Hz V^2). With precision of 1/1000000, this - * becomes fW/(Hz V^2), which are the units of coeff_mul. However, - * kbase_scale_dynamic_power() expects units of pW/(Hz V^2), so divide - * by 1000. - * Range: 0 <= coeff_mul < 2^47 - */ - coeff_mul = div_u64(coeff_mul, 1000u); - -err0: - /* Clamp to a sensible range - 2^16 gives about 14W at 400MHz/750mV */ - *coeffp = clamp(coeff_mul, (u64)0, (u64)1 << 16); - return err; -} - -void kbase_ipa_vinstr_reset_data(struct kbase_ipa_model *model) -{ - CSTD_UNUSED(model); - /* Currently not implemented */ - WARN_ON_ONCE(1); -} - -int kbase_ipa_vinstr_common_model_init(struct kbase_ipa_model *model, - const struct kbase_ipa_group *ipa_groups_def, - size_t ipa_group_size, - kbase_ipa_get_active_cycles_callback get_active_cycles, - s32 reference_voltage) -{ - int err = 0; - size_t i; - struct kbase_ipa_model_vinstr_data *model_data; - - if (!model || !ipa_groups_def || !ipa_group_size || !get_active_cycles) - return -EINVAL; - - model_data = kzalloc(sizeof(*model_data), GFP_KERNEL); - if (!model_data) - return -ENOMEM; - - model_data->kbdev = model->kbdev; - model_data->groups_def = ipa_groups_def; - model_data->groups_def_num = ipa_group_size; - model_data->get_active_cycles = get_active_cycles; - - model->model_data = (void *)model_data; - - for (i = 0; i < model_data->groups_def_num; ++i) { - const struct kbase_ipa_group *group = &model_data->groups_def[i]; - - model_data->group_values[i] = group->default_value; - err = kbase_ipa_model_add_param_s32(model, group->name, - &model_data->group_values[i], 1, false); - if (err) - goto exit; - } - - model_data->scaling_factor = DEFAULT_SCALING_FACTOR; - err = kbase_ipa_model_add_param_s32(model, "scale", &model_data->scaling_factor, 1, false); - if (err) - goto exit; - - model_data->min_sample_cycles = DEFAULT_MIN_SAMPLE_CYCLES; - err = kbase_ipa_model_add_param_s32(model, "min_sample_cycles", - &model_data->min_sample_cycles, 1, false); - if (err) - goto exit; - - model_data->reference_voltage = reference_voltage; - err = kbase_ipa_model_add_param_s32(model, "reference_voltage", - &model_data->reference_voltage, 1, false); - if (err) - goto exit; - - err = kbase_ipa_attach_vinstr(model_data); - -exit: - if (err) { - kbase_ipa_model_param_free_all(model); - kfree(model_data); - } - return err; -} - -void kbase_ipa_vinstr_common_model_term(struct kbase_ipa_model *model) -{ - struct kbase_ipa_model_vinstr_data *model_data = - (struct kbase_ipa_model_vinstr_data *)model->model_data; - - kbase_ipa_detach_vinstr(model_data); - kfree(model_data); -} diff --git a/drivers/gpu/arm/valhall/ipa/backend/mali_kbase_ipa_counter_common_jm.h b/drivers/gpu/arm/valhall/ipa/backend/mali_kbase_ipa_counter_common_jm.h deleted file mode 100644 index d1d1f7d3b57f..000000000000 --- a/drivers/gpu/arm/valhall/ipa/backend/mali_kbase_ipa_counter_common_jm.h +++ /dev/null @@ -1,227 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2017-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#ifndef _KBASE_IPA_COUNTER_COMMON_JM_H_ -#define _KBASE_IPA_COUNTER_COMMON_JM_H_ - -#include "mali_kbase.h" -#include "hwcnt/mali_kbase_hwcnt_virtualizer.h" -#include "hwcnt/mali_kbase_hwcnt_types.h" - -/* Maximum number of IPA groups for an IPA model. */ -#define KBASE_IPA_MAX_GROUP_DEF_NUM 16 - -/* Number of bytes per hardware counter in a vinstr_buffer. */ -#define KBASE_IPA_NR_BYTES_PER_CNT (sizeof(u64)) - -/* Number of hardware counters per block in a vinstr_buffer. */ -#define KBASE_IPA_NR_CNT_PER_BLOCK 64 - -/* Number of bytes per block in a vinstr_buffer. */ -#define KBASE_IPA_NR_BYTES_PER_BLOCK (KBASE_IPA_NR_CNT_PER_BLOCK * KBASE_IPA_NR_BYTES_PER_CNT) - -struct kbase_ipa_model_vinstr_data; - -typedef u32 kbase_ipa_get_active_cycles_callback(struct kbase_ipa_model_vinstr_data *); - -/** - * struct kbase_ipa_model_vinstr_data - IPA context per device - * @kbdev: pointer to kbase device - * @group_values: values of coefficients for IPA groups - * @groups_def: Array of IPA groups. - * @groups_def_num: Number of elements in the array of IPA groups. - * @get_active_cycles: Callback to return number of active cycles during - * counter sample period - * @hvirt_cli: hardware counter virtualizer client handle - * @dump_buf: buffer to dump hardware counters onto - * @reference_voltage: voltage, in mV, of the operating point used when - * deriving the power model coefficients. Range approx - * 0.1V - 5V (~= 8V): 2^7 <= reference_voltage <= 2^13 - * @scaling_factor: User-specified power scaling factor. This is an - * integer, which is multiplied by the power coefficient - * just before OPP scaling. - * Range approx 0-32: 0 < scaling_factor < 2^5 - * @min_sample_cycles: If the value of the GPU_ACTIVE counter (the number of - * cycles the GPU was working) is less than - * min_sample_cycles, the counter model will return an - * error, causing the IPA framework to approximate using - * the cached simple model results instead. This may be - * more accurate than extrapolating using a very small - * counter dump. - */ -struct kbase_ipa_model_vinstr_data { - struct kbase_device *kbdev; - s32 group_values[KBASE_IPA_MAX_GROUP_DEF_NUM]; - const struct kbase_ipa_group *groups_def; - size_t groups_def_num; - kbase_ipa_get_active_cycles_callback *get_active_cycles; - struct kbase_hwcnt_virtualizer_client *hvirt_cli; - struct kbase_hwcnt_dump_buffer dump_buf; - s32 reference_voltage; - s32 scaling_factor; - s32 min_sample_cycles; -}; - -/** - * struct kbase_ipa_group - represents a single IPA group - * @name: name of the IPA group - * @default_value: default value of coefficient for IPA group. - * Coefficients are interpreted as fractions where the - * denominator is 1000000. - * @op: which operation to be performed on the counter values - * @counter_block_offset: block offset in bytes of the counter used to calculate energy for IPA group - */ -struct kbase_ipa_group { - const char *name; - s32 default_value; - s64 (*op)(struct kbase_ipa_model_vinstr_data *model_data, s32 coeff, - u32 counter_block_offset); - u32 counter_block_offset; -}; - -/** - * kbase_ipa_sum_all_shader_cores() - sum a counter over all cores - * @model_data: pointer to model data - * @coeff: model coefficient. Unity is ~2^20, so range approx - * +/- 4.0: -2^22 < coeff < 2^22 - * @counter: offset in bytes of the counter used to calculate energy - * for IPA group - * - * Calculate energy estimation based on hardware counter `counter' - * across all shader cores. - * - * Return: Sum of counter values. Range: -2^54 < ret < 2^54 - */ -s64 kbase_ipa_sum_all_shader_cores(struct kbase_ipa_model_vinstr_data *model_data, s32 coeff, - u32 counter); - -/** - * kbase_ipa_sum_all_memsys_blocks() - sum a counter over all mem system blocks - * @model_data: pointer to model data - * @coeff: model coefficient. Unity is ~2^20, so range approx - * +/- 4.0: -2^22 < coeff < 2^22 - * @counter: offset in bytes of the counter used to calculate energy - * for IPA group - * - * Calculate energy estimation based on hardware counter `counter' across all - * memory system blocks. - * - * Return: Sum of counter values. Range: -2^51 < ret < 2^51 - */ -s64 kbase_ipa_sum_all_memsys_blocks(struct kbase_ipa_model_vinstr_data *model_data, s32 coeff, - u32 counter); - -/** - * kbase_ipa_single_counter() - sum a single counter - * @model_data: pointer to model data - * @coeff: model coefficient. Unity is ~2^20, so range approx - * +/- 4.0: -2^22 < coeff < 2^22 - * @counter: offset in bytes of the counter used to calculate energy - * for IPA group - * - * Calculate energy estimation based on hardware counter `counter'. - * - * Return: Counter value. Range: -2^49 < ret < 2^49 - */ -s64 kbase_ipa_single_counter(struct kbase_ipa_model_vinstr_data *model_data, s32 coeff, - u32 counter); - -/** - * kbase_ipa_attach_vinstr() - attach a vinstr_buffer to an IPA model. - * @model_data: pointer to model data - * - * Attach a vinstr_buffer to an IPA model. The vinstr_buffer - * allows access to the hardware counters used to calculate - * energy consumption. - * - * Return: 0 on success, or an error code. - */ -int kbase_ipa_attach_vinstr(struct kbase_ipa_model_vinstr_data *model_data); - -/** - * kbase_ipa_detach_vinstr() - detach a vinstr_buffer from an IPA model. - * @model_data: pointer to model data - * - * Detach a vinstr_buffer from an IPA model. - */ -void kbase_ipa_detach_vinstr(struct kbase_ipa_model_vinstr_data *model_data); - -/** - * kbase_ipa_vinstr_dynamic_coeff() - calculate dynamic power based on HW counters - * @model: pointer to instantiated model - * @coeffp: pointer to location where calculated power, in - * pW/(Hz V^2), is stored. - * - * This is a GPU-agnostic implementation of the get_dynamic_coeff() - * function of an IPA model. It relies on the model being populated - * with GPU-specific attributes at initialization time. - * - * Return: 0 on success, or an error code. - */ -int kbase_ipa_vinstr_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp); - -/** - * kbase_ipa_vinstr_reset_data() - Reset the counters data used for dynamic - * power estimation - * @model: pointer to instantiated model - * - * Currently it is not implemented for JM GPUs. - * When implemented it is expected to retrieve the accumulated value of HW - * counters from the Vinstr component, without doing any processing, which is - * effectively a reset as the next call to kbase_ipa_counter_dynamic_coeff() - * will see the increment in counter values from this point onwards. - */ -void kbase_ipa_vinstr_reset_data(struct kbase_ipa_model *model); - -/** - * kbase_ipa_vinstr_common_model_init() - initialize ipa power model - * @model: ipa power model to initialize - * @ipa_groups_def: array of ipa groups which sets coefficients for - * the corresponding counters used in the ipa model - * @ipa_group_size: number of elements in the array @ipa_groups_def - * @get_active_cycles: callback to return the number of cycles the GPU was - * active during the counter sample period. - * @reference_voltage: voltage, in mV, of the operating point used when - * deriving the power model coefficients. - * - * This initialization function performs initialization steps common - * for ipa models based on counter values. In each call, the model - * passes its specific coefficient values per ipa counter group via - * @ipa_groups_def array. - * - * Return: 0 on success, error code otherwise - */ -int kbase_ipa_vinstr_common_model_init(struct kbase_ipa_model *model, - const struct kbase_ipa_group *ipa_groups_def, - size_t ipa_group_size, - kbase_ipa_get_active_cycles_callback *get_active_cycles, - s32 reference_voltage); - -/** - * kbase_ipa_vinstr_common_model_term() - terminate ipa power model - * @model: ipa power model to terminate - * - * This function performs all necessary steps to terminate ipa power model - * including clean up of resources allocated to hold model data. - */ -void kbase_ipa_vinstr_common_model_term(struct kbase_ipa_model *model); - -#endif /* _KBASE_IPA_COUNTER_COMMON_JM_H_ */ diff --git a/drivers/gpu/arm/valhall/ipa/backend/mali_kbase_ipa_counter_jm.c b/drivers/gpu/arm/valhall/ipa/backend/mali_kbase_ipa_counter_jm.c deleted file mode 100644 index b19608a71b54..000000000000 --- a/drivers/gpu/arm/valhall/ipa/backend/mali_kbase_ipa_counter_jm.c +++ /dev/null @@ -1,536 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2016-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include - -#include "mali_kbase_ipa_counter_common_jm.h" -#include "mali_kbase.h" -#include - -/* Performance counter blocks base offsets */ -#define JM_BASE (0 * KBASE_IPA_NR_BYTES_PER_BLOCK) -#define MEMSYS_BASE (2 * KBASE_IPA_NR_BYTES_PER_BLOCK) - -/* JM counter block offsets */ -#define JM_GPU_ACTIVE (KBASE_IPA_NR_BYTES_PER_CNT * 6) - -/* MEMSYS counter block offsets */ -#define MEMSYS_L2_ANY_LOOKUP (KBASE_IPA_NR_BYTES_PER_CNT * 25) - -/* SC counter block offsets */ -#define SC_EXEC_INSTR_FMA (KBASE_IPA_NR_BYTES_PER_CNT * 27) -#define SC_EXEC_INSTR_COUNT (KBASE_IPA_NR_BYTES_PER_CNT * 28) -#define SC_EXEC_INSTR_MSG (KBASE_IPA_NR_BYTES_PER_CNT * 30) -#define SC_TEX_FILT_NUM_OPERATIONS (KBASE_IPA_NR_BYTES_PER_CNT * 39) -#define SC_TEX_COORD_ISSUE (KBASE_IPA_NR_BYTES_PER_CNT * 40) -#define SC_TEX_TFCH_NUM_OPERATIONS (KBASE_IPA_NR_BYTES_PER_CNT * 42) -#define SC_VARY_INSTR (KBASE_IPA_NR_BYTES_PER_CNT * 49) -#define SC_BEATS_WR_TIB (KBASE_IPA_NR_BYTES_PER_CNT * 62) - -/** - * kbase_g7x_power_model_get_jm_counter() - get performance counter offset - * inside the Job Manager block - * @model_data: pointer to GPU model data. - * @counter_block_offset: offset in bytes of the performance counter inside - * the Job Manager block. - * - * Return: Block offset in bytes of the required performance counter. - */ -static u32 kbase_g7x_power_model_get_jm_counter(struct kbase_ipa_model_vinstr_data *model_data, - u32 counter_block_offset) -{ - CSTD_UNUSED(model_data); - return JM_BASE + counter_block_offset; -} - -/** - * kbase_g7x_power_model_get_memsys_counter() - get performance counter offset - * inside the Memory System block - * @model_data: pointer to GPU model data. - * @counter_block_offset: offset in bytes of the performance counter inside - * the (first) Memory System block. - * - * Return: Block offset in bytes of the required performance counter. - */ -static u32 kbase_g7x_power_model_get_memsys_counter(struct kbase_ipa_model_vinstr_data *model_data, - u32 counter_block_offset) -{ - CSTD_UNUSED(model_data); - /* The base address of Memory System performance counters is always the same, although their number - * may vary based on the number of cores. For the moment it's ok to return a constant. - */ - return MEMSYS_BASE + counter_block_offset; -} - -/** - * kbase_g7x_power_model_get_sc_counter() - get performance counter offset - * inside the Shader Cores block - * @model_data: pointer to GPU model data. - * @counter_block_offset: offset in bytes of the performance counter inside - * the (first) Shader Cores block. - * - * Return: Block offset in bytes of the required performance counter. - */ -static u32 kbase_g7x_power_model_get_sc_counter(struct kbase_ipa_model_vinstr_data *model_data, - u32 counter_block_offset) -{ -#if IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI) - const u32 sc_base = - MEMSYS_BASE + (KBASE_DUMMY_MODEL_MAX_MEMSYS_BLOCKS * KBASE_IPA_NR_BYTES_PER_BLOCK); -#else - const u32 sc_base = MEMSYS_BASE + (model_data->kbdev->gpu_props.num_l2_slices * - KBASE_IPA_NR_BYTES_PER_BLOCK); -#endif - return sc_base + counter_block_offset; -} - -/** - * kbase_g7x_sum_all_memsys_blocks() - calculate energy for a single Memory - * System performance counter. - * @model_data: pointer to GPU model data. - * @coeff: default value of coefficient for IPA group. - * @counter_block_offset: offset in bytes of the counter inside the block it - * belongs to. - * - * Return: Energy estimation for a single Memory System performance counter. - */ -static s64 kbase_g7x_sum_all_memsys_blocks(struct kbase_ipa_model_vinstr_data *model_data, - s32 coeff, u32 counter_block_offset) -{ - u32 counter; - - counter = kbase_g7x_power_model_get_memsys_counter(model_data, counter_block_offset); - return kbase_ipa_sum_all_memsys_blocks(model_data, coeff, counter); -} - -/** - * kbase_g7x_sum_all_shader_cores() - calculate energy for a Shader Cores - * performance counter for all cores. - * @model_data: pointer to GPU model data. - * @coeff: default value of coefficient for IPA group. - * @counter_block_offset: offset in bytes of the counter inside the block it - * belongs to. - * - * Return: Energy estimation for a Shader Cores performance counter for all - * cores. - */ -static s64 kbase_g7x_sum_all_shader_cores(struct kbase_ipa_model_vinstr_data *model_data, s32 coeff, - u32 counter_block_offset) -{ - u32 counter; - - counter = kbase_g7x_power_model_get_sc_counter(model_data, counter_block_offset); - return kbase_ipa_sum_all_shader_cores(model_data, coeff, counter); -} - -/** - * kbase_g7x_jm_single_counter() - calculate energy for a single Job Manager performance counter. - * @model_data: pointer to GPU model data. - * @coeff: default value of coefficient for IPA group. - * @counter_block_offset: offset in bytes of the counter inside the block it belongs to. - * - * Return: Energy estimation for a single Job Manager performance counter. - */ -static s64 kbase_g7x_jm_single_counter(struct kbase_ipa_model_vinstr_data *model_data, s32 coeff, - u32 counter_block_offset) -{ - u32 counter; - - counter = kbase_g7x_power_model_get_jm_counter(model_data, counter_block_offset); - return kbase_ipa_single_counter(model_data, coeff, counter); -} - -/** - * kbase_g7x_get_active_cycles() - return the GPU_ACTIVE counter - * @model_data: pointer to GPU model data. - * - * Return: the number of cycles the GPU was active during the counter sampling - * period. - */ -static u32 kbase_g7x_get_active_cycles(struct kbase_ipa_model_vinstr_data *model_data) -{ - u32 counter = kbase_g7x_power_model_get_jm_counter(model_data, JM_GPU_ACTIVE); - - /* Counters are only 32-bit, so we can safely multiply by 1 then cast - * the 64-bit result back to a u32. - */ - return kbase_ipa_single_counter(model_data, 1, counter); -} - -/* Table of IPA group definitions. - * - * For each IPA group, this table defines a function to access the given performance block counter (or counters, - * if the operation needs to be iterated on multiple blocks) and calculate energy estimation. - */ - -static const struct kbase_ipa_group ipa_groups_def_g71[] = { - { - .name = "l2_access", - .default_value = 526300, - .op = kbase_g7x_sum_all_memsys_blocks, - .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, - }, - { - .name = "exec_instr_count", - .default_value = 301100, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_EXEC_INSTR_COUNT, - }, - { - .name = "tex_issue", - .default_value = 197400, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_TEX_COORD_ISSUE, - }, - { - .name = "tile_wb", - .default_value = -156400, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_BEATS_WR_TIB, - }, - { - .name = "gpu_active", - .default_value = 115800, - .op = kbase_g7x_jm_single_counter, - .counter_block_offset = JM_GPU_ACTIVE, - }, -}; - -static const struct kbase_ipa_group ipa_groups_def_g72[] = { - { - .name = "l2_access", - .default_value = 393000, - .op = kbase_g7x_sum_all_memsys_blocks, - .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, - }, - { - .name = "exec_instr_count", - .default_value = 227000, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_EXEC_INSTR_COUNT, - }, - { - .name = "tex_issue", - .default_value = 181900, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_TEX_COORD_ISSUE, - }, - { - .name = "tile_wb", - .default_value = -120200, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_BEATS_WR_TIB, - }, - { - .name = "gpu_active", - .default_value = 133100, - .op = kbase_g7x_jm_single_counter, - .counter_block_offset = JM_GPU_ACTIVE, - }, -}; - -static const struct kbase_ipa_group ipa_groups_def_g76[] = { - { - .name = "gpu_active", - .default_value = 122000, - .op = kbase_g7x_jm_single_counter, - .counter_block_offset = JM_GPU_ACTIVE, - }, - { - .name = "exec_instr_count", - .default_value = 488900, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_EXEC_INSTR_COUNT, - }, - { - .name = "vary_instr", - .default_value = 212100, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_VARY_INSTR, - }, - { - .name = "tex_tfch_num_operations", - .default_value = 288000, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_TEX_TFCH_NUM_OPERATIONS, - }, - { - .name = "l2_access", - .default_value = 378100, - .op = kbase_g7x_sum_all_memsys_blocks, - .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, - }, -}; - -static const struct kbase_ipa_group ipa_groups_def_g52_r1[] = { - { - .name = "gpu_active", - .default_value = 224200, - .op = kbase_g7x_jm_single_counter, - .counter_block_offset = JM_GPU_ACTIVE, - }, - { - .name = "exec_instr_count", - .default_value = 384700, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_EXEC_INSTR_COUNT, - }, - { - .name = "vary_instr", - .default_value = 271900, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_VARY_INSTR, - }, - { - .name = "tex_tfch_num_operations", - .default_value = 477700, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_TEX_TFCH_NUM_OPERATIONS, - }, - { - .name = "l2_access", - .default_value = 551400, - .op = kbase_g7x_sum_all_memsys_blocks, - .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, - }, -}; - -static const struct kbase_ipa_group ipa_groups_def_g51[] = { - { - .name = "gpu_active", - .default_value = 201400, - .op = kbase_g7x_jm_single_counter, - .counter_block_offset = JM_GPU_ACTIVE, - }, - { - .name = "exec_instr_count", - .default_value = 392700, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_EXEC_INSTR_COUNT, - }, - { - .name = "vary_instr", - .default_value = 274000, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_VARY_INSTR, - }, - { - .name = "tex_tfch_num_operations", - .default_value = 528000, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_TEX_TFCH_NUM_OPERATIONS, - }, - { - .name = "l2_access", - .default_value = 506400, - .op = kbase_g7x_sum_all_memsys_blocks, - .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, - }, -}; - -static const struct kbase_ipa_group ipa_groups_def_g77[] = { - { - .name = "l2_access", - .default_value = 710800, - .op = kbase_g7x_sum_all_memsys_blocks, - .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, - }, - { - .name = "exec_instr_msg", - .default_value = 2375300, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_EXEC_INSTR_MSG, - }, - { - .name = "exec_instr_fma", - .default_value = 656100, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_EXEC_INSTR_FMA, - }, - { - .name = "tex_filt_num_operations", - .default_value = 318800, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_TEX_FILT_NUM_OPERATIONS, - }, - { - .name = "gpu_active", - .default_value = 172800, - .op = kbase_g7x_jm_single_counter, - .counter_block_offset = JM_GPU_ACTIVE, - }, -}; - -static const struct kbase_ipa_group ipa_groups_def_tbex[] = { - { - .name = "l2_access", - .default_value = 599800, - .op = kbase_g7x_sum_all_memsys_blocks, - .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, - }, - { - .name = "exec_instr_msg", - .default_value = 1830200, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_EXEC_INSTR_MSG, - }, - { - .name = "exec_instr_fma", - .default_value = 407300, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_EXEC_INSTR_FMA, - }, - { - .name = "tex_filt_num_operations", - .default_value = 224500, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_TEX_FILT_NUM_OPERATIONS, - }, - { - .name = "gpu_active", - .default_value = 153800, - .op = kbase_g7x_jm_single_counter, - .counter_block_offset = JM_GPU_ACTIVE, - }, -}; - -static const struct kbase_ipa_group ipa_groups_def_tbax[] = { - { - .name = "l2_access", - .default_value = 599800, - .op = kbase_g7x_sum_all_memsys_blocks, - .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, - }, - { - .name = "exec_instr_msg", - .default_value = 1830200, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_EXEC_INSTR_MSG, - }, - { - .name = "exec_instr_fma", - .default_value = 407300, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_EXEC_INSTR_FMA, - }, - { - .name = "tex_filt_num_operations", - .default_value = 224500, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_TEX_FILT_NUM_OPERATIONS, - }, - { - .name = "gpu_active", - .default_value = 153800, - .op = kbase_g7x_jm_single_counter, - .counter_block_offset = JM_GPU_ACTIVE, - }, -}; - -#define IPA_POWER_MODEL_OPS(gpu, init_token) \ - static const struct kbase_ipa_model_ops kbase_##gpu##_ipa_model_ops = { \ - .name = "mali-" #gpu "-power-model", \ - .init = kbase_##init_token##_power_model_init, \ - .term = kbase_ipa_vinstr_common_model_term, \ - .get_dynamic_coeff = kbase_ipa_vinstr_dynamic_coeff, \ - .reset_counter_data = kbase_ipa_vinstr_reset_data, \ - } - -#define STANDARD_POWER_MODEL(gpu, reference_voltage) \ - static int kbase_##gpu##_power_model_init(struct kbase_ipa_model *model) \ - { \ - BUILD_BUG_ON(ARRAY_SIZE(ipa_groups_def_##gpu) > KBASE_IPA_MAX_GROUP_DEF_NUM); \ - return kbase_ipa_vinstr_common_model_init(model, ipa_groups_def_##gpu, \ - ARRAY_SIZE(ipa_groups_def_##gpu), \ - kbase_g7x_get_active_cycles, \ - (reference_voltage)); \ - } \ - IPA_POWER_MODEL_OPS(gpu, gpu) - -#define ALIAS_POWER_MODEL(gpu, as_gpu) IPA_POWER_MODEL_OPS(gpu, as_gpu) - -STANDARD_POWER_MODEL(g71, 800); -STANDARD_POWER_MODEL(g72, 800); -STANDARD_POWER_MODEL(g76, 800); -STANDARD_POWER_MODEL(g52_r1, 1000); -STANDARD_POWER_MODEL(g51, 1000); -STANDARD_POWER_MODEL(g77, 1000); -STANDARD_POWER_MODEL(tbex, 1000); -STANDARD_POWER_MODEL(tbax, 1000); - -/* g52 is an alias of g76 (TNOX) for IPA */ -ALIAS_POWER_MODEL(g52, g76); -/* tnax is an alias of g77 (TTRX) for IPA */ -ALIAS_POWER_MODEL(tnax, g77); - -static const struct kbase_ipa_model_ops *ipa_counter_model_ops[] = { - &kbase_g71_ipa_model_ops, &kbase_g72_ipa_model_ops, &kbase_g76_ipa_model_ops, - &kbase_g52_ipa_model_ops, &kbase_g52_r1_ipa_model_ops, &kbase_g51_ipa_model_ops, - &kbase_g77_ipa_model_ops, &kbase_tnax_ipa_model_ops, &kbase_tbex_ipa_model_ops, - &kbase_tbax_ipa_model_ops -}; - -const struct kbase_ipa_model_ops *kbase_ipa_counter_model_ops_find(struct kbase_device *kbdev, - const char *name) -{ - size_t i; - - for (i = 0; i < ARRAY_SIZE(ipa_counter_model_ops); ++i) { - const struct kbase_ipa_model_ops *ops = ipa_counter_model_ops[i]; - - if (!strcmp(ops->name, name)) - return ops; - } - - dev_err(kbdev->dev, "power model \'%s\' not found\n", name); - - return NULL; -} - -const char *kbase_ipa_counter_model_name_from_id(struct kbase_gpu_id_props *gpu_id) -{ - switch (gpu_id->product_model) { - case GPU_ID_PRODUCT_TMIX: - return "mali-g71-power-model"; - case GPU_ID_PRODUCT_THEX: - return "mali-g72-power-model"; - case GPU_ID_PRODUCT_TNOX: - return "mali-g76-power-model"; - case GPU_ID_PRODUCT_TSIX: - return "mali-g51-power-model"; - case GPU_ID_PRODUCT_TGOX: - if (gpu_id->version_major == 0) - /* g52 aliased to g76 power-model's ops */ - return "mali-g52-power-model"; - else - return "mali-g52_r1-power-model"; - case GPU_ID_PRODUCT_TNAX: - return "mali-tnax-power-model"; - case GPU_ID_PRODUCT_TTRX: - return "mali-g77-power-model"; - case GPU_ID_PRODUCT_TBEX: - return "mali-tbex-power-model"; - case GPU_ID_PRODUCT_TBAX: - return "mali-tbax-power-model"; - default: - return NULL; - } -} diff --git a/drivers/gpu/arm/valhall/ipa/mali_kbase_ipa_simple.c b/drivers/gpu/arm/valhall/ipa/mali_kbase_ipa_simple.c index d8a356c3e8c2..b2dce96f4922 100644 --- a/drivers/gpu/arm/valhall/ipa/mali_kbase_ipa_simple.c +++ b/drivers/gpu/arm/valhall/ipa/mali_kbase_ipa_simple.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2016-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2016-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -34,8 +34,6 @@ #include "mali_kbase_ipa_simple.h" #include "mali_kbase_ipa_debugfs.h" -#if MALI_USE_CSF - /* This is used if the dynamic power for top-level is estimated separately * through the counter model. To roughly match the contribution of top-level * power in the total dynamic power, when calculated through counter model, @@ -46,8 +44,6 @@ */ #define TOP_LEVEL_DYN_COEFF_SCALER (3) -#endif /* MALI_USE_CSF */ - #if MALI_UNIT_TEST static int dummy_temp; @@ -208,7 +204,6 @@ static int model_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp) struct kbase_ipa_model_simple_data *model_data = (struct kbase_ipa_model_simple_data *)model->model_data; -#if MALI_USE_CSF /* On CSF GPUs, the dynamic power for top-level and shader cores is * estimated separately. Currently there is a single dynamic * coefficient value provided in the device tree for simple model. @@ -219,9 +214,6 @@ static int model_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp) coeffp[KBASE_IPA_BLOCK_TYPE_TOP_LEVEL] = model_data->dynamic_coefficient / TOP_LEVEL_DYN_COEFF_SCALER; coeffp[KBASE_IPA_BLOCK_TYPE_SHADER_CORES] = model_data->dynamic_coefficient; -#else - *coeffp = model_data->dynamic_coefficient; -#endif return 0; } diff --git a/drivers/gpu/arm/valhall/jm/mali_kbase_jm_defs.h b/drivers/gpu/arm/valhall/jm/mali_kbase_jm_defs.h deleted file mode 100644 index e9b388fabbda..000000000000 --- a/drivers/gpu/arm/valhall/jm/mali_kbase_jm_defs.h +++ /dev/null @@ -1,770 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Definitions (types, defines, etcs) specific to Job Manager Kbase. - * They are placed here to allow the hierarchy of header files to work. - */ - -#ifndef _KBASE_JM_DEFS_H_ -#define _KBASE_JM_DEFS_H_ - -#include "mali_kbase_js_defs.h" - -#include - -/* Dump Job slot trace on error (only active if KBASE_KTRACE_ENABLE != 0) */ -#define KBASE_KTRACE_DUMP_ON_JOB_SLOT_ERROR 1 - -/* - * Number of milliseconds before resetting the GPU when a job cannot be "zapped" - * from the hardware. Note that the time is actually - * ZAP_TIMEOUT+SOFT_STOP_RESET_TIMEOUT between the context zap starting and - * the GPU actually being reset to give other contexts time for their jobs - * to be soft-stopped and removed from the hardware before resetting. - */ -#define ZAP_TIMEOUT 1000 - -/* - * Prevent soft-stops from occurring in scheduling situations - * - * This is not due to HW issues, but when scheduling is desired to be more - * predictable. - * - * Therefore, soft stop may still be disabled due to HW issues. - * - * Soft stop will still be used for non-scheduling purposes e.g. when - * terminating a context. - * - * if not in use, define this value to 0 instead of being undefined. - */ -#define KBASE_DISABLE_SCHEDULING_SOFT_STOPS 0 - -/* - * Prevent hard-stops from occurring in scheduling situations - * - * This is not due to HW issues, but when scheduling is desired to be more - * predictable. - * - * Hard stop will still be used for non-scheduling purposes e.g. when - * terminating a context. - * - * if not in use, define this value to 0 instead of being undefined. - */ -#define KBASE_DISABLE_SCHEDULING_HARD_STOPS 0 - -/* Atom has been previously soft-stopped */ -#define KBASE_KATOM_FLAG_BEEN_SOFT_STOPPED (1U << 1) -/* Atom has been previously retried to execute */ -#define KBASE_KATOM_FLAGS_RERUN (1U << 2) -/* Atom submitted with JOB_CHAIN_FLAG bit set in JS_CONFIG_NEXT register, helps - * to disambiguate short-running job chains during soft/hard stopping of jobs - */ -#define KBASE_KATOM_FLAGS_JOBCHAIN (1U << 3) -/* Atom has been previously hard-stopped. */ -#define KBASE_KATOM_FLAG_BEEN_HARD_STOPPED (1U << 4) -/* Atom has caused us to enter disjoint state */ -#define KBASE_KATOM_FLAG_IN_DISJOINT (1U << 5) -/* Atom blocked on cross-slot dependency */ -#define KBASE_KATOM_FLAG_X_DEP_BLOCKED (1U << 7) -/* Atom has fail dependency on cross-slot dependency */ -#define KBASE_KATOM_FLAG_FAIL_BLOCKER (1U << 8) -/* Atom is currently in the list of atoms blocked on cross-slot dependencies */ -#define KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST (1U << 9) -/* Atom requires GPU to be in protected mode */ -#define KBASE_KATOM_FLAG_PROTECTED (1U << 11) -/* Atom has been stored in runnable_tree */ -#define KBASE_KATOM_FLAG_JSCTX_IN_TREE (1U << 12) -/* Atom is waiting for L2 caches to power up in order to enter protected mode */ -#define KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT (1U << 13) - -/* SW related flags about types of JS_COMMAND action - * NOTE: These must be masked off by JS_COMMAND_MASK - */ - -/* This command causes a disjoint event */ -#define JS_COMMAND_SW_CAUSES_DISJOINT 0x100 - -/* Bitmask of all SW related flags */ -#define JS_COMMAND_SW_BITS (JS_COMMAND_SW_CAUSES_DISJOINT) - -#if (JS_COMMAND_SW_BITS & JS_COMMAND_MASK) -#error "JS_COMMAND_SW_BITS not masked off by JS_COMMAND_MASK." \ - "Must update JS_COMMAND_SW_<..> bitmasks" -#endif - -/* Soft-stop command that causes a Disjoint event. This of course isn't - * entirely masked off by JS_COMMAND_MASK - */ -#define JS_COMMAND_SOFT_STOP_WITH_SW_DISJOINT (JS_COMMAND_SW_CAUSES_DISJOINT | JS_COMMAND_SOFT_STOP) - -#define KBASEP_ATOM_ID_INVALID BASE_JD_ATOM_COUNT - -/* Serialize atoms within a slot (ie only one atom per job slot) */ -#define KBASE_SERIALIZE_INTRA_SLOT (1 << 0) -/* Serialize atoms between slots (ie only one job slot running at any time) */ -#define KBASE_SERIALIZE_INTER_SLOT (1 << 1) -/* Reset the GPU after each atom completion */ -#define KBASE_SERIALIZE_RESET (1 << 2) - -/** - * enum kbase_timeout_selector - The choice of which timeout to get scaled - * using the lowest GPU frequency. - * @MMU_AS_INACTIVE_WAIT_TIMEOUT: Maximum waiting time in ms for the completion - * of a MMU operation - * @JM_DEFAULT_JS_FREE_TIMEOUT: Maximum timeout to wait for JS_COMMAND_NEXT - * to be updated on HW side so a Job Slot is - * considered free. - * @KBASE_PRFCNT_ACTIVE_TIMEOUT: Waiting time for prfcnt to be ready. - * @KBASE_CLEAN_CACHE_TIMEOUT: Waiting time for cache flush to complete. - * @KBASE_AS_INACTIVE_TIMEOUT: Waiting time for MCU address space to become inactive. - * @KBASE_TIMEOUT_SELECTOR_COUNT: Number of timeout selectors. - * @KBASE_DEFAULT_TIMEOUT: Fallthrough in case an invalid timeout is - * passed. - */ -enum kbase_timeout_selector { - MMU_AS_INACTIVE_WAIT_TIMEOUT, - JM_DEFAULT_JS_FREE_TIMEOUT, - KBASE_PRFCNT_ACTIVE_TIMEOUT, - KBASE_CLEAN_CACHE_TIMEOUT, - KBASE_AS_INACTIVE_TIMEOUT, - /* Must be the last in the enum */ - KBASE_TIMEOUT_SELECTOR_COUNT, - KBASE_DEFAULT_TIMEOUT = JM_DEFAULT_JS_FREE_TIMEOUT -}; - -#if IS_ENABLED(CONFIG_DEBUG_FS) -/** - * struct base_job_fault_event - keeps track of the atom which faulted or which - * completed after the faulty atom but before the - * debug data for faulty atom was dumped. - * - * @event_code: event code for the atom, should != BASE_JD_EVENT_DONE for - * the atom which faulted. - * @katom: pointer to the atom for which job fault occurred or which - * completed after the faulty atom. - * @job_fault_work: work item, queued only for the faulty atom, which waits for - * the dumping to get completed and then does the bottom half - * of job done for the atoms which followed the faulty atom. - * @head: List head used to store the atom in the global list of - * faulty atoms or context specific list of atoms which got - * completed during the dump. - * @reg_offset: offset of the register to be dumped next, only applicable - * for the faulty atom. - */ -struct base_job_fault_event { - u32 event_code; - struct kbase_jd_atom *katom; - struct work_struct job_fault_work; - struct list_head head; - int reg_offset; -}; -#endif - -/** - * struct kbase_jd_atom_dependency - Contains the dependency info for an atom. - * @atom: pointer to the dependee atom. - * @dep_type: type of dependency on the dependee @atom, i.e. order or data - * dependency. BASE_JD_DEP_TYPE_INVALID indicates no dependency. - */ -struct kbase_jd_atom_dependency { - struct kbase_jd_atom *atom; - u8 dep_type; -}; - -/** - * kbase_jd_katom_dep_atom - Retrieves a read-only reference to the - * dependee atom. - * @dep: pointer to the dependency info structure. - * - * Return: readonly reference to dependee atom. - */ -static inline const struct kbase_jd_atom * -kbase_jd_katom_dep_atom(const struct kbase_jd_atom_dependency *dep) -{ - return (const struct kbase_jd_atom *)(dep->atom); -} - -/** - * kbase_jd_katom_dep_type - Retrieves the dependency type info - * - * @dep: pointer to the dependency info structure. - * - * Return: the type of dependency there is on the dependee atom. - */ -static inline u8 kbase_jd_katom_dep_type(const struct kbase_jd_atom_dependency *dep) -{ - return dep->dep_type; -} - -/** - * kbase_jd_katom_dep_set - sets up the dependency info structure - * as per the values passed. - * @const_dep: pointer to the dependency info structure to be setup. - * @a: pointer to the dependee atom. - * @type: type of dependency there is on the dependee atom. - */ -static inline void kbase_jd_katom_dep_set(const struct kbase_jd_atom_dependency *const_dep, - struct kbase_jd_atom *a, u8 type) -{ - struct kbase_jd_atom_dependency *dep; - - dep = (struct kbase_jd_atom_dependency *)const_dep; - - dep->atom = a; - dep->dep_type = type; -} - -/** - * kbase_jd_katom_dep_clear - resets the dependency info structure - * - * @const_dep: pointer to the dependency info structure to be setup. - */ -static inline void kbase_jd_katom_dep_clear(const struct kbase_jd_atom_dependency *const_dep) -{ - struct kbase_jd_atom_dependency *dep; - - dep = (struct kbase_jd_atom_dependency *)const_dep; - - dep->atom = NULL; - dep->dep_type = BASE_JD_DEP_TYPE_INVALID; -} - -/** - * enum kbase_atom_gpu_rb_state - The state of an atom, pertinent after it - * becomes runnable, with respect to job slot - * ringbuffer/fifo. - * @KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: Atom not currently present in slot fifo, - * which implies that either atom has not become - * runnable due to dependency or has completed - * the execution on GPU. - * @KBASE_ATOM_GPU_RB_WAITING_BLOCKED: Atom has been added to slot fifo but is - * blocked due to cross slot dependency, - * can't be submitted to GPU. - * @KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: Atom has been added to slot - * fifo but is waiting for the completion of - * previously added atoms in current & other - * slots, as their protected mode requirements - * do not match with the current atom. - * @KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: Atom is in slot fifo - * and is waiting for completion of protected - * mode transition, needed before the atom is - * submitted to GPU. - * @KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: Atom is in slot fifo but is - * waiting for the cores, which are needed to - * execute the job chain represented by the atom, - * to become available - * @KBASE_ATOM_GPU_RB_READY: Atom is in slot fifo and can be submitted to - * GPU. - * @KBASE_ATOM_GPU_RB_SUBMITTED: Atom is in slot fifo and has been submitted - * to GPU. - * @KBASE_ATOM_GPU_RB_RETURN_TO_JS: Atom must be returned to JS due to some - * failure, but only after the previously added - * atoms in fifo have completed or have also - * been returned to JS. - */ -enum kbase_atom_gpu_rb_state { - KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB, - KBASE_ATOM_GPU_RB_WAITING_BLOCKED, - KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV, - KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION, - KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE, - KBASE_ATOM_GPU_RB_READY, - KBASE_ATOM_GPU_RB_SUBMITTED, - KBASE_ATOM_GPU_RB_RETURN_TO_JS = -1 -}; - -/** - * enum kbase_atom_enter_protected_state - The state of an atom with respect to - * the preparation for GPU's entry into protected mode, - * becomes pertinent only after atom's state with respect - * to slot ringbuffer is - * KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION - * @KBASE_ATOM_ENTER_PROTECTED_CHECK: Starting state. Check if there are any - * atoms currently submitted to GPU and protected mode - * transition is not already in progress. - * @KBASE_ATOM_ENTER_PROTECTED_HWCNT: Wait for hardware counter context to - * become disabled before entry into protected mode. - * @KBASE_ATOM_ENTER_PROTECTED_IDLE_L2: Wait for the L2 to become idle in - * preparation for the coherency change. L2 shall be - * powered down and GPU shall come out of fully - * coherent mode before entering protected mode. - * @KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY: Prepare coherency change; - * for KBASE_HW_ISSUE_TGOX_R1_1234 also request L2 power on - * so that coherency register contains correct value when - * GPU enters protected mode. - * @KBASE_ATOM_ENTER_PROTECTED_FINISHED: End state; for - * KBASE_HW_ISSUE_TGOX_R1_1234 check - * that L2 is powered up and switch GPU to protected mode. - */ -enum kbase_atom_enter_protected_state { - /* - * NOTE: The integer value of this must match - * KBASE_ATOM_EXIT_PROTECTED_CHECK. - */ - KBASE_ATOM_ENTER_PROTECTED_CHECK = 0, - KBASE_ATOM_ENTER_PROTECTED_HWCNT, - KBASE_ATOM_ENTER_PROTECTED_IDLE_L2, - KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY, - KBASE_ATOM_ENTER_PROTECTED_FINISHED, -}; - -/** - * enum kbase_atom_exit_protected_state - The state of an atom with respect to - * the preparation for GPU's exit from protected mode, - * becomes pertinent only after atom's state with respect - * to slot ngbuffer is - * KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION - * @KBASE_ATOM_EXIT_PROTECTED_CHECK: Starting state. Check if there are any - * atoms currently submitted to GPU and protected mode - * transition is not already in progress. - * @KBASE_ATOM_EXIT_PROTECTED_IDLE_L2: Wait for the L2 to become idle in - * preparation for the reset, as exiting protected mode - * requires a reset. - * @KBASE_ATOM_EXIT_PROTECTED_RESET: Issue the reset to trigger exit from - * protected mode - * @KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT: End state, Wait for the reset to - * complete - */ -enum kbase_atom_exit_protected_state { - /* - * NOTE: The integer value of this must match - * KBASE_ATOM_ENTER_PROTECTED_CHECK. - */ - KBASE_ATOM_EXIT_PROTECTED_CHECK = 0, - KBASE_ATOM_EXIT_PROTECTED_IDLE_L2, - KBASE_ATOM_EXIT_PROTECTED_RESET, - KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT, -}; - -/** - * struct kbase_jd_atom - object representing the atom, containing the complete - * state and attributes of an atom. - * @work: work item for the bottom half processing of the atom, - * by JD or JS, after it got executed on GPU or the - * input fence got signaled - * @start_timestamp: time at which the atom was submitted to the GPU, by - * updating the JS_HEAD_NEXTn register. - * @udata: copy of the user data sent for the atom in - * base_jd_submit. - * @kctx: Pointer to the base context with which the atom is - * associated. - * @dep_head: Array of 2 list heads, pointing to the two list of - * atoms - * which are blocked due to dependency on this atom. - * @dep_item: Array of 2 list heads, used to store the atom in the - * list of other atoms depending on the same dependee - * atom. - * @dep: Array containing the dependency info for the 2 atoms - * on which the atom depends upon. - * @jd_item: List head used during job dispatch job_done - * processing - as dependencies may not be entirely - * resolved at this point, - * we need to use a separate list head. - * @in_jd_list: flag set to true if atom's @jd_item is currently on - * a list, prevents atom being processed twice. - * @jit_ids: Zero-terminated array of IDs of just-in-time memory - * allocations written to by the atom. When the atom - * completes, the value stored at the - * &struct_base_jit_alloc_info.heap_info_gpu_addr of - * each allocation is read in order to enforce an - * overall physical memory usage limit. - * @nr_extres: number of external resources referenced by the atom. - * @extres: Pointer to @nr_extres VA regions containing the external - * resource allocation and other information. - * @nr_extres external resources referenced by the atom. - * @device_nr: indicates the coregroup with which the atom is - * associated, when - * BASE_JD_REQ_SPECIFIC_COHERENT_GROUP specified. - * @jc: GPU address of the job-chain. - * @softjob_data: Copy of data read from the user space buffer that @jc - * points to. - * @dma_fence: object containing pointers to both input & output - * fences and other related members used for explicit - * sync through soft jobs and for the implicit - * synchronization required on access to external - * resources. - * @dma_fence.fence_in: Points to the dma-buf input fence for this atom. - * The atom would complete only after the fence is - * signaled. - * @dma_fence.fence: Points to the dma-buf output fence for this atom. - * @dma_fence.fence_cb: The object that is passed at the time of adding the - * callback that gets invoked when @dma_fence.fence_in - * is signaled. - * @dma_fence.fence_cb_added: Flag to keep a track if the callback was successfully - * added for @dma_fence.fence_in, which is supposed to be - * invoked on the signaling of fence. - * @dma_fence.context: The dma-buf fence context number for this atom. A - * unique context number is allocated to each katom in - * the context on context creation. - * @dma_fence.seqno: The dma-buf fence sequence number for this atom. This - * is increased every time this katom uses dma-buf fence - * @event_code: Event code for the job chain represented by the atom, - * both HW and low-level SW events are represented by - * event codes. - * @core_req: bitmask of BASE_JD_REQ_* flags specifying either - * Hw or Sw requirements for the job chain represented - * by the atom. - * @ticks: Number of scheduling ticks for which atom has been - * running on the GPU. - * @sched_priority: Priority of the atom for Job scheduling, as per the - * KBASE_JS_ATOM_SCHED_PRIO_*. - * @completed: Wait queue to wait upon for the completion of atom. - * @status: Indicates at high level at what stage the atom is in, - * as per KBASE_JD_ATOM_STATE_*, that whether it is not - * in use or its queued in JD or given to JS or - * submitted to Hw or it completed the execution on Hw. - * @slot_nr: Job slot chosen for the atom. - * @atom_flags: bitmask of KBASE_KATOM_FLAG* flags capturing the - * excat low level state of the atom. - * @gpu_rb_state: bitmnask of KBASE_ATOM_GPU_RB_* flags, precisely - * tracking atom's state after it has entered - * Job scheduler on becoming runnable. Atom - * could be blocked due to cross slot dependency - * or waiting for the shader cores to become available - * or waiting for protected mode transitions to - * complete. - * @need_cache_flush_cores_retained: flag indicating that manual flush of GPU - * cache is needed for the atom and the shader cores - * used for atom have been kept on. - * @blocked: flag indicating that atom's resubmission to GPU is - * blocked till the work item is scheduled to return the - * atom to JS. - * @seq_nr: user-space sequence number, to order atoms in some - * temporal order - * @pre_dep: Pointer to atom that this atom has same-slot - * dependency on - * @post_dep: Pointer to atom that has same-slot dependency on - * this atom - * @x_pre_dep: Pointer to atom that this atom has cross-slot - * dependency on - * @x_post_dep: Pointer to atom that has cross-slot dependency on - * this atom - * @flush_id: The GPU's flush count recorded at the time of - * submission, - * used for the cache flush optimization - * @fault_event: Info for dumping the debug data on Job fault. - * @queue: List head used for 4 different purposes : - * Adds atom to the list of dma-buf fence waiting atoms. - * Adds atom to the list of atoms blocked due to cross - * slot dependency. - * Adds atom to the list of softjob atoms for which JIT - * allocation has been deferred - * Adds atom to the list of softjob atoms waiting for - * the signaling of fence. - * @jit_node: Used to keep track of all JIT free/alloc jobs in - * submission order - * @jit_blocked: Flag indicating that JIT allocation requested through - * softjob atom will be reattempted after the impending - * free of other active JIT allocations. - * @will_fail_event_code: If non-zero, this indicates that the atom will fail - * with the set event_code when the atom is processed. - * Used for special handling of atoms, which have a data - * dependency on the failed atoms. - * @protected_state: State of the atom, as per - * KBASE_ATOM_(ENTER|EXIT)_PROTECTED_*, - * when transitioning into or out of protected mode. - * Atom will be either entering or exiting the - * protected mode. - * @protected_state.enter: entering the protected mode. - * @protected_state.exit: exiting the protected mode. - * @runnable_tree_node: The node added to context's job slot specific rb tree - * when the atom becomes runnable. - * @age: Age of atom relative to other atoms in the context, - * is snapshot of the age_count counter in kbase - * context. - * @jobslot: Job slot to use when BASE_JD_REQ_JOB_SLOT is specified. - */ -struct kbase_jd_atom { - struct work_struct work; - ktime_t start_timestamp; - - struct base_jd_udata udata; - struct kbase_context *kctx; - - struct list_head dep_head[2]; - struct list_head dep_item[2]; - const struct kbase_jd_atom_dependency dep[2]; - struct list_head jd_item; - bool in_jd_list; - -#if MALI_JIT_PRESSURE_LIMIT_BASE - u8 jit_ids[2]; -#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - - u16 nr_extres; - struct kbase_va_region **extres; - - u32 device_nr; - u64 jc; - void *softjob_data; -#if IS_ENABLED(CONFIG_SYNC_FILE) - struct { - /* Use the functions/API defined in mali_kbase_fence.h to - * when working with this sub struct - */ - struct dma_fence *fence_in; - /* This points to the dma-buf output fence for this atom. If - * this is NULL then there is no fence for this atom and the - * following fields related to dma_fence may have invalid data. - * - * The context and seqno fields contain the details for this - * fence. - * - * This fence is signaled when the katom is completed, - * regardless of the event_code of the katom (signal also on - * failure). - */ - struct dma_fence *fence; - - /* This is the callback object that is registered for the fence_in. - * The callback is invoked when the fence_in is signaled. - */ - struct dma_fence_cb fence_cb; - bool fence_cb_added; - - unsigned int context; - atomic_t seqno; - } dma_fence; -#endif /* CONFIG_SYNC_FILE */ - - /* Note: refer to kbasep_js_atom_retained_state, which will take a copy - * of some of the following members - */ - enum base_jd_event_code event_code; - base_jd_core_req core_req; - u8 jobslot; - - u32 ticks; - int sched_priority; - - wait_queue_head_t completed; - enum kbase_jd_atom_state status; - unsigned int slot_nr; - - u32 atom_flags; - - enum kbase_atom_gpu_rb_state gpu_rb_state; - - bool need_cache_flush_cores_retained; - - atomic_t blocked; - - u64 seq_nr; - - struct kbase_jd_atom *pre_dep; - struct kbase_jd_atom *post_dep; - - struct kbase_jd_atom *x_pre_dep; - struct kbase_jd_atom *x_post_dep; - - u32 flush_id; - -#if IS_ENABLED(CONFIG_DEBUG_FS) - struct base_job_fault_event fault_event; -#endif - struct list_head queue; - - struct list_head jit_node; - bool jit_blocked; - - enum base_jd_event_code will_fail_event_code; - - union { - enum kbase_atom_enter_protected_state enter; - enum kbase_atom_exit_protected_state exit; - } protected_state; - - struct rb_node runnable_tree_node; - - u32 age; -}; - -static inline bool kbase_jd_katom_is_protected(const struct kbase_jd_atom *katom) -{ - return (bool)(katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED); -} - -/** - * kbase_jd_atom_is_younger - query if one atom is younger by age than another - * - * @katom_a: the first atom - * @katom_b: the second atom - * - * Return: true if the first atom is strictly younger than the second, - * false otherwise. - */ -static inline bool kbase_jd_atom_is_younger(const struct kbase_jd_atom *katom_a, - const struct kbase_jd_atom *katom_b) -{ - return ((s32)(katom_a->age - katom_b->age) < 0); -} - -/** - * kbase_jd_atom_is_earlier - Check whether the first atom has been submitted - * earlier than the second one - * - * @katom_a: the first atom - * @katom_b: the second atom - * - * Return: true if the first atom has been submitted earlier than the - * second atom. It is used to understand if an atom that is ready has been - * submitted earlier than the currently running atom, so that the currently - * running atom should be preempted to allow the ready atom to run. - */ -static inline bool kbase_jd_atom_is_earlier(const struct kbase_jd_atom *katom_a, - const struct kbase_jd_atom *katom_b) -{ - /* No seq_nr set? */ - if (!katom_a->seq_nr || !katom_b->seq_nr) - return false; - - /* Efficiently handle the unlikely case of wrapping. - * The following code assumes that the delta between the sequence number - * of the two atoms is less than INT64_MAX. - * In the extremely unlikely case where the delta is higher, the comparison - * defaults for no preemption. - * The code also assumes that the conversion from unsigned to signed types - * works because the signed integers are 2's complement. - */ - return (s64)(katom_a->seq_nr - katom_b->seq_nr) < 0; -} - -/* - * Theory of operations: - * - * Atom objects are statically allocated within the context structure. - * - * Each atom is the head of two lists, one for the "left" set of dependencies, - * one for the "right" set. - */ - -#define KBASE_JD_DEP_QUEUE_SIZE 256 - -/** - * struct kbase_jd_context - per context object encapsulating all the - * Job dispatcher related state. - * @lock: lock to serialize the updates made to the - * Job dispatcher state and kbase_jd_atom objects. - * @sched_info: Structure encapsulating all the Job scheduling - * info. - * @atoms: Array of the objects representing atoms, - * containing the complete state and attributes - * of an atom. - * @job_nr: Tracks the number of atoms being processed by the - * kbase. This includes atoms that are not tracked by - * scheduler: 'not ready to run' & 'dependency-only' - * jobs. - * @zero_jobs_wait: Waitq that reflects whether there are no jobs - * (including SW-only dependency jobs). This is set - * when no jobs are present on the ctx, and clear - * when there are jobs. - * This must be updated atomically with @job_nr. - * note: Job Dispatcher knows about more jobs than - * the Job Scheduler as it is unaware of jobs that - * are blocked on dependencies and SW-only dependency - * jobs. This waitq can be waited upon to find out - * when the context jobs are all done/cancelled - * (including those that might've been blocked - * on dependencies) - and so, whether it can be - * terminated. However, it should only be terminated - * once it is not present in the run-pool. - * Since the waitq is only set under @lock, - * the waiter should also briefly obtain and drop - * @lock to guarantee that the setter has completed - * its work on the kbase_context - * @job_done_wq: Workqueue to which the per atom work item is - * queued for bottom half processing when the - * atom completes - * execution on GPU or the input fence get signaled. - * @tb_lock: Lock to serialize the write access made to @tb to - * store the register access trace messages. - * @tb: Pointer to the Userspace accessible buffer storing - * the trace messages for register read/write - * accesses made by the Kbase. The buffer is filled - * in circular fashion. - * @tb_wrap_offset: Offset to the end location in the trace buffer, - * the write pointer is moved to the beginning on - * reaching this offset. - * @jit_atoms_head: A list of the just-in-time memory soft-jobs, both - * allocate & free, in submission order, protected - * by kbase_jd_context.lock. - * @jit_pending_alloc: A list of just-in-time memory allocation - * soft-jobs which will be reattempted after the - * impending free of other active allocations. - * @max_priority: Max priority level allowed for this context. - */ -struct kbase_jd_context { - struct mutex lock; - struct kbasep_js_kctx_info sched_info; - struct kbase_jd_atom atoms[BASE_JD_ATOM_COUNT]; - struct workqueue_struct *job_done_wq; - - wait_queue_head_t zero_jobs_wait; - spinlock_t tb_lock; - u32 *tb; - u32 job_nr; - size_t tb_wrap_offset; - - struct list_head jit_atoms_head; - struct list_head jit_pending_alloc; - int max_priority; -}; - -/** - * struct jsctx_queue - JS context atom queue - * @runnable_tree: Root of RB-tree containing currently runnable atoms on this - * job slot. - * @x_dep_head: Head item of the linked list of atoms blocked on cross-slot - * dependencies. Atoms on this list will be moved to the - * runnable_tree when the blocking atom completes. - * - * hwaccess_lock must be held when accessing this structure. - */ -struct jsctx_queue { - struct rb_root runnable_tree; - struct list_head x_dep_head; -}; - -/** - * struct kbase_as - Object representing an address space of GPU. - * @number: Index at which this address space structure is present - * in an array of address space structures embedded inside - * the &struct kbase_device. - * @pf_wq: Workqueue for processing work items related to - * Page fault and Bus fault handling. - * @work_pagefault: Work item for the Page fault handling. - * @work_busfault: Work item for the Bus fault handling. - * @pf_data: Data relating to Page fault. - * @bf_data: Data relating to Bus fault. - * @current_setup: Stores the MMU configuration for this address space. - */ -struct kbase_as { - unsigned int number; - struct workqueue_struct *pf_wq; - struct work_struct work_pagefault; - struct work_struct work_busfault; - struct kbase_fault pf_data; - struct kbase_fault bf_data; - struct kbase_mmu_setup current_setup; -}; - -#endif /* _KBASE_JM_DEFS_H_ */ diff --git a/drivers/gpu/arm/valhall/jm/mali_kbase_jm_js.h b/drivers/gpu/arm/valhall/jm/mali_kbase_jm_js.h deleted file mode 100644 index 65b54c68d8c7..000000000000 --- a/drivers/gpu/arm/valhall/jm/mali_kbase_jm_js.h +++ /dev/null @@ -1,1002 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Job Scheduler Interface. - * These interfaces are Internal to KBase. - */ - -#ifndef _KBASE_JM_JS_H_ -#define _KBASE_JM_JS_H_ - -#include "mali_kbase_js_ctx_attr.h" - -#define JS_MAX_RUNNING_JOBS 8 - -/** - * kbasep_js_devdata_init - Initialize the Job Scheduler - * @kbdev: The kbase_device to operate on - * - * The struct kbasep_js_device_data sub-structure of kbdev must be zero - * initialized before passing to the kbasep_js_devdata_init() function. This is - * to give efficient error path code. - * - * Return: 0 on success, error code otherwise. - */ -int kbasep_js_devdata_init(struct kbase_device *const kbdev); - -/** - * kbasep_js_devdata_halt - Halt the Job Scheduler. - * @kbdev: The kbase_device to operate on - * - * It is safe to call this on kbdev even if it the kbasep_js_device_data - * sub-structure was never initialized/failed initialization, to give efficient - * error-path code. - * - * For this to work, the struct kbasep_js_device_data sub-structure of kbdev - * must be zero initialized before passing to the kbasep_js_devdata_init() - * function. This is to give efficient error path code. - * - * It is a programming error to call this whilst there are still kbase_context - * structures registered with this scheduler. - * - */ -void kbasep_js_devdata_halt(struct kbase_device *kbdev); - -/** - * kbasep_js_devdata_term - Terminate the Job Scheduler - * @kbdev: The kbase_device to operate on - * - * It is safe to call this on kbdev even if it the kbasep_js_device_data - * sub-structure was never initialized/failed initialization, to give efficient - * error-path code. - * - * For this to work, the struct kbasep_js_device_data sub-structure of kbdev - * must be zero initialized before passing to the kbasep_js_devdata_init() - * function. This is to give efficient error path code. - * - * It is a programming error to call this whilst there are still kbase_context - * structures registered with this scheduler. - */ -void kbasep_js_devdata_term(struct kbase_device *kbdev); - -/** - * kbasep_js_kctx_init - Initialize the Scheduling Component of a - * struct kbase_context on the Job Scheduler. - * @kctx: The kbase_context to operate on - * - * This effectively registers a struct kbase_context with a Job Scheduler. - * - * It does not register any jobs owned by the struct kbase_context with - * the scheduler. Those must be separately registered by kbasep_js_add_job(). - * - * The struct kbase_context must be zero initialized before passing to the - * kbase_js_init() function. This is to give efficient error path code. - * - * Return: 0 on success, error code otherwise. - */ -int kbasep_js_kctx_init(struct kbase_context *const kctx); - -/** - * kbasep_js_kctx_term - Terminate the Scheduling Component of a - * struct kbase_context on the Job Scheduler - * @kctx: The kbase_context to operate on - * - * This effectively de-registers a struct kbase_context from its Job Scheduler - * - * It is safe to call this on a struct kbase_context that has never had or - * failed initialization of its jctx.sched_info member, to give efficient - * error-path code. - * - * For this to work, the struct kbase_context must be zero intitialized before - * passing to the kbase_js_init() function. - * - * It is a Programming Error to call this whilst there are still jobs - * registered with this context. - */ -void kbasep_js_kctx_term(struct kbase_context *kctx); - -/* kbase_jsctx_slot_prio_blocked_set - Set a context as being blocked for a job - * slot at and below a given priority level - * @kctx: The kbase_context - * @js: The job slot - * @sched_prio: The priority levels that the context is blocked at for @js (all - * priority levels at this level and below will be blocked) - * - * To preserve ordering and dependencies of atoms on soft-stopping (both within - * an between priority levels), a context must be marked as blocked for that - * atom's job slot, for all priority levels at or below the atom's priority. - * - * This must only be called due to an atom that was pulled from the context, - * otherwise there will be no way of unblocking the context when the atom is - * completed/unpulled. - * - * Atoms of higher priority might still be able to be pulled from the context - * on @js. This helps with starting a high priority atom as soon as possible. - */ -static inline void kbase_jsctx_slot_prio_blocked_set(struct kbase_context *kctx, unsigned int js, - int sched_prio) -{ - struct kbase_jsctx_slot_tracking *slot_tracking = &kctx->slot_tracking[js]; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - WARN(!slot_tracking->atoms_pulled_pri[sched_prio], - "When marking slot %u as blocked for priority %d on a kctx, no atoms were pulled - the slot cannot become unblocked", - js, sched_prio); - - slot_tracking->blocked |= ((kbase_js_prio_bitmap_t)1) << sched_prio; - KBASE_KTRACE_ADD_JM_SLOT_INFO(kctx->kbdev, JS_SLOT_PRIO_BLOCKED, kctx, NULL, 0, js, - (unsigned int)sched_prio); -} - -/* kbase_jsctx_atoms_pulled - Return number of atoms pulled on a context - * @kctx: The kbase_context - * - * Having atoms pulled indicates the context is not idle. - * - * Return: the number of atoms pulled on @kctx - */ -static inline int kbase_jsctx_atoms_pulled(struct kbase_context *kctx) -{ - return atomic_read(&kctx->atoms_pulled_all_slots); -} - -/** - * kbasep_js_add_job - Add a job chain to the Job Scheduler, - * and take necessary actions to - * schedule the context/run the job. - * @kctx: The kbase_context to operate on - * @atom: Atom to add - * - * This atomically does the following: - * * Update the numbers of jobs information - * * Add the job to the run pool if necessary (part of init_job) - * - * Once this is done, then an appropriate action is taken: - * * If the ctx is scheduled, it attempts to start the next job (which might be - * this added job) - * * Otherwise, and if this is the first job on the context, it enqueues it on - * the Policy Queue - * - * The Policy's Queue can be updated by this in the following ways: - * * In the above case that this is the first job on the context - * * If the context is high priority and the context is not scheduled, then it - * could cause the Policy to schedule out a low-priority context, allowing - * this context to be scheduled in. - * - * If the context is already scheduled on the RunPool, then adding a job to it - * is guaranteed not to update the Policy Queue. And so, the caller is - * guaranteed to not need to try scheduling a context from the Run Pool - it - * can safely assert that the result is false. - * - * It is a programming error to have more than U32_MAX jobs in flight at a time. - * - * The following locking conditions are made on the caller: - * * it must not hold kbasep_js_kctx_info::ctx::jsctx_mutex. - * * it must not hold hwaccess_lock (as this will be obtained internally) - * * it must not hold kbasep_js_device_data::runpool_mutex (as this will be - * obtained internally) - * * it must not hold kbasep_jd_device_data::queue_mutex (again, it's used - * internally). - * - * Return: true indicates that the Policy Queue was updated, and so the - * caller will need to try scheduling a context onto the Run Pool, - * false indicates that no updates were made to the Policy Queue, - * so no further action is required from the caller. This is always returned - * when the context is currently scheduled. - */ -bool kbasep_js_add_job(struct kbase_context *kctx, struct kbase_jd_atom *atom); - -/** - * kbasep_js_remove_job - Remove a job chain from the Job Scheduler, - * except for its 'retained state'. - * @kbdev: The kbase_device to operate on - * @kctx: The kbase_context to operate on - * @atom: Atom to remove - * - * Completely removing a job requires several calls: - * * kbasep_js_copy_atom_retained_state(), to capture the 'retained state' of - * the atom - * * kbasep_js_remove_job(), to partially remove the atom from the Job Scheduler - * * kbasep_js_runpool_release_ctx_and_katom_retained_state(), to release the - * remaining state held as part of the job having been run. - * - * In the common case of atoms completing normally, this set of actions is more - * optimal for spinlock purposes than having kbasep_js_remove_job() handle all - * of the actions. - * - * In the case of canceling atoms, it is easier to call - * kbasep_js_remove_cancelled_job(), which handles all the necessary actions. - * - * It is a programming error to call this when: - * * a atom is not a job belonging to kctx. - * * a atom has already been removed from the Job Scheduler. - * * a atom is still in the runpool - * - * Do not use this for removing jobs being killed by kbase_jd_cancel() - use - * kbasep_js_remove_cancelled_job() instead. - * - * The following locking conditions are made on the caller: - * * it must hold kbasep_js_kctx_info::ctx::jsctx_mutex. - * - */ -void kbasep_js_remove_job(struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbase_jd_atom *atom); - -/** - * kbasep_js_remove_cancelled_job - Completely remove a job chain from the - * Job Scheduler, in the case - * where the job chain was cancelled. - * @kbdev: The kbase_device to operate on - * @kctx: The kbase_context to operate on - * @katom: Atom to remove - * - * This is a variant of kbasep_js_remove_job() that takes care of removing all - * of the retained state too. This is generally useful for cancelled atoms, - * which need not be handled in an optimal way. - * - * It is a programming error to call this when: - * * a atom is not a job belonging to kctx. - * * a atom has already been removed from the Job Scheduler. - * * a atom is still in the runpool: - * * it is not being killed with kbasep_jd_cancel() - * - * The following locking conditions are made on the caller: - * * it must hold kbasep_js_kctx_info::ctx::jsctx_mutex. - * * it must not hold the hwaccess_lock, (as this will be obtained - * internally) - * * it must not hold kbasep_js_device_data::runpool_mutex (as this could be - * obtained internally) - * - * Return: true indicates that ctx attributes have changed and the caller - * should call kbase_js_sched_all() to try to run more jobs and - * false otherwise. - */ -bool kbasep_js_remove_cancelled_job(struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbase_jd_atom *katom); - -/** - * kbasep_js_runpool_requeue_or_kill_ctx - Handling the requeuing/killing of a - * context that was evicted from the - * policy queue or runpool. - * @kbdev: The kbase_device to operate on - * @kctx: The kbase_context to operate on - * @has_pm_ref: tells whether to release Power Manager active reference - * - * This should be used whenever handing off a context that has been evicted - * from the policy queue or the runpool: - * * If the context is not dying and has jobs, it gets re-added to the policy - * queue - * * Otherwise, it is not added - * - * In addition, if the context is dying the jobs are killed asynchronously. - * - * In all cases, the Power Manager active reference is released - * (kbase_pm_context_idle()) whenever the has_pm_ref parameter is true. - * has_pm_ref must be set to false whenever the context was not previously in - * the runpool and does not hold a Power Manager active refcount. Note that - * contexts in a rollback of kbasep_js_try_schedule_head_ctx() might have an - * active refcount even though they weren't in the runpool. - * - * The following locking conditions are made on the caller: - * * it must hold kbasep_js_kctx_info::ctx::jsctx_mutex. - * * it must not hold kbasep_jd_device_data::queue_mutex (as this will be - * obtained internally) - */ -void kbasep_js_runpool_requeue_or_kill_ctx(struct kbase_device *kbdev, struct kbase_context *kctx, - bool has_pm_ref); - -/** - * kbasep_js_runpool_release_ctx - Release a refcount of a context being busy, - * allowing it to be scheduled out. - * @kbdev: The kbase_device to operate on - * @kctx: The kbase_context to operate on - * - * When the refcount reaches zero and the context might be scheduled out - * (depending on whether the Scheduling Policy has deemed it so, or if it has - * run out of jobs). - * - * If the context does get scheduled out, then The following actions will be - * taken as part of deschduling a context: - * For the context being descheduled: - * * If the context is in the processing of dying (all the jobs are being - * removed from it), then descheduling also kills off any jobs remaining in the - * context. - * * If the context is not dying, and any jobs remain after descheduling the - * context then it is re-enqueued to the Policy's Queue. - * * Otherwise, the context is still known to the scheduler, but remains absent - * from the Policy Queue until a job is next added to it. - * * In all descheduling cases, the Power Manager active reference (obtained - * during kbasep_js_try_schedule_head_ctx()) is released - * (kbase_pm_context_idle()). - * - * Whilst the context is being descheduled, this also handles actions that - * cause more atoms to be run: - * * Attempt submitting atoms when the Context Attributes on the Runpool have - * changed. This is because the context being scheduled out could mean that - * there are more opportunities to run atoms. - * * Attempt submitting to a slot that was previously blocked due to affinity - * restrictions. This is usually only necessary when releasing a context - * happens as part of completing a previous job, but is harmless nonetheless. - * * Attempt scheduling in a new context (if one is available), and if - * necessary, running a job from that new context. - * - * Unlike retaining a context in the runpool, this function cannot be called - * from IRQ context. - * - * It is a programming error to call this on a kctx that is not currently - * scheduled, or that already has a zero refcount. - * - * The following locking conditions are made on the caller: - * * it must not hold the hwaccess_lock, because it will be used internally. - * * it must not hold kbasep_js_kctx_info::ctx::jsctx_mutex. - * * it must not hold kbasep_js_device_data::runpool_mutex (as this will be - * obtained internally) - * * it must not hold the kbase_device::mmu_hw_mutex (as this will be - * obtained internally) - * * it must not hold kbasep_jd_device_data::queue_mutex (as this will be - * obtained internally) - * - */ -void kbasep_js_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); - -/** - * kbasep_js_runpool_release_ctx_and_katom_retained_state - Variant of - * kbasep_js_runpool_release_ctx() that handles additional - * actions from completing an atom. - * - * @kbdev: KBase device - * @kctx: KBase context - * @katom_retained_state: Retained state from the atom - * - * This is usually called as part of completing an atom and releasing the - * refcount on the context held by the atom. - * - * Therefore, the extra actions carried out are part of handling actions queued - * on a completed atom, namely: - * * Releasing the atom's context attributes - * * Retrying the submission on a particular slot, because we couldn't submit - * on that slot from an IRQ handler. - * - * The locking conditions of this function are the same as those for - * kbasep_js_runpool_release_ctx() - */ -void kbasep_js_runpool_release_ctx_and_katom_retained_state( - struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbasep_js_atom_retained_state *katom_retained_state); - -/** - * kbasep_js_runpool_release_ctx_nolock - Variant of kbase_js_runpool_release_ctx() - * without locks - * @kbdev: KBase device - * @kctx: KBase context - * - * Variant of kbase_js_runpool_release_ctx() that assumes that - * kbasep_js_device_data::runpool_mutex and - * kbasep_js_kctx_info::ctx::jsctx_mutex are held by the caller, and does not - * attempt to schedule new contexts. - */ -void kbasep_js_runpool_release_ctx_nolock(struct kbase_device *kbdev, struct kbase_context *kctx); - -/** - * kbasep_js_schedule_privileged_ctx - Schedule in a privileged context - * - * @kbdev: KBase device - * @kctx: KBase context - * - * This schedules a context in regardless of the context priority. - * If the runpool is full, a context will be forced out of the runpool and the - * function will wait for the new context to be scheduled in. - * The context will be kept scheduled in (and the corresponding address space - * reserved) until kbasep_js_release_privileged_ctx is called). - * - * The following locking conditions are made on the caller: - * * it must not hold the hwaccess_lock, because it will be used internally. - * * it must not hold kbasep_js_device_data::runpool_mutex (as this will be - * obtained internally) - * * it must not hold the kbase_device::mmu_hw_mutex (as this will be - * obtained internally) - * * it must not hold kbasep_jd_device_data::queue_mutex (again, it's used - * internally). - * * it must not hold kbasep_js_kctx_info::ctx::jsctx_mutex, because it will - * be used internally. - * - */ -void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); - -/** - * kbasep_js_release_privileged_ctx - Release a privileged context, - * allowing it to be scheduled out. - * @kbdev: KBase device - * @kctx: KBase context - * - * See kbasep_js_runpool_release_ctx for potential side effects. - * - * The following locking conditions are made on the caller: - * * it must not hold the hwaccess_lock, because it will be used internally. - * * it must not hold kbasep_js_kctx_info::ctx::jsctx_mutex. - * * it must not hold kbasep_js_device_data::runpool_mutex (as this will be - * obtained internally) - * * it must not hold the kbase_device::mmu_hw_mutex (as this will be - * obtained internally) - * - */ -void kbasep_js_release_privileged_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); - -/** - * kbase_js_try_run_jobs - Try to submit the next job on each slot - * @kbdev: KBase device - * - * The following locks may be used: - * * kbasep_js_device_data::runpool_mutex - * * hwaccess_lock - */ -void kbase_js_try_run_jobs(struct kbase_device *kbdev); - -/** - * kbasep_js_suspend - Suspend the job scheduler during a Power Management - * Suspend event. - * @kbdev: KBase device - * - * Causes all contexts to be removed from the runpool, and prevents any - * contexts from (re)entering the runpool. - * - * This does not handle suspending the one privileged context: the caller must - * instead do this by suspending the GPU HW Counter Instrumentation. - * - * This will eventually cause all Power Management active references held by - * contexts on the runpool to be released, without running any more atoms. - * - * The caller must then wait for all Power Management active refcount to become - * zero before completing the suspend. - * - * The emptying mechanism may take some time to complete, since it can wait for - * jobs to complete naturally instead of forcing them to end quickly. However, - * this is bounded by the Job Scheduler's Job Timeouts. Hence, this - * function is guaranteed to complete in a finite time. - */ -void kbasep_js_suspend(struct kbase_device *kbdev); - -/** - * kbasep_js_resume - Resume the Job Scheduler after a Power Management - * Resume event. - * @kbdev: KBase device - * - * This restores the actions from kbasep_js_suspend(): - * * Schedules contexts back into the runpool - * * Resumes running atoms on the GPU - */ -void kbasep_js_resume(struct kbase_device *kbdev); - -/** - * kbase_js_dep_resolved_submit - Submit an atom to the job scheduler. - * - * @kctx: Context pointer - * @katom: Pointer to the atom to submit - * - * The atom is enqueued on the context's ringbuffer. The caller must have - * ensured that all dependencies can be represented in the ringbuffer. - * - * Caller must hold jctx->lock - * - * Return: true if the context requires to be enqueued, otherwise false. - */ -bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, struct kbase_jd_atom *katom); - -/** - * kbase_js_pull - Pull an atom from a context in the job scheduler for - * execution. - * - * @kctx: Context to pull from - * @js: Job slot to pull from - * - * The atom will not be removed from the ringbuffer at this stage. - * - * The HW access lock must be held when calling this function. - * - * Return: a pointer to an atom, or NULL if there are no atoms for this - * slot that can be currently run. - */ -struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, unsigned int js); - -/** - * kbase_js_unpull - Return an atom to the job scheduler ringbuffer. - * - * @kctx: Context pointer - * @katom: Pointer to the atom to unpull - * - * An atom is 'unpulled' if execution is stopped but intended to be returned to - * later. The most common reason for this is that the atom has been - * soft-stopped. Another reason is if an end-of-renderpass atom completed - * but will need to be run again as part of the same renderpass. - * - * Note that if multiple atoms are to be 'unpulled', they must be returned in - * the reverse order to which they were originally pulled. It is a programming - * error to return atoms in any other order. - * - * The HW access lock must be held when calling this function. - * - */ -void kbase_js_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom); - -/** - * kbase_js_complete_atom_wq - Complete an atom from jd_done_worker(), - * removing it from the job - * scheduler ringbuffer. - * @kctx: Context pointer - * @katom: Pointer to the atom to complete - * - * If the atom failed then all dependee atoms marked for failure propagation - * will also fail. - * - * Return: true if the context is now idle (no jobs pulled) false otherwise. - */ -bool kbase_js_complete_atom_wq(struct kbase_context *kctx, struct kbase_jd_atom *katom); - -/** - * kbase_js_complete_atom - Complete an atom. - * - * @katom: Pointer to the atom to complete - * @end_timestamp: The time that the atom completed (may be NULL) - * - * Most of the work required to complete an atom will be performed by - * jd_done_worker(). - * - * The HW access lock must be held when calling this function. - * - * Return: a atom that has now been unblocked and can now be run, or NULL - * if none - */ -struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, ktime_t *end_timestamp); - -/** - * kbase_js_sched - Submit atoms from all available contexts. - * - * @kbdev: Device pointer - * @js_mask: Mask of job slots to submit to - * - * This will attempt to submit as many jobs as possible to the provided job - * slots. It will exit when either all job slots are full, or all contexts have - * been used. - * - */ -void kbase_js_sched(struct kbase_device *kbdev, unsigned int js_mask); - -/** - * kbase_js_zap_context - Attempt to deschedule a context that is being - * destroyed - * @kctx: Context pointer - * - * This will attempt to remove a context from any internal job scheduler queues - * and perform any other actions to ensure a context will not be submitted - * from. - * - * If the context is currently scheduled, then the caller must wait for all - * pending jobs to complete before taking any further action. - */ -void kbase_js_zap_context(struct kbase_context *kctx); - -/** - * kbase_js_is_atom_valid - Validate an atom - * - * @kbdev: Device pointer - * @katom: Atom to validate - * - * This will determine whether the atom can be scheduled onto the GPU. Atoms - * with invalid combinations of core requirements will be rejected. - * - * Return: true if atom is valid false otherwise. - */ -bool kbase_js_is_atom_valid(struct kbase_device *kbdev, struct kbase_jd_atom *katom); - -/** - * kbase_js_set_timeouts - update all JS timeouts with user specified data - * - * @kbdev: Device pointer - * - * Timeouts are specified through the 'js_timeouts' sysfs file. If a timeout is - * set to a positive number then that becomes the new value used, if a timeout - * is negative then the default is set. - */ -void kbase_js_set_timeouts(struct kbase_device *kbdev); - -/** - * kbase_js_set_ctx_priority - set the context priority - * - * @kctx: Context pointer - * @new_priority: New priority value for the Context - * - * The context priority is set to a new value and it is moved to the - * pullable/unpullable list as per the new priority. - */ -void kbase_js_set_ctx_priority(struct kbase_context *kctx, int new_priority); - -/** - * kbase_js_update_ctx_priority - update the context priority - * - * @kctx: Context pointer - * - * The context priority gets updated as per the priority of atoms currently in - * use for that context, but only if system priority mode for context scheduling - * is being used. - */ -void kbase_js_update_ctx_priority(struct kbase_context *kctx); - -/* - * Helpers follow - */ - -/** - * kbasep_js_is_submit_allowed - Check that a context is allowed to submit - * jobs on this policy - * @js_devdata: KBase Job Scheduler Device Data - * @kctx: KBase context - * - * The purpose of this abstraction is to hide the underlying data size, - * and wrap up the long repeated line of code. - * - * As with any bool, never test the return value with true. - * - * The caller must hold hwaccess_lock. - * - * Return: true if the context is allowed to submit jobs, false otherwise. - */ -static inline bool kbasep_js_is_submit_allowed(struct kbasep_js_device_data *js_devdata, - struct kbase_context *kctx) -{ - u16 test_bit; - bool is_allowed; - - /* Ensure context really is scheduled in */ - if (WARN((kctx->as_nr == KBASEP_AS_NR_INVALID) || !kbase_ctx_flag(kctx, KCTX_SCHEDULED), - "%s: kctx %pK has assigned AS %d and context flag %d\n", __func__, (void *)kctx, - kctx->as_nr, atomic_read(&kctx->flags))) - return false; - - test_bit = (u16)(1u << kctx->as_nr); - - is_allowed = (bool)(js_devdata->runpool_irq.submit_allowed & test_bit); - dev_dbg(kctx->kbdev->dev, "JS: submit %s allowed on %pK (as=%d)", - is_allowed ? "is" : "isn't", (void *)kctx, kctx->as_nr); - return is_allowed; -} - -/** - * kbasep_js_set_submit_allowed - Allow a context to submit jobs on this policy - * @js_devdata: KBase Job Scheduler Device Data - * @kctx: KBase context - * - * The purpose of this abstraction is to hide the underlying data size, - * and wrap up the long repeated line of code. - * - * The caller must hold hwaccess_lock. - */ -static inline void kbasep_js_set_submit_allowed(struct kbasep_js_device_data *js_devdata, - struct kbase_context *kctx) -{ - u16 set_bit; - - /* Ensure context really is scheduled in */ - if (WARN((kctx->as_nr == KBASEP_AS_NR_INVALID) || !kbase_ctx_flag(kctx, KCTX_SCHEDULED), - "%s: kctx %pK has assigned AS %d and context flag %d\n", __func__, (void *)kctx, - kctx->as_nr, atomic_read(&kctx->flags))) - return; - - set_bit = (u16)(1u << kctx->as_nr); - - dev_dbg(kctx->kbdev->dev, "JS: Setting Submit Allowed on %pK (as=%d)", kctx, kctx->as_nr); - - js_devdata->runpool_irq.submit_allowed |= set_bit; -} - -/** - * kbasep_js_clear_submit_allowed - Prevent a context from submitting more - * jobs on this policy - * @js_devdata: KBase Job Scheduler Device Data - * @kctx: KBase context - * - * The purpose of this abstraction is to hide the underlying data size, - * and wrap up the long repeated line of code. - * - * The caller must hold hwaccess_lock. - */ -static inline void kbasep_js_clear_submit_allowed(struct kbasep_js_device_data *js_devdata, - struct kbase_context *kctx) -{ - u16 clear_bit; - u16 clear_mask; - - /* Ensure context really is scheduled in */ - if (WARN((kctx->as_nr == KBASEP_AS_NR_INVALID) || !kbase_ctx_flag(kctx, KCTX_SCHEDULED), - "%s: kctx %pK has assigned AS %d and context flag %d\n", __func__, (void *)kctx, - kctx->as_nr, atomic_read(&kctx->flags))) - return; - - clear_bit = (u16)(1u << kctx->as_nr); - clear_mask = ~clear_bit; - - dev_dbg(kctx->kbdev->dev, "JS: Clearing Submit Allowed on %pK (as=%d)", kctx, kctx->as_nr); - - js_devdata->runpool_irq.submit_allowed &= clear_mask; -} - -/** - * kbasep_js_atom_retained_state_init_invalid - Create an initial 'invalid' - * atom retained state - * - * @retained_state: pointer where to create and initialize the state - * - * Create an initial 'invalid' atom retained state, that requires no - * atom-related work to be done on releasing with - * kbasep_js_runpool_release_ctx_and_katom_retained_state() - */ -static inline void -kbasep_js_atom_retained_state_init_invalid(struct kbasep_js_atom_retained_state *retained_state) -{ - retained_state->event_code = BASE_JD_EVENT_NOT_STARTED; - retained_state->core_req = KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID; -} - -/** - * kbasep_js_atom_retained_state_copy() - Copy atom state - * @retained_state: where to copy - * @katom: where to copy from - * - * Copy atom state that can be made available after kbase_jd_done_nolock() is called - * on that atom. - */ -static inline void -kbasep_js_atom_retained_state_copy(struct kbasep_js_atom_retained_state *retained_state, - const struct kbase_jd_atom *katom) -{ - retained_state->event_code = katom->event_code; - retained_state->core_req = katom->core_req; - retained_state->sched_priority = katom->sched_priority; - retained_state->device_nr = katom->device_nr; -} - -/** - * kbasep_js_has_atom_finished - Determine whether an atom has finished - * (given its retained state), - * and so should be given back to - * userspace/removed from the system. - * - * @katom_retained_state: the retained state of the atom to check - * - * Reasons for an atom not finishing include: - * * Being soft-stopped (and so, the atom should be resubmitted sometime later) - * * It is an end of renderpass atom that was run to consume the output of a - * start-of-renderpass atom that was soft-stopped because it used too much - * memory. In this case, it will have to be run again later. - * - * Return: false if the atom has not finished, true otherwise. - */ -static inline bool -kbasep_js_has_atom_finished(const struct kbasep_js_atom_retained_state *katom_retained_state) -{ - return (bool)(katom_retained_state->event_code != BASE_JD_EVENT_STOPPED && - katom_retained_state->event_code != BASE_JD_EVENT_REMOVED_FROM_NEXT); -} - -/** - * kbasep_js_atom_retained_state_is_valid - Determine whether a struct - * kbasep_js_atom_retained_state - * is valid - * @katom_retained_state: the atom's retained state to check - * - * An invalid struct kbasep_js_atom_retained_state is allowed, and indicates - * that the code should just ignore it. - * - * Return: false if the retained state is invalid, true otherwise. - */ -static inline bool kbasep_js_atom_retained_state_is_valid( - const struct kbasep_js_atom_retained_state *katom_retained_state) -{ - return (bool)(katom_retained_state->core_req != - KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID); -} - -/** - * kbase_js_runpool_inc_context_count - Increment number of running contexts. - * @kbdev: KBase device - * @kctx: KBase context - * - * The following locking conditions are made on the caller: - * * The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex. - * * The caller must hold the kbasep_js_device_data::runpool_mutex - */ -static inline void kbase_js_runpool_inc_context_count(struct kbase_device *kbdev, - struct kbase_context *kctx) -{ - struct kbasep_js_device_data *js_devdata; - struct kbasep_js_kctx_info *js_kctx_info; - - js_devdata = &kbdev->js_data; - js_kctx_info = &kctx->jctx.sched_info; - - lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); - lockdep_assert_held(&js_devdata->runpool_mutex); - - /* Track total contexts */ - WARN_ON_ONCE(js_devdata->nr_all_contexts_running >= JS_MAX_RUNNING_JOBS); - ++(js_devdata->nr_all_contexts_running); - - if (!kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { - /* Track contexts that can submit jobs */ - WARN_ON_ONCE(js_devdata->nr_user_contexts_running >= JS_MAX_RUNNING_JOBS); - ++(js_devdata->nr_user_contexts_running); - } -} - -/** - * kbase_js_runpool_dec_context_count - decrement number of running contexts. - * - * @kbdev: KBase device - * @kctx: KBase context - * The following locking conditions are made on the caller: - * * The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex. - * * The caller must hold the kbasep_js_device_data::runpool_mutex - */ -static inline void kbase_js_runpool_dec_context_count(struct kbase_device *kbdev, - struct kbase_context *kctx) -{ - struct kbasep_js_device_data *js_devdata; - struct kbasep_js_kctx_info *js_kctx_info; - - js_devdata = &kbdev->js_data; - js_kctx_info = &kctx->jctx.sched_info; - - lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); - lockdep_assert_held(&js_devdata->runpool_mutex); - - /* Track total contexts */ - --(js_devdata->nr_all_contexts_running); - WARN_ON_ONCE(js_devdata->nr_all_contexts_running < 0); - - if (!kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { - /* Track contexts that can submit jobs */ - --(js_devdata->nr_user_contexts_running); - WARN_ON_ONCE(js_devdata->nr_user_contexts_running < 0); - } -} - -/** - * kbase_js_sched_all - Submit atoms from all available contexts to all - * job slots. - * - * @kbdev: Device pointer - * - * This will attempt to submit as many jobs as possible. It will exit when - * either all job slots are full, or all contexts have been used. - */ -static inline void kbase_js_sched_all(struct kbase_device *kbdev) -{ - kbase_js_sched(kbdev, (1U << kbdev->gpu_props.num_job_slots) - 1U); -} - -extern const int kbasep_js_atom_priority_to_relative[BASE_JD_NR_PRIO_LEVELS]; - -extern const base_jd_prio kbasep_js_relative_priority_to_atom[KBASE_JS_ATOM_SCHED_PRIO_COUNT]; - -/** - * kbasep_js_atom_prio_to_sched_prio - Convert atom priority (base_jd_prio) - * to relative ordering. - * @atom_prio: Priority ID to translate. - * - * Atom priority values for @ref base_jd_prio cannot be compared directly to - * find out which are higher or lower. - * - * This function will convert base_jd_prio values for successively lower - * priorities into a monotonically increasing sequence. That is, the lower the - * base_jd_prio priority, the higher the value produced by this function. This - * is in accordance with how the rest of the kernel treats priority. - * - * The mapping is 1:1 and the size of the valid input range is the same as the - * size of the valid output range, i.e. - * KBASE_JS_ATOM_SCHED_PRIO_COUNT == BASE_JD_NR_PRIO_LEVELS - * - * Note This must be kept in sync with BASE_JD_PRIO_<...> definitions - * - * Return: On success: a value in the inclusive range - * 0..KBASE_JS_ATOM_SCHED_PRIO_COUNT-1. On failure: - * KBASE_JS_ATOM_SCHED_PRIO_INVALID - */ -static inline int kbasep_js_atom_prio_to_sched_prio(base_jd_prio atom_prio) -{ - if (atom_prio >= BASE_JD_NR_PRIO_LEVELS) - return KBASE_JS_ATOM_SCHED_PRIO_INVALID; - - return kbasep_js_atom_priority_to_relative[atom_prio]; -} - -/** - * kbasep_js_sched_prio_to_atom_prio - Convert relative scheduler priority - * to atom priority (base_jd_prio). - * - * @kbdev: Device pointer - * @sched_prio: Relative scheduler priority to translate. - * - * This function will convert relative scheduler priority back into base_jd_prio - * values. It takes values which priorities are monotonically increasing - * and converts them to the corresponding base_jd_prio values. If an invalid number is - * passed in (i.e. not within the expected range) an error code is returned instead. - * - * The mapping is 1:1 and the size of the valid input range is the same as the - * size of the valid output range, i.e. - * KBASE_JS_ATOM_SCHED_PRIO_COUNT == BASE_JD_NR_PRIO_LEVELS - * - * Return: On success: a value in the inclusive range - * 0..BASE_JD_NR_PRIO_LEVELS-1. On failure: BASE_JD_PRIO_INVALID. - */ -static inline base_jd_prio kbasep_js_sched_prio_to_atom_prio(struct kbase_device *kbdev, - int sched_prio) -{ - if (likely(sched_prio >= 0 && sched_prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT)) - return kbasep_js_relative_priority_to_atom[sched_prio]; - /* Invalid priority value if reached here */ - dev_warn(kbdev->dev, "Unknown JS scheduling priority %d", sched_prio); - return BASE_JD_PRIO_INVALID; -} - -/** - * kbase_js_priority_check - Check the priority requested - * - * @kbdev: Device pointer - * @priority: Requested priority - * - * This will determine whether the requested priority can be satisfied. - * - * Return: The same or lower priority than requested. - */ -base_jd_prio kbase_js_priority_check(struct kbase_device *kbdev, base_jd_prio priority); - -/** - * kbase_js_atom_runs_before - determine if atoms for the same slot have an - * ordering relation - * @kbdev: kbase device - * @katom_a: the first atom - * @katom_b: the second atom. - * @order_flags: combination of KBASE_ATOM_ORDERING_FLAG_<...> for the ordering - * relation - * - * This is for making consistent decisions about the ordering of atoms when we - * need to do pre-emption on a slot, which includes stopping existing atoms - * when a new atom is ready to run, and also which other atoms to remove from - * the slot when the atom in JS_HEAD is being pre-empted. - * - * This only handles @katom_a and @katom_b being for the same job slot, as - * pre-emption only operates within a slot. - * - * Note: there is currently no use-case for this as a sorting comparison - * functions, hence only a boolean returned instead of int -1, 0, +1 return. If - * required in future, a modification to do so would be better than calling - * twice with katom_a and katom_b swapped. - * - * Return: - * true if @katom_a should run before @katom_b, false otherwise. - * A false return value does not distinguish between "no ordering relation" and - * "@katom_a should run after @katom_b". - */ -bool kbase_js_atom_runs_before(struct kbase_device *kbdev, const struct kbase_jd_atom *katom_a, - const struct kbase_jd_atom *katom_b, - const kbase_atom_ordering_flag_t order_flags); - -#endif /* _KBASE_JM_JS_H_ */ diff --git a/drivers/gpu/arm/valhall/jm/mali_kbase_jm_mem_flags.h b/drivers/gpu/arm/valhall/jm/mali_kbase_jm_mem_flags.h deleted file mode 100644 index 98767fc06527..000000000000 --- a/drivers/gpu/arm/valhall/jm/mali_kbase_jm_mem_flags.h +++ /dev/null @@ -1,53 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#ifndef _KBASE_JM_MEM_FLAGS_H_ -#define _KBASE_JM_MEM_FLAGS_H_ - -#include - -/* Kernel-side only flags allocated from 63 bit downwards */ - -/* Use the GPU VA chosen by the kernel client */ -#define BASEP_MEM_FLAG_MAP_FIXED ((base_mem_alloc_flags)1 << 61) - -/* Force trimming of JIT allocations when creating a new allocation */ -#define BASEP_MEM_PERFORM_JIT_TRIM ((base_mem_alloc_flags)1 << 60) - -/* Region belongs to a shrinker. - * - * This can either mean that it is part of the JIT/Ephemeral or tiler heap - * shrinker paths. Should be removed only after making sure that there are - * no references remaining to it in these paths, as it may cause the physical - * backing of the region to disappear during use. - */ -#define BASEP_MEM_DONT_NEED ((base_mem_alloc_flags)1 << 59) - -/* Allocation is actively used for JIT memory */ -#define BASEP_MEM_ACTIVE_JIT_ALLOC ((base_mem_alloc_flags)1 << 58) - -/* The first available bit which can be used to define new memory flag - * if needed. It should be decremented by one once new flag is added - * and BASEP_MEM_FLAGS_NR_BITS should be incremented accordingly - */ -#define BASEP_MEM_FIRST_FREE_FLAG ((base_mem_alloc_flags)1 << 57) - -#endif /* _KBASE_JM_MEM_FLAGS_H_ */ diff --git a/drivers/gpu/arm/valhall/jm/mali_kbase_js_defs.h b/drivers/gpu/arm/valhall/jm/mali_kbase_js_defs.h deleted file mode 100644 index c81af50918fd..000000000000 --- a/drivers/gpu/arm/valhall/jm/mali_kbase_js_defs.h +++ /dev/null @@ -1,466 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2011-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/** - * DOC: Job Scheduler Type Definitions - */ - -#ifndef _KBASE_JS_DEFS_H_ -#define _KBASE_JS_DEFS_H_ - -/* Forward decls */ -struct kbase_device; -struct kbase_jd_atom; - -typedef u32 kbase_context_flags; - -/* - * typedef kbasep_js_ctx_job_cb - Callback function run on all of a context's - * jobs registered with the Job Scheduler - */ -typedef void kbasep_js_ctx_job_cb(struct kbase_device *kbdev, struct kbase_jd_atom *katom); - -/* - * @brief Maximum number of jobs that can be submitted to a job slot whilst - * inside the IRQ handler. - * - * This is important because GPU NULL jobs can complete whilst the IRQ handler - * is running. Otherwise, it potentially allows an unlimited number of GPU NULL - * jobs to be submitted inside the IRQ handler, which increases IRQ latency. - */ -#define KBASE_JS_MAX_JOB_SUBMIT_PER_SLOT_PER_IRQ 2 - -/** - * enum kbasep_js_ctx_attr - Context attributes - * @KBASEP_JS_CTX_ATTR_COMPUTE: Attribute indicating a context that contains - * Compute jobs. - * @KBASEP_JS_CTX_ATTR_NON_COMPUTE: Attribute indicating a context that contains - * Non-Compute jobs. - * @KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES: Attribute indicating that a context - * contains compute-job atoms that aren't - * restricted to a coherent group, - * and can run on all cores. - * @KBASEP_JS_CTX_ATTR_COUNT: Must be the last in the enum - * - * Each context attribute can be thought of as a boolean value that caches some - * state information about either the runpool, or the context: - * - In the case of the runpool, it is a cache of "Do any contexts owned by - * the runpool have attribute X?" - * - In the case of a context, it is a cache of "Do any atoms owned by the - * context have attribute X?" - * - * The boolean value of the context attributes often affect scheduling - * decisions, such as affinities to use and job slots to use. - * - * To accomodate changes of state in the context, each attribute is refcounted - * in the context, and in the runpool for all running contexts. Specifically: - * - The runpool holds a refcount of how many contexts in the runpool have this - * attribute. - * - The context holds a refcount of how many atoms have this attribute. - * - * KBASEP_JS_CTX_ATTR_COMPUTE: - * Attribute indicating a context that contains Compute jobs. That is, - * the context has jobs of type @ref BASE_JD_REQ_ONLY_COMPUTE - * - * @note A context can be both 'Compute' and 'Non Compute' if it contains - * both types of jobs. - * - * KBASEP_JS_CTX_ATTR_NON_COMPUTE: - * Attribute indicating a context that contains Non-Compute jobs. That is, - * the context has some jobs that are \b not of type @ref - * BASE_JD_REQ_ONLY_COMPUTE. - * - * @note A context can be both 'Compute' and 'Non Compute' if it contains - * both types of jobs. - * - * KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES: - * Attribute indicating that a context contains compute-job atoms that - * aren't restricted to a coherent group, and can run on all cores. - * - * Specifically, this is when the atom's \a core_req satisfy: - * - (\a core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T) // uses slot 1 or slot 2 - * - && !(\a core_req & BASE_JD_REQ_COHERENT_GROUP) // not restricted to coherent groups - * - * Such atoms could be blocked from running if one of the coherent groups - * is being used by another job slot, so tracking this context attribute - * allows us to prevent such situations. - * - * @note This doesn't take into account the 1-coregroup case, where all - * compute atoms would effectively be able to run on 'all cores', but - * contexts will still not always get marked with this attribute. Instead, - * it is the caller's responsibility to take into account the number of - * coregroups when interpreting this attribute. - * - * @note Whilst Tiler atoms are normally combined with - * BASE_JD_REQ_COHERENT_GROUP, it is possible to send such atoms without - * BASE_JD_REQ_COHERENT_GROUP set. This is an unlikely case, but it's easy - * enough to handle anyway. - * - */ -enum kbasep_js_ctx_attr { - KBASEP_JS_CTX_ATTR_COMPUTE, - KBASEP_JS_CTX_ATTR_NON_COMPUTE, - KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES, - KBASEP_JS_CTX_ATTR_COUNT -}; - -enum { - /* - * Bit indicating that new atom should be started because this atom - * completed - */ - KBASE_JS_ATOM_DONE_START_NEW_ATOMS = (1u << 0), - /* - * Bit indicating that the atom was evicted from the JS_NEXT registers - */ - KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT = (1u << 1) -}; - -/** - * typedef kbasep_js_atom_done_code - Combination of KBASE_JS_ATOM_DONE_<...> - * bits - */ -typedef u32 kbasep_js_atom_done_code; - -/* - * Context scheduling mode defines for kbase_device::js_ctx_scheduling_mode - */ -enum { - /* - * In this mode, higher priority atoms will be scheduled first, - * regardless of the context they belong to. Newly-runnable higher - * priority atoms can preempt lower priority atoms currently running on - * the GPU, even if they belong to a different context. - */ - KBASE_JS_SYSTEM_PRIORITY_MODE = 0, - - /* - * In this mode, the highest-priority atom will be chosen from each - * context in turn using a round-robin algorithm, so priority only has - * an effect within the context an atom belongs to. Newly-runnable - * higher priority atoms can preempt the lower priority atoms currently - * running on the GPU, but only if they belong to the same context. - */ - KBASE_JS_PROCESS_LOCAL_PRIORITY_MODE, - - /* Must be the last in the enum */ - KBASE_JS_PRIORITY_MODE_COUNT, -}; - -/* - * Internal atom priority defines for kbase_jd_atom::sched_prio - */ -enum { - KBASE_JS_ATOM_SCHED_PRIO_FIRST = 0, - KBASE_JS_ATOM_SCHED_PRIO_REALTIME = KBASE_JS_ATOM_SCHED_PRIO_FIRST, - KBASE_JS_ATOM_SCHED_PRIO_HIGH, - KBASE_JS_ATOM_SCHED_PRIO_MED, - KBASE_JS_ATOM_SCHED_PRIO_LOW, - KBASE_JS_ATOM_SCHED_PRIO_COUNT, -}; - -/* Invalid priority for kbase_jd_atom::sched_prio */ -#define KBASE_JS_ATOM_SCHED_PRIO_INVALID -1 - -/* Default priority in the case of contexts with no atoms, or being lenient - * about invalid priorities from userspace. - */ -#define KBASE_JS_ATOM_SCHED_PRIO_DEFAULT KBASE_JS_ATOM_SCHED_PRIO_MED - -/* Atom priority bitmaps, where bit 0 is the highest priority, and higher bits - * indicate successively lower KBASE_JS_ATOM_SCHED_PRIO_<...> levels. - * - * Must be strictly larger than the number of bits to represent a bitmap of - * priorities, so that we can do calculations such as: - * (1 << KBASE_JS_ATOM_SCHED_PRIO_COUNT) - 1 - * ...without causing undefined behavior due to a shift beyond the width of the - * type - * - * If KBASE_JS_ATOM_SCHED_PRIO_COUNT starts requiring 32 bits, then it's worth - * moving to DECLARE_BITMAP() - */ -typedef u8 kbase_js_prio_bitmap_t; - -/* Ordering modification for kbase_js_atom_runs_before() */ -typedef u32 kbase_atom_ordering_flag_t; - -/* Atoms of the same context and priority should have their ordering decided by - * their seq_nr instead of their age. - * - * seq_nr is used as a more slowly changing variant of age - it increases once - * per group of related atoms, as determined by user-space. Hence, it can be - * used to limit re-ordering decisions (such as pre-emption) to only re-order - * between such groups, rather than re-order within those groups of atoms. - */ -#define KBASE_ATOM_ORDERING_FLAG_SEQNR (((kbase_atom_ordering_flag_t)1) << 0) - -/** - * struct kbasep_js_device_data - KBase Device Data Job Scheduler sub-structure - * @runpool_irq: Sub-structure to collect together Job Scheduling data used in - * IRQ context. The hwaccess_lock must be held when accessing. - * @runpool_irq.submit_allowed: Bitvector indicating whether a currently - * scheduled context is allowed to submit jobs. - * When bit 'N' is set in this, it indicates whether - * the context bound to address space 'N' is - * allowed to submit jobs. - * @runpool_irq.ctx_attr_ref_count: Array of Context Attributes Ref_counters: - * Each is large enough to hold a refcount of the number of contexts - * that can fit into the runpool. This is currently BASE_MAX_NR_AS. - * Note that when BASE_MAX_NR_AS==16 we need 5 bits (not 4) to store - * the refcount. Hence, it's not worthwhile reducing this to - * bit-manipulation on u32s to save space (where in contrast, 4 bit - * sub-fields would be easy to do and would save space). - * Whilst this must not become negative, the sign bit is used for: - * - error detection in debug builds - * - Optimization: it is undefined for a signed int to overflow, and so - * the compiler can optimize for that never happening (thus, no masking - * is required on updating the variable) - * @runpool_irq.slot_affinities: Affinity management and tracking. Bitvector - * to aid affinity checking. - * Element 'n' bit 'i' indicates that slot 'n' - * is using core i (i.e. slot_affinity_refcount[n][i] > 0) - * @runpool_irq.slot_affinity_refcount: Array of fefcount for each core owned - * by each slot. Used to generate the slot_affinities array of bitvectors. - * The value of the refcount will not exceed BASE_JM_SUBMIT_SLOTS, - * because it is refcounted only when a job is definitely about to be - * submitted to a slot, and is de-refcounted immediately after a job - * finishes - * @schedule_sem: Scheduling semaphore. This must be held when calling - * kbase_jm_kick() - * @ctx_list_pullable: List of contexts that can currently be pulled from - * @ctx_list_unpullable: List of contexts that can not currently be pulled - * from, but have jobs currently running. - * @nr_user_contexts_running: Number of currently scheduled user contexts - * (excluding ones that are not submitting jobs) - * @nr_all_contexts_running: Number of currently scheduled contexts (including - * ones that are not submitting jobs) - * @js_reqs: Core Requirements to match up with base_js_atom's core_req memeber - * @note This is a write-once member, and so no locking is required to - * read - * @scheduling_period_ns: Value for JS_SCHEDULING_PERIOD_NS - * @soft_stop_ticks: Value for JS_SOFT_STOP_TICKS - * @soft_stop_ticks_cl: Value for JS_SOFT_STOP_TICKS_CL - * @hard_stop_ticks_ss: Value for JS_HARD_STOP_TICKS_SS - * @hard_stop_ticks_cl: Value for JS_HARD_STOP_TICKS_CL - * @hard_stop_ticks_dumping: Value for JS_HARD_STOP_TICKS_DUMPING - * @gpu_reset_ticks_ss: Value for JS_RESET_TICKS_SS - * @gpu_reset_ticks_cl: Value for JS_RESET_TICKS_CL - * @gpu_reset_ticks_dumping: Value for JS_RESET_TICKS_DUMPING - * @ctx_timeslice_ns: Value for JS_CTX_TIMESLICE_NS - * @suspended_soft_jobs_list: List of suspended soft jobs - * @softstop_always: Support soft-stop on a single context - * @init_status:The initialized-flag is placed at the end, to avoid - * cache-pollution (we should only be using this during init/term paths). - * @note This is a write-once member, and so no locking is required to - * read - * @nr_contexts_pullable:Number of contexts that can currently be pulled from - * @nr_contexts_runnable:Number of contexts that can either be pulled from or - * arecurrently running - * @soft_job_timeout_ms:Value for JS_SOFT_JOB_TIMEOUT - * @js_free_wait_time_ms: Maximum waiting time in ms for a Job Slot to be seen free. - * @queue_mutex: Queue Lock, used to access the Policy's queue of contexts - * independently of the Run Pool. - * Of course, you don't need the Run Pool lock to access this. - * @runpool_mutex: Run Pool mutex, for managing contexts within the runpool. - * - * This encapsulates the current context of the Job Scheduler on a particular - * device. This context is global to the device, and is not tied to any - * particular struct kbase_context running on the device. - * - * nr_contexts_running and as_free are optimized for packing together (by making - * them smaller types than u32). The operations on them should rarely involve - * masking. The use of signed types for arithmetic indicates to the compiler - * that the value will not rollover (which would be undefined behavior), and so - * under the Total License model, it is free to make optimizations based on - * that (i.e. to remove masking). - */ -struct kbasep_js_device_data { - struct runpool_irq { - u16 submit_allowed; - s8 ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT]; - u64 slot_affinities[BASE_JM_MAX_NR_SLOTS]; - s8 slot_affinity_refcount[BASE_JM_MAX_NR_SLOTS][64]; - } runpool_irq; - struct semaphore schedule_sem; - struct list_head ctx_list_pullable[BASE_JM_MAX_NR_SLOTS][KBASE_JS_ATOM_SCHED_PRIO_COUNT]; - struct list_head ctx_list_unpullable[BASE_JM_MAX_NR_SLOTS][KBASE_JS_ATOM_SCHED_PRIO_COUNT]; - s8 nr_user_contexts_running; - s8 nr_all_contexts_running; - base_jd_core_req js_reqs[BASE_JM_MAX_NR_SLOTS]; - - u32 scheduling_period_ns; - u32 soft_stop_ticks; - u32 soft_stop_ticks_cl; - u32 hard_stop_ticks_ss; - u32 hard_stop_ticks_cl; - u32 hard_stop_ticks_dumping; - u32 gpu_reset_ticks_ss; - u32 gpu_reset_ticks_cl; - u32 gpu_reset_ticks_dumping; - u32 ctx_timeslice_ns; - - struct list_head suspended_soft_jobs_list; - -#ifdef CONFIG_MALI_VALHALL_DEBUG - bool softstop_always; -#endif /* CONFIG_MALI_VALHALL_DEBUG */ - int init_status; - u32 nr_contexts_pullable; - atomic_t nr_contexts_runnable; - atomic_t soft_job_timeout_ms; - u32 js_free_wait_time_ms; - - struct mutex queue_mutex; - /* - * Run Pool mutex, for managing contexts within the runpool. - * Unless otherwise specified, you must hold this lock whilst accessing - * any members that follow - * - * In addition, this is used to access: - * * the kbasep_js_kctx_info::runpool substructure - */ - struct mutex runpool_mutex; - -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) - /** - * @gpu_metrics_timer: High-resolution timer used to periodically emit the GPU metrics - * tracepoints for applications that are using the GPU. The timer is - * needed for the long duration handling so that the length of work - * period is within the allowed limit. - */ - struct hrtimer gpu_metrics_timer; - - /** - * @gpu_metrics_timer_needed: Flag to indicate if the @gpu_metrics_timer is needed. - * The timer won't be started after the expiry if the flag - * isn't set. - */ - bool gpu_metrics_timer_needed; - - /** - * @gpu_metrics_timer_running: Flag to indicate if the @gpu_metrics_timer is running. - * The flag is set to false when the timer is cancelled or - * is not restarted after the expiry. - */ - bool gpu_metrics_timer_running; -#endif -}; - -/** - * struct kbasep_js_kctx_info - KBase Context Job Scheduling information - * structure - * @ctx: Job Scheduler Context information sub-structure.Its members are - * accessed regardless of whether the context is: - * - In the Policy's Run Pool - * - In the Policy's Queue - * - Not queued nor in the Run Pool. - * You must obtain the @ctx.jsctx_mutex before accessing any other members - * of this substructure. - * You may not access any of its members from IRQ context. - * @ctx.jsctx_mutex: Job Scheduler Context lock - * @ctx.nr_jobs: Number of jobs ready to run - does \em not include - * the jobs waiting in the dispatcher, and dependency-only - * jobs. See kbase_jd_context::job_nr for such jobs - * @ctx.ctx_attr_ref_count: Context Attributes ref count. Each is large enough - * to hold a refcount of the number of atoms on the context. - * @ctx.is_scheduled_wait: Wait queue to wait for KCTX_SHEDULED flag state - * changes. - * @ctx.ctx_list_entry: Link implementing JS queues. Context can be present on - * one list per job slot. - * @init_status: The initalized-flag is placed at the end, to avoid - * cache-pollution (we should only be using this during init/term paths) - * - * This is a substructure in the struct kbase_context that encapsulates all the - * scheduling information. - */ -struct kbasep_js_kctx_info { - struct kbase_jsctx { - struct mutex jsctx_mutex; - - u32 nr_jobs; - u32 ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT]; - wait_queue_head_t is_scheduled_wait; - struct list_head ctx_list_entry[BASE_JM_MAX_NR_SLOTS]; - } ctx; - int init_status; -}; - -/** - * struct kbasep_js_atom_retained_state - Subset of atom state. - * @event_code: to determine whether the atom has finished - * @core_req: core requirements - * @sched_priority: priority - * @device_nr: Core group atom was executed on - * - * Subset of atom state that can be available after kbase_jd_done_nolock() is called - * on that atom. A copy must be taken via kbasep_js_atom_retained_state_copy(), - * because the original atom could disappear. - */ -struct kbasep_js_atom_retained_state { - /* Event code - to determine whether the atom has finished */ - enum base_jd_event_code event_code; - /* core requirements */ - base_jd_core_req core_req; - /* priority */ - int sched_priority; - /* Core group atom was executed on */ - u32 device_nr; -}; - -/* - * Value signifying 'no retry on a slot required' for: - * - kbase_js_atom_retained_state::retry_submit_on_slot - * - kbase_jd_atom::retry_submit_on_slot - */ -#define KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID (-1) - -/* - * base_jd_core_req value signifying 'invalid' for a - * kbase_jd_atom_retained_state. See kbase_atom_retained_state_is_valid() - */ -#define KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID BASE_JD_REQ_DEP - -/* - * The JS timer resolution, in microseconds - * Any non-zero difference in time will be at least this size. - */ -#define KBASEP_JS_TICK_RESOLUTION_US 1 - -/** - * struct kbase_jsctx_slot_tracking - Job Scheduling tracking of a context's - * use of a job slot - * @blocked: bitmap of priorities that this slot is blocked at - * @atoms_pulled: counts of atoms that have been pulled from this slot, - * across all priority levels - * @atoms_pulled_pri: counts of atoms that have been pulled from this slot, per - * priority level - * - * Controls how a slot from the &struct kbase_context's jsctx_queue is managed, - * for example to ensure correct ordering of atoms when atoms of different - * priorities are unpulled. - */ -struct kbase_jsctx_slot_tracking { - kbase_js_prio_bitmap_t blocked; - atomic_t atoms_pulled; - int atoms_pulled_pri[KBASE_JS_ATOM_SCHED_PRIO_COUNT]; -}; - -#endif /* _KBASE_JS_DEFS_H_ */ diff --git a/drivers/gpu/arm/valhall/mali_csffw.bin b/drivers/gpu/arm/valhall/mali_csffw.bin index d95a7e530576..5d6480e8c70f 100644 Binary files a/drivers/gpu/arm/valhall/mali_csffw.bin and b/drivers/gpu/arm/valhall/mali_csffw.bin differ diff --git a/drivers/gpu/arm/valhall/mali_kbase.h b/drivers/gpu/arm/valhall/mali_kbase.h index b260be8077a8..da34a4225975 100644 --- a/drivers/gpu/arm/valhall/mali_kbase.h +++ b/drivers/gpu/arm/valhall/mali_kbase.h @@ -45,20 +45,11 @@ #include "mali_kbase_mem_profile_debugfs.h" #include "mali_kbase_gpuprops.h" #include -#if !MALI_USE_CSF -#include "mali_kbase_debug_job_fault.h" -#include "mali_kbase_jd_debugfs.h" -#include "mali_kbase_jm.h" -#include "mali_kbase_js.h" -#else /* !MALI_USE_CSF */ #include "csf/mali_kbase_debug_csf_fault.h" -#endif /* MALI_USE_CSF */ #include "ipa/mali_kbase_ipa.h" -#if MALI_USE_CSF #include "csf/mali_kbase_csf.h" -#endif #include "mali_linux_trace.h" @@ -84,7 +75,6 @@ #define KBASE_DRV_NAME "mali" #define KBASE_TIMELINE_NAME KBASE_DRV_NAME ".timeline" -#if MALI_USE_CSF /* Physical memory group ID for CSF user I/O. */ #define KBASE_MEM_GROUP_CSF_IO BASE_MEM_GROUP_DEFAULT @@ -92,7 +82,6 @@ /* Physical memory group ID for CSF firmware. */ #define KBASE_MEM_GROUP_CSF_FW BASE_MEM_GROUP_DEFAULT -#endif /* Physical memory group ID for a special page which can alias several regions. */ @@ -118,13 +107,6 @@ struct kbase_device *kbase_device_alloc(void); */ int kbase_device_misc_init(struct kbase_device *kbdev); void kbase_device_misc_term(struct kbase_device *kbdev); - -#if !MALI_USE_CSF -void kbase_enable_quick_reset(struct kbase_device *kbdev); -void kbase_disable_quick_reset(struct kbase_device *kbdev); -bool kbase_is_quick_reset_enabled(struct kbase_device *kbdev); -#endif - void kbase_device_free(struct kbase_device *kbdev); int kbase_device_has_feature(struct kbase_device *kbdev, u32 feature); @@ -242,162 +224,7 @@ void registers_unmap(struct kbase_device *kbdev); int kbase_device_coherency_init(struct kbase_device *kbdev); -#if !MALI_USE_CSF -/** - * kbase_jd_init() - Initialize kbase context for job dispatcher. - * @kctx: Pointer to the kbase context to be initialized. - * - * This function must be called only when a kbase context is instantiated. - * - * Return: 0 on success. - */ -int kbase_jd_init(struct kbase_context *kctx); -void kbase_jd_exit(struct kbase_context *kctx); - -/** - * kbase_jd_submit - Submit atoms to the job dispatcher - * - * @kctx: The kbase context to submit to - * @user_addr: The address in user space of the struct base_jd_atom array - * @nr_atoms: The number of atoms in the array - * @stride: sizeof(struct base_jd_atom) - * @uk6_atom: true if the atoms are legacy atoms (struct base_jd_atom_v2_uk6) - * - * Return: 0 on success or error code - */ -int kbase_jd_submit(struct kbase_context *kctx, void __user *user_addr, u32 nr_atoms, u32 stride, - bool uk6_atom); - -/** - * kbase_jd_done_worker - Handle a job completion - * @data: a &struct work_struct - * - * This function requeues the job from the runpool (if it was soft-stopped or - * removed from NEXT registers). - * - * Removes it from the system if it finished/failed/was cancelled. - * - * Resolves dependencies to add dependent jobs to the context, potentially - * starting them if necessary (which may add more references to the context) - * - * Releases the reference to the context from the no-longer-running job. - * - * Handles retrying submission outside of IRQ context if it failed from within - * IRQ context. - */ -void kbase_jd_done_worker(struct work_struct *data); - -void kbase_jd_done(struct kbase_jd_atom *katom, unsigned int slot_nr, ktime_t *end_timestamp, - kbasep_js_atom_done_code done_code); -void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom); -void kbase_jd_zap_context(struct kbase_context *kctx); - -/* - * kbase_jd_done_nolock - Perform the necessary handling of an atom that has completed - * the execution. - * - * @katom: Pointer to the atom that completed the execution - * @post_immediately: Flag indicating that completion event can be posted - * immediately for @katom and the other atoms depdendent - * on @katom which also completed execution. The flag is - * false only for the case where the function is called by - * kbase_jd_done_worker() on the completion of atom running - * on the GPU. - * - * Note that if this is a soft-job that has had kbase_prepare_soft_job called on it then the caller - * is responsible for calling kbase_finish_soft_job *before* calling this function. - * - * The caller must hold the kbase_jd_context.lock. - */ -bool kbase_jd_done_nolock(struct kbase_jd_atom *katom, bool post_immediately); - -void kbase_jd_free_external_resources(struct kbase_jd_atom *katom); -void kbase_jd_dep_clear_locked(struct kbase_jd_atom *katom); - -/** - * kbase_job_done - Process completed jobs from job interrupt - * @kbdev: Pointer to the kbase device. - * @done: Bitmask of done or failed jobs, from JOB_IRQ_STAT register - * - * This function processes the completed, or failed, jobs from the GPU job - * slots, for the bits set in the @done bitmask. - * - * The hwaccess_lock must be held when calling this function. - */ -void kbase_job_done(struct kbase_device *kbdev, u32 done); - -/** - * kbase_job_slot_ctx_priority_check_locked(): - Check for lower priority atoms - * and soft stop them - * @kctx: Pointer to context to check. - * @katom: Pointer to priority atom. - * - * Atoms from @kctx on the same job slot as @katom, which have lower priority - * than @katom will be soft stopped and put back in the queue, so that atoms - * with higher priority can run. - * - * The hwaccess_lock must be held when calling this function. - */ -void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx, - struct kbase_jd_atom *katom); - -/** - * kbase_job_slot_softstop - Soft-stop the specified job slot - * - * @kbdev: The kbase device - * @js: The job slot to soft-stop - * @target_katom: The job that should be soft-stopped (or NULL for any job) - * Context: - * The job slot lock must be held when calling this function. - * The job slot must not already be in the process of being soft-stopped. - * - * Where possible any job in the next register is evicted before the soft-stop. - */ -void kbase_job_slot_softstop(struct kbase_device *kbdev, unsigned int js, - struct kbase_jd_atom *target_katom); - -void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, unsigned int js, - struct kbase_jd_atom *target_katom, u32 sw_flags); - -/** - * kbase_job_check_enter_disjoint - potentiall enter disjoint mode - * @kbdev: kbase device - * @action: the event which has occurred - * @core_reqs: core requirements of the atom - * @target_katom: the atom which is being affected - * - * For a certain soft-stop action, work out whether to enter disjoint - * state. - * - * This does not register multiple disjoint events if the atom has already - * started a disjoint period - * - * @core_reqs can be supplied as 0 if the atom had not started on the hardware - * (and so a 'real' soft/hard-stop was not required, but it still interrupted - * flow, perhaps on another context) - * - * kbase_job_check_leave_disjoint() should be used to end the disjoint - * state when the soft/hard-stop action is complete - */ -void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action, - base_jd_core_req core_reqs, struct kbase_jd_atom *target_katom); - -/** - * kbase_job_check_leave_disjoint - potentially leave disjoint state - * @kbdev: kbase device - * @target_katom: atom which is finishing - * - * Work out whether to leave disjoint state when finishing an atom that was - * originated by kbase_job_check_enter_disjoint(). - */ -void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, struct kbase_jd_atom *target_katom); - -#endif /* !MALI_USE_CSF */ - void kbase_event_post(struct kbase_context *kctx, struct kbase_jd_atom *event); -#if !MALI_USE_CSF -int kbase_event_dequeue(struct kbase_context *kctx, struct base_jd_event_v2 *uevent); -#endif /* !MALI_USE_CSF */ int kbase_event_pending(struct kbase_context *kctx); int kbase_event_init(struct kbase_context *kctx); void kbase_event_close(struct kbase_context *kctx); @@ -449,22 +276,6 @@ static inline void kbase_free_user_buffer(struct kbase_debug_copy_buffer *buffer } } -#if !MALI_USE_CSF -int kbase_process_soft_job(struct kbase_jd_atom *katom); -int kbase_prepare_soft_job(struct kbase_jd_atom *katom); -void kbase_finish_soft_job(struct kbase_jd_atom *katom); -void kbase_cancel_soft_job(struct kbase_jd_atom *katom); -void kbase_resume_suspended_soft_jobs(struct kbase_device *kbdev); -void kbasep_remove_waiting_soft_job(struct kbase_jd_atom *katom); -#if IS_ENABLED(CONFIG_SYNC_FILE) -void kbase_soft_event_wait_callback(struct kbase_jd_atom *katom); -#endif -int kbase_soft_event_update(struct kbase_context *kctx, u64 event, unsigned char new_status); - -void kbasep_soft_job_timeout_worker(struct timer_list *timer); -void kbasep_complete_triggered_soft_events(struct kbase_context *kctx, u64 evt); -#endif /* !MALI_USE_CSF */ - void kbasep_as_do_poke(struct work_struct *work); /** @@ -549,7 +360,6 @@ void kbase_pm_metrics_start(struct kbase_device *kbdev); */ void kbase_pm_metrics_stop(struct kbase_device *kbdev); -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) /** * kbase_pm_handle_runtime_suspend - Handle the runtime suspend of GPU * @@ -587,43 +397,6 @@ int kbase_pm_handle_runtime_suspend(struct kbase_device *kbdev); * Return: 0 if the wake up was successful. */ int kbase_pm_force_mcu_wakeup_after_sleep(struct kbase_device *kbdev); -#endif - -#if !MALI_USE_CSF -/** - * kbase_jd_atom_id - Return the atom's ID, as was originally supplied by userspace in - * base_jd_atom::atom_number - * @kctx: KBase context pointer - * @katom: Atome for which to return ID - * - * Return: the atom's ID. - */ -static inline unsigned int kbase_jd_atom_id(struct kbase_context *kctx, - const struct kbase_jd_atom *katom) -{ - unsigned int result; - - KBASE_DEBUG_ASSERT(kctx); - KBASE_DEBUG_ASSERT(katom); - KBASE_DEBUG_ASSERT(katom->kctx == kctx); - - result = katom - &kctx->jctx.atoms[0]; - KBASE_DEBUG_ASSERT(result <= BASE_JD_ATOM_COUNT); - return result; -} - -/** - * kbase_jd_atom_from_id - Return the atom structure for the given atom ID - * @kctx: Context pointer - * @id: ID of atom to retrieve - * - * Return: Pointer to struct kbase_jd_atom associated with the supplied ID - */ -static inline struct kbase_jd_atom *kbase_jd_atom_from_id(struct kbase_context *kctx, int id) -{ - return &kctx->jctx.atoms[id]; -} -#endif /* !MALI_USE_CSF */ /** * kbase_disjoint_init - Initialize the disjoint state @@ -730,8 +503,6 @@ int kbase_device_pcm_dev_init(struct kbase_device *const kbdev); */ void kbase_device_pcm_dev_term(struct kbase_device *const kbdev); -#if MALI_USE_CSF - /** * kbasep_adjust_prioritized_process() - Adds or removes the specified PID from * the list of prioritized processes. @@ -744,8 +515,6 @@ void kbase_device_pcm_dev_term(struct kbase_device *const kbdev); */ bool kbasep_adjust_prioritized_process(struct kbase_device *kbdev, bool add, uint32_t tgid); -#endif /* MALI_USE_CSF */ - /** * KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD - If a job is soft stopped * and the number of contexts is >= this value it is reported as a disjoint event diff --git a/drivers/gpu/arm/valhall/mali_kbase_config.c b/drivers/gpu/arm/valhall/mali_kbase_config.c index 37dbca129f86..c90952dd9d44 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_config.c +++ b/drivers/gpu/arm/valhall/mali_kbase_config.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2011-2015, 2017, 2020-2021 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -62,43 +62,3 @@ void kbasep_platform_device_late_term(struct kbase_device *kbdev) if (platform_funcs_p && platform_funcs_p->platform_late_term_func) platform_funcs_p->platform_late_term_func(kbdev); } - -#if !MALI_USE_CSF -int kbasep_platform_context_init(struct kbase_context *kctx) -{ - struct kbase_platform_funcs_conf *platform_funcs_p; - - platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS; - if (platform_funcs_p && platform_funcs_p->platform_handler_context_init_func) - return platform_funcs_p->platform_handler_context_init_func(kctx); - - return 0; -} - -void kbasep_platform_context_term(struct kbase_context *kctx) -{ - struct kbase_platform_funcs_conf *platform_funcs_p; - - platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS; - if (platform_funcs_p && platform_funcs_p->platform_handler_context_term_func) - platform_funcs_p->platform_handler_context_term_func(kctx); -} - -void kbasep_platform_event_atom_submit(struct kbase_jd_atom *katom) -{ - struct kbase_platform_funcs_conf *platform_funcs_p; - - platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS; - if (platform_funcs_p && platform_funcs_p->platform_handler_atom_submit_func) - platform_funcs_p->platform_handler_atom_submit_func(katom); -} - -void kbasep_platform_event_atom_complete(struct kbase_jd_atom *katom) -{ - struct kbase_platform_funcs_conf *platform_funcs_p; - - platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS; - if (platform_funcs_p && platform_funcs_p->platform_handler_atom_complete_func) - platform_funcs_p->platform_handler_atom_complete_func(katom); -} -#endif diff --git a/drivers/gpu/arm/valhall/mali_kbase_config.h b/drivers/gpu/arm/valhall/mali_kbase_config.h index 2f9e28aaec9a..aefa8e631650 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_config.h +++ b/drivers/gpu/arm/valhall/mali_kbase_config.h @@ -33,14 +33,6 @@ /* Forward declaration of struct kbase_device */ struct kbase_device; -#if !MALI_USE_CSF -/* Forward declaration of struct kbase_context */ -struct kbase_context; - -/* Forward declaration of struct kbase_atom */ -struct kbase_jd_atom; -#endif - /** * struct kbase_platform_funcs_conf - Specifies platform integration function * pointers for DDK events such as device init and term. @@ -103,57 +95,6 @@ struct kbase_platform_funcs_conf { * can be accessed (and possibly terminated) in here. */ void (*platform_late_term_func)(struct kbase_device *kbdev); - -#if !MALI_USE_CSF - /** - * @platform_handler_context_init_func: platform specific handler for - * when a new kbase_context is created. - * @kctx - kbase_context pointer - * - * Returns 0 on success, negative error code otherwise. - * - * Function pointer for platform specific initialization of a kernel - * context or NULL if not required. Called at the last stage of kernel - * context initialization. - */ - int (*platform_handler_context_init_func)(struct kbase_context *kctx); - /** - * @platform_handler_context_term_func: platform specific handler for - * when a kbase_context is terminated. - * @kctx - kbase_context pointer - * - * Function pointer for platform specific termination of a kernel - * context or NULL if not required. Called at the first stage of kernel - * context termination. - */ - void (*platform_handler_context_term_func)(struct kbase_context *kctx); - /** - * @platform_handler_atom_submit_func: platform specific handler for - * when a kbase_jd_atom is submitted. - * @katom - kbase_jd_atom pointer - * - * Function pointer for platform specific handling at the point when an - * atom is submitted to the GPU or set to NULL if not required. The - * function cannot assume that it is running in a process context. - * - * Context: The caller must hold the hwaccess_lock. Function must be - * runnable in an interrupt context. - */ - void (*platform_handler_atom_submit_func)(struct kbase_jd_atom *katom); - /** - * @platform_handler_atom_complete_func: platform specific handler for - * when a kbase_jd_atom completes. - * @katom - kbase_jd_atom pointer - * - * Function pointer for platform specific handling at the point when an - * atom stops running on the GPU or set to NULL if not required. The - * function cannot assume that it is running in a process context. - * - * Context: The caller must hold the hwaccess_lock. Function must be - * runnable in an interrupt context. - */ - void (*platform_handler_atom_complete_func)(struct kbase_jd_atom *katom); -#endif }; /* @@ -232,7 +173,7 @@ struct kbase_pm_callback_conf { * to the OS from within this function. * The runtime calls can be triggered by calls from @ref power_off_callback * and @ref power_on_callback. - * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. + * Note: for linux the kernel must have CONFIG_PM enabled to use this feature. * * @return 0 on success, else int error code. */ @@ -243,21 +184,21 @@ struct kbase_pm_callback_conf { * The runtime power management callbacks @ref power_runtime_off_callback * and @ref power_runtime_on_callback should no longer be called by the * OS on completion of this function. - * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. + * Note: for linux the kernel must have CONFIG_PM enabled to use this feature. */ void (*power_runtime_term_callback)(struct kbase_device *kbdev); /** Callback for runtime power-off power management callback * * For linux this callback will be called by the kernel runtime_suspend callback. - * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. + * Note: for linux the kernel must have CONFIG_PM enabled to use this feature. */ void (*power_runtime_off_callback)(struct kbase_device *kbdev); /** Callback for runtime power-on power management callback * * For linux this callback will be called by the kernel runtime_resume callback. - * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. + * Note: for linux the kernel must have CONFIG_PM enabled to use this feature. * * @return 0 on success, else OS error code. */ @@ -281,7 +222,7 @@ struct kbase_pm_callback_conf { * This callback is optional and if not provided regular autosuspend * will be triggered. * - * Note: The Linux kernel must have CONFIG_PM_RUNTIME enabled to use + * Note: The Linux kernel must have CONFIG_PM enabled to use * this feature. * * Return 0 if GPU can be suspended, positive value if it can not be @@ -316,7 +257,7 @@ struct kbase_pm_callback_conf { * GPU registers still remain accessible until @power_off_callback gets * invoked later on the expiry of auto-suspend timer. * - * Note: The Linux kernel must have CONFIG_PM_RUNTIME enabled to use + * Note: The Linux kernel must have CONFIG_PM enabled to use * this feature. */ void (*power_runtime_gpu_idle_callback)(struct kbase_device *kbdev); @@ -339,7 +280,7 @@ struct kbase_pm_callback_conf { * to avoid a potential deadlock due to the runtime suspend happening * simultaneously from some other thread. * - * Note: The Linux kernel must have CONFIG_PM_RUNTIME enabled to use + * Note: The Linux kernel must have CONFIG_PM enabled to use * this feature. */ void (*power_runtime_gpu_active_callback)(struct kbase_device *kbdev); @@ -514,56 +455,6 @@ int kbasep_platform_device_late_init(struct kbase_device *kbdev); */ void kbasep_platform_device_late_term(struct kbase_device *kbdev); -#if !MALI_USE_CSF -/** - * kbasep_platform_context_init - Platform specific callback when a kernel - * context is created - * @kctx: kbase_context pointer - * - * Function calls a platform defined routine if specified in the configuration - * attributes. The routine can initialize any per kernel context structures - * that are required for the GPU block to function. - * - * Return: 0 if no errors were encountered. Negative error code otherwise. - */ -int kbasep_platform_context_init(struct kbase_context *kctx); - -/** - * kbasep_platform_context_term - Platform specific callback when a kernel - * context is terminated - * @kctx: kbase_context pointer - * - * Function calls a platform defined routine if specified in the configuration - * attributes. The routine should terminate any per kernel context structures - * created as part of &kbasep_platform_context_init. - * - */ -void kbasep_platform_context_term(struct kbase_context *kctx); - -/** - * kbasep_platform_event_atom_submit - Platform specific callback when an atom - * is submitted to the GPU - * @katom: kbase_jd_atom pointer - * - * Function calls a platform defined routine if specified in the configuration - * attributes. The routine should not assume that it is in a process context. - * - * Return: 0 if no errors were encountered. Negative error code otherwise. - */ -void kbasep_platform_event_atom_submit(struct kbase_jd_atom *katom); - -/** - * kbasep_platform_event_atom_complete - Platform specific callback when an atom - * has stopped running on the GPU - * @katom: kbase_jd_atom pointer - * - * Function calls a platform defined routine if specified in the configuration - * attributes. The routine should not assume that it is in a process context. - * - */ -void kbasep_platform_event_atom_complete(struct kbase_jd_atom *katom); -#endif - #ifndef CONFIG_OF /** * kbase_platform_register - Register a platform device for the GPU diff --git a/drivers/gpu/arm/valhall/mali_kbase_config_defaults.h b/drivers/gpu/arm/valhall/mali_kbase_config_defaults.h index 06298c5d1509..76b2ce2c6895 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_config_defaults.h +++ b/drivers/gpu/arm/valhall/mali_kbase_config_defaults.h @@ -314,13 +314,6 @@ enum { /* MAX allowed timeout value(ms) on host side, should be less than ANR timeout */ #define MAX_TIMEOUT_MS (4500) -#if !MALI_USE_CSF -/* Default number of milliseconds given for other jobs on the GPU to be - * soft-stopped when the GPU needs to be reset. - */ -#define JM_DEFAULT_RESET_TIMEOUT_MS (1) /* 1 ms */ -#endif /* !MALI_USE_CSF */ - /* Timeout for polling the GPU PRFCNT_ACTIVE bit in clock cycles. * * Based on 120s timeout at 100MHz, based on original MAX_LOOPS value. diff --git a/drivers/gpu/arm/valhall/mali_kbase_core_linux.c b/drivers/gpu/arm/valhall/mali_kbase_core_linux.c index bc7c487611e9..081752a4bf60 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_core_linux.c +++ b/drivers/gpu/arm/valhall/mali_kbase_core_linux.c @@ -41,28 +41,20 @@ #include "mali_kbase_debugfs_helper.h" #include #include -#if !MALI_USE_CSF -#include -#endif /* !MALI_USE_CSF */ #ifdef CONFIG_MALI_VALHALL_PRFCNT_SET_SELECT_VIA_DEBUG_FS #include #endif #include #include -#if !MALI_USE_CSF -#include "mali_kbase_kinstr_jm.h" -#endif #include "hwcnt/mali_kbase_hwcnt_context.h" #include "hwcnt/mali_kbase_hwcnt_virtualizer.h" #include "mali_kbase_kinstr_prfcnt.h" -#if MALI_USE_CSF #include "csf/mali_kbase_csf_firmware.h" #include "csf/mali_kbase_csf_tiler_heap.h" #include "csf/mali_kbase_csf_csg_debugfs.h" #include "csf/mali_kbase_csf_cpu_queue.h" #include "csf/mali_kbase_csf_event.h" #include "csf/mali_kbase_csf_ne_debugfs.h" -#endif #include "arbiter/mali_kbase_arbiter_pm.h" #include "mali_kbase_cs_experimental.h" @@ -150,7 +142,6 @@ struct mali_kbase_capability_def { * in the table. Less efficient but potentially safer. */ static const struct mali_kbase_capability_def kbase_caps_table[MALI_KBASE_NUM_CAPS] = { -#if MALI_USE_CSF { 1, 0 }, /* SYSTEM_MONITOR */ { 1, 0 }, /* JIT_PRESSURE_LIMIT */ { 1, 22 }, /* QUERY_MEM_DONT_NEED */ @@ -169,26 +160,6 @@ static const struct mali_kbase_capability_def kbase_caps_table[MALI_KBASE_NUM_CA { 1, 32 }, /* REJECT_ALLOC_MEM_UNUSED_BIT_7 */ { U16_MAX, U16_MAX }, /* REJECT_ALLOC_MEM_UNUSED_BIT_29 */ { 1, 35 } /* CSG_CS_USER_PAGE_ALLOCATION */ -#else - { 11, 15 }, /* SYSTEM_MONITOR */ - { 11, 25 }, /* JIT_PRESSURE_LIMIT */ - { 11, 40 }, /* QUERY_MEM_DONT_NEED */ - { 11, 2 }, /* QUERY_MEM_GROW_ON_GPF */ - { 11, 2 }, /* QUERY_MEM_PROTECTED */ - { 11, 43 }, /* QUERY_MEM_IMPORT_SYNC_ON_MAP_UNMAP */ - { 11, 43 }, /* QUERY_MEM_KERNEL_SYNC */ - { 11, 44 }, /* QUERY_MEM_SAME_VA */ - { 11, 46 }, /* REJECT_ALLOC_MEM_DONT_NEED */ - { 11, 46 }, /* REJECT_ALLOC_MEM_PROTECTED_IN_UNPROTECTED_ALLOCS */ - { 11, 46 }, /* REJECT_ALLOC_MEM_UNUSED_BIT_8 */ - { 11, 46 }, /* REJECT_ALLOC_MEM_UNUSED_BIT_19 */ - { U16_MAX, U16_MAX }, /* REJECT_ALLOC_MEM_UNUSED_BIT_20 */ - { 11, 48 }, /* REJECT_ALLOC_MEM_UNUSED_BIT_27 */ - { 11, 48 }, /* REJECT_ALLOC_MEM_UNUSED_BIT_5 */ - { 11, 48 }, /* REJECT_ALLOC_MEM_UNUSED_BIT_7 */ - { 11, 48 }, /* REJECT_ALLOC_MEM_UNUSED_BIT_29 */ - { U16_MAX, U16_MAX } /* CSG_CS_USER_PAGE_ALLOCATION */ -#endif }; #if (KERNEL_VERSION(5, 3, 0) <= LINUX_VERSION_CODE) @@ -496,7 +467,6 @@ static int get_irqs(struct kbase_device *kbdev, struct platform_device *pdev) return 0; } -#if MALI_USE_CSF /** * get_irq_irqaw - Get interrupt information from the device tree. * @@ -531,7 +501,6 @@ static int get_irq_irqaw(struct kbase_device *kbdev, struct platform_device *pde return 0; } -#endif /* MALI_USE_CSF */ int kbase_get_irqs(struct kbase_device *kbdev) { @@ -543,7 +512,6 @@ int kbase_get_irqs(struct kbase_device *kbdev) if (!result) return result; -#if MALI_USE_CSF /* return error if any one of the GPU, JOB, MMU * interrupts missing. Don't lookup for IRQAW. */ @@ -553,7 +521,6 @@ int kbase_get_irqs(struct kbase_device *kbdev) } result = get_irq_irqaw(kbdev, pdev); -#endif /* MALI_USE_CSF */ if (result) dev_err(kbdev->dev, "Invalid or No interrupt resources"); @@ -811,11 +778,6 @@ static int kbase_api_set_flags(struct kbase_file *kfile, struct kbase_ioctl_set_ if (mali_kbase_supports_system_monitor(api_version)) { err = kbase_file_create_kctx(kfile, flags->create_flags); } else { -#if !MALI_USE_CSF - struct kbasep_js_kctx_info *js_kctx_info = NULL; - unsigned long irq_flags = 0; -#endif - /* If setup is incomplete (e.g. because the API version * wasn't set) then we have to give up. */ @@ -823,35 +785,15 @@ static int kbase_api_set_flags(struct kbase_file *kfile, struct kbase_ioctl_set_ if (unlikely(!kctx)) return -EPERM; -#if MALI_USE_CSF - /* On CSF GPUs Job Manager interface isn't used to submit jobs + /* On CSF GPUs Job Manager interface isn't used to submit jobs * (there are no job slots). So the legacy job manager path to * submit jobs needs to remain disabled for CSF GPUs. */ -#else - js_kctx_info = &kctx->jctx.sched_info; - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, irq_flags); - /* Translate the flags */ - if ((flags->create_flags & BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED) == 0) - kbase_ctx_flag_clear(kctx, KCTX_SUBMIT_DISABLED); - - spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, irq_flags); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); -#endif } return err; } -#if !MALI_USE_CSF -static int kbase_api_job_submit(struct kbase_context *kctx, struct kbase_ioctl_job_submit *submit) -{ - return kbase_jd_submit(kctx, u64_to_user_ptr(submit->addr), submit->nr_atoms, - submit->stride, false); -} -#endif /* !MALI_USE_CSF */ - static int kbase_api_get_gpuprops(struct kbase_file *kfile, struct kbase_ioctl_get_gpuprops *get_props) { @@ -875,15 +817,6 @@ static int kbase_api_get_gpuprops(struct kbase_file *kfile, return (int)kprops->prop_buffer_size; } -#if !MALI_USE_CSF -static int kbase_api_post_term(struct kbase_context *kctx) -{ - kbase_event_close(kctx); - return 0; -} -#endif /* !MALI_USE_CSF */ - -#if MALI_USE_CSF static int kbase_api_mem_alloc_ex(struct kbase_context *kctx, union kbase_ioctl_mem_alloc_ex *alloc_ex) { @@ -994,46 +927,6 @@ static int kbase_api_mem_alloc(struct kbase_context *kctx, union kbase_ioctl_mem return ret; } -#else -static int kbase_api_mem_alloc(struct kbase_context *kctx, union kbase_ioctl_mem_alloc *alloc) -{ - struct kbase_va_region *reg; - base_mem_alloc_flags flags = alloc->in.flags; - u64 gpu_va; - - /* Calls to this function are inherently asynchronous, with respect to - * MMU operations. - */ - const enum kbase_caller_mmu_sync_info mmu_sync_info = CALLER_MMU_ASYNC; - - if (!kbase_mem_allow_alloc(kctx)) - return -EINVAL; - - if (flags & BASE_MEM_FLAGS_KERNEL_ONLY) - return -ENOMEM; - - /* Force SAME_VA if a 64-bit client. - * The only exception is GPU-executable memory if an EXEC_VA zone - * has been initialized. In that case, GPU-executable memory may - * or may not be SAME_VA. - */ - if ((!kbase_ctx_flag(kctx, KCTX_COMPAT)) && kbase_ctx_flag(kctx, KCTX_FORCE_SAME_VA)) { - if (!(flags & BASE_MEM_PROT_GPU_EX) || !kbase_has_exec_va_zone(kctx)) - flags |= BASE_MEM_SAME_VA; - } - - reg = kbase_mem_alloc(kctx, alloc->in.va_pages, alloc->in.commit_pages, alloc->in.extension, - &flags, &gpu_va, mmu_sync_info); - - if (!reg) - return -ENOMEM; - - alloc->out.flags = flags; - alloc->out.gpu_va = gpu_va; - - return 0; -} -#endif /* MALI_USE_CSF */ static int kbase_api_mem_query(struct kbase_context *kctx, union kbase_ioctl_mem_query *query) { @@ -1045,13 +938,6 @@ static int kbase_api_mem_free(struct kbase_context *kctx, struct kbase_ioctl_mem return kbase_mem_free(kctx, free->gpu_addr); } -#if !MALI_USE_CSF -static int kbase_api_kinstr_jm_fd(struct kbase_context *kctx, union kbase_kinstr_jm_fd *arg) -{ - return kbase_kinstr_jm_get_fd(kctx->kinstr_jm, arg); -} -#endif - static int kbase_api_get_cpu_gpu_timeinfo(struct kbase_context *kctx, union kbase_ioctl_get_cpu_gpu_timeinfo *timeinfo) { @@ -1060,6 +946,9 @@ static int kbase_api_get_cpu_gpu_timeinfo(struct kbase_context *kctx, u64 timestamp = 0; u64 cycle_cnt = 0; + if (kbase_io_is_aw_removed(kctx->kbdev)) + return -ENODEV; + kbase_pm_context_active(kctx->kbdev); kbase_backend_get_gpu_time(kctx->kbdev, @@ -1137,7 +1026,7 @@ static int kbase_api_mem_exec_init(struct kbase_context *kctx, static int kbase_api_mem_sync(struct kbase_context *kctx, struct kbase_ioctl_mem_sync *sync) { - struct basep_syncset sset = { .mem_handle.basep.handle = sync->handle, + struct basep_syncset sset = { .gpu_va = sync->gpu_va, .user_addr = sync->user_addr, .size = sync->size, .type = sync->type }; @@ -1327,17 +1216,6 @@ static int kbase_api_mem_profile_add(struct kbase_context *kctx, return kbasep_mem_profile_debugfs_insert(kctx, buf, data->len); } -#if !MALI_USE_CSF -static int kbase_api_soft_event_update(struct kbase_context *kctx, - struct kbase_ioctl_soft_event_update *update) -{ - if (update->flags != 0) - return -EINVAL; - - return kbase_soft_event_update(kctx, update->event, update->new_status); -} -#endif /* !MALI_USE_CSF */ - static int kbase_api_sticky_resource_map(struct kbase_context *kctx, struct kbase_ioctl_sticky_resource_map *map) { @@ -1426,7 +1304,6 @@ static int kbase_api_tlstream_stats(struct kbase_context *kctx, } #endif /* MALI_UNIT_TEST */ -#if MALI_USE_CSF static int kbasep_cs_event_signal(struct kbase_context *kctx) { kbase_csf_event_signal_notify_gpu(kctx); @@ -1703,8 +1580,16 @@ static int kbasep_ioctl_cs_cpu_queue_dump(struct kbase_context *kctx, return kbase_csf_cpu_queue_dump_buffer(kctx, cpu_queue_info->buffer, cpu_queue_info->size); } +#if MALI_UNIT_TEST +int kbase_ioctl_read_user_page(struct kbase_context *kctx, + union kbase_ioctl_read_user_page *user_page); + +int kbase_ioctl_read_user_page(struct kbase_context *kctx, + union kbase_ioctl_read_user_page *user_page) +#else static int kbase_ioctl_read_user_page(struct kbase_context *kctx, union kbase_ioctl_read_user_page *user_page) +#endif { struct kbase_device *kbdev = kctx->kbdev; unsigned long flags; @@ -1714,7 +1599,7 @@ static int kbase_ioctl_read_user_page(struct kbase_context *kctx, return -EINVAL; spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - if (!kbase_io_is_gpu_powered(kbdev) || kbase_io_is_aw_removed(kbdev)) + if (!kbase_io_has_gpu(kbdev)) user_page->out.val_lo = POWER_DOWN_LATEST_FLUSH_VALUE; else user_page->out.val_lo = kbase_reg_read32(kbdev, USER_ENUM(LATEST_FLUSH)); @@ -1723,19 +1608,15 @@ static int kbase_ioctl_read_user_page(struct kbase_context *kctx, return 0; } -#endif /* MALI_USE_CSF */ +#if MALI_UNIT_TEST +KBASE_EXPORT_TEST_API(kbase_ioctl_read_user_page); +#endif static int kbasep_ioctl_context_priority_check(struct kbase_context *kctx, struct kbase_ioctl_context_priority_check *priority_check) { -#if MALI_USE_CSF priority_check->priority = kbase_csf_priority_check(kctx->kbdev, priority_check->priority); -#else - base_jd_prio req_priority = (base_jd_prio)priority_check->priority; - - priority_check->priority = (u8)kbase_js_priority_check(kctx->kbdev, req_priority); -#endif return 0; } @@ -1811,27 +1692,14 @@ static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) /* Normal ioctls */ switch (cmd) { -#if !MALI_USE_CSF - case KBASE_IOCTL_JOB_SUBMIT: - KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_JOB_SUBMIT, kbase_api_job_submit, - struct kbase_ioctl_job_submit, kctx); - break; -#endif /* !MALI_USE_CSF */ -#if !MALI_USE_CSF - case KBASE_IOCTL_POST_TERM: - KBASE_HANDLE_IOCTL(KBASE_IOCTL_POST_TERM, kbase_api_post_term, kctx); - break; -#endif /* !MALI_USE_CSF */ case KBASE_IOCTL_MEM_ALLOC: KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_ALLOC, kbase_api_mem_alloc, union kbase_ioctl_mem_alloc, kctx); break; -#if MALI_USE_CSF case KBASE_IOCTL_MEM_ALLOC_EX: KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_ALLOC_EX, kbase_api_mem_alloc_ex, union kbase_ioctl_mem_alloc_ex, kctx); break; -#endif case KBASE_IOCTL_MEM_QUERY: KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_QUERY, kbase_api_mem_query, union kbase_ioctl_mem_query, kctx); @@ -1910,13 +1778,6 @@ static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) struct kbase_ioctl_mem_profile_add, kctx); break; -#if !MALI_USE_CSF - case KBASE_IOCTL_SOFT_EVENT_UPDATE: - KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_SOFT_EVENT_UPDATE, kbase_api_soft_event_update, - struct kbase_ioctl_soft_event_update, kctx); - break; -#endif /* !MALI_USE_CSF */ - case KBASE_IOCTL_STICKY_RESOURCE_MAP: KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_STICKY_RESOURCE_MAP, kbase_api_sticky_resource_map, @@ -1929,12 +1790,6 @@ static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) break; /* Instrumentation. */ -#if !MALI_USE_CSF - case KBASE_IOCTL_KINSTR_JM_FD: - KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_KINSTR_JM_FD, kbase_api_kinstr_jm_fd, - union kbase_kinstr_jm_fd, kctx); - break; -#endif case KBASE_IOCTL_GET_CPU_GPU_TIMEINFO: KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_GET_CPU_GPU_TIMEINFO, kbase_api_get_cpu_gpu_timeinfo, @@ -1946,7 +1801,6 @@ static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) struct kbase_ioctl_hwcnt_values, kctx); break; #endif /* CONFIG_MALI_VALHALL_NO_MALI */ -#if MALI_USE_CSF case KBASE_IOCTL_CS_EVENT_SIGNAL: KBASE_HANDLE_IOCTL(KBASE_IOCTL_CS_EVENT_SIGNAL, kbasep_cs_event_signal, kctx); break; @@ -2042,7 +1896,6 @@ static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_READ_USER_PAGE, kbase_ioctl_read_user_page, union kbase_ioctl_read_user_page, kctx); break; -#endif /* MALI_USE_CSF */ #if MALI_UNIT_TEST case KBASE_IOCTL_TLSTREAM_STATS: KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_TLSTREAM_STATS, kbase_api_tlstream_stats, @@ -2066,7 +1919,6 @@ static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) return -ENOIOCTLCMD; } -#if MALI_USE_CSF static ssize_t kbase_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos) { struct kbase_file *const kfile = filp->private_data; @@ -2110,54 +1962,6 @@ static ssize_t kbase_read(struct file *filp, char __user *buf, size_t count, lof return data_size; } -#else /* MALI_USE_CSF */ -static ssize_t kbase_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos) -{ - struct kbase_file *const kfile = filp->private_data; - struct kbase_context *const kctx = kbase_file_get_kctx_if_setup_complete(kfile); - struct base_jd_event_v2 uevent; - int out_count = 0; - - CSTD_UNUSED(f_pos); - - if (unlikely(!kctx)) - return -EPERM; - - if (count < sizeof(uevent)) - return -ENOBUFS; - - memset(&uevent, 0, sizeof(uevent)); - - do { - while (kbase_event_dequeue(kctx, &uevent)) { - if (out_count > 0) - goto out; - - if (filp->f_flags & O_NONBLOCK) - return -EAGAIN; - - if (wait_event_interruptible(kctx->event_queue, - kbase_event_pending(kctx)) != 0) - return -ERESTARTSYS; - } - if (uevent.event_code == BASE_JD_EVENT_DRV_TERMINATED) { - if (out_count == 0) - return -EPIPE; - goto out; - } - - if (copy_to_user(buf, &uevent, sizeof(uevent)) != 0) - return -EFAULT; - - buf += sizeof(uevent); - out_count++; - count -= sizeof(uevent); - } while (count >= sizeof(uevent)); - -out: - return out_count * sizeof(uevent); -} -#endif /* MALI_USE_CSF */ static __poll_t kbase_poll(struct file *filp, poll_table *wait) { @@ -2193,7 +1997,6 @@ void kbase_event_wakeup(struct kbase_context *kctx) KBASE_EXPORT_TEST_API(kbase_event_wakeup); -#if MALI_USE_CSF int kbase_event_pending(struct kbase_context *kctx) { KBASE_DEBUG_ASSERT(kctx); @@ -2204,17 +2007,6 @@ int kbase_event_pending(struct kbase_context *kctx) return (atomic_read(&kctx->event_count) != 0) || kbase_csf_event_error_pending(kctx) || kbase_csf_cpu_queue_dump_needed(kctx); } -#else -int kbase_event_pending(struct kbase_context *kctx) -{ - KBASE_DEBUG_ASSERT(kctx); - - if (unlikely(!kctx)) - return -EPERM; - - return (atomic_read(&kctx->event_count) != 0) || (atomic_read(&kctx->event_closed) != 0); -} -#endif KBASE_EXPORT_TEST_API(kbase_event_pending); @@ -2396,10 +2188,9 @@ static ssize_t core_mask_show(struct device *dev, struct device_attribute *attr, { struct kbase_device *kbdev; unsigned long flags; + u64 debug_mask; + u64 ca_mask; ssize_t ret = 0; -#if !MALI_USE_CSF - size_t i; -#endif CSTD_UNUSED(attr); @@ -2410,35 +2201,30 @@ static ssize_t core_mask_show(struct device *dev, struct device_attribute *attr, spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -#if MALI_USE_CSF - ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), "Current debug core mask : 0x%llX\n", - kbdev->pm.debug_core_mask); - ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), - "Current desired core mask : 0x%llX\n", kbase_pm_ca_get_core_mask(kbdev)); - ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), - "Current in use core mask : 0x%llX\n", kbdev->pm.backend.shaders_avail); -#else - for (i = 0; i < BASE_JM_MAX_NR_SLOTS; i++) { - if (PAGE_SIZE < ret) - goto out_unlock; - - ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), - "Current core mask (JS%zu) : 0x%llX\n", i, - kbdev->pm.debug_core_mask[i]); + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) { + ca_mask = kbase_pm_ca_get_gov_core_mask(kbdev); + debug_mask = kbase_pm_ca_get_sysfs_gov_core_mask(kbdev); + } else { + ca_mask = kbase_pm_ca_get_core_mask(kbdev); + debug_mask = kbase_pm_ca_get_debug_core_mask(kbdev); } -#endif /* MALI_USE_CSF */ + ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), "Current debug core mask : 0x%llX\n", + debug_mask); + ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), + "Current desired core mask : 0x%llX\n", ca_mask); + if (!kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) + ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), + "Current in use core mask : 0x%llX\n", + kbdev->pm.backend.shaders_avail); ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), "Available core mask : 0x%llX\n", kbdev->gpu_props.shader_present); -#if !MALI_USE_CSF -out_unlock: -#endif + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); return ret; } -#if MALI_USE_CSF struct kbase_core_mask { u64 new_core_mask; }; @@ -2458,6 +2244,8 @@ static int core_mask_set(struct kbase_device *kbdev, struct kbase_core_mask *con { u64 new_core_mask = new_mask->new_core_mask; u64 shader_present; + u64 ca_mask; + u64 debug_mask; unsigned long flags; int ret = 0; @@ -2465,6 +2253,14 @@ static int core_mask_set(struct kbase_device *kbdev, struct kbase_core_mask *con kbase_pm_lock(kbdev); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) { + ca_mask = kbdev->pm.backend.ca_gov_cores_enabled; + debug_mask = kbase_pm_ca_get_sysfs_gov_core_mask(kbdev); + } else { + ca_mask = kbdev->pm.backend.ca_cores_enabled; + debug_mask = kbase_pm_ca_get_debug_core_mask(kbdev); + } + shader_present = kbdev->gpu_props.shader_present; if ((new_core_mask & shader_present) != new_core_mask) { @@ -2473,13 +2269,16 @@ static int core_mask_set(struct kbase_device *kbdev, struct kbase_core_mask *con new_core_mask, shader_present); ret = -EINVAL; goto exit; - } else if (!(new_core_mask & shader_present & kbdev->pm.backend.ca_cores_enabled)) { - dev_err(kbdev->dev, - "Invalid requested core mask 0x%llX: No intersection with currently available cores (present = 0x%llX, CA enabled = 0x%llX)", - new_core_mask, kbdev->gpu_props.shader_present, - kbdev->pm.backend.ca_cores_enabled); - ret = -EINVAL; - goto exit; + } else if (!(new_core_mask & shader_present & ca_mask)) { + if (!kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT) || + (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT) && + new_core_mask != 0)) { + dev_err(kbdev->dev, + "Invalid requested core mask 0x%llX: No intersection with currently available cores (present = 0x%llX, CA enabled = 0x%llX)", + new_core_mask, kbdev->gpu_props.shader_present, ca_mask); + ret = -EINVAL; + goto exit; + } } if (kbase_csf_dev_has_ne(kbdev)) { @@ -2502,7 +2301,7 @@ static int core_mask_set(struct kbase_device *kbdev, struct kbase_core_mask *con } } - if (kbdev->pm.debug_core_mask != new_core_mask) + if (debug_mask != new_core_mask) kbase_pm_set_debug_core_mask(kbdev, new_core_mask); exit: @@ -2512,97 +2311,6 @@ exit: return ret; } -#else -struct kbase_core_mask { - u64 new_core_mask[BASE_JM_MAX_NR_SLOTS]; -}; - -static int core_mask_parse(struct kbase_device *const kbdev, const char *const buf, - struct kbase_core_mask *const mask) -{ - int items; - - items = sscanf(buf, "%llx %llx %llx", &mask->new_core_mask[0], &mask->new_core_mask[1], - &mask->new_core_mask[2]); - - if (items != 1 && items != BASE_JM_MAX_NR_SLOTS) { - dev_err(kbdev->dev, "Couldn't process core mask write operation.\n" - "Use format \n" - "or \n"); - return -EINVAL; - } - - /* If only one value was provided, set all other core masks equal to the value. */ - if (items == 1) { - size_t i; - - for (i = 1; i < BASE_JM_MAX_NR_SLOTS; i++) - mask->new_core_mask[i] = mask->new_core_mask[0]; - } - - return 0; -} - -static int core_mask_set(struct kbase_device *kbdev, struct kbase_core_mask *const new_mask) -{ - u64 shader_present = kbdev->gpu_props.shader_present; - u64 group_core_mask = kbdev->gpu_props.coherency_info.group.core_mask; - u64 *new_core_mask; - unsigned long flags; - int ret = 0; - size_t i; - - kbase_pm_lock(kbdev); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - new_core_mask = &new_mask->new_core_mask[0]; - - for (i = 0; i < BASE_JM_MAX_NR_SLOTS; ++i) { - if ((new_core_mask[i] & shader_present) != new_core_mask[i]) { - dev_err(kbdev->dev, - "Invalid core mask 0x%llX for JS %zu: Includes non-existent cores (present = 0x%llX)", - new_core_mask[i], i, shader_present); - ret = -EINVAL; - goto exit; - - } else if (!(new_core_mask[i] & shader_present & - kbdev->pm.backend.ca_cores_enabled)) { - dev_err(kbdev->dev, - "Invalid core mask 0x%llX for JS %zu: No intersection with currently available cores (present = 0x%llX, CA enabled = 0x%llX)", - new_core_mask[i], i, kbdev->gpu_props.shader_present, - kbdev->pm.backend.ca_cores_enabled); - ret = -EINVAL; - goto exit; - } else if (!(new_core_mask[i] & group_core_mask)) { - dev_err(kbdev->dev, - "Invalid core mask 0x%llX for JS %zu: No intersection with group 0 core mask 0x%llX", - new_core_mask[i], i, group_core_mask); - ret = -EINVAL; - goto exit; - } else if (!(new_core_mask[i] & kbdev->gpu_props.curr_config.shader_present)) { - dev_err(kbdev->dev, - "Invalid core mask 0x%llX for JS %zu: No intersection with current core mask 0x%llX", - new_core_mask[i], i, kbdev->gpu_props.curr_config.shader_present); - ret = -EINVAL; - goto exit; - } - } - - for (i = 0; i < BASE_JM_MAX_NR_SLOTS; i++) { - if (kbdev->pm.debug_core_mask[i] != new_core_mask[i]) { - kbase_pm_set_debug_core_mask(kbdev, new_core_mask, BASE_JM_MAX_NR_SLOTS); - break; - } - } - -exit: - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - kbase_pm_unlock(kbdev); - - return ret; -} - -#endif /** * core_mask_store - Store callback for the core_mask sysfs file. @@ -2652,446 +2360,6 @@ static ssize_t core_mask_store(struct device *dev, struct device_attribute *attr */ static DEVICE_ATTR_RW(core_mask); -#if !MALI_USE_CSF -/** - * soft_job_timeout_store - Store callback for the soft_job_timeout sysfs - * file. - * - * @dev: The device this sysfs file is for. - * @attr: The attributes of the sysfs file. - * @buf: The value written to the sysfs file. - * @count: The number of bytes to write to the sysfs file. - * - * This allows setting the timeout for software jobs. Waiting soft event wait - * jobs will be cancelled after this period expires, while soft fence wait jobs - * will print debug information if the fence debug feature is enabled. - * - * This is expressed in milliseconds. - * - * Return: count if the function succeeded. An error code on failure. - */ -static ssize_t soft_job_timeout_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct kbase_device *kbdev; - int soft_job_timeout_ms; - - CSTD_UNUSED(attr); - - kbdev = to_kbase_device(dev); - if (!kbdev) - return -ENODEV; - - if ((kstrtoint(buf, 0, &soft_job_timeout_ms) != 0) || (soft_job_timeout_ms <= 0)) - return -EINVAL; - - atomic_set(&kbdev->js_data.soft_job_timeout_ms, soft_job_timeout_ms); - - return (ssize_t)count; -} - -/** - * soft_job_timeout_show - Show callback for the soft_job_timeout sysfs - * file. - * - * @dev: The device this sysfs file is for. - * @attr: The attributes of the sysfs file. - * @buf: The output buffer for the sysfs file contents. - * - * This will return the timeout for the software jobs. - * - * Return: The number of bytes output to buf. - */ -static ssize_t soft_job_timeout_show(struct device *dev, struct device_attribute *attr, - char *const buf) -{ - struct kbase_device *kbdev; - - CSTD_UNUSED(attr); - - kbdev = to_kbase_device(dev); - if (!kbdev) - return -ENODEV; - - return scnprintf(buf, PAGE_SIZE, "%i\n", atomic_read(&kbdev->js_data.soft_job_timeout_ms)); -} - -static DEVICE_ATTR_RW(soft_job_timeout); - -static u32 timeout_ms_to_ticks(struct kbase_device *kbdev, long timeout_ms, int default_ticks, - u32 old_ticks) -{ - if (timeout_ms > 0) { - u64 ticks = (u64)timeout_ms * 1000000ULL; - - do_div(ticks, kbdev->js_data.scheduling_period_ns); - if (!ticks) - return 1; - return ticks; - } else if (timeout_ms < 0) { - return (u32)default_ticks; - } else { - return old_ticks; - } -} - -/** - * js_timeouts_store - Store callback for the js_timeouts sysfs file. - * - * @dev: The device with sysfs file is for - * @attr: The attributes of the sysfs file - * @buf: The value written to the sysfs file - * @count: The number of bytes to write to the sysfs file - * - * This function is called to get the contents of the js_timeouts sysfs - * file. This file contains five values separated by whitespace. The values - * are basically the same as %JS_SOFT_STOP_TICKS, %JS_HARD_STOP_TICKS_SS, - * %JS_HARD_STOP_TICKS_DUMPING, %JS_RESET_TICKS_SS, %JS_RESET_TICKS_DUMPING - * configuration values (in that order), with the difference that the js_timeout - * values are expressed in MILLISECONDS. - * - * The js_timeouts sysfile file allows the current values in - * use by the job scheduler to get override. Note that a value needs to - * be other than 0 for it to override the current job scheduler value. - * - * Return: @count if the function succeeded. An error code on failure. - */ -static ssize_t js_timeouts_store(struct device *dev, struct device_attribute *attr, const char *buf, - size_t count) -{ - struct kbase_device *kbdev; - int items; - long js_soft_stop_ms; - long js_soft_stop_ms_cl; - long js_hard_stop_ms_ss; - long js_hard_stop_ms_cl; - long js_hard_stop_ms_dumping; - long js_reset_ms_ss; - long js_reset_ms_cl; - long js_reset_ms_dumping; - - CSTD_UNUSED(attr); - - kbdev = to_kbase_device(dev); - if (!kbdev) - return -ENODEV; - - items = sscanf(buf, "%ld %ld %ld %ld %ld %ld %ld %ld", &js_soft_stop_ms, - &js_soft_stop_ms_cl, &js_hard_stop_ms_ss, &js_hard_stop_ms_cl, - &js_hard_stop_ms_dumping, &js_reset_ms_ss, &js_reset_ms_cl, - &js_reset_ms_dumping); - - if (items == 8) { - struct kbasep_js_device_data *js_data = &kbdev->js_data; - unsigned long flags; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - -#define UPDATE_TIMEOUT(ticks_name, ms_name, default) \ - do { \ - js_data->ticks_name = \ - timeout_ms_to_ticks(kbdev, ms_name, default, js_data->ticks_name); \ - dev_dbg(kbdev->dev, "Overriding " #ticks_name " with %lu ticks (%lu ms)\n", \ - (unsigned long)js_data->ticks_name, ms_name); \ - } while (0) - - UPDATE_TIMEOUT(soft_stop_ticks, js_soft_stop_ms, DEFAULT_JS_SOFT_STOP_TICKS); - UPDATE_TIMEOUT(soft_stop_ticks_cl, js_soft_stop_ms_cl, - DEFAULT_JS_SOFT_STOP_TICKS_CL); - UPDATE_TIMEOUT(hard_stop_ticks_ss, js_hard_stop_ms_ss, - DEFAULT_JS_HARD_STOP_TICKS_SS); - UPDATE_TIMEOUT(hard_stop_ticks_cl, js_hard_stop_ms_cl, - DEFAULT_JS_HARD_STOP_TICKS_CL); - UPDATE_TIMEOUT(hard_stop_ticks_dumping, js_hard_stop_ms_dumping, - DEFAULT_JS_HARD_STOP_TICKS_DUMPING); - UPDATE_TIMEOUT(gpu_reset_ticks_ss, js_reset_ms_ss, DEFAULT_JS_RESET_TICKS_SS); - UPDATE_TIMEOUT(gpu_reset_ticks_cl, js_reset_ms_cl, DEFAULT_JS_RESET_TICKS_CL); - UPDATE_TIMEOUT(gpu_reset_ticks_dumping, js_reset_ms_dumping, - DEFAULT_JS_RESET_TICKS_DUMPING); - - kbase_js_set_timeouts(kbdev); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - return (ssize_t)count; - } - - dev_err(kbdev->dev, - "Couldn't process js_timeouts write operation.\n" - "Use format \n" - "Write 0 for no change, -1 to restore default timeout\n"); - return -EINVAL; -} - -static unsigned long get_js_timeout_in_ms(u32 scheduling_period_ns, u32 ticks) -{ - u64 ms = (u64)ticks * scheduling_period_ns; - - do_div(ms, 1000000UL); - return ms; -} - -/** - * js_timeouts_show - Show callback for the js_timeouts sysfs file. - * - * @dev: The device this sysfs file is for - * @attr: The attributes of the sysfs file - * @buf: The output buffer for the sysfs file contents - * - * This function is called to get the contents of the js_timeouts sysfs - * file. It returns the last set values written to the js_timeouts sysfs file. - * If the file didn't get written yet, the values will be current setting in - * use. - * - * Return: The number of bytes output to @buf. - */ -static ssize_t js_timeouts_show(struct device *dev, struct device_attribute *attr, char *const buf) -{ - struct kbase_device *kbdev; - ssize_t ret; - unsigned long js_soft_stop_ms; - unsigned long js_soft_stop_ms_cl; - unsigned long js_hard_stop_ms_ss; - unsigned long js_hard_stop_ms_cl; - unsigned long js_hard_stop_ms_dumping; - unsigned long js_reset_ms_ss; - unsigned long js_reset_ms_cl; - unsigned long js_reset_ms_dumping; - u32 scheduling_period_ns; - - CSTD_UNUSED(attr); - - kbdev = to_kbase_device(dev); - if (!kbdev) - return -ENODEV; - - scheduling_period_ns = kbdev->js_data.scheduling_period_ns; - -#define GET_TIMEOUT(name) get_js_timeout_in_ms(scheduling_period_ns, kbdev->js_data.name) - - js_soft_stop_ms = GET_TIMEOUT(soft_stop_ticks); - js_soft_stop_ms_cl = GET_TIMEOUT(soft_stop_ticks_cl); - js_hard_stop_ms_ss = GET_TIMEOUT(hard_stop_ticks_ss); - js_hard_stop_ms_cl = GET_TIMEOUT(hard_stop_ticks_cl); - js_hard_stop_ms_dumping = GET_TIMEOUT(hard_stop_ticks_dumping); - js_reset_ms_ss = GET_TIMEOUT(gpu_reset_ticks_ss); - js_reset_ms_cl = GET_TIMEOUT(gpu_reset_ticks_cl); - js_reset_ms_dumping = GET_TIMEOUT(gpu_reset_ticks_dumping); - -#undef GET_TIMEOUT - - ret = scnprintf(buf, PAGE_SIZE, "%lu %lu %lu %lu %lu %lu %lu %lu\n", js_soft_stop_ms, - js_soft_stop_ms_cl, js_hard_stop_ms_ss, js_hard_stop_ms_cl, - js_hard_stop_ms_dumping, js_reset_ms_ss, js_reset_ms_cl, - js_reset_ms_dumping); - - if (ret >= (ssize_t)PAGE_SIZE) { - buf[PAGE_SIZE - 2] = '\n'; - buf[PAGE_SIZE - 1] = '\0'; - ret = PAGE_SIZE - 1; - } - - return ret; -} - -/* - * The sysfs file js_timeouts. - * - * This is used to override the current job scheduler values for - * JS_STOP_STOP_TICKS_SS - * JS_STOP_STOP_TICKS_CL - * JS_HARD_STOP_TICKS_SS - * JS_HARD_STOP_TICKS_CL - * JS_HARD_STOP_TICKS_DUMPING - * JS_RESET_TICKS_SS - * JS_RESET_TICKS_CL - * JS_RESET_TICKS_DUMPING. - */ -static DEVICE_ATTR_RW(js_timeouts); - -static u32 get_new_js_timeout(u32 old_period, u32 old_ticks, u32 new_scheduling_period_ns) -{ - u64 ticks = (u64)old_period * (u64)old_ticks; - - do_div(ticks, new_scheduling_period_ns); - return ticks ? ticks : 1; -} - -/** - * js_scheduling_period_store - Store callback for the js_scheduling_period sysfs - * file - * @dev: The device the sysfs file is for - * @attr: The attributes of the sysfs file - * @buf: The value written to the sysfs file - * @count: The number of bytes to write to the sysfs file - * - * This function is called when the js_scheduling_period sysfs file is written - * to. It checks the data written, and if valid updates the js_scheduling_period - * value - * - * Return: @count if the function succeeded. An error code on failure. - */ -static ssize_t js_scheduling_period_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct kbase_device *kbdev; - int ret; - unsigned int js_scheduling_period; - u32 new_scheduling_period_ns; - u32 old_period; - struct kbasep_js_device_data *js_data; - unsigned long flags; - - CSTD_UNUSED(attr); - - kbdev = to_kbase_device(dev); - if (!kbdev) - return -ENODEV; - - js_data = &kbdev->js_data; - - ret = kstrtouint(buf, 0, &js_scheduling_period); - if (ret || !js_scheduling_period) { - dev_err(kbdev->dev, "Couldn't process js_scheduling_period write operation.\n" - "Use format \n"); - return -EINVAL; - } - - new_scheduling_period_ns = js_scheduling_period * 1000000; - - /* Update scheduling timeouts */ - mutex_lock(&js_data->runpool_mutex); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - /* If no contexts have been scheduled since js_timeouts was last written - * to, the new timeouts might not have been latched yet. So check if an - * update is pending and use the new values if necessary. - */ - - /* Use previous 'new' scheduling period as a base if present. */ - old_period = js_data->scheduling_period_ns; - -#define SET_TIMEOUT(name) \ - (js_data->name = \ - get_new_js_timeout(old_period, kbdev->js_data.name, new_scheduling_period_ns)) - - SET_TIMEOUT(soft_stop_ticks); - SET_TIMEOUT(soft_stop_ticks_cl); - SET_TIMEOUT(hard_stop_ticks_ss); - SET_TIMEOUT(hard_stop_ticks_cl); - SET_TIMEOUT(hard_stop_ticks_dumping); - SET_TIMEOUT(gpu_reset_ticks_ss); - SET_TIMEOUT(gpu_reset_ticks_cl); - SET_TIMEOUT(gpu_reset_ticks_dumping); - -#undef SET_TIMEOUT - - js_data->scheduling_period_ns = new_scheduling_period_ns; - - kbase_js_set_timeouts(kbdev); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&js_data->runpool_mutex); - - dev_dbg(kbdev->dev, "JS scheduling period: %dms\n", js_scheduling_period); - - return (ssize_t)count; -} - -/** - * js_scheduling_period_show - Show callback for the js_scheduling_period sysfs - * entry. - * @dev: The device this sysfs file is for. - * @attr: The attributes of the sysfs file. - * @buf: The output buffer to receive the GPU information. - * - * This function is called to get the current period used for the JS scheduling - * period. - * - * Return: The number of bytes output to @buf. - */ -static ssize_t js_scheduling_period_show(struct device *dev, struct device_attribute *attr, - char *const buf) -{ - struct kbase_device *kbdev; - u32 period; - ssize_t ret; - - CSTD_UNUSED(attr); - - kbdev = to_kbase_device(dev); - if (!kbdev) - return -ENODEV; - - period = kbdev->js_data.scheduling_period_ns; - - ret = scnprintf(buf, PAGE_SIZE, "%d\n", period / 1000000); - - return ret; -} - -static DEVICE_ATTR_RW(js_scheduling_period); - -#ifdef CONFIG_MALI_VALHALL_DEBUG -static ssize_t js_softstop_always_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct kbase_device *kbdev; - int ret; - int softstop_always; - - CSTD_UNUSED(attr); - - kbdev = to_kbase_device(dev); - if (!kbdev) - return -ENODEV; - - ret = kstrtoint(buf, 0, &softstop_always); - if (ret || ((softstop_always != 0) && (softstop_always != 1))) { - dev_err(kbdev->dev, "Couldn't process js_softstop_always write operation.\n" - "Use format \n"); - return -EINVAL; - } - - kbdev->js_data.softstop_always = (bool)softstop_always; - dev_dbg(kbdev->dev, "Support for softstop on a single context: %s\n", - (kbdev->js_data.softstop_always) ? "Enabled" : "Disabled"); - return (ssize_t)count; -} - -static ssize_t js_softstop_always_show(struct device *dev, struct device_attribute *attr, - char *const buf) -{ - struct kbase_device *kbdev; - ssize_t ret; - - CSTD_UNUSED(attr); - - kbdev = to_kbase_device(dev); - if (!kbdev) - return -ENODEV; - - ret = scnprintf(buf, PAGE_SIZE, "%d\n", kbdev->js_data.softstop_always); - - if (ret >= (ssize_t)PAGE_SIZE) { - buf[PAGE_SIZE - 2] = '\n'; - buf[PAGE_SIZE - 1] = '\0'; - ret = PAGE_SIZE - 1; - } - - return ret; -} - -/* - * By default, soft-stops are disabled when only a single context is present. - * The ability to enable soft-stop when only a single context is present can be - * used for debug and unit-testing purposes. - * (see CL t6xx_stress_1 unit-test as an example whereby this feature is used.) - */ -static DEVICE_ATTR_RW(js_softstop_always); -#endif /* CONFIG_MALI_VALHALL_DEBUG */ -#endif /* !MALI_USE_CSF */ - #ifdef CONFIG_MALI_VALHALL_DEBUG typedef void kbasep_debug_command_func(struct kbase_device *); @@ -3276,7 +2544,6 @@ static ssize_t gpuinfo_show(struct device *dev, struct device_attribute *attr, c } } -#if MALI_USE_CSF if (product_model == GPU_ID_PRODUCT_TTUX) { const bool rt_supported = gpu_props->gpu_features.ray_intersection; const u8 nr_cores = gpu_props->num_cores; @@ -3355,7 +2622,6 @@ static ssize_t gpuinfo_show(struct device *dev, struct device_attribute *attr, c dev_dbg(kbdev->dev, "GPU ID_Name: %s (ID: 0x%x), nr_cores(%u)\n", product_name, product_id, nr_cores); } -#endif /* MALI_USE_CSF */ return scnprintf(buf, PAGE_SIZE, "%s %d cores r%dp%d 0x%08X\n", product_name, kbdev->gpu_props.num_cores, gpu_props->gpu_id.version_major, @@ -3603,11 +2869,7 @@ static ssize_t reset_timeout_store(struct device *dev, struct device_attribute * return -EINVAL; } -#if MALI_USE_CSF default_reset_timeout = kbase_get_timeout_ms(kbdev, CSF_GPU_RESET_TIMEOUT); -#else /* MALI_USE_CSF */ - default_reset_timeout = JM_DEFAULT_RESET_TIMEOUT_MS; -#endif /* !MALI_USE_CSF */ if (reset_timeout < default_reset_timeout) dev_warn(kbdev->dev, "requested reset_timeout(%u) is smaller than default(%u)", @@ -3648,689 +2910,6 @@ static ssize_t reset_timeout_show(struct device *dev, struct device_attribute *a static DEVICE_ATTR_RW(reset_timeout); -static ssize_t mem_pool_size_show(struct device *dev, struct device_attribute *attr, - char *const buf) -{ - struct kbase_device *const kbdev = to_kbase_device(dev); - - CSTD_UNUSED(attr); - - if (!kbdev) - return -ENODEV; - - return kbase_debugfs_helper_get_attr_to_string(buf, PAGE_SIZE, kbdev->mem_pools.small, - MEMORY_GROUP_MANAGER_NR_GROUPS, - kbase_mem_pool_debugfs_size); -} - -static ssize_t mem_pool_size_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct kbase_device *const kbdev = to_kbase_device(dev); - ssize_t err; - - CSTD_UNUSED(attr); - - if (!kbdev) - return -ENODEV; - - err = kbase_debugfs_helper_set_attr_from_string(buf, kbdev->mem_pools.small, - MEMORY_GROUP_MANAGER_NR_GROUPS, - kbase_mem_pool_debugfs_trim); - - return err ? err : (ssize_t)count; -} - -static DEVICE_ATTR_RW(mem_pool_size); - -static ssize_t mem_pool_max_size_show(struct device *dev, struct device_attribute *attr, - char *const buf) -{ - struct kbase_device *const kbdev = to_kbase_device(dev); - - CSTD_UNUSED(attr); - - if (!kbdev) - return -ENODEV; - - return kbase_debugfs_helper_get_attr_to_string(buf, PAGE_SIZE, kbdev->mem_pools.small, - MEMORY_GROUP_MANAGER_NR_GROUPS, - kbase_mem_pool_debugfs_max_size); -} - -static ssize_t mem_pool_max_size_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct kbase_device *const kbdev = to_kbase_device(dev); - ssize_t err; - - CSTD_UNUSED(attr); - - if (!kbdev) - return -ENODEV; - - err = kbase_debugfs_helper_set_attr_from_string(buf, kbdev->mem_pools.small, - MEMORY_GROUP_MANAGER_NR_GROUPS, - kbase_mem_pool_debugfs_set_max_size); - - return err ? err : (ssize_t)count; -} - -static DEVICE_ATTR_RW(mem_pool_max_size); - -/** - * lp_mem_pool_size_show - Show size of the large memory pages pool. - * @dev: The device this sysfs file is for. - * @attr: The attributes of the sysfs file. - * @buf: The output buffer to receive the pool size. - * - * This function is called to get the number of large memory pages which currently populate the kbdev pool. - * - * Return: The number of bytes output to @buf. - */ -static ssize_t lp_mem_pool_size_show(struct device *dev, struct device_attribute *attr, - char *const buf) -{ - struct kbase_device *const kbdev = to_kbase_device(dev); - - CSTD_UNUSED(attr); - - if (!kbdev) - return -ENODEV; - - return kbase_debugfs_helper_get_attr_to_string(buf, PAGE_SIZE, kbdev->mem_pools.large, - MEMORY_GROUP_MANAGER_NR_GROUPS, - kbase_mem_pool_debugfs_size); -} - -/** - * lp_mem_pool_size_store - Set size of the large memory pages pool. - * @dev: The device this sysfs file is for. - * @attr: The attributes of the sysfs file. - * @buf: The value written to the sysfs file. - * @count: The number of bytes written to the sysfs file. - * - * This function is called to set the number of large memory pages which should populate the kbdev pool. - * This may cause existing pages to be removed from the pool, or new pages to be created and then added to the pool. - * - * Return: @count if the function succeeded. An error code on failure. - */ -static ssize_t lp_mem_pool_size_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct kbase_device *const kbdev = to_kbase_device(dev); - ssize_t err; - - CSTD_UNUSED(attr); - - if (!kbdev) - return -ENODEV; - - err = kbase_debugfs_helper_set_attr_from_string(buf, kbdev->mem_pools.large, - MEMORY_GROUP_MANAGER_NR_GROUPS, - kbase_mem_pool_debugfs_trim); - - return err ? err : (ssize_t)count; -} - -static DEVICE_ATTR_RW(lp_mem_pool_size); - -/** - * lp_mem_pool_max_size_show - Show maximum size of the large memory pages pool. - * @dev: The device this sysfs file is for. - * @attr: The attributes of the sysfs file. - * @buf: The output buffer to receive the pool size. - * - * This function is called to get the maximum number of large memory pages that the kbdev pool can possibly contain. - * - * Return: The number of bytes output to @buf. - */ -static ssize_t lp_mem_pool_max_size_show(struct device *dev, struct device_attribute *attr, - char *const buf) -{ - struct kbase_device *const kbdev = to_kbase_device(dev); - - CSTD_UNUSED(attr); - - if (!kbdev) - return -ENODEV; - - return kbase_debugfs_helper_get_attr_to_string(buf, PAGE_SIZE, kbdev->mem_pools.large, - MEMORY_GROUP_MANAGER_NR_GROUPS, - kbase_mem_pool_debugfs_max_size); -} - -/** - * lp_mem_pool_max_size_store - Set maximum size of the large memory pages pool. - * @dev: The device this sysfs file is for. - * @attr: The attributes of the sysfs file. - * @buf: The value written to the sysfs file. - * @count: The number of bytes written to the sysfs file. - * - * This function is called to set the maximum number of large memory pages that the kbdev pool can possibly contain. - * - * Return: @count if the function succeeded. An error code on failure. - */ -static ssize_t lp_mem_pool_max_size_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct kbase_device *const kbdev = to_kbase_device(dev); - ssize_t err; - - CSTD_UNUSED(attr); - - if (!kbdev) - return -ENODEV; - - err = kbase_debugfs_helper_set_attr_from_string(buf, kbdev->mem_pools.large, - MEMORY_GROUP_MANAGER_NR_GROUPS, - kbase_mem_pool_debugfs_set_max_size); - - return err ? err : (ssize_t)count; -} - -static DEVICE_ATTR_RW(lp_mem_pool_max_size); - -/** - * show_simplified_mem_pool_max_size - Show the maximum size for the memory - * pool 0 of small (4KiB/16KiB/64KiB) pages. - * @dev: The device this sysfs file is for. - * @attr: The attributes of the sysfs file. - * @buf: The output buffer to receive the max size. - * - * This function is called to get the maximum size for the memory pool 0 of - * small pages. It is assumed that the maximum size value is same for - * all the pools. - * - * Return: The number of bytes output to @buf. - */ -static ssize_t show_simplified_mem_pool_max_size(struct device *dev, struct device_attribute *attr, - char *const buf) -{ - struct kbase_device *const kbdev = to_kbase_device(dev); - - CSTD_UNUSED(attr); - - if (!kbdev) - return -ENODEV; - - return kbase_debugfs_helper_get_attr_to_string(buf, PAGE_SIZE, kbdev->mem_pools.small, 1, - kbase_mem_pool_debugfs_max_size); -} - -/** - * set_simplified_mem_pool_max_size - Set the same maximum size for all the - * memory pools of small (4KiB/16KiB/64KiB) pages. - * @dev: The device with sysfs file is for - * @attr: The attributes of the sysfs file - * @buf: The value written to the sysfs file - * @count: The number of bytes written to the sysfs file - * - * This function is called to set the same maximum size for all the memory - * pools of small pages. - * - * Return: The number of bytes output to @buf. - */ -static ssize_t set_simplified_mem_pool_max_size(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct kbase_device *const kbdev = to_kbase_device(dev); - unsigned long new_size; - size_t gid; - int err; - - CSTD_UNUSED(attr); - - if (!kbdev) - return -ENODEV; - - err = kstrtoul(buf, 0, &new_size); - if (err) - return -EINVAL; - - for (gid = 0; gid < MEMORY_GROUP_MANAGER_NR_GROUPS; ++gid) - kbase_mem_pool_debugfs_set_max_size(kbdev->mem_pools.small, gid, (size_t)new_size); - - return (ssize_t)count; -} - -static DEVICE_ATTR(max_size, 0600, show_simplified_mem_pool_max_size, - set_simplified_mem_pool_max_size); - -/** - * show_simplified_lp_mem_pool_max_size - Show the maximum size for the memory - * pool 0 of large (2MiB) pages. - * @dev: The device this sysfs file is for. - * @attr: The attributes of the sysfs file. - * @buf: The output buffer to receive the total current pool size. - * - * This function is called to get the maximum size for the memory pool 0 of - * large (2MiB) pages. It is assumed that the maximum size value is same for - * all the pools. - * - * Return: The number of bytes output to @buf. - */ -static ssize_t show_simplified_lp_mem_pool_max_size(struct device *dev, - struct device_attribute *attr, char *const buf) -{ - struct kbase_device *const kbdev = to_kbase_device(dev); - - CSTD_UNUSED(attr); - - if (!kbdev) - return -ENODEV; - - return kbase_debugfs_helper_get_attr_to_string(buf, PAGE_SIZE, kbdev->mem_pools.large, 1, - kbase_mem_pool_debugfs_max_size); -} - -/** - * set_simplified_lp_mem_pool_max_size - Set the same maximum size for all the - * memory pools of large (2MiB) pages. - * @dev: The device with sysfs file is for - * @attr: The attributes of the sysfs file - * @buf: The value written to the sysfs file - * @count: The number of bytes written to the sysfs file - * - * This function is called to set the same maximum size for all the memory - * pools of large (2MiB) pages. - * - * Return: The number of bytes output to @buf. - */ -static ssize_t set_simplified_lp_mem_pool_max_size(struct device *dev, - struct device_attribute *attr, const char *buf, - size_t count) -{ - struct kbase_device *const kbdev = to_kbase_device(dev); - unsigned long new_size; - size_t gid; - int err; - - CSTD_UNUSED(attr); - - if (!kbdev) - return -ENODEV; - - err = kstrtoul(buf, 0, &new_size); - if (err) - return -EINVAL; - - for (gid = 0; gid < MEMORY_GROUP_MANAGER_NR_GROUPS; ++gid) - kbase_mem_pool_debugfs_set_max_size(kbdev->mem_pools.large, gid, (size_t)new_size); - - return (ssize_t)count; -} - -static DEVICE_ATTR(lp_max_size, 0600, show_simplified_lp_mem_pool_max_size, - set_simplified_lp_mem_pool_max_size); - -/** - * show_simplified_ctx_default_max_size - Show the default maximum size for the - * memory pool 0 of small (4KiB/16KiB/64KiB) pages. - * @dev: The device this sysfs file is for. - * @attr: The attributes of the sysfs file. - * @buf: The output buffer to receive the pool size. - * - * This function is called to get the default ctx maximum size for the memory - * pool 0 of small pages. It is assumed that maximum size value is same - * for all the pools. The maximum size for the pool of large (2MiB) pages will - * be same as max size of the pool of small pages in terms of bytes. - * - * Return: The number of bytes output to @buf. - */ -static ssize_t show_simplified_ctx_default_max_size(struct device *dev, - struct device_attribute *attr, char *const buf) -{ - struct kbase_device *kbdev = to_kbase_device(dev); - size_t max_size; - - CSTD_UNUSED(attr); - - if (!kbdev) - return -ENODEV; - - max_size = kbase_mem_pool_config_debugfs_max_size(kbdev->mem_pool_defaults.small, 0); - - return scnprintf(buf, PAGE_SIZE, "%zu\n", max_size); -} - -/** - * set_simplified_ctx_default_max_size - Set the same default maximum size for - * all the pools created for new - * contexts. This covers the pool of - * large pages as well and its max size - * will be same as max size of the pool - * of small pages in terms of bytes. - * @dev: The device this sysfs file is for. - * @attr: The attributes of the sysfs file. - * @buf: The value written to the sysfs file. - * @count: The number of bytes written to the sysfs file. - * - * This function is called to set the same maximum size for all pools created - * for new contexts. - * - * Return: @count if the function succeeded. An error code on failure. - */ -static ssize_t set_simplified_ctx_default_max_size(struct device *dev, - struct device_attribute *attr, const char *buf, - size_t count) -{ - struct kbase_device *kbdev; - unsigned long new_size; - int err; - - CSTD_UNUSED(attr); - - kbdev = to_kbase_device(dev); - if (!kbdev) - return -ENODEV; - - err = kstrtoul(buf, 0, &new_size); - if (err) - return -EINVAL; - - kbase_mem_pool_group_config_set_max_size(&kbdev->mem_pool_defaults, (size_t)new_size); - - return (ssize_t)count; -} - -static DEVICE_ATTR(ctx_default_max_size, 0600, show_simplified_ctx_default_max_size, - set_simplified_ctx_default_max_size); - -#if !MALI_USE_CSF -/** - * js_ctx_scheduling_mode_show - Show callback for js_ctx_scheduling_mode sysfs - * entry. - * @dev: The device this sysfs file is for. - * @attr: The attributes of the sysfs file. - * @buf: The output buffer to receive the context scheduling mode information. - * - * This function is called to get the context scheduling mode being used by JS. - * - * Return: The number of bytes output to @buf. - */ -static ssize_t js_ctx_scheduling_mode_show(struct device *dev, struct device_attribute *attr, - char *const buf) -{ - struct kbase_device *kbdev; - - CSTD_UNUSED(attr); - - kbdev = to_kbase_device(dev); - if (!kbdev) - return -ENODEV; - - return scnprintf(buf, PAGE_SIZE, "%u\n", kbdev->js_ctx_scheduling_mode); -} - -/** - * js_ctx_scheduling_mode_store - Set callback for js_ctx_scheduling_mode sysfs - * entry. - * @dev: The device this sysfs file is for. - * @attr: The attributes of the sysfs file. - * @buf: The value written to the sysfs file. - * @count: The number of bytes written to the sysfs file. - * - * This function is called when the js_ctx_scheduling_mode sysfs file is written - * to. It checks the data written, and if valid updates the ctx scheduling mode - * being by JS. - * - * Return: @count if the function succeeded. An error code on failure. - */ -static ssize_t js_ctx_scheduling_mode_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct kbase_context *kctx; - u32 new_js_ctx_scheduling_mode; - struct kbase_device *kbdev; - unsigned long flags; - int ret; - - CSTD_UNUSED(attr); - - kbdev = to_kbase_device(dev); - if (!kbdev) - return -ENODEV; - - ret = kstrtouint(buf, 0, &new_js_ctx_scheduling_mode); - if (ret || new_js_ctx_scheduling_mode >= KBASE_JS_PRIORITY_MODE_COUNT) { - dev_err(kbdev->dev, "Couldn't process js_ctx_scheduling_mode" - " write operation.\n" - "Use format \n"); - return -EINVAL; - } - - if (new_js_ctx_scheduling_mode == kbdev->js_ctx_scheduling_mode) - return (ssize_t)count; - - mutex_lock(&kbdev->kctx_list_lock); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - /* Update the context priority mode */ - kbdev->js_ctx_scheduling_mode = new_js_ctx_scheduling_mode; - - /* Adjust priority of all the contexts as per the new mode */ - list_for_each_entry(kctx, &kbdev->kctx_list, kctx_list_link) - kbase_js_update_ctx_priority(kctx); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kbdev->kctx_list_lock); - - dev_dbg(kbdev->dev, "JS ctx scheduling mode: %u\n", new_js_ctx_scheduling_mode); - - return (ssize_t)count; -} - -static DEVICE_ATTR_RW(js_ctx_scheduling_mode); - -/* Number of entries in serialize_jobs_settings[] */ -#define NR_SERIALIZE_JOBS_SETTINGS 5 -/* Maximum string length in serialize_jobs_settings[].name */ -#define MAX_SERIALIZE_JOBS_NAME_LEN 16 - -static struct { - char *name; - u8 setting; -} serialize_jobs_settings[NR_SERIALIZE_JOBS_SETTINGS] = { - { "none", 0 }, - { "intra-slot", KBASE_SERIALIZE_INTRA_SLOT }, - { "inter-slot", KBASE_SERIALIZE_INTER_SLOT }, - { "full", KBASE_SERIALIZE_INTRA_SLOT | KBASE_SERIALIZE_INTER_SLOT }, - { "full-reset", - KBASE_SERIALIZE_INTRA_SLOT | KBASE_SERIALIZE_INTER_SLOT | KBASE_SERIALIZE_RESET } -}; - -/** - * update_serialize_jobs_setting - Update the serialization setting for the - * submission of GPU jobs. - * - * @kbdev: An instance of the GPU platform device, allocated from the probe - * method of the driver. - * @buf: Buffer containing the value written to the sysfs/debugfs file. - * @count: The number of bytes to write to the sysfs/debugfs file. - * - * This function is called when the serialize_jobs sysfs/debugfs file is - * written to. It matches the requested setting against the available settings - * and if a matching setting is found updates kbdev->serialize_jobs. - * - * Return: @count if the function succeeded. An error code on failure. - */ -static ssize_t update_serialize_jobs_setting(struct kbase_device *kbdev, const char *buf, - size_t count) -{ - int i; - bool valid = false; - - for (i = 0; i < NR_SERIALIZE_JOBS_SETTINGS; i++) { - if (sysfs_streq(serialize_jobs_settings[i].name, buf)) { - kbdev->serialize_jobs = serialize_jobs_settings[i].setting; - valid = true; - break; - } - } - - if (!valid) { - dev_err(kbdev->dev, "serialize_jobs: invalid setting"); - return -EINVAL; - } - - return (ssize_t)count; -} - -#if IS_ENABLED(CONFIG_DEBUG_FS) -/** - * kbasep_serialize_jobs_seq_debugfs_show - Show callback for the serialize_jobs - * debugfs file - * @sfile: seq_file pointer - * @data: Private callback data - * - * This function is called to get the contents of the serialize_jobs debugfs - * file. This is a list of the available settings with the currently active one - * surrounded by square brackets. - * - * Return: 0 on success, or an error code on error - */ -static int kbasep_serialize_jobs_seq_debugfs_show(struct seq_file *sfile, void *data) -{ - struct kbase_device *kbdev = sfile->private; - int i; - - CSTD_UNUSED(data); - - for (i = 0; i < NR_SERIALIZE_JOBS_SETTINGS; i++) { - if (kbdev->serialize_jobs == serialize_jobs_settings[i].setting) - seq_printf(sfile, "[%s] ", serialize_jobs_settings[i].name); - else - seq_printf(sfile, "%s ", serialize_jobs_settings[i].name); - } - - seq_puts(sfile, "\n"); - - return 0; -} - -/** - * kbasep_serialize_jobs_debugfs_write - Store callback for the serialize_jobs - * debugfs file. - * @file: File pointer - * @ubuf: User buffer containing data to store - * @count: Number of bytes in user buffer - * @ppos: File position - * - * This function is called when the serialize_jobs debugfs file is written to. - * It matches the requested setting against the available settings and if a - * matching setting is found updates kbdev->serialize_jobs. - * - * Return: @count if the function succeeded. An error code on failure. - */ -static ssize_t kbasep_serialize_jobs_debugfs_write(struct file *file, const char __user *ubuf, - size_t count, loff_t *ppos) -{ - struct seq_file *s = file->private_data; - struct kbase_device *kbdev = s->private; - char buf[MAX_SERIALIZE_JOBS_NAME_LEN]; - - CSTD_UNUSED(ppos); - - count = min_t(size_t, sizeof(buf) - 1, count); - if (copy_from_user(buf, ubuf, count)) - return -EFAULT; - - buf[count] = 0; - - return update_serialize_jobs_setting(kbdev, buf, count); -} - -/** - * kbasep_serialize_jobs_debugfs_open - Open callback for the serialize_jobs - * debugfs file - * @in: inode pointer - * @file: file pointer - * - * Return: Zero on success, error code on failure - */ -static int kbasep_serialize_jobs_debugfs_open(struct inode *in, struct file *file) -{ - return single_open(file, kbasep_serialize_jobs_seq_debugfs_show, in->i_private); -} - -static const struct file_operations kbasep_serialize_jobs_debugfs_fops = { - .owner = THIS_MODULE, - .open = kbasep_serialize_jobs_debugfs_open, - .read = seq_read, - .write = kbasep_serialize_jobs_debugfs_write, - .llseek = seq_lseek, - .release = single_release, -}; - -#endif /* CONFIG_DEBUG_FS */ - -/** - * show_serialize_jobs_sysfs - Show callback for serialize_jobs sysfs file. - * - * @dev: The device this sysfs file is for - * @attr: The attributes of the sysfs file - * @buf: The output buffer for the sysfs file contents - * - * This function is called to get the contents of the serialize_jobs sysfs - * file. This is a list of the available settings with the currently active - * one surrounded by square brackets. - * - * Return: The number of bytes output to @buf. - */ -static ssize_t show_serialize_jobs_sysfs(struct device *dev, struct device_attribute *attr, - char *buf) -{ - struct kbase_device *kbdev = to_kbase_device(dev); - ssize_t ret = 0; - int i; - - CSTD_UNUSED(attr); - - for (i = 0; i < NR_SERIALIZE_JOBS_SETTINGS; i++) { - if (kbdev->serialize_jobs == serialize_jobs_settings[i].setting) - ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), "[%s]", - serialize_jobs_settings[i].name); - else - ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), "%s ", - serialize_jobs_settings[i].name); - } - - if (ret < (ssize_t)(PAGE_SIZE - 1)) { - ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), "\n"); - } else { - buf[PAGE_SIZE - 2] = '\n'; - buf[PAGE_SIZE - 1] = '\0'; - ret = PAGE_SIZE - 1; - } - - return ret; -} - -/** - * store_serialize_jobs_sysfs - Store callback for serialize_jobs sysfs file. - * - * @dev: The device this sysfs file is for - * @attr: The attributes of the sysfs file - * @buf: The value written to the sysfs file - * @count: The number of bytes to write to the sysfs file - * - * This function is called when the serialize_jobs sysfs file is written to. - * It matches the requested setting against the available settings and if a - * matching setting is found updates kbdev->serialize_jobs. - * - * Return: @count if the function succeeded. An error code on failure. - */ -static ssize_t store_serialize_jobs_sysfs(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - CSTD_UNUSED(attr); - return update_serialize_jobs_setting(to_kbase_device(dev), buf, count); -} - -static DEVICE_ATTR(serialize_jobs, 0600, show_serialize_jobs_sysfs, store_serialize_jobs_sysfs); -#endif /* !MALI_USE_CSF */ - static void kbasep_protected_mode_hwcnt_disable_worker(struct work_struct *data) { struct kbase_device *kbdev = @@ -4340,11 +2919,7 @@ static void kbasep_protected_mode_hwcnt_disable_worker(struct work_struct *data) bool do_disable; -#if MALI_USE_CSF backend_lock = &kbdev->csf.scheduler.interrupt_lock; -#else - backend_lock = &kbdev->hwaccess_lock; -#endif spin_lock_irqsave(backend_lock, flags); do_disable = !kbdev->protected_mode_hwcnt_desired && !kbdev->protected_mode_hwcnt_disabled; @@ -4364,9 +2939,6 @@ static void kbasep_protected_mode_hwcnt_disable_worker(struct work_struct *data) * the state machine. */ kbdev->protected_mode_hwcnt_disabled = true; -#if !MALI_USE_CSF - kbase_backend_slot_update(kbdev); -#endif /* !MALI_USE_CSF */ } else { /* Protected mode state was updated while we were doing the * disable, so we need to undo the disable we just performed. @@ -4565,25 +3137,6 @@ int kbase_device_backend_init(struct kbase_device *kbdev) err = kbase_arbiter_pm_early_init(kbdev); if (err == 0) { -#if !MALI_USE_CSF - u32 product_model; - - /* - * Attempt to obtain and parse gpu_id in the event an external AW module - * is used for messaging. We should have access to GPU at this point. - */ - if (kbdev->gpu_props.gpu_id.arch_major == 0) - kbase_gpuprops_parse_gpu_id(&kbdev->gpu_props.gpu_id, - kbase_reg_get_gpu_id(kbdev)); - - product_model = kbdev->gpu_props.gpu_id.product_model; - if (product_model != GPU_ID_PRODUCT_TGOX && product_model != GPU_ID_PRODUCT_TNOX && - product_model != GPU_ID_PRODUCT_TBAX) { - kbase_arbiter_pm_early_term(kbdev); - dev_dbg(kbdev->dev, "GPU platform not suitable for arbitration"); - return -EPERM; - } -#endif /* !MALI_USE_CSF */ dev_info(kbdev->dev, "Arbitration interface enabled"); } #endif /* defined(CONFIG_OF) */ @@ -4787,9 +3340,7 @@ MAKE_QUIRK_ACCESSORS(sc); MAKE_QUIRK_ACCESSORS(tiler); MAKE_QUIRK_ACCESSORS(mmu); MAKE_QUIRK_ACCESSORS(gpu); -#if MALI_USE_CSF MAKE_QUIRK_ACCESSORS(ne); -#endif /** * kbase_device_debugfs_reset_write() - Reset the GPU @@ -4859,36 +3410,52 @@ static const struct file_operations fops_protected_debug_mode = { .llseek = default_llseek, }; -static int kbase_device_debugfs_mem_pool_max_size_show(struct seq_file *sfile, void *data) +static int kbase_device_debugfs_heap_reclaim_offslot_show(struct seq_file *sfile, void *data) { + struct kbase_csf_heap_reclaim_offslot *const setting = sfile->private; + CSTD_UNUSED(data); - return kbase_debugfs_helper_seq_read(sfile, MEMORY_GROUP_MANAGER_NR_GROUPS, - kbase_mem_pool_config_debugfs_max_size); + + seq_printf(sfile, "%u %u\n", setting->timeout_ms, setting->pages); + + return 0; } -static ssize_t kbase_device_debugfs_mem_pool_max_size_write(struct file *file, - const char __user *ubuf, size_t count, - loff_t *ppos) +static ssize_t kbase_device_debugfs_heap_reclaim_offslot_write(struct file *file, + const char __user *ubuf, + size_t count, loff_t *ppos) { - ssize_t err = 0; + const struct seq_file *const sfile = (struct seq_file *)file->private_data; + struct kbase_csf_heap_reclaim_offslot *const setting = sfile->private; + u32 read_len, timeout_ms, pages; + char kbuf[32]; CSTD_UNUSED(ppos); - err = kbase_debugfs_helper_seq_write(file, ubuf, count, MEMORY_GROUP_MANAGER_NR_GROUPS, - kbase_mem_pool_config_debugfs_set_max_size); - return err ? err : (ssize_t)count; + read_len = count < (sizeof(kbuf) - 1) ? count : (sizeof(kbuf) - 1); + if (copy_from_user(kbuf, ubuf, read_len)) + return -EFAULT; + kbuf[read_len] = '\0'; + + if (sscanf(kbuf, "%u %u", &timeout_ms, &pages) != 2) + return -EINVAL; + + setting->timeout_ms = timeout_ms; + setting->pages = pages; + + return count; } -static int kbase_device_debugfs_mem_pool_max_size_open(struct inode *in, struct file *file) +static int kbase_device_debugfs_heap_reclaim_offslot_open(struct inode *in, struct file *file) { - return single_open(file, kbase_device_debugfs_mem_pool_max_size_show, in->i_private); + return single_open(file, kbase_device_debugfs_heap_reclaim_offslot_show, in->i_private); } -static const struct file_operations kbase_device_debugfs_mem_pool_max_size_fops = { +static const struct file_operations kbase_device_debugfs_heap_reclaim_offslot_fops = { .owner = THIS_MODULE, - .open = kbase_device_debugfs_mem_pool_max_size_open, + .open = kbase_device_debugfs_heap_reclaim_offslot_open, .read = seq_read, - .write = kbase_device_debugfs_mem_pool_max_size_write, + .write = kbase_device_debugfs_heap_reclaim_offslot_write, .llseek = seq_lseek, .release = single_release, }; @@ -4916,20 +3483,6 @@ static struct dentry *debugfs_ctx_defaults_init(struct kbase_device *const kbdev debugfs_create_bool("infinite_cache", mode, debugfs_ctx_defaults_directory, &kbdev->infinite_cache_active_default); - dentry = debugfs_create_file("mem_pool_max_size", mode, debugfs_ctx_defaults_directory, - &kbdev->mem_pool_defaults.small, - &kbase_device_debugfs_mem_pool_max_size_fops); - if (IS_ERR_OR_NULL(dentry)) { - dev_err(kbdev->dev, "Unable to create mem_pool_max_size debugfs entry\n"); - return dentry; - } - - dentry = debugfs_create_file("lp_mem_pool_max_size", mode, debugfs_ctx_defaults_directory, - &kbdev->mem_pool_defaults.large, - &kbase_device_debugfs_mem_pool_max_size_fops); - if (IS_ERR_OR_NULL(dentry)) - dev_err(kbdev->dev, "Unable to create lp_mem_pool_max_size debugfs entry\n"); - return dentry; } @@ -4965,11 +3518,7 @@ static struct dentry *init_debugfs(struct kbase_device *kbdev) kbasep_regs_history_debugfs_init(kbdev); -#if MALI_USE_CSF kbase_debug_csf_fault_debugfs_init(kbdev); -#else /* MALI_USE_CSF */ - kbase_debug_job_fault_debugfs_init(kbdev); -#endif /* !MALI_USE_CSF */ kbasep_gpu_memory_debugfs_init(kbdev); kbase_as_fault_debugfs_init(kbdev); @@ -4977,6 +3526,7 @@ static struct dentry *init_debugfs(struct kbase_device *kbdev) kbase_instr_backend_debugfs_init(kbdev); #endif kbase_pbha_debugfs_init(kbdev); + kbase_gpu_timestamp_offset_debugfs_init(kbdev); /* fops_* variables created by invocations of macro * MAKE_QUIRK_ACCESSORS() above. @@ -5009,7 +3559,6 @@ static struct dentry *init_debugfs(struct kbase_device *kbdev) return dentry; } -#if MALI_USE_CSF if (kbase_csf_dev_has_ne(kbdev)) { dentry = debugfs_create_file("quirks_ne", 0644, kbdev->mali_debugfs_directory, kbdev, &fops_ne_quirks); @@ -5021,7 +3570,6 @@ static struct dentry *init_debugfs(struct kbase_device *kbdev) if (kbase_csf_ne_control_debugfs_init(kbdev)) return NULL; } -#endif dentry = debugfs_ctx_defaults_init(kbdev); if (IS_ERR_OR_NULL(dentry)) @@ -5045,6 +3593,20 @@ static struct dentry *init_debugfs(struct kbase_device *kbdev) return dentry; } + { + struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; + struct kbase_csf_sched_heap_reclaim_mgr *const mgr = &scheduler->reclaim_mgr; + + dentry = debugfs_create_file("heap_reclaim_offslot", 0644, + kbdev->mali_debugfs_directory, &mgr->offslot_setting, + &kbase_device_debugfs_heap_reclaim_offslot_fops); + if (IS_ERR_OR_NULL(dentry)) { + dev_err(kbdev->dev, + "Unable to create heap_reclaim_offslot debugfs entry\n"); + return dentry; + } + } + kbase_ktrace_debugfs_init(kbdev); #ifdef CONFIG_MALI_VALHALL_DEVFREQ @@ -5054,15 +3616,6 @@ static struct dentry *init_debugfs(struct kbase_device *kbdev) #endif /* CONFIG_DEVFREQ_THERMAL */ #endif /* CONFIG_MALI_VALHALL_DEVFREQ */ -#if !MALI_USE_CSF - dentry = debugfs_create_file("serialize_jobs", 0644, kbdev->mali_debugfs_directory, kbdev, - &kbasep_serialize_jobs_debugfs_fops); - if (IS_ERR_OR_NULL(dentry)) { - dev_err(kbdev->dev, "Unable to create serialize_jobs debugfs entry\n"); - return dentry; - } - kbase_timeline_io_debugfs_init(kbdev); -#endif kbase_dvfs_status_debugfs_init(kbdev); @@ -5140,7 +3693,7 @@ static bool kbase_device_supports_coherency_mode(struct kbase_device *kbdev, u32 * on CSF GPUs. */ if (coherency_mode == COHERENCY_ACE) { - if (IS_ENABLED(MALI_USE_CSF) && !IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI)) { + if (!IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI)) { dev_err(kbdev->dev, "ACE coherency not supported on CSF, wrong DT configuration"); return false; @@ -5208,8 +3761,6 @@ early_exit: } -#if MALI_USE_CSF - bool kbasep_adjust_prioritized_process(struct kbase_device *kbdev, bool add, uint32_t tgid) { struct kbase_context *kctx; @@ -5723,33 +4274,16 @@ static ssize_t mcu_shader_pwroff_timeout_ns_store(struct device *dev, struct dev static DEVICE_ATTR_RW(mcu_shader_pwroff_timeout_ns); -#endif /* MALI_USE_CSF */ - -static struct attribute *kbase_scheduling_attrs[] = { -#if !MALI_USE_CSF - &dev_attr_serialize_jobs.attr, -#endif /* !MALI_USE_CSF */ - NULL -}; +static struct attribute *kbase_scheduling_attrs[] = { NULL }; static struct attribute *kbase_attrs[] = { #ifdef CONFIG_MALI_VALHALL_DEBUG &dev_attr_debug_command.attr, -#if !MALI_USE_CSF - &dev_attr_js_softstop_always.attr, -#endif /* !MALI_USE_CSF */ #endif -#if !MALI_USE_CSF - &dev_attr_js_timeouts.attr, - &dev_attr_soft_job_timeout.attr, -#endif /* !MALI_USE_CSF */ &dev_attr_gpuinfo.attr, &dev_attr_dvfs_period.attr, &dev_attr_pm_poweroff.attr, &dev_attr_reset_timeout.attr, -#if !MALI_USE_CSF - &dev_attr_js_scheduling_period.attr, -#else &dev_attr_csg_scheduling_period.attr, &dev_attr_add_prioritized_process.attr, &dev_attr_remove_prioritized_process.attr, @@ -5758,35 +4292,17 @@ static struct attribute *kbase_attrs[] = { &dev_attr_idle_hysteresis_time_ns.attr, &dev_attr_mcu_shader_pwroff_timeout.attr, &dev_attr_mcu_shader_pwroff_timeout_ns.attr, -#endif /* !MALI_USE_CSF */ &dev_attr_power_policy.attr, &dev_attr_core_mask.attr, - &dev_attr_mem_pool_size.attr, - &dev_attr_mem_pool_max_size.attr, - &dev_attr_lp_mem_pool_size.attr, - &dev_attr_lp_mem_pool_max_size.attr, -#if !MALI_USE_CSF - &dev_attr_js_ctx_scheduling_mode.attr, -#endif /* !MALI_USE_CSF */ NULL }; -static struct attribute *kbase_mempool_attrs[] = { &dev_attr_max_size.attr, - &dev_attr_lp_max_size.attr, - &dev_attr_ctx_default_max_size.attr, NULL }; - #define SYSFS_SCHEDULING_GROUP "scheduling" static const struct attribute_group kbase_scheduling_attr_group = { .name = SYSFS_SCHEDULING_GROUP, .attrs = kbase_scheduling_attrs, }; -#define SYSFS_MEMPOOL_GROUP "mempool" -static const struct attribute_group kbase_mempool_attr_group = { - .name = SYSFS_MEMPOOL_GROUP, - .attrs = kbase_mempool_attrs, -}; - static const struct attribute_group kbase_attr_group = { .attrs = kbase_attrs, }; @@ -5812,19 +4328,11 @@ int kbase_sysfs_init(struct kbase_device *kbdev) return err; } - err = sysfs_create_group(&kbdev->dev->kobj, &kbase_mempool_attr_group); - if (err) { - dev_err(kbdev->dev, "Creation of %s sysfs group failed", SYSFS_MEMPOOL_GROUP); - sysfs_remove_group(&kbdev->dev->kobj, &kbase_scheduling_attr_group); - sysfs_remove_group(&kbdev->dev->kobj, &kbase_attr_group); - } - return err; } void kbase_sysfs_term(struct kbase_device *kbdev) { - sysfs_remove_group(&kbdev->dev->kobj, &kbase_mempool_attr_group); sysfs_remove_group(&kbdev->dev->kobj, &kbase_scheduling_attr_group); sysfs_remove_group(&kbdev->dev->kobj, &kbase_attr_group); put_device(kbdev->dev); @@ -6013,10 +4521,6 @@ static int kbase_device_resume(struct device *dev) kbase_devfreq_enqueue_work(kbdev, DEVFREQ_WORK_RESUME); #endif -#if !MALI_USE_CSF - kbase_enable_quick_reset(kbdev); -#endif - return 0; } @@ -6031,23 +4535,23 @@ static int kbase_device_resume(struct device *dev) * * Return: A standard Linux error code */ -#ifdef KBASE_PM_RUNTIME static int kbase_device_runtime_suspend(struct device *dev) { struct kbase_device *kbdev = to_kbase_device(dev); int ret = 0; + if (!IS_ENABLED(CONFIG_PM)) + return 0; + if (!kbdev) return -ENODEV; dev_dbg(dev, "Callback %s\n", __func__); KBASE_KTRACE_ADD(kbdev, PM_RUNTIME_SUSPEND_CALLBACK, NULL, 0); -#if MALI_USE_CSF ret = kbase_pm_handle_runtime_suspend(kbdev); if (ret) return ret; -#endif #ifdef CONFIG_MALI_VALHALL_DVFS kbase_pm_metrics_stop(kbdev); @@ -6063,9 +4567,8 @@ static int kbase_device_runtime_suspend(struct device *dev) kbdev->is_runtime_resumed = false; dev_dbg(dev, "runtime suspend\n"); } - return ret; + return 0; } -#endif /* KBASE_PM_RUNTIME */ /** * kbase_device_runtime_resume - Runtime resume callback from the OS. @@ -6077,12 +4580,14 @@ static int kbase_device_runtime_suspend(struct device *dev) * Return: A standard Linux error code */ -#ifdef KBASE_PM_RUNTIME static int kbase_device_runtime_resume(struct device *dev) { int ret = 0; struct kbase_device *kbdev = to_kbase_device(dev); + if (!IS_ENABLED(CONFIG_PM)) + return 0; + if (!kbdev) return -ENODEV; @@ -6105,9 +4610,7 @@ static int kbase_device_runtime_resume(struct device *dev) return ret; } -#endif /* KBASE_PM_RUNTIME */ -#ifdef KBASE_PM_RUNTIME /** * kbase_device_runtime_idle - Runtime idle callback from the OS. * @dev: The device to suspend @@ -6122,6 +4625,9 @@ static int kbase_device_runtime_idle(struct device *dev) { struct kbase_device *kbdev = to_kbase_device(dev); + if (!IS_ENABLED(CONFIG_PM)) + return 0; + if (!kbdev) return -ENODEV; @@ -6136,18 +4642,15 @@ static int kbase_device_runtime_idle(struct device *dev) pm_runtime_mark_last_busy(kbdev->dev); return 0; } -#endif /* KBASE_PM_RUNTIME */ /* The power management operations for the platform driver. */ static const struct dev_pm_ops kbase_pm_ops = { SYSTEM_SLEEP_PM_OPS(kbase_device_suspend, kbase_device_resume) -#ifdef KBASE_PM_RUNTIME .runtime_suspend = kbase_device_runtime_suspend, .runtime_resume = kbase_device_runtime_resume, .runtime_idle = kbase_device_runtime_idle, -#endif /* KBASE_PM_RUNTIME */ }; #if IS_ENABLED(CONFIG_OF) diff --git a/drivers/gpu/arm/valhall/mali_kbase_ctx_sched.c b/drivers/gpu/arm/valhall/mali_kbase_ctx_sched.c index e64a5715b126..f9b3069e7128 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_ctx_sched.c +++ b/drivers/gpu/arm/valhall/mali_kbase_ctx_sched.c @@ -24,11 +24,7 @@ #include #include "mali_kbase_ctx_sched.h" #include "tl/mali_kbase_tracepoints.h" -#if MALI_USE_CSF #include "mali_kbase_reset_gpu.h" -#else -#include -#endif /* Helper for ktrace */ #if KBASE_KTRACE_ENABLE @@ -152,19 +148,12 @@ void kbase_ctx_sched_retain_ctx_refcount(struct kbase_context *kctx) struct kbase_device *const kbdev = kctx->kbdev; lockdep_assert_held(&kbdev->hwaccess_lock); -#if MALI_USE_CSF /* We expect the context to be active when this function is called, * except for the case where a page fault is reported for it during * the GPU reset sequence, in which case we can expect the refcount * to be 0. */ WARN_ON(!atomic_read(&kctx->refcount) && !kbase_reset_gpu_is_active(kbdev)); -#else - /* We expect the context to be active (and thus refcount should be non-zero) - * when this function is called - */ - WARN_ON(!atomic_read(&kctx->refcount)); -#endif if (likely((kctx->as_nr >= 0) && (kctx->as_nr < BASE_MAX_NR_AS))) WARN_ON(kbdev->as_to_kctx[kctx->as_nr] != kctx); else @@ -188,9 +177,6 @@ void kbase_ctx_sched_release_ctx(struct kbase_context *kctx) kbdev->as_to_kctx[kctx->as_nr] = NULL; kctx->as_nr = KBASEP_AS_NR_INVALID; kbase_ctx_flag_clear(kctx, KCTX_AS_DISABLED_ON_FAULT); -#if !MALI_USE_CSF - kbase_backend_slot_kctx_purge_locked(kbdev, kctx); -#endif } } } @@ -206,6 +192,19 @@ void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx) mutex_lock(&kbdev->mmu_hw_mutex); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_ctx_sched_remove_ctx_nolock(kctx); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); +} + +void kbase_ctx_sched_remove_ctx_nolock(struct kbase_context *kctx) +{ + struct kbase_device *const kbdev = kctx->kbdev; + + lockdep_assert_held(&kbdev->mmu_hw_mutex); + lockdep_assert_held(&kbdev->hwaccess_lock); + WARN_ON(atomic_read(&kctx->refcount) != 0); if ((kctx->as_nr >= 0) && (kctx->as_nr < BASE_MAX_NR_AS)) { @@ -215,9 +214,6 @@ void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx) kbdev->as_to_kctx[kctx->as_nr] = NULL; kctx->as_nr = KBASEP_AS_NR_INVALID; } - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kbdev->mmu_hw_mutex); } void kbase_ctx_sched_restore_all_as(struct kbase_device *kbdev) @@ -234,12 +230,10 @@ void kbase_ctx_sched_restore_all_as(struct kbase_device *kbdev) for (i = 0; i != kbdev->nr_hw_address_spaces; ++i) { struct kbase_context *kctx; -#if MALI_USE_CSF if ((i == MCU_AS_NR) && kbdev->csf.firmware_inited) { kbase_mmu_update(kbdev, &kbdev->csf.mcu_mmu, MCU_AS_NR); continue; } -#endif kctx = kbdev->as_to_kctx[i]; if (kctx) { if (atomic_read(&kctx->refcount)) { @@ -378,7 +372,6 @@ void kbase_ctx_sched_release_ctx_lock(struct kbase_context *kctx) spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); } -#if MALI_USE_CSF bool kbase_ctx_sched_inc_refcount_if_as_valid(struct kbase_context *kctx) { struct kbase_device *kbdev; @@ -412,4 +405,3 @@ bool kbase_ctx_sched_inc_refcount_if_as_valid(struct kbase_context *kctx) return added_ref; } -#endif diff --git a/drivers/gpu/arm/valhall/mali_kbase_ctx_sched.h b/drivers/gpu/arm/valhall/mali_kbase_ctx_sched.h index fd1b82471d26..bdc9e8fa4784 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_ctx_sched.h +++ b/drivers/gpu/arm/valhall/mali_kbase_ctx_sched.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2017-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2017-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -130,6 +130,20 @@ void kbase_ctx_sched_release_ctx(struct kbase_context *kctx); */ void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx); +/** + * kbase_ctx_sched_remove_ctx_nolock - Unassign previously assigned address space + * @kctx: The context to be removed + * + * The following lock must be held by the caller: + * kbase_device::mmu_hw_mutex + * kbase_device::hwaccess_lock + * + * This function should be called when a context is being destroyed. The + * context must no longer have any reference. If it has been assigned an + * address space before then the AS will be unprogrammed. + */ +void kbase_ctx_sched_remove_ctx_nolock(struct kbase_context *kctx); + /** * kbase_ctx_sched_restore_all_as - Reprogram all address spaces * @kbdev: The device for which address spaces to be reprogrammed @@ -228,7 +242,6 @@ bool kbase_ctx_sched_inc_refcount(struct kbase_context *kctx); */ void kbase_ctx_sched_release_ctx_lock(struct kbase_context *kctx); -#if MALI_USE_CSF /** * kbase_ctx_sched_inc_refcount_if_as_valid - Refcount the context if it has GPU * address space slot assigned to it. @@ -244,6 +257,5 @@ void kbase_ctx_sched_release_ctx_lock(struct kbase_context *kctx); * was not assigned). */ bool kbase_ctx_sched_inc_refcount_if_as_valid(struct kbase_context *kctx); -#endif #endif /* _KBASE_CTX_SCHED_H_ */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_debug_job_fault.c b/drivers/gpu/arm/valhall/mali_kbase_debug_job_fault.c deleted file mode 100644 index 679dd75187db..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_debug_job_fault.c +++ /dev/null @@ -1,544 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2012-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include -#include -#include - -#if IS_ENABLED(CONFIG_DEBUG_FS) - -static bool kbase_is_job_fault_event_pending(struct kbase_device *kbdev) -{ - struct list_head *event_list = &kbdev->job_fault_event_list; - unsigned long flags; - bool ret; - - spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); - ret = !list_empty(event_list); - spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); - - return ret; -} - -static void kbase_ctx_remove_pending_event(struct kbase_context *kctx) -{ - struct list_head *event_list = &kctx->kbdev->job_fault_event_list; - struct base_job_fault_event *event; - unsigned long flags; - - spin_lock_irqsave(&kctx->kbdev->job_fault_event_lock, flags); - list_for_each_entry(event, event_list, head) { - if (event->katom->kctx == kctx) { - list_del(&event->head); - WARN_ON_ONCE(&event->job_fault_work != kctx->job_fault_work); - wake_up(&kctx->kbdev->job_fault_resume_wq); - /* job_fault_event_list can only have a single atom for - * each context. - */ - break; - } - } - spin_unlock_irqrestore(&kctx->kbdev->job_fault_event_lock, flags); - if (kctx->job_fault_work) - flush_work(kctx->job_fault_work); -} - -static bool kbase_ctx_has_no_event_pending(struct kbase_context *kctx) -{ - struct kbase_device *kbdev = kctx->kbdev; - struct list_head *event_list = &kctx->kbdev->job_fault_event_list; - struct base_job_fault_event *event; - unsigned long flags; - - spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); - if (list_empty(event_list)) { - spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); - return true; - } - list_for_each_entry(event, event_list, head) { - if (event->katom->kctx == kctx) { - spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); - return false; - } - } - spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); - return true; -} - -static int wait_for_job_fault(struct kbase_device *kbdev) -{ -#if KERNEL_VERSION(4, 15, 0) > LINUX_VERSION_CODE - int ret = wait_event_interruptible_timeout(kbdev->job_fault_wq, - kbase_is_job_fault_event_pending(kbdev), - msecs_to_jiffies(2000)); - if (ret == 0) - return -EAGAIN; - else if (ret > 0) - return 0; - else - return ret; -#else - return wait_event_interruptible(kbdev->job_fault_wq, - kbase_is_job_fault_event_pending(kbdev)); -#endif -} - -/* wait until the fault happen and copy the event */ -static int kbase_job_fault_event_wait(struct kbase_device *kbdev, - struct base_job_fault_event *event) -{ - struct list_head *event_list = &kbdev->job_fault_event_list; - struct base_job_fault_event *event_in; - unsigned long flags; - - spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); - while (list_empty(event_list)) { - int err; - - spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); - - err = wait_for_job_fault(kbdev); - if (err) - return err; - - spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); - } - - event_in = list_entry(event_list->next, struct base_job_fault_event, head); - event->event_code = event_in->event_code; - event->katom = event_in->katom; - - spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); - - return 0; -} - -/* remove the event from the queue */ -static struct base_job_fault_event *kbase_job_fault_event_dequeue(struct kbase_device *kbdev, - struct list_head *event_list) -{ - struct base_job_fault_event *event; - - CSTD_UNUSED(kbdev); - - event = list_entry(event_list->next, struct base_job_fault_event, head); - list_del(event_list->next); - - return event; -} - -/* Remove all the following atoms after the failed atom in the same context - * Call the postponed bottom half of job done. - * Then, this context could be rescheduled. - */ -static void kbase_job_fault_resume_event_cleanup(struct kbase_context *kctx) -{ - struct list_head *event_list = &kctx->job_fault_resume_event_list; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - while (!list_empty(event_list)) { - struct base_job_fault_event *event; - - event = kbase_job_fault_event_dequeue(kctx->kbdev, - &kctx->job_fault_resume_event_list); - WARN_ON(work_pending(&event->katom->work)); - INIT_WORK(&event->katom->work, kbase_jd_done_worker); - queue_work(kctx->jctx.job_done_wq, &event->katom->work); - } -} - -static void kbase_job_fault_resume_worker(struct work_struct *data) -{ - struct base_job_fault_event *event = - container_of(data, struct base_job_fault_event, job_fault_work); - struct kbase_device *kbdev; - struct kbase_context *kctx; - struct kbase_jd_atom *katom; - unsigned long flags; - - katom = event->katom; - kctx = katom->kctx; - kbdev = kctx->kbdev; - - dev_info(kbdev->dev, "Job dumping wait\n"); - - /* When it was waked up, it need to check if queue is empty or the - * failed atom belongs to different context. If yes, wake up. Both - * of them mean the failed job has been dumped. Please note, it - * should never happen that the job_fault_event_list has the two - * atoms belong to the same context. - */ - wait_event(kbdev->job_fault_resume_wq, kbase_ctx_has_no_event_pending(kctx)); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - atomic_set(&kctx->job_fault_count, 0); - WARN_ON(work_pending(&katom->work)); - INIT_WORK(&katom->work, kbase_jd_done_worker); - queue_work(kctx->jctx.job_done_wq, &katom->work); - - /* In case the following atoms were scheduled during failed job dump - * the job_done_worker was held. We need to rerun it after the dump - * was finished - */ - kbase_job_fault_resume_event_cleanup(kctx); - dev_info(kbdev->dev, "Job dumping finish, resume scheduler\n"); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -} - -static struct base_job_fault_event *kbase_job_fault_event_queue(struct list_head *event_list, - struct kbase_jd_atom *atom, - u32 completion_code) -{ - struct base_job_fault_event *event; - - event = &atom->fault_event; - - event->katom = atom; - event->event_code = completion_code; - - list_add_tail(&event->head, event_list); - - return event; -} - -static void kbase_job_fault_event_post(struct kbase_device *kbdev, struct kbase_jd_atom *katom, - u32 completion_code) -{ - struct base_job_fault_event *event; - unsigned long flags; - - spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); - event = kbase_job_fault_event_queue(&kbdev->job_fault_event_list, katom, completion_code); - INIT_WORK(&event->job_fault_work, kbase_job_fault_resume_worker); - katom->kctx->job_fault_work = &event->job_fault_work; - wake_up_interruptible(&kbdev->job_fault_wq); - queue_work(kbdev->job_fault_resume_workq, &event->job_fault_work); - spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); - - dev_info(katom->kctx->kbdev->dev, "Job fault happen, start dump: %d_%d", katom->kctx->tgid, - katom->kctx->id); -} - -/* - * This function will process the job fault - * Get the register copy - * Send the failed job dump event - * Create a Wait queue to wait until the job dump finish - */ - -bool kbase_debug_job_fault_process(struct kbase_jd_atom *katom, u32 completion_code) -{ - struct kbase_context *kctx = katom->kctx; - - /* Check if dumping is in the process - * only one atom of each context can be dumped at the same time - * If the atom belongs to different context, it can be dumped - */ - if (atomic_read(&kctx->job_fault_count) > 0) { - kbase_job_fault_event_queue(&kctx->job_fault_resume_event_list, katom, - completion_code); - dev_info(kctx->kbdev->dev, "queue:%d\n", kbase_jd_atom_id(kctx, katom)); - return true; - } - - if (kbase_ctx_flag(kctx, KCTX_DYING)) - return false; - - if (atomic_read(&kctx->kbdev->job_fault_debug) > 0) { - if (completion_code != BASE_JD_EVENT_DONE) { - if (kbase_job_fault_get_reg_snapshot(kctx) == false) { - dev_warn(kctx->kbdev->dev, "get reg dump failed\n"); - return false; - } - - kbase_job_fault_event_post(kctx->kbdev, katom, completion_code); - atomic_inc(&kctx->job_fault_count); - dev_info(kctx->kbdev->dev, "post:%d\n", kbase_jd_atom_id(kctx, katom)); - return true; - } - } - return false; -} - -static int debug_job_fault_show(struct seq_file *m, void *v) -{ - struct kbase_device *kbdev = m->private; - struct base_job_fault_event *event = (struct base_job_fault_event *)v; - struct kbase_context *kctx = event->katom->kctx; - int i; - - dev_info(kbdev->dev, "debug job fault seq show:%d_%d, %d", kctx->tgid, kctx->id, - event->reg_offset); - - if (kctx->reg_dump == NULL) { - dev_warn(kbdev->dev, "reg dump is NULL"); - return -1; - } - - if (kctx->reg_dump[event->reg_offset] == REGISTER_DUMP_TERMINATION_FLAG) { - /* Return the error here to stop the read. And the - * following next() will not be called. The stop can - * get the real event resource and release it - */ - return -1; - } - - if (event->reg_offset == 0) - seq_printf(m, "%d_%d\n", kctx->tgid, kctx->id); - - for (i = 0; i < 50; i++) { - if (kctx->reg_dump[event->reg_offset] == REGISTER_DUMP_TERMINATION_FLAG) { - break; - } - seq_printf(m, "%08x: %08x\n", kctx->reg_dump[event->reg_offset], - kctx->reg_dump[1 + event->reg_offset]); - event->reg_offset += 2; - } - - return 0; -} -static void *debug_job_fault_next(struct seq_file *m, void *v, loff_t *pos) -{ - struct kbase_device *kbdev = m->private; - struct base_job_fault_event *event = (struct base_job_fault_event *)v; - - dev_info(kbdev->dev, "debug job fault seq next:%d, %d", event->reg_offset, (int)*pos); - - return event; -} - -static void *debug_job_fault_start(struct seq_file *m, loff_t *pos) -{ - struct kbase_device *kbdev = m->private; - struct base_job_fault_event *event; - - dev_info(kbdev->dev, "fault job seq start:%d", (int)*pos); - - /* The condition is trick here. It needs make sure the - * fault hasn't happened and the dumping hasn't been started, - * or the dumping has finished - */ - if (*pos == 0) { - event = kmalloc(sizeof(*event), GFP_KERNEL); - if (!event) - return NULL; - event->reg_offset = 0; - if (kbase_job_fault_event_wait(kbdev, event)) { - kfree(event); - return NULL; - } - - /* The cache flush workaround is called in bottom half of - * job done but we delayed it. Now we should clean cache - * earlier. Then the GPU memory dump should be correct. - */ - kbase_backend_cache_clean(kbdev, event->katom); - } else - return NULL; - - return event; -} - -static void debug_job_fault_stop(struct seq_file *m, void *v) -{ - struct kbase_device *kbdev = m->private; - - /* here we wake up the kbase_jd_done_worker after stop, it needs - * get the memory dump before the register dump in debug daemon, - * otherwise, the memory dump may be incorrect. - */ - - if (v != NULL) { - kfree(v); - dev_info(kbdev->dev, "debug job fault seq stop stage 1"); - - } else { - unsigned long flags; - - spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); - if (!list_empty(&kbdev->job_fault_event_list)) { - kbase_job_fault_event_dequeue(kbdev, &kbdev->job_fault_event_list); - wake_up(&kbdev->job_fault_resume_wq); - } - spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); - dev_info(kbdev->dev, "debug job fault seq stop stage 2"); - } -} - -static const struct seq_operations ops = { - .start = debug_job_fault_start, - .next = debug_job_fault_next, - .stop = debug_job_fault_stop, - .show = debug_job_fault_show, -}; - -static int debug_job_fault_open(struct inode *in, struct file *file) -{ - struct kbase_device *kbdev = in->i_private; - - if (atomic_cmpxchg(&kbdev->job_fault_debug, 0, 1) == 1) { - dev_warn(kbdev->dev, "debug job fault is busy, only a single client is allowed"); - return -EBUSY; - } - - seq_open(file, &ops); - - ((struct seq_file *)file->private_data)->private = kbdev; - dev_info(kbdev->dev, "debug job fault seq open"); - - return 0; -} - -static int debug_job_fault_release(struct inode *in, struct file *file) -{ - struct kbase_device *kbdev = in->i_private; - struct list_head *event_list = &kbdev->job_fault_event_list; - unsigned long flags; - - seq_release(in, file); - - spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); - - /* Disable job fault dumping. This will let kbase run jobs as normal, - * without blocking waiting for a job_fault client to read failed jobs. - * - * After this a new client may open the file, and may re-enable job - * fault dumping, but the job_fault_event_lock we hold here will block - * that from interfering until after we've completed the cleanup. - */ - atomic_dec(&kbdev->job_fault_debug); - - /* Clean the unprocessed job fault. After that, all the suspended - * contexts could be rescheduled. Remove all the failed atoms that - * belong to different contexts Resume all the contexts that were - * suspend due to failed job. - */ - while (!list_empty(event_list)) { - kbase_job_fault_event_dequeue(kbdev, event_list); - spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); - wake_up(&kbdev->job_fault_resume_wq); - spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); - } - - spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); - - dev_info(kbdev->dev, "debug job fault seq close"); - - return 0; -} - -static const struct file_operations kbasep_debug_job_fault_fops = { - .owner = THIS_MODULE, - .open = debug_job_fault_open, - .read = seq_read, - .llseek = seq_lseek, - .release = debug_job_fault_release, -}; - -/* - * Initialize debugfs entry for job fault dump - */ -void kbase_debug_job_fault_debugfs_init(struct kbase_device *kbdev) -{ - debugfs_create_file("job_fault", 0400, kbdev->mali_debugfs_directory, kbdev, - &kbasep_debug_job_fault_fops); -} - -int kbase_debug_job_fault_dev_init(struct kbase_device *kbdev) -{ - INIT_LIST_HEAD(&kbdev->job_fault_event_list); - - init_waitqueue_head(&(kbdev->job_fault_wq)); - init_waitqueue_head(&(kbdev->job_fault_resume_wq)); - spin_lock_init(&kbdev->job_fault_event_lock); - - kbdev->job_fault_resume_workq = - alloc_workqueue("kbase_job_fault_resume_work_queue", WQ_MEM_RECLAIM, 1); - if (!kbdev->job_fault_resume_workq) - return -ENOMEM; - - return 0; -} - -/* - * Release the relevant resource per device - */ -void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev) -{ - destroy_workqueue(kbdev->job_fault_resume_workq); -} - -/* - * Initialize the relevant data structure per context - */ -int kbase_debug_job_fault_context_init(struct kbase_context *kctx) -{ - /* We need allocate double size register range - * Because this memory will keep the register address and value - */ - kctx->reg_dump = vmalloc(0x4000 * 2); - if (kctx->reg_dump != NULL) { - if (kbase_debug_job_fault_reg_snapshot_init(kctx, 0x4000) == false) { - vfree(kctx->reg_dump); - kctx->reg_dump = NULL; - } - INIT_LIST_HEAD(&kctx->job_fault_resume_event_list); - } - - return 0; -} - -/* - * release the relevant resource per context - */ -void kbase_debug_job_fault_context_term(struct kbase_context *kctx) -{ - vfree(kctx->reg_dump); -} - -void kbase_debug_job_fault_kctx_unblock(struct kbase_context *kctx) -{ - WARN_ON(!kbase_ctx_flag(kctx, KCTX_DYING)); - - /* Return early if the job fault part of the kbase_device is not - * initialized yet. An error can happen during the device probe after - * the privileged Kbase context was created for the HW counter dumping - * but before the job fault part is initialized. - */ - if (!kctx->kbdev->job_fault_resume_workq) - return; - - kbase_ctx_remove_pending_event(kctx); -} - -#else /* CONFIG_DEBUG_FS */ - -int kbase_debug_job_fault_dev_init(struct kbase_device *kbdev) -{ - return 0; -} - -void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev) -{ -} - -#endif /* CONFIG_DEBUG_FS */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_debug_job_fault.h b/drivers/gpu/arm/valhall/mali_kbase_debug_job_fault.h deleted file mode 100644 index ee1228155621..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_debug_job_fault.h +++ /dev/null @@ -1,125 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2012-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#ifndef _KBASE_DEBUG_JOB_FAULT_H -#define _KBASE_DEBUG_JOB_FAULT_H - -#include - -struct kbase_context; -struct kbase_device; - -#define REGISTER_DUMP_TERMINATION_FLAG 0xFFFFFFFF - -/** - * kbase_debug_job_fault_dev_init - Create the fault event wait queue - * per device and initialize the required lists. - * @kbdev: Device pointer - * - * This function must be called only when a kbase device is initialized. - * - * Return: Zero on success or a negative error code. - */ -int kbase_debug_job_fault_dev_init(struct kbase_device *kbdev); - -/** - * kbase_debug_job_fault_debugfs_init - Initialize job fault debug sysfs - * @kbdev: Device pointer - */ -void kbase_debug_job_fault_debugfs_init(struct kbase_device *kbdev); - -/** - * kbase_debug_job_fault_dev_term - Clean up resources created in - * kbase_debug_job_fault_dev_init. - * @kbdev: Device pointer - */ -void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev); - -/** - * kbase_debug_job_fault_context_init - Initialize the relevant data structure per context - * @kctx: KBase context pointer - * - * This function must be called only when a kbase context is instantiated. - * - * Return: 0 on success - */ -int kbase_debug_job_fault_context_init(struct kbase_context *kctx); - -/** - * kbase_debug_job_fault_context_term - Release the relevant - * resource per context - * @kctx: KBase context pointer - */ -void kbase_debug_job_fault_context_term(struct kbase_context *kctx); - -/** - * kbase_debug_job_fault_kctx_unblock - Unblock the atoms blocked on job fault - * dumping on context termination. - * - * @kctx: KBase context pointer - * - * This function is called during context termination to unblock the atom for - * which the job fault occurred and also the atoms following it. This is needed - * otherwise the wait for zero jobs could timeout (leading to an assertion - * failure, kernel panic in debug builds) in the pathological case where - * although the thread/daemon capturing the job fault events is running, - * but for some reasons has stopped consuming the events. - */ -void kbase_debug_job_fault_kctx_unblock(struct kbase_context *kctx); - -/** - * kbase_debug_job_fault_process - Process the failed job. - * - * @katom: The failed atom pointer - * @completion_code: the job status - * - * It will send a event and wake up the job fault waiting queue - * Then create a work queue to wait for job dump finish - * This function should be called in the interrupt handler and before - * jd_done that make sure the jd_done_worker will be delayed until the - * job dump finish - * - * Return: true if dump is going on - */ -bool kbase_debug_job_fault_process(struct kbase_jd_atom *katom, u32 completion_code); - -/** - * kbase_debug_job_fault_reg_snapshot_init - Set the interested registers - * address during the job fault process, the relevant registers will - * be saved when a job fault happen - * @kctx: KBase context pointer - * @reg_range: Maximum register address space - * - * Return: true if initializing successfully - */ -bool kbase_debug_job_fault_reg_snapshot_init(struct kbase_context *kctx, int reg_range); - -/** - * kbase_job_fault_get_reg_snapshot - Read the interested registers for - * failed job dump - * - * @kctx: KBase context pointer - * - * Return: true if getting registers successfully - */ -bool kbase_job_fault_get_reg_snapshot(struct kbase_context *kctx); - -#endif /*_KBASE_DEBUG_JOB_FAULT_H*/ diff --git a/drivers/gpu/arm/valhall/mali_kbase_debug_mem_zones.c b/drivers/gpu/arm/valhall/mali_kbase_debug_mem_zones.c index 01267b599d89..3964b5505bdd 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_debug_mem_zones.c +++ b/drivers/gpu/arm/valhall/mali_kbase_debug_mem_zones.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2022-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2022-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -63,7 +63,6 @@ static int debug_mem_zones_show(struct seq_file *sfile, void *data) reg_zone->va_size_pages); } } -#if MALI_USE_CSF reg_zone = &kctx->kbdev->csf.mcu_shared_zone; if (reg_zone && reg_zone->base_pfn) { @@ -71,7 +70,6 @@ static int debug_mem_zones_show(struct seq_file *sfile, void *data) kbase_reg_zone_get_name(MCU_SHARED_ZONE), MCU_SHARED_ZONE, reg_zone->base_pfn, reg_zone->va_size_pages); } -#endif kbase_gpu_vm_unlock(kctx); return 0; diff --git a/drivers/gpu/arm/valhall/mali_kbase_defs.h b/drivers/gpu/arm/valhall/mali_kbase_defs.h index 978e76c3fdcf..ccfeacae678b 100755 --- a/drivers/gpu/arm/valhall/mali_kbase_defs.h +++ b/drivers/gpu/arm/valhall/mali_kbase_defs.h @@ -39,12 +39,7 @@ #include #include -#if MALI_USE_CSF #include -#else -#include -#include -#endif #include "debug/mali_kbase_debug_ktrace_defs.h" @@ -126,7 +121,6 @@ */ #define KBASE_HWCNT_GPU_VIRTUALIZER_DUMP_THRESHOLD_NS (200 * NSEC_PER_USEC) -#if MALI_USE_CSF /* The buffer count of CSF hwcnt backend ring buffer, which is used when CSF * hwcnt backend allocate the ring buffer to communicate with CSF firmware for * HWC dump samples. @@ -134,7 +128,6 @@ * CSF hwcnt backend creation will be failed. */ #define KBASE_HWCNT_BACKEND_CSF_RING_BUFFER_COUNT (128) -#endif /* Maximum number of clock/regulator pairs that may be referenced by * the device node. @@ -150,7 +143,6 @@ struct kbase_context; struct kbase_device; struct kbase_as; struct kbase_mmu_setup; -struct kbase_kinstr_jm; struct kbase_io; #if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) @@ -358,11 +350,7 @@ struct kbase_mmu_table { #endif }; -#if MALI_USE_CSF #include "csf/mali_kbase_csf_defs.h" -#else -#include "jm/mali_kbase_jm_defs.h" -#endif #include "mali_kbase_hwaccess_time.h" @@ -459,6 +447,12 @@ struct kbase_clk_rate_trace_manager { * has already been called from the power_on callback, then * the call to it from runtime_gpu_active callback can be * skipped. + * @runtime_suspend_result: Stores the return value of + * kbase_pm_handle_runtime_suspend() to be passed back + * to the kernel. This is consumed in + * kbase_device_runtime_suspend() once the runtime + * suspend event has been handled by the CSF scheduler + * kthread. * @gpu_lost: Flag indicating gpu lost * This structure contains data for the power management framework. * There is one instance of this structure per device in the system. @@ -466,8 +460,8 @@ struct kbase_clk_rate_trace_manager { * @resume_wait: Wait queue to wait for the System suspend/resume of GPU device. * @debug_core_mask: Bit masks identifying the available shader cores that are * specified via sysfs. One mask per job slot. - * @debug_core_mask_all: Bit masks identifying the available shader cores that - * are specified via sysfs. + * @sysfs_gov_core_mask: Bit masks identifying the available shader cores that are + * specified via sysfs when writing to GOV_CORE_MASK. * @callback_power_runtime_init: Callback for initializing the runtime power * management. Return 0 on success, else error code * @callback_power_runtime_term: Callback for terminating the runtime power @@ -485,20 +479,14 @@ struct kbase_pm_device_data { int active_count; bool suspending; bool resuming; -#if MALI_USE_CSF bool runtime_active; -#endif + int runtime_suspend_result; atomic_t gpu_lost; wait_queue_head_t zero_active_count_wait; wait_queue_head_t resume_wait; -#if MALI_USE_CSF u64 debug_core_mask; -#else - /* One mask per job slot. */ - u64 debug_core_mask[BASE_JM_MAX_NR_SLOTS]; - u64 debug_core_mask_all; -#endif /* MALI_USE_CSF */ + u64 sysfs_gov_core_mask; int (*callback_power_runtime_init)(struct kbase_device *kbdev); void (*callback_power_runtime_term)(struct kbase_device *kbdev); @@ -530,13 +518,12 @@ struct kbase_pm_device_data { * @isolation_in_progress_cnt: Number of pages in pool undergoing page isolation. * This is used to avoid race condition between pool termination * and page isolation for page migration. - * @next_pool: Pointer to next pool where pages can be allocated when this - * pool is empty. Pages will spill over to the next pool when - * this pool is full. Can be NULL if there is no next pool. * @dying: true if the pool is being terminated, and any ongoing * operations should be abandoned - * @dont_reclaim: true if the shrinker is forbidden from reclaiming memory from - * this pool, eg during a grow operation + * @pool_supports_reclaim: Whether this pool supports page reclaiming. + * @reclaim_allowed: true if the shrinker is currently allowed to reclaim from this + * pool. Otherwise, false: the shrinker is forbidden from reclaiming + * memory from it - eg during a grow operation. */ struct kbase_mem_pool { struct kbase_device *kbdev; @@ -549,10 +536,9 @@ struct kbase_mem_pool { DEFINE_KBASE_SHRINKER reclaim; atomic_t isolation_in_progress_cnt; - struct kbase_mem_pool *next_pool; - bool dying; - bool dont_reclaim; + bool pool_supports_reclaim; + bool reclaim_allowed; }; /** @@ -599,6 +585,20 @@ struct kbase_mem_pool_group_config { struct kbase_mem_pool_config large[MEMORY_GROUP_MANAGER_NR_GROUPS]; }; +/** + * struct kbase_fw_mem_pool_group - a set of physical memory pools for fw load. + * + * @small: Object containing the state for pool of small size + * physical pages. + * @large: Object containing the state for pool of large size + * physical pages. + * + */ +struct kbase_fw_mem_pool_group { + struct kbase_mem_pool small; + struct kbase_mem_pool large; +}; + /** * struct kbase_devfreq_opp - Lookup table for converting between nominal OPP * frequency, real frequencies and core mask @@ -819,12 +819,11 @@ struct kbase_mem_migrate { * framework. * @fw_load_lock: Mutex to protect firmware loading in @ref kbase_open. * @csf: CSF object for the GPU device. - * @js_data: Per device object encapsulating the current context of - * Job Scheduler, which is global to the device and is not - * tied to any particular struct kbase_context running on - * the device - * @mem_pools: Global pools of free physical memory pages which can - * be used by all the contexts. + * @pgd_mem_pool: Global pool of free physical memory pages dedicated to + * allocation of pgd pages that are not associated with + * contexts. + * @fw_mem_pools: Global pools of free physical memory pages which can + * be used by firmware load subroutine. * @memdev: keeps track of the in use physical pages allocated by * the Driver. * @mmu_mode: Pointer to the object containing methods for programming @@ -857,17 +856,6 @@ struct kbase_mem_migrate { * @nr_user_address_spaces: Number of address spaces available to user contexts * @hwcnt_backend_csf_if_fw: Firmware interface to access CSF GPU performance * counters. - * @hwcnt: Structure used for instrumentation and HW counters - * dumping - * @hwcnt.lock: The lock should be used when accessing any of the - * following members - * @hwcnt.kctx: kbase context - * @hwcnt.addr: HW counter address - * @hwcnt.addr_bytes: HW counter size in bytes - * @hwcnt.backend: Kbase instrumentation backend - * @hwcnt_gpu_jm_backend: Job manager GPU backend interface, used as superclass reference - * pointer by hwcnt_gpu_iface, which wraps this implementation in - * order to extend it with periodic dumping functionality. * @hwcnt_gpu_iface: Backend interface for GPU hardware counter access. * @hwcnt_watchdog_timer: Watchdog interface, used by the GPU backend hwcnt_gpu_iface to * perform periodic dumps in order to prevent hardware counter value @@ -955,26 +943,12 @@ struct kbase_mem_migrate { * @previous_frequency: Previous frequency of GPU clock used for * KBASE_HW_ISSUE_GPU2017_1336 workaround, This clock is * restored when L2 is powered on. - * @job_fault_debug: Flag to control the dumping of debug data for job faults, - * set when the 'job_fault' debugfs file is opened. * @mali_debugfs_directory: Root directory for the debugfs files created by the driver * @debugfs_ctx_directory: Directory inside the @mali_debugfs_directory containing * a sub-directory for every context. * @debugfs_instr_directory: Instrumentation debugfs directory * @debugfs_as_read_bitmap: bitmap of address spaces for which the bus or page fault * has occurred. - * @job_fault_wq: Waitqueue to block the job fault dumping daemon till the - * occurrence of a job fault. - * @job_fault_resume_wq: Waitqueue on which every context with a faulty job wait - * for the job fault dumping to complete before they can - * do bottom half of job done for the atoms which followed - * the faulty atom. - * @job_fault_resume_workq: workqueue to process the work items queued for the faulty - * atoms, whereby the work item function waits for the dumping - * to get completed. - * @job_fault_event_list: List of atoms, each belonging to a different context, which - * generated a job fault. - * @job_fault_event_lock: Lock to protect concurrent accesses to @job_fault_event_list * @regs_dump_debugfs_data: Contains the offset of register to be read through debugfs * file "read_register". * @regs_dump_debugfs_data.reg_offset: Contains the offset of register to be @@ -995,8 +969,6 @@ struct kbase_mem_migrate { * power on for GPU is started. * @infinite_cache_active_default: Set to enable using infinite cache for all the * allocations of a new context. - * @mem_pool_defaults: Default configuration for the group of memory pools - * created for a new context. * @current_gpu_coherency_mode: coherency mode in use, which can be different * from @system_coherency, when using protected mode. * @system_coherency: coherency mode as retrieved from the device tree. @@ -1029,10 +1001,6 @@ struct kbase_mem_migrate { * the updates made to Job dispatcher + scheduler states. * @mmu_hw_mutex: Protects access to MMU operations and address space * related state. - * @serialize_jobs: Currently used mode for serialization of jobs, both - * intra & inter slots serialization is supported. - * @js_ctx_scheduling_mode: Context scheduling mode currently being used by - * Job Scheduler * @l2_size_override: Used to set L2 cache size via device tree blob * @l2_hash_override: Used to set L2 cache hash via device tree blob * @l2_hash_values_override: true if @l2_hash_values is valid. @@ -1086,9 +1054,7 @@ struct kbase_device { u32 hw_quirks_tiler; u32 hw_quirks_mmu; u32 hw_quirks_gpu; -#if MALI_USE_CSF u32 hw_quirks_ne; -#endif struct list_head entry; struct device *dev; @@ -1128,7 +1094,8 @@ struct kbase_device { #endif /* !IS_ENABLED(CONFIG_MALI_VALHALL_REAL_HW) */ struct kbase_pm_device_data pm; - struct kbase_mem_pool_group mem_pools; + struct kbase_mem_pool pgd_mem_pool; + struct kbase_fw_mem_pool_group fw_mem_pools; struct kbasep_mem_device memdev; struct kbase_mmu_mode const *mmu_mode; @@ -1166,21 +1133,7 @@ struct kbase_device { */ u32 mma_wa_id; -#if MALI_USE_CSF struct kbase_hwcnt_backend_csf_if hwcnt_backend_csf_if_fw; -#else - struct kbase_hwcnt { - spinlock_t lock; - - struct kbase_context *kctx; - u64 addr; - u64 addr_bytes; - - struct kbase_instr_backend backend; - } hwcnt; - - struct kbase_hwcnt_backend_interface hwcnt_gpu_jm_backend; -#endif struct kbase_hwcnt_backend_interface hwcnt_gpu_iface; struct kbase_hwcnt_watchdog_interface hwcnt_watchdog_timer; @@ -1253,10 +1206,6 @@ struct kbase_device { #endif /* CONFIG_MALI_VALHALL_DEVFREQ */ unsigned long previous_frequency; -#if !MALI_USE_CSF - atomic_t job_fault_debug; -#endif /* !MALI_USE_CSF */ - struct dentry *mali_debugfs_directory; struct dentry *debugfs_ctx_directory; struct dentry *debugfs_instr_directory; @@ -1265,14 +1214,6 @@ struct kbase_device { u64 debugfs_as_read_bitmap; #endif /* CONFIG_MALI_VALHALL_DEBUG */ -#if !MALI_USE_CSF - wait_queue_head_t job_fault_wq; - wait_queue_head_t job_fault_resume_wq; - struct workqueue_struct *job_fault_resume_workq; - struct list_head job_fault_event_list; - spinlock_t job_fault_event_lock; -#endif /* !MALI_USE_CSF */ - #if !MALI_CUSTOMER_RELEASE struct { u32 reg_offset; @@ -1294,8 +1235,6 @@ struct kbase_device { bool infinite_cache_active_default; - struct kbase_mem_pool_group_config mem_pool_defaults; - u32 current_gpu_coherency_mode; u32 system_coherency; @@ -1335,19 +1274,8 @@ struct kbase_device { u32 sysc_alloc[GPU_SYSC_ALLOC_COUNT]; struct mutex fw_load_lock; -#if MALI_USE_CSF /* CSF object for the GPU device. */ struct kbase_csf_device csf; -#else - struct kbasep_js_device_data js_data; - - /* See KBASE_JS_*_PRIORITY_MODE for details. */ - u32 js_ctx_scheduling_mode; - - /* See KBASE_SERIALIZE_* for details */ - u8 serialize_jobs; - -#endif /* MALI_USE_CSF */ struct rb_root process_root; struct rb_root dma_buf_root; @@ -1370,22 +1298,9 @@ struct kbase_device { struct notifier_block oom_notifier_block; -#if !MALI_USE_CSF - spinlock_t quick_reset_lock; - bool quick_reset_enabled; - /* - * 进入 quck_reset_mode 后 (quick_reset_enabled 为 true), - * 对已经进入 KBASE_JD_ATOM_STATE_HW_COMPLETED 状态的 atom 的计数. - * - * 若 num_of_atoms_hw_completed 达到一定值, 将退出 quck_reset_mode. - * 见 kbase_js_complete_atom() 对 num_of_atoms_hw_completed 的引用. - */ - u32 num_of_atoms_hw_completed; -#endif - struct kbase_mem_migrate mem_migrate; -#if MALI_USE_CSF && IS_ENABLED(CONFIG_SYNC_FILE) +#if IS_ENABLED(CONFIG_SYNC_FILE) atomic_t live_fence_metadata; #endif struct kmem_cache *va_region_slab; @@ -1397,9 +1312,7 @@ struct kbase_device { */ struct kbase_gpu_metrics gpu_metrics; #endif -#if MALI_USE_CSF atomic_t fence_signal_timeout_enabled; -#endif struct notifier_block pcm_prioritized_process_nb; @@ -1652,9 +1565,6 @@ struct kbase_sub_alloc { * events to Userspace. * @event_mutex: Lock to protect the concurrent access to @event_list & * @event_mutex. - * @event_closed: Flag set through POST_TERM ioctl, indicates that Driver - * should stop posting events and also inform event handling - * thread that context termination is in progress. * @event_workq: Workqueue for processing work items corresponding to atoms * that do not return an event to userspace. * @event_count: Count of the posted events to be consumed by Userspace. @@ -1710,8 +1620,6 @@ struct kbase_sub_alloc { * an application becomes foreground or goes to the * background. * @csf: kbase csf context - * @jctx: object encapsulating all the Job dispatcher related state, - * including the array of atoms. * @used_pages: Keeps a track of the number of small physical pages in use * for the context. * @nonmapped_pages: Updated in the same way as @used_pages, except for the case @@ -1719,6 +1627,8 @@ struct kbase_sub_alloc { * is reset to 0. * @permanent_mapped_pages: Usage count of permanently mapped memory * @mem_pools: Context-specific pools of free physical memory pages. + * Note that context-specific PGDs are not allocated from this. + * @pgd_mem_pool: Context-specific pool dedicated to allocation of PGDs. * @reclaim: Shrinker object registered with the kernel containing * the pointer to callback function which is invoked under * low memory conditions. In the callback function Driver @@ -1787,23 +1697,6 @@ struct kbase_sub_alloc { * output of "mem_view" debugfs file. * @job_fault_work: Tracking the latest fault dump work item for assisting the * operation of the job-fault-dump debug process. - * @jsctx_queue: Per slot & priority arrays of object containing the root - * of RB-tree holding currently runnable atoms on the job slot - * and the head item of the linked list of atoms blocked on - * cross-slot dependencies. - * @slot_tracking: Tracking and control of this context's use of all job - * slots - * @atoms_pulled_all_slots: Total number of atoms currently pulled from the - * context, across all slots. - * @slots_pullable: Bitmask of slots, indicating the slots for which the - * context has pullable atoms in the runnable tree. - * @completed_jobs: List containing completed atoms for which base_jd_event is - * to be posted. - * @work_count: Number of work items, corresponding to atoms, currently - * pending on job_done workqueue of @jctx. - * @soft_job_timeout: Timer object used for failing/cancelling the waiting - * soft-jobs which have been blocked for more than the - * timeout value used for the soft-jobs * @jit_alloc: Array of 256 pointers to GPU memory regions, used for * just-in-time memory allocations. * @jit_max_allocations: Maximum allowed number of in-flight @@ -1858,30 +1751,22 @@ struct kbase_sub_alloc { * @ext_res_meta_head: A list of sticky external resources which were requested to * be mapped on GPU side, through a softjob atom of type * EXT_RES_MAP or STICKY_RESOURCE_MAP ioctl. - * @age_count: Counter incremented on every call to jd_submit_atom, - * atom is assigned the snapshot of this counter, which - * is used to determine the atom's age when it is added to - * the runnable RB-tree. * @trim_level: Level of JIT allocation trimming to perform on free (0-100%) * @kprcs: Reference to @struct kbase_process that the current * kbase_context belongs to. * @kprcs_link: List link for the list of kbase context maintained * under kbase_process. - * @priority: Indicates the context priority. Used along with @atoms_count - * for context scheduling, protected by hwaccess_lock. - * @atoms_count: Number of GPU atoms currently in use, per priority * @create_flags: Flags used in context creation. - * @kinstr_jm: Kernel job manager instrumentation context handle * @tl_kctx_list_node: List item into the device timeline's list of * contexts, for timeline summarization. * @limited_core_mask: The mask that is applied to the affinity in case of atoms * marked with BASE_JD_REQ_LIMITED_CORE_MASK. - * @platform_data: Pointer to platform specific per-context data. * @task: Pointer to the task structure of the main thread of the process * that created the Kbase context. It would be set only for the * contexts created by the Userspace and not for the contexts * created internally by the Kbase. * @comm: Record the process name + * @offslot_ts: System time in ns when all CSGs belonged to this kctx become offslot. * * A kernel base context is an entity among which the GPU is scheduled. * Each context has its own GPU address space. @@ -1900,9 +1785,6 @@ struct kbase_context { struct list_head event_list; struct list_head event_coalesce_list; struct mutex event_mutex; -#if !MALI_USE_CSF - atomic_t event_closed; -#endif struct workqueue_struct *event_workq; atomic_t event_count; int event_coalesce_count; @@ -1915,29 +1797,11 @@ struct kbase_context { struct list_head mem_partials; struct mutex reg_lock; -#if MALI_USE_CSF atomic64_t num_fixable_allocs; atomic64_t num_fixed_allocs; -#endif struct kbase_reg_zone reg_zone[CONTEXT_ZONE_MAX]; -#if MALI_USE_CSF struct kbase_csf_context csf; -#else - struct kbase_jd_context jctx; - struct jsctx_queue jsctx_queue[KBASE_JS_ATOM_SCHED_PRIO_COUNT][BASE_JM_MAX_NR_SLOTS]; - struct kbase_jsctx_slot_tracking slot_tracking[BASE_JM_MAX_NR_SLOTS]; - atomic_t atoms_pulled_all_slots; - - struct list_head completed_jobs; - atomic_t work_count; - struct timer_list soft_job_timeout; - - int priority; - s16 atoms_count[KBASE_JS_ATOM_SCHED_PRIO_COUNT]; - u32 slots_pullable; - u32 age_count; -#endif /* MALI_USE_CSF */ DECLARE_BITMAP(cookies, BITS_PER_LONG); struct kbase_va_region *pending_regions[BITS_PER_LONG]; @@ -1951,6 +1815,7 @@ struct kbase_context { atomic_t permanent_mapped_pages; struct kbase_mem_pool_group mem_pools; + struct kbase_mem_pool pgd_mem_pool; DEFINE_KBASE_SHRINKER reclaim; @@ -1966,11 +1831,9 @@ struct kbase_context { struct mm_struct *process_mm; u64 gpu_va_end; -#if MALI_USE_CSF u32 running_total_tiler_heap_nr_chunks; u64 running_total_tiler_heap_memory; u64 peak_total_tiler_heap_memory; -#endif bool jit_va; #if IS_ENABLED(CONFIG_DEBUG_FS) @@ -2011,17 +1874,10 @@ struct kbase_context { base_context_create_flags create_flags; -#if !MALI_USE_CSF - struct kbase_kinstr_jm *kinstr_jm; -#endif struct list_head tl_kctx_list_node; u64 limited_core_mask; -#if !MALI_USE_CSF - void *platform_data; -#endif - struct task_struct *task; #if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) @@ -2033,6 +1889,8 @@ struct kbase_context { #endif char comm[TASK_COMM_LEN]; + + u64 offslot_ts; }; /** diff --git a/drivers/gpu/arm/valhall/mali_kbase_dummy_job_wa.c b/drivers/gpu/arm/valhall/mali_kbase_dummy_job_wa.c deleted file mode 100644 index 10bbb64f7def..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_dummy_job_wa.c +++ /dev/null @@ -1,404 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Implementation of the dummy job execution workaround for the GPU hang issue. - */ - -#include -#include -#include -#include - -#include -#include - -#define DUMMY_JOB_WA_BINARY_NAME "valhall-1691526.wa" - -struct wa_header { - u16 signature; - u16 version; - u32 info_offset; -} __packed; - -struct wa_v2_info { - u64 jc; - u32 js; - u32 blob_offset; - u64 flags; -} __packed; - -struct wa_blob { - u64 base; - u32 size; - u32 map_flags; - u32 payload_offset; - u32 blob_offset; -} __packed; - -static bool within_range(const u8 *base, const u8 *end, off_t off, size_t sz) -{ - return !((size_t)(end - base - off) < sz); -} - -static u32 wait_any(struct kbase_device *kbdev, off_t offset, u32 bits) -{ - int loop; - const int timeout = 100; - u32 val; - - for (loop = 0; loop < timeout; loop++) { - val = kbase_reg_read32(kbdev, offset); - if (val & bits) - break; - udelay(10); - } - - if (loop == timeout) { - dev_err(kbdev->dev, - "Timeout reading register 0x%lx, bits 0x%lx, last read was 0x%lx\n", - (unsigned long)offset, (unsigned long)bits, (unsigned long)val); - } - - return (val & bits); -} - -static inline int run_job(struct kbase_device *kbdev, int as, u32 slot, u64 cores, u64 jc) -{ - u32 done; - - /* setup job */ - kbase_reg_write64(kbdev, JOB_SLOT_OFFSET(slot, HEAD_NEXT), jc); - kbase_reg_write64(kbdev, JOB_SLOT_OFFSET(slot, AFFINITY_NEXT), cores); - kbase_reg_write32(kbdev, JOB_SLOT_OFFSET(slot, CONFIG_NEXT), - JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK | (unsigned int)as); - - /* go */ - kbase_reg_write32(kbdev, JOB_SLOT_OFFSET(slot, COMMAND_NEXT), JS_COMMAND_START); - - /* wait for the slot to finish (done, error) */ - done = wait_any(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_RAWSTAT), - (1ul << (16 + slot)) | (1ul << slot)); - kbase_reg_write32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_CLEAR), done); - - if (done != (1ul << slot)) { - dev_err(kbdev->dev, "Failed to run WA job on slot %u cores 0x%llx: done 0x%lx\n", - slot, (unsigned long long)cores, (unsigned long)done); - dev_err(kbdev->dev, "JS_STATUS on failure: 0x%x\n", - kbase_reg_read32(kbdev, JOB_SLOT_OFFSET(slot, STATUS))); - - return -EFAULT; - } else { - return 0; - } -} - -/* To be called after power up & MMU init, but before everything else */ -int kbase_dummy_job_wa_execute(struct kbase_device *kbdev, u64 cores) -{ - int as; - u32 slot; - u64 jc; - int failed = 0; - int runs = 0; - u32 old_gpu_mask; - u32 old_job_mask; - u64 val; - const u32 timeout_us = 10000; - - if (!kbdev) - return -EFAULT; - - if (!kbdev->dummy_job_wa.kctx) - return -EFAULT; - - as = kbdev->dummy_job_wa.kctx->as_nr; - slot = kbdev->dummy_job_wa.slot; - jc = kbdev->dummy_job_wa.jc; - - /* mask off all but MMU IRQs */ - old_gpu_mask = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_MASK)); - old_job_mask = kbase_reg_read32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_MASK)); - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_MASK), 0); - kbase_reg_write32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_MASK), 0); - - /* power up requested cores */ - kbase_reg_write64(kbdev, GPU_CONTROL_ENUM(SHADER_PWRON), cores); - - if (kbdev->dummy_job_wa.flags & KBASE_DUMMY_JOB_WA_FLAG_WAIT_POWERUP) { - /* wait for power-ups */ - kbase_reg_poll64_timeout(kbdev, GPU_CONTROL_ENUM(SHADER_READY), val, - (val & cores) == cores, 10, timeout_us, false); - } - - if (kbdev->dummy_job_wa.flags & KBASE_DUMMY_JOB_WA_FLAG_SERIALIZE) { - size_t i; - - /* do for each requested core */ - for (i = 0; i < sizeof(cores) * 8; i++) { - u64 affinity; - - affinity = 1ull << i; - - if (!(cores & affinity)) - continue; - - if (run_job(kbdev, as, slot, affinity, jc)) - failed++; - runs++; - } - - } else { - if (run_job(kbdev, as, slot, cores, jc)) - failed++; - runs++; - } - - if (kbdev->dummy_job_wa.flags & KBASE_DUMMY_JOB_WA_FLAG_LOGICAL_SHADER_POWER) { - /* power off shader cores (to reduce any dynamic leakage) */ - kbase_reg_write64(kbdev, GPU_CONTROL_ENUM(SHADER_PWROFF), cores); - - /* wait for power off complete */ - kbase_reg_poll64_timeout(kbdev, GPU_CONTROL_ENUM(SHADER_READY), val, !(val & cores), - 10, timeout_us, false); - kbase_reg_poll64_timeout(kbdev, GPU_CONTROL_ENUM(SHADER_PWRTRANS), val, - !(val & cores), 10, timeout_us, false); - - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_CLEAR), U32_MAX); - } - - /* restore IRQ masks */ - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_MASK), old_gpu_mask); - kbase_reg_write32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_MASK), old_job_mask); - - if (failed) - dev_err(kbdev->dev, "WA complete with %d failures out of %d runs\n", failed, runs); - - return failed ? -EFAULT : 0; -} - -static ssize_t dummy_job_wa_info_show(struct device *const dev, struct device_attribute *const attr, - char *const buf) -{ - struct kbase_device *const kbdev = dev_get_drvdata(dev); - int err; - - CSTD_UNUSED(attr); - - if (!kbdev || !kbdev->dummy_job_wa.kctx) - return -ENODEV; - - err = scnprintf(buf, PAGE_SIZE, "slot %u flags %llx\n", kbdev->dummy_job_wa.slot, - kbdev->dummy_job_wa.flags); - - return err; -} - -static DEVICE_ATTR_RO(dummy_job_wa_info); - -static bool wa_blob_load_needed(struct kbase_device *kbdev) -{ - if (kbdev->gpu_props.impl_tech >= THREAD_FEATURES_IMPLEMENTATION_TECHNOLOGY_FPGA) - return false; - - if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TTRX_3485)) - return true; - - return false; -} - -int kbase_dummy_job_wa_load(struct kbase_device *kbdev) -{ - const struct firmware *firmware; - static const char wa_name[] = DUMMY_JOB_WA_BINARY_NAME; - const u32 signature = 0x4157; - const u32 version = 2; - const u8 *fw_end; - const u8 *fw; - const struct wa_header *header; - const struct wa_v2_info *v2_info; - u32 blob_offset; - int err; - struct kbase_context *kctx; - - /* Calls to this function are inherently asynchronous, with respect to - * MMU operations. - */ - const enum kbase_caller_mmu_sync_info mmu_sync_info = CALLER_MMU_ASYNC; - - lockdep_assert_held(&kbdev->fw_load_lock); - - if (!wa_blob_load_needed(kbdev)) - return 0; - - /* load the wa */ - err = request_firmware(&firmware, wa_name, kbdev->dev); - - if (err) { - dev_err(kbdev->dev, - "WA blob missing. Please refer to the Arm Mali DDK Valhall Release Notes, " - "Part number DC-06002 or contact support-mali@arm.com - driver probe will be failed"); - return -ENODEV; - } - - kctx = kbase_create_context(kbdev, true, BASE_CONTEXT_CREATE_FLAG_NONE, 0, NULL); - - if (!kctx) { - dev_err(kbdev->dev, "Failed to create WA context\n"); - goto no_ctx; - } - - fw = firmware->data; - fw_end = fw + firmware->size; - - dev_dbg(kbdev->dev, "Loaded firmware of size %zu bytes\n", firmware->size); - - if (!within_range(fw, fw_end, 0, sizeof(*header))) { - dev_err(kbdev->dev, "WA too small\n"); - goto bad_fw; - } - - header = (const struct wa_header *)(fw + 0); - - if (header->signature != signature) { - dev_err(kbdev->dev, "WA signature failure: 0x%lx\n", - (unsigned long)header->signature); - goto bad_fw; - } - - if (header->version != version) { - dev_err(kbdev->dev, "WA version 0x%lx not supported\n", - (unsigned long)header->version); - goto bad_fw; - } - - if (!within_range(fw, fw_end, header->info_offset, sizeof(*v2_info))) { - dev_err(kbdev->dev, "WA info offset out of bounds\n"); - goto bad_fw; - } - - v2_info = (const struct wa_v2_info *)(fw + header->info_offset); - - if (v2_info->flags & ~KBASE_DUMMY_JOB_WA_FLAGS) { - dev_err(kbdev->dev, "Unsupported WA flag(s): 0x%llx\n", - (unsigned long long)v2_info->flags); - goto bad_fw; - } - - kbdev->dummy_job_wa.slot = v2_info->js; - kbdev->dummy_job_wa.jc = v2_info->jc; - kbdev->dummy_job_wa.flags = v2_info->flags; - - blob_offset = v2_info->blob_offset; - - while (blob_offset) { - const struct wa_blob *blob; - size_t nr_pages; - base_mem_alloc_flags flags; - u64 gpu_va; - struct kbase_va_region *va_region; - - if (!within_range(fw, fw_end, blob_offset, sizeof(*blob))) { - dev_err(kbdev->dev, "Blob offset out-of-range: 0x%lx\n", - (unsigned long)blob_offset); - goto bad_fw; - } - - blob = (const struct wa_blob *)(fw + blob_offset); - if (!within_range(fw, fw_end, blob->payload_offset, blob->size)) { - dev_err(kbdev->dev, "Payload out-of-bounds\n"); - goto bad_fw; - } - - gpu_va = blob->base; - if (PAGE_ALIGN(gpu_va) != gpu_va) { - dev_err(kbdev->dev, "blob not page aligned\n"); - goto bad_fw; - } - nr_pages = PFN_UP(blob->size); - flags = blob->map_flags | BASEP_MEM_FLAG_MAP_FIXED; - - va_region = kbase_mem_alloc(kctx, nr_pages, nr_pages, 0, &flags, &gpu_va, - mmu_sync_info); - - if (!va_region) { - dev_err(kbdev->dev, "Failed to allocate for blob\n"); - } else { - struct kbase_vmap_struct vmap = { 0 }; - const u8 *payload; - void *dst; - - /* copy the payload, */ - payload = fw + blob->payload_offset; - - dst = kbase_vmap(kctx, va_region->start_pfn << PAGE_SHIFT, - nr_pages << PAGE_SHIFT, &vmap); - - if (dst) { - memcpy(dst, payload, blob->size); - kbase_vunmap(kctx, &vmap); - } else { - dev_err(kbdev->dev, "Failed to copy payload\n"); - } - } - blob_offset = blob->blob_offset; /* follow chain */ - } - - release_firmware(firmware); - - kbasep_js_schedule_privileged_ctx(kbdev, kctx); - - kbdev->dummy_job_wa.kctx = kctx; - - err = sysfs_create_file(&kbdev->dev->kobj, &dev_attr_dummy_job_wa_info.attr); - if (err) - dev_err(kbdev->dev, "SysFS file creation for dummy job wa failed\n"); - - return 0; - -bad_fw: - kbase_destroy_context(kctx); -no_ctx: - release_firmware(firmware); - return -EFAULT; -} - -void kbase_dummy_job_wa_cleanup(struct kbase_device *kbdev) -{ - struct kbase_context *wa_kctx; - - /* return if the dummy job has not been loaded */ - if (kbdev->dummy_job_wa_loaded == false) - return; - - /* Can be safely called even if the file wasn't created on probe */ - sysfs_remove_file(&kbdev->dev->kobj, &dev_attr_dummy_job_wa_info.attr); - - wa_kctx = READ_ONCE(kbdev->dummy_job_wa.kctx); - WRITE_ONCE(kbdev->dummy_job_wa.kctx, NULL); - /* make this write visible before we tear down the ctx */ - smp_mb(); - - if (wa_kctx) { - kbasep_js_release_privileged_ctx(kbdev, wa_kctx); - kbase_destroy_context(wa_kctx); - } -} diff --git a/drivers/gpu/arm/valhall/mali_kbase_dummy_job_wa.h b/drivers/gpu/arm/valhall/mali_kbase_dummy_job_wa.h deleted file mode 100644 index b46197aa6df3..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_dummy_job_wa.h +++ /dev/null @@ -1,72 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#ifndef _KBASE_DUMMY_JOB_WORKAROUND_ -#define _KBASE_DUMMY_JOB_WORKAROUND_ - -#define KBASE_DUMMY_JOB_WA_FLAG_SERIALIZE (1ull << 0) -#define KBASE_DUMMY_JOB_WA_FLAG_WAIT_POWERUP (1ull << 1) -#define KBASE_DUMMY_JOB_WA_FLAG_LOGICAL_SHADER_POWER (1ull << 2) - -#define KBASE_DUMMY_JOB_WA_FLAGS \ - (KBASE_DUMMY_JOB_WA_FLAG_SERIALIZE | KBASE_DUMMY_JOB_WA_FLAG_WAIT_POWERUP | \ - KBASE_DUMMY_JOB_WA_FLAG_LOGICAL_SHADER_POWER) - -#if MALI_USE_CSF - -static inline int kbase_dummy_job_wa_load(struct kbase_device *kbdev) -{ - CSTD_UNUSED(kbdev); - return 0; -} - -static inline void kbase_dummy_job_wa_cleanup(struct kbase_device *kbdev) -{ - CSTD_UNUSED(kbdev); -} - -static inline int kbase_dummy_job_wa_execute(struct kbase_device *kbdev, u64 cores) -{ - CSTD_UNUSED(kbdev); - CSTD_UNUSED(cores); - return 0; -} - -static inline bool kbase_dummy_job_wa_enabled(struct kbase_device *kbdev) -{ - CSTD_UNUSED(kbdev); - return false; -} - -#else - -int kbase_dummy_job_wa_load(struct kbase_device *kbdev); -void kbase_dummy_job_wa_cleanup(struct kbase_device *kbdev); -int kbase_dummy_job_wa_execute(struct kbase_device *kbdev, u64 cores); - -static inline bool kbase_dummy_job_wa_enabled(struct kbase_device *kbdev) -{ - return (kbdev->dummy_job_wa.kctx != NULL); -} - -#endif /* MALI_USE_CSF */ - -#endif /* _KBASE_DUMMY_JOB_WORKAROUND_ */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_dvfs_debugfs.c b/drivers/gpu/arm/valhall/mali_kbase_dvfs_debugfs.c index e912178b3324..f33a484a6796 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_dvfs_debugfs.c +++ b/drivers/gpu/arm/valhall/mali_kbase_dvfs_debugfs.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -38,16 +38,10 @@ static int kbasep_dvfs_utilization_debugfs_show(struct seq_file *file, void *dat struct kbase_device *kbdev = file->private; CSTD_UNUSED(data); -#if MALI_USE_CSF seq_printf(file, "busy_time: %u idle_time: %u protm_time: %u\n", kbdev->pm.backend.metrics.values.time_busy, kbdev->pm.backend.metrics.values.time_idle, kbdev->pm.backend.metrics.values.time_in_protm); -#else - seq_printf(file, "busy_time: %u idle_time: %u\n", - kbdev->pm.backend.metrics.values.time_busy, - kbdev->pm.backend.metrics.values.time_idle); -#endif return 0; } diff --git a/drivers/gpu/arm/valhall/mali_kbase_event.c b/drivers/gpu/arm/valhall/mali_kbase_event.c deleted file mode 100644 index 3c59b20913f2..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_event.c +++ /dev/null @@ -1,265 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include -#include -#include -#include - -static struct base_jd_udata kbase_event_process(struct kbase_context *kctx, - struct kbase_jd_atom *katom) -{ - struct base_jd_udata data; - struct kbase_device *kbdev; - - lockdep_assert_held(&kctx->jctx.lock); - - KBASE_DEBUG_ASSERT(kctx != NULL); - KBASE_DEBUG_ASSERT(katom != NULL); - KBASE_DEBUG_ASSERT(katom->status == KBASE_JD_ATOM_STATE_COMPLETED); - - kbdev = kctx->kbdev; - data = katom->udata; - - KBASE_TLSTREAM_TL_NRET_ATOM_CTX(kbdev, katom, kctx); - KBASE_TLSTREAM_TL_DEL_ATOM(kbdev, katom); - - katom->status = KBASE_JD_ATOM_STATE_UNUSED; - dev_dbg(kbdev->dev, "Atom %pK status to unused\n", (void *)katom); - wake_up(&katom->completed); - - return data; -} - -int kbase_event_dequeue(struct kbase_context *kctx, struct base_jd_event_v2 *uevent) -{ - struct kbase_jd_atom *atom; - - KBASE_DEBUG_ASSERT(kctx); - - mutex_lock(&kctx->event_mutex); - - if (list_empty(&kctx->event_list)) { - if (!atomic_read(&kctx->event_closed)) { - mutex_unlock(&kctx->event_mutex); - return -1; - } - - /* generate the BASE_JD_EVENT_DRV_TERMINATED message on the fly */ - mutex_unlock(&kctx->event_mutex); - uevent->event_code = BASE_JD_EVENT_DRV_TERMINATED; - memset(&uevent->udata, 0, sizeof(uevent->udata)); - dev_dbg(kctx->kbdev->dev, - "event system closed, returning BASE_JD_EVENT_DRV_TERMINATED(0x%X)\n", - BASE_JD_EVENT_DRV_TERMINATED); - return 0; - } - - /* normal event processing */ - atomic_dec(&kctx->event_count); - atom = list_entry(kctx->event_list.next, struct kbase_jd_atom, dep_item[0]); - list_del(kctx->event_list.next); - - mutex_unlock(&kctx->event_mutex); - - dev_dbg(kctx->kbdev->dev, "event dequeuing %pK\n", (void *)atom); - uevent->event_code = atom->event_code; - - uevent->atom_number = (atom - kctx->jctx.atoms); - - if (atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) - kbase_jd_free_external_resources(atom); - - mutex_lock(&kctx->jctx.lock); - uevent->udata = kbase_event_process(kctx, atom); - mutex_unlock(&kctx->jctx.lock); - - return 0; -} - -KBASE_EXPORT_TEST_API(kbase_event_dequeue); - -/** - * kbase_event_process_noreport_worker - Worker for processing atoms that do not - * return an event but do have external - * resources - * @data: Work structure - */ -static void kbase_event_process_noreport_worker(struct work_struct *data) -{ - struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work); - struct kbase_context *kctx = katom->kctx; - - if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) - kbase_jd_free_external_resources(katom); - - mutex_lock(&kctx->jctx.lock); - kbase_event_process(kctx, katom); - mutex_unlock(&kctx->jctx.lock); -} - -/** - * kbase_event_process_noreport - Process atoms that do not return an event - * @kctx: Context pointer - * @katom: Atom to be processed - * - * Atoms that do not have external resources will be processed immediately. - * Atoms that do have external resources will be processed on a workqueue, in - * order to avoid locking issues. - */ -static void kbase_event_process_noreport(struct kbase_context *kctx, struct kbase_jd_atom *katom) -{ - if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { - INIT_WORK(&katom->work, kbase_event_process_noreport_worker); - queue_work(kctx->event_workq, &katom->work); - } else { - kbase_event_process(kctx, katom); - } -} - -/** - * kbase_event_coalesce - Move pending events to the main event list - * @kctx: Context pointer - * - * kctx->event_list and kctx->event_coalesce_count must be protected - * by a lock unless this is the last thread using them - * (and we're about to terminate the lock). - * - * Return: The number of pending events moved to the main event list - */ -static int kbase_event_coalesce(struct kbase_context *kctx) -{ - const int event_count = kctx->event_coalesce_count; - - /* Join the list of pending events onto the tail of the main list - * and reset it - */ - list_splice_tail_init(&kctx->event_coalesce_list, &kctx->event_list); - kctx->event_coalesce_count = 0; - - /* Return the number of events moved */ - return event_count; -} - -void kbase_event_post(struct kbase_context *kctx, struct kbase_jd_atom *atom) -{ - struct kbase_device *kbdev = kctx->kbdev; - - dev_dbg(kbdev->dev, "Posting event for atom %pK\n", (void *)atom); - - if (WARN_ON(atom->status != KBASE_JD_ATOM_STATE_COMPLETED)) { - dev_warn(kbdev->dev, "%s: Atom %d (%pK) not completed (status %d)\n", __func__, - kbase_jd_atom_id(atom->kctx, atom), atom->kctx, atom->status); - return; - } - - if (atom->core_req & BASE_JD_REQ_EVENT_ONLY_ON_FAILURE) { - if (atom->event_code == BASE_JD_EVENT_DONE) { - dev_dbg(kbdev->dev, "Suppressing event (atom done)\n"); - kbase_event_process_noreport(kctx, atom); - return; - } - } - - if (atom->core_req & BASEP_JD_REQ_EVENT_NEVER) { - dev_dbg(kbdev->dev, "Suppressing event (never)\n"); - kbase_event_process_noreport(kctx, atom); - return; - } - KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(kbdev, atom, TL_ATOM_STATE_POSTED); - if (atom->core_req & BASE_JD_REQ_EVENT_COALESCE) { - /* Don't report the event until other event(s) have completed */ - dev_dbg(kbdev->dev, "Deferring event (coalesced)\n"); - mutex_lock(&kctx->event_mutex); - list_add_tail(&atom->dep_item[0], &kctx->event_coalesce_list); - ++kctx->event_coalesce_count; - mutex_unlock(&kctx->event_mutex); - } else { - /* Report the event and any pending events now */ - int event_count = 1; - - mutex_lock(&kctx->event_mutex); - event_count += kbase_event_coalesce(kctx); - list_add_tail(&atom->dep_item[0], &kctx->event_list); - atomic_add(event_count, &kctx->event_count); - mutex_unlock(&kctx->event_mutex); - dev_dbg(kbdev->dev, "Reporting %d events\n", event_count); - - kbase_event_wakeup(kctx); - - /* Post-completion latency */ - trace_sysgraph(SGR_POST, kctx->id, kbase_jd_atom_id(kctx, atom)); - } -} -KBASE_EXPORT_TEST_API(kbase_event_post); - -void kbase_event_close(struct kbase_context *kctx) -{ - mutex_lock(&kctx->event_mutex); - atomic_set(&kctx->event_closed, true); - mutex_unlock(&kctx->event_mutex); - kbase_event_wakeup(kctx); -} - -int kbase_event_init(struct kbase_context *kctx) -{ - KBASE_DEBUG_ASSERT(kctx); - - INIT_LIST_HEAD(&kctx->event_list); - INIT_LIST_HEAD(&kctx->event_coalesce_list); - mutex_init(&kctx->event_mutex); - kctx->event_workq = alloc_workqueue("kbase_event", WQ_MEM_RECLAIM, 1); - - if (kctx->event_workq == NULL) - return -EINVAL; - - return 0; -} - -KBASE_EXPORT_TEST_API(kbase_event_init); - -void kbase_event_cleanup(struct kbase_context *kctx) -{ - int event_count; - - KBASE_DEBUG_ASSERT(kctx); - KBASE_DEBUG_ASSERT(kctx->event_workq); - - flush_workqueue(kctx->event_workq); - destroy_workqueue(kctx->event_workq); - - /* We use kbase_event_dequeue to remove the remaining events as that - * deals with all the cleanup needed for the atoms. - * - * Note: use of kctx->event_list without a lock is safe because this must be the last - * thread using it (because we're about to terminate the lock) - */ - event_count = kbase_event_coalesce(kctx); - atomic_add(event_count, &kctx->event_count); - - while (!list_empty(&kctx->event_list)) { - struct base_jd_event_v2 event; - - kbase_event_dequeue(kctx, &event); - } -} - -KBASE_EXPORT_TEST_API(kbase_event_cleanup); diff --git a/drivers/gpu/arm/valhall/mali_kbase_fence.c b/drivers/gpu/arm/valhall/mali_kbase_fence.c deleted file mode 100644 index 8b0505a800a4..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_fence.c +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2011-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include -#include -#include -#include -#include - -#include - -/* Spin lock protecting all Mali fences as fence->lock. */ -static DEFINE_SPINLOCK(kbase_fence_lock); - -struct dma_fence *kbase_fence_out_new(struct kbase_jd_atom *katom) -{ - struct dma_fence *fence; - - WARN_ON(katom->dma_fence.fence); - - fence = kzalloc(sizeof(*fence), GFP_KERNEL); - if (!fence) - return NULL; - - dma_fence_init(fence, &kbase_fence_ops, &kbase_fence_lock, katom->dma_fence.context, - (u64)atomic_inc_return(&katom->dma_fence.seqno)); - - katom->dma_fence.fence = fence; - - return fence; -} diff --git a/drivers/gpu/arm/valhall/mali_kbase_fence.h b/drivers/gpu/arm/valhall/mali_kbase_fence.h index 9fe6c67b464c..f15f4048e922 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_fence.h +++ b/drivers/gpu/arm/valhall/mali_kbase_fence.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -34,7 +34,6 @@ #include #include -#if MALI_USE_CSF /* Number of digits needed to express the max value of given unsigned type. * * Details: The number of digits needed to express the max value of given type is log10(t_max) + 1 @@ -91,12 +90,13 @@ struct kbase_kcpu_dma_fence_meta { * * @base: Fence object itself. * @metadata: Pointer to metadata structure. + * @module: Pointer to the mali_kbase kernel module. */ struct kbase_kcpu_dma_fence { struct dma_fence base; struct kbase_kcpu_dma_fence_meta *metadata; + struct module *module; }; -#endif extern const struct dma_fence_ops kbase_fence_ops; @@ -123,86 +123,6 @@ struct dma_fence *kbase_fence_out_new(struct kbase_jd_atom *katom); } while (0) #endif -#if !MALI_USE_CSF -/** - * kbase_fence_out_remove() - Removes the output fence from atom - * @katom: Atom to remove output fence for - * - * This will also release the reference to this fence which the atom keeps - */ -static inline void kbase_fence_out_remove(struct kbase_jd_atom *katom) -{ - if (katom->dma_fence.fence) { - dma_fence_put(katom->dma_fence.fence); - katom->dma_fence.fence = NULL; - } -} - -#if IS_ENABLED(CONFIG_SYNC_FILE) -/** - * kbase_fence_in_remove() - Removes the input fence from atom - * @katom: Atom to remove input fence for - * - * This will also release the reference to this fence which the atom keeps - */ -static inline void kbase_fence_in_remove(struct kbase_jd_atom *katom) -{ - if (katom->dma_fence.fence_in) { - dma_fence_put(katom->dma_fence.fence_in); - katom->dma_fence.fence_in = NULL; - } -} -#endif - -/** - * kbase_fence_out_is_ours() - Check if atom has a valid fence created by us - * @katom: Atom to check output fence for - * - * Return: true if fence exists and is valid, otherwise false - */ -static inline bool kbase_fence_out_is_ours(struct kbase_jd_atom *katom) -{ - return katom->dma_fence.fence && katom->dma_fence.fence->ops == &kbase_fence_ops; -} - -/** - * kbase_fence_out_signal() - Signal output fence of atom - * @katom: Atom to signal output fence for - * @status: Status to signal with (0 for success, < 0 for error) - * - * Return: 0 on success, < 0 on error - */ -static inline int kbase_fence_out_signal(struct kbase_jd_atom *katom, int status) -{ - if (status) - dma_fence_set_error_helper(katom->dma_fence.fence, status); - return dma_fence_signal(katom->dma_fence.fence); -} - -#if IS_ENABLED(CONFIG_SYNC_FILE) -/** - * kbase_fence_in_get() - Retrieve input fence for atom. - * @katom: Atom to get input fence from - * - * A ref will be taken for the fence, so use @kbase_fence_put() to release it - * - * Return: The fence, or NULL if there is no input fence for atom - */ -#define kbase_fence_in_get(katom) dma_fence_get((katom)->dma_fence.fence_in) -#endif - -/** - * kbase_fence_out_get() - Retrieve output fence for atom. - * @katom: Atom to get output fence from - * - * A ref will be taken for the fence, so use @kbase_fence_put() to release it - * - * Return: The fence, or NULL if there is no output fence for atom - */ -#define kbase_fence_out_get(katom) dma_fence_get((katom)->dma_fence.fence) - -#endif /* !MALI_USE_CSF */ - /** * kbase_fence_get() - Retrieve fence for a KCPUQ fence command. * @fence_info: KCPUQ fence command @@ -213,7 +133,6 @@ static inline int kbase_fence_out_signal(struct kbase_jd_atom *katom, int status */ #define kbase_fence_get(fence_info) dma_fence_get((fence_info)->fence) -#if MALI_USE_CSF static inline struct kbase_kcpu_dma_fence *kbase_kcpu_dma_fence_get(struct dma_fence *fence) { if (fence->ops == &kbase_fence_ops) @@ -237,7 +156,6 @@ static inline void kbase_kcpu_dma_fence_put(struct dma_fence *fence) if (kcpu_fence) kbase_kcpu_dma_fence_meta_put(kcpu_fence->metadata); } -#endif /* MALI_USE_CSF */ /** * kbase_fence_put() - Releases a reference to a fence diff --git a/drivers/gpu/arm/valhall/mali_kbase_fence_ops.c b/drivers/gpu/arm/valhall/mali_kbase_fence_ops.c index b10e69cea565..faaf4f1a82dc 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_fence_ops.c +++ b/drivers/gpu/arm/valhall/mali_kbase_fence_ops.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -31,21 +31,12 @@ static const char *kbase_fence_get_driver_name(struct dma_fence *fence) return KBASE_DRV_NAME; } -#if MALI_USE_CSF static const char *kbase_fence_get_timeline_name(struct dma_fence *fence) { struct kbase_kcpu_dma_fence *kcpu_fence = (struct kbase_kcpu_dma_fence *)fence; return kcpu_fence->metadata->timeline_name; } -#else -static const char *kbase_fence_get_timeline_name(struct dma_fence *fence) -{ - CSTD_UNUSED(fence); - - return KBASE_TIMELINE_NAME; -} -#endif /* MALI_USE_CSF */ static bool kbase_fence_enable_signaling(struct dma_fence *fence) { @@ -67,12 +58,13 @@ static void kbase_fence_fence_value_str(struct dma_fence *fence, char *str, int pr_err("Fail to encode fence seqno to string"); } -#if MALI_USE_CSF static void kbase_fence_release(struct dma_fence *fence) { struct kbase_kcpu_dma_fence *kcpu_fence = (struct kbase_kcpu_dma_fence *)fence; kbase_kcpu_dma_fence_meta_put(kcpu_fence->metadata); + if (likely(kcpu_fence->module)) + module_put(kcpu_fence->module); kfree(kcpu_fence); } @@ -83,13 +75,5 @@ const struct dma_fence_ops kbase_fence_ops = { .wait = dma_fence_default_wait, .enable_signaling = kbase_fence_enable_signaling, .fence_value_str = kbase_fence_fence_value_str, .release = kbase_fence_release }; -#else -extern const struct dma_fence_ops kbase_fence_ops; /* silence checker warning */ -const struct dma_fence_ops kbase_fence_ops = { .wait = dma_fence_default_wait, - .get_driver_name = kbase_fence_get_driver_name, - .get_timeline_name = kbase_fence_get_timeline_name, - .enable_signaling = kbase_fence_enable_signaling, - .fence_value_str = kbase_fence_fence_value_str }; -#endif /* MALI_USE_CSF */ KBASE_EXPORT_TEST_API(kbase_fence_ops); diff --git a/drivers/gpu/arm/valhall/mali_kbase_gpuprops.c b/drivers/gpu/arm/valhall/mali_kbase_gpuprops.c index 55a7a796c58a..fd73c86e35a6 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_gpuprops.c +++ b/drivers/gpu/arm/valhall/mali_kbase_gpuprops.c @@ -239,11 +239,7 @@ static int kbase_gpuprops_get_props(struct kbase_device *kbdev) else gpu_props->max_threads = regdump->thread_max_threads; -#if MALI_USE_CSF gpu_props->impl_tech = KBASE_UBFX32(regdump->thread_features, 22U, 2); -#else /* MALI_USE_CSF */ - gpu_props->impl_tech = KBASE_UBFX32(regdump->thread_features, 30U, 2); -#endif /* MALI_USE_CSF */ if (IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI)) gpu_props->impl_tech = THREAD_FEATURES_IMPLEMENTATION_TECHNOLOGY_NO_MALI; @@ -678,9 +674,6 @@ static void kbase_populate_user_data(struct kbase_device *kbdev, struct gpu_prop * GPUs like tTIx have additional fields like LSC_SIZE that are * otherwise reserved/RAZ on older GPUs. */ -#if !MALI_USE_CSF - data->core_props.num_exec_engines = KBASE_UBFX64(regdump->core_features, 0, 4); -#endif data->l2_props.log2_cache_size = KBASE_UBFX64(regdump->l2_features, 16U, 8); data->coherency_info.coherency = regdump->mem_features; @@ -707,15 +700,9 @@ static void kbase_populate_user_data(struct kbase_device *kbdev, struct gpu_prop else data->thread_props.tls_alloc = regdump->thread_tls_alloc; -#if MALI_USE_CSF data->thread_props.max_registers = KBASE_UBFX32(regdump->thread_features, 0U, 22); data->thread_props.max_task_queue = KBASE_UBFX32(regdump->thread_features, 24U, 8); data->thread_props.max_thread_group_split = 0; -#else - data->thread_props.max_registers = KBASE_UBFX32(regdump->thread_features, 0U, 16); - data->thread_props.max_task_queue = KBASE_UBFX32(regdump->thread_features, 16U, 8); - data->thread_props.max_thread_group_split = KBASE_UBFX32(regdump->thread_features, 24U, 6); -#endif if (data->thread_props.max_registers == 0) { data->thread_props.max_registers = THREAD_MR_DEFAULT; diff --git a/drivers/gpu/arm/valhall/mali_kbase_hwaccess_defs.h b/drivers/gpu/arm/valhall/mali_kbase_hwaccess_defs.h index 96aeb6188a05..fba612771e33 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_hwaccess_defs.h +++ b/drivers/gpu/arm/valhall/mali_kbase_hwaccess_defs.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -40,10 +40,6 @@ * @backend: GPU backend specific data for HW access layer */ struct kbase_hwaccess_data { -#if !MALI_USE_CSF - struct kbase_context *active_kctx[BASE_JM_MAX_NR_SLOTS]; -#endif - struct kbase_backend_data backend; }; diff --git a/drivers/gpu/arm/valhall/mali_kbase_hwaccess_jm.h b/drivers/gpu/arm/valhall/mali_kbase_hwaccess_jm.h deleted file mode 100644 index ed91019ff74c..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_hwaccess_jm.h +++ /dev/null @@ -1,308 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * HW access job manager common APIs - */ - -#ifndef _KBASE_HWACCESS_JM_H_ -#define _KBASE_HWACCESS_JM_H_ - -/** - * kbase_backend_run_atom() - Run an atom on the GPU - * @kbdev: Device pointer - * @katom: Atom to run - * - * Caller must hold the HW access lock - */ -void kbase_backend_run_atom(struct kbase_device *kbdev, struct kbase_jd_atom *katom); - -/** - * kbase_backend_slot_update - Update state based on slot ringbuffers - * - * @kbdev: Device pointer - * - * Inspect the jobs in the slot ringbuffers and update state. - * - * This will cause jobs to be submitted to hardware if they are unblocked - */ -void kbase_backend_slot_update(struct kbase_device *kbdev); - -/** - * kbase_backend_find_and_release_free_address_space() - Release a free AS - * @kbdev: Device pointer - * @kctx: Context pointer - * - * This function can evict an idle context from the runpool, freeing up the - * address space it was using. - * - * The address space is marked as in use. The caller must either assign a - * context using kbase_gpu_use_ctx(), or release it using - * kbase_ctx_sched_release() - * - * Return: Number of free address space, or KBASEP_AS_NR_INVALID if none - * available - */ -int kbase_backend_find_and_release_free_address_space(struct kbase_device *kbdev, - struct kbase_context *kctx); - -/** - * kbase_backend_use_ctx() - Activate a currently unscheduled context, using the - * provided address space. - * @kbdev: Device pointer - * @kctx: Context pointer. May be NULL - * @as_nr: Free address space to use - * - * kbase_gpu_next_job() will pull atoms from the active context. - * - * Return: true if successful, false if ASID not assigned. - */ -bool kbase_backend_use_ctx(struct kbase_device *kbdev, struct kbase_context *kctx, int as_nr); - -/** - * kbase_backend_use_ctx_sched() - Activate a context. - * @kbdev: Device pointer - * @kctx: Context pointer - * @js: Job slot to activate context on - * - * kbase_gpu_next_job() will pull atoms from the active context. - * - * The context must already be scheduled and assigned to an address space. If - * the context is not scheduled, then kbase_gpu_use_ctx() should be used - * instead. - * - * Caller must hold hwaccess_lock - * - * Return: true if context is now active, false otherwise (ie if context does - * not have an address space assigned) - */ -bool kbase_backend_use_ctx_sched(struct kbase_device *kbdev, struct kbase_context *kctx, - unsigned int js); - -/** - * kbase_backend_release_ctx_irq - Release a context from the GPU. This will - * de-assign the assigned address space. - * @kbdev: Device pointer - * @kctx: Context pointer - * - * Caller must hold kbase_device->mmu_hw_mutex and hwaccess_lock - */ -void kbase_backend_release_ctx_irq(struct kbase_device *kbdev, struct kbase_context *kctx); - -/** - * kbase_backend_release_ctx_noirq - Release a context from the GPU. This will - * de-assign the assigned address space. - * @kbdev: Device pointer - * @kctx: Context pointer - * - * Caller must hold kbase_device->mmu_hw_mutex - * - * This function must perform any operations that could not be performed in IRQ - * context by kbase_backend_release_ctx_irq(). - */ -void kbase_backend_release_ctx_noirq(struct kbase_device *kbdev, struct kbase_context *kctx); - -/** - * kbase_backend_cache_clean - Perform a cache clean if the given atom requires - * one - * @kbdev: Device pointer - * @katom: Pointer to the failed atom - * - * On some GPUs, the GPU cache must be cleaned following a failed atom. This - * function performs a clean if it is required by @katom. - */ -void kbase_backend_cache_clean(struct kbase_device *kbdev, struct kbase_jd_atom *katom); - -/** - * kbase_backend_complete_wq() - Perform backend-specific actions required on - * completing an atom. - * @kbdev: Device pointer - * @katom: Pointer to the atom to complete - * - * This function should only be called from kbase_jd_done_worker() or - * js_return_worker(). - * - * Return: true if atom has completed, false if atom should be re-submitted - */ -void kbase_backend_complete_wq(struct kbase_device *kbdev, struct kbase_jd_atom *katom); - -#if !MALI_USE_CSF -/** - * kbase_backend_complete_wq_post_sched - Perform backend-specific actions - * required on completing an atom, after - * any scheduling has taken place. - * @kbdev: Device pointer - * @core_req: Core requirements of atom - * - * This function should only be called from kbase_jd_done_worker() or - * js_return_worker(). - */ -void kbase_backend_complete_wq_post_sched(struct kbase_device *kbdev, base_jd_core_req core_req); -#endif /* !MALI_USE_CSF */ - -/** - * kbase_backend_reset() - The GPU is being reset. Cancel all jobs on the GPU - * and remove any others from the ringbuffers. - * @kbdev: Device pointer - * @end_timestamp: Timestamp of reset - */ -void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp); - -/** - * kbase_backend_inspect_tail - Return the atom currently at the tail of slot - * @js - * @kbdev: Device pointer - * @js: Job slot to inspect - * - * Return: Atom currently at the head of slot @js, or NULL - */ -struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev, unsigned int js); - -/** - * kbase_backend_nr_atoms_on_slot() - Return the number of atoms currently on a - * slot. - * @kbdev: Device pointer - * @js: Job slot to inspect - * - * Return: Number of atoms currently on slot - */ -int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, unsigned int js); - -/** - * kbase_backend_nr_atoms_submitted() - Return the number of atoms on a slot - * that are currently on the GPU. - * @kbdev: Device pointer - * @js: Job slot to inspect - * - * Return: Number of atoms currently on slot @js that are currently on the GPU. - */ -u32 kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, unsigned int js); - -/** - * kbase_backend_ctx_count_changed() - Number of contexts ready to submit jobs - * has changed. - * @kbdev: Device pointer - * - * Perform any required backend-specific actions (eg starting/stopping - * scheduling timers). - */ -void kbase_backend_ctx_count_changed(struct kbase_device *kbdev); - -/** - * kbase_backend_timeouts_changed() - Job Scheduler timeouts have changed. - * @kbdev: Device pointer - * - * Perform any required backend-specific actions (eg updating timeouts of - * currently running atoms). - */ -void kbase_backend_timeouts_changed(struct kbase_device *kbdev); - -/** - * kbase_backend_slot_free() - Return the number of jobs that can be currently - * submitted to slot @js. - * @kbdev: Device pointer - * @js: Job slot to inspect - * - * Return: Number of jobs that can be submitted. - */ -int kbase_backend_slot_free(struct kbase_device *kbdev, unsigned int js); - -/** - * kbase_job_check_leave_disjoint - potentially leave disjoint state - * @kbdev: kbase device - * @target_katom: atom which is finishing - * - * Work out whether to leave disjoint state when finishing an atom that was - * originated by kbase_job_check_enter_disjoint(). - */ -void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, struct kbase_jd_atom *target_katom); - -/** - * kbase_backend_jm_kill_running_jobs_from_kctx - Kill all jobs that are - * currently running on GPU from a context - * @kctx: Context pointer - * - * This is used in response to a page fault to remove all jobs from the faulting - * context from the hardware. - * - * Caller must hold hwaccess_lock. - */ -void kbase_backend_jm_kill_running_jobs_from_kctx(struct kbase_context *kctx); - -/** - * kbase_jm_wait_for_zero_jobs - Wait for context to have zero jobs running, and - * to be descheduled. - * @kctx: Context pointer - * - * This should be called following kbase_js_zap_context(), to ensure the context - * can be safely destroyed. - */ -void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx); - -/** - * kbase_backend_get_current_flush_id - Return the current flush ID - * - * @kbdev: Device pointer - * - * Return: the current flush ID to be recorded for each job chain - */ -u32 kbase_backend_get_current_flush_id(struct kbase_device *kbdev); - -/** - * kbase_job_slot_hardstop - Hard-stop the specified job slot - * @kctx: The kbase context that contains the job(s) that should - * be hard-stopped - * @js: The job slot to hard-stop - * @target_katom: The job that should be hard-stopped (or NULL for all - * jobs from the context) - * Context: - * The job slot lock must be held when calling this function. - */ -void kbase_job_slot_hardstop(struct kbase_context *kctx, unsigned int js, - struct kbase_jd_atom *target_katom); - -/** - * kbase_gpu_atoms_submitted_any() - Inspect whether there are any atoms - * currently on the GPU - * @kbdev: Device pointer - * - * Return: true if there are any atoms on the GPU, false otherwise - */ -bool kbase_gpu_atoms_submitted_any(struct kbase_device *kbdev); - -/** - * kbase_backend_slot_kctx_purge_locked - Perform a purge on the slot_rb tracked - * kctx - * - * @kbdev: Device pointer - * @kctx: The kbase context that needs to be purged from slot_rb[] - * - * For JM GPUs, the L1 read only caches may need a start_flush invalidation, - * potentially on all slots (even if the kctx was only using a single slot), - * following a context termination or address-space ID recycle. This function - * performs a clean-up purge on the given kctx which if it has been tracked by - * slot_rb[] objects. - * - * Caller must hold kbase_device->hwaccess_lock. - */ -void kbase_backend_slot_kctx_purge_locked(struct kbase_device *kbdev, struct kbase_context *kctx); - -#endif /* _KBASE_HWACCESS_JM_H_ */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_hwaccess_pm.h b/drivers/gpu/arm/valhall/mali_kbase_hwaccess_pm.h index 982547d16022..a396f02b59ed 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_hwaccess_pm.h +++ b/drivers/gpu/arm/valhall/mali_kbase_hwaccess_pm.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -114,7 +114,6 @@ void kbase_hwaccess_pm_gpu_active(struct kbase_device *kbdev); */ void kbase_hwaccess_pm_gpu_idle(struct kbase_device *kbdev); -#if MALI_USE_CSF /** * kbase_pm_set_debug_core_mask - Set the debug core mask. * @@ -124,20 +123,6 @@ void kbase_hwaccess_pm_gpu_idle(struct kbase_device *kbdev); * This determines which cores the power manager is allowed to use. */ void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, u64 new_core_mask); -#else -/** - * kbase_pm_set_debug_core_mask - Set the debug core mask. - * - * @kbdev: The kbase device structure for the device (must be a valid pointer) - * @new_core_mask: The core mask to use, as an array where each element refers - * to a job slot. - * @new_core_mask_size: Number of elements in the core mask array. - * - * This determines which cores the power manager is allowed to use. - */ -void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, u64 *new_core_mask, - size_t new_core_mask_size); -#endif /* MALI_USE_CSF */ /** * kbase_pm_ca_get_policy - Get the current policy. diff --git a/drivers/gpu/arm/valhall/mali_kbase_hwaccess_time.h b/drivers/gpu/arm/valhall/mali_kbase_hwaccess_time.h index 222ff2001e56..2df4cad514f8 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_hwaccess_time.h +++ b/drivers/gpu/arm/valhall/mali_kbase_hwaccess_time.h @@ -53,15 +53,12 @@ * */ struct kbase_backend_time { -#if MALI_USE_CSF u64 multiplier; u64 divisor; s64 gpu_timestamp_offset; -#endif unsigned int device_scaled_timeouts[KBASE_TIMEOUT_SELECTOR_COUNT]; }; -#if MALI_USE_CSF /** * kbase_backend_time_convert_gpu_to_cpu() - Convert GPU timestamp to CPU timestamp. * @@ -105,7 +102,6 @@ void kbase_backend_invalidate_gpu_timestamp_offset(struct kbase_device *kbdev); */ u64 kbase_backend_read_gpu_timestamp_offset_reg(struct kbase_device *kbdev); #endif -#endif /** * kbase_backend_get_gpu_time() - Get current GPU time @@ -205,4 +201,11 @@ u64 kbase_arch_timer_get_cntfrq(struct kbase_device *kbdev); */ int kbase_backend_time_init(struct kbase_device *kbdev); +/** + * kbase_gpu_timestamp_offset_debugfs_init - Define debugfs entry to read the GPU timestamp offset. + * + * @kbdev: kbase device pointer + * + */ +void kbase_gpu_timestamp_offset_debugfs_init(struct kbase_device *kbdev); #endif /* _KBASE_BACKEND_TIME_H_ */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_io.c b/drivers/gpu/arm/valhall/mali_kbase_io.c index 33acab795f75..49e1ea5175d6 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_io.c +++ b/drivers/gpu/arm/valhall/mali_kbase_io.c @@ -36,25 +36,6 @@ struct kbase_io { struct kbase_device *kbdev; }; -/** - * kbase_io_is_gpu_removed() - Has the GPU been removed. - * @kbdev: Kbase device pointer - * - * When Kbase takes too long to give up the GPU, the Arbiter - * can remove it. This will then be followed by a GPU lost event. - * This function will return true if the GPU has been removed. - * When this happens register reads will be zero. A zero GPU_ID is - * invalid so this is used to detect when GPU is removed. - * - * Return: True if GPU removed - */ -static bool kbase_io_is_gpu_removed(struct kbase_device *kbdev) -{ - if (!kbase_has_arbiter(kbdev)) - return false; - return (KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(GPU_ID)) == 0); -} - void kbase_io_set_status(struct kbase_io *io, enum kbase_io_status_bits status_bit) { set_bit(status_bit, io->status); @@ -81,21 +62,13 @@ KBASE_EXPORT_TEST_API(kbase_io_is_gpu_powered); bool kbase_io_is_aw_removed(struct kbase_device *kbdev) { - return (kbdev->arb.arb_if && test_bit(KBASE_IO_STATUS_AW_REMOVED, kbdev->io->status)); + return test_bit(KBASE_IO_STATUS_AW_REMOVED, kbdev->io->status); } KBASE_EXPORT_TEST_API(kbase_io_is_aw_removed); bool kbase_io_has_gpu(struct kbase_device *kbdev) { - if (kbase_io_test_status(kbdev, KBASE_IO_STATUS_AW_REMOVED)) - return false; - - if (kbase_io_is_gpu_removed(kbdev)) { - kbase_io_set_status(kbdev->io, KBASE_IO_STATUS_AW_REMOVED); - return false; - } - - return true; + return kbase_io_is_gpu_powered(kbdev) && !kbase_io_is_aw_removed(kbdev); } KBASE_EXPORT_TEST_API(kbase_io_has_gpu); diff --git a/drivers/gpu/arm/valhall/mali_kbase_io.h b/drivers/gpu/arm/valhall/mali_kbase_io.h index 3a352fcb92b4..0f1cf8a6264b 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_io.h +++ b/drivers/gpu/arm/valhall/mali_kbase_io.h @@ -32,14 +32,12 @@ struct kbase_io; /** * enum kbase_io_status_bits - Status bits for kbase I/O interface. * - * @KBASE_IO_STATUS_GPU_SUSPENDED: The GPU is suspended. * @KBASE_IO_STATUS_GPU_OFF: The GPU is OFF. * @KBASE_IO_STATUS_AW_REMOVED: The GPU access window has been removed. * @KBASE_IO_STATUS_NUM_BITS: Number of bits used to encode the status. */ enum kbase_io_status_bits { - KBASE_IO_STATUS_GPU_SUSPENDED = 0, - KBASE_IO_STATUS_GPU_OFF, + KBASE_IO_STATUS_GPU_OFF = 0, KBASE_IO_STATUS_AW_REMOVED, KBASE_IO_STATUS_NUM_BITS, }; diff --git a/drivers/gpu/arm/valhall/mali_kbase_ioctl_helpers.h b/drivers/gpu/arm/valhall/mali_kbase_ioctl_helpers.h index d8d144c46f16..cfecc2de6b92 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_ioctl_helpers.h +++ b/drivers/gpu/arm/valhall/mali_kbase_ioctl_helpers.h @@ -286,8 +286,6 @@ check_padding_KBASE_IOCTL_KINSTR_PRFCNT_SETUP(union kbase_ioctl_kinstr_prfcnt_se #if MALI_UNIT_TEST #endif /* MALI_UNIT_TEST */ -#if MALI_USE_CSF - static inline int check_padding_KBASE_IOCTL_CS_QUEUE_REGISTER(struct kbase_ioctl_cs_queue_register *p) { @@ -518,31 +516,4 @@ check_padding_KBASE_IOCTL_CS_TILER_HEAP_SIZE(union kbase_ioctl_cs_tiler_heap_siz return 0; } -#else /* MALI_USE_CSF */ - -static inline int check_padding_KBASE_IOCTL_JOB_SUBMIT(struct kbase_ioctl_job_submit *p) -{ - return 0; -} - -static inline int -check_padding_KBASE_IOCTL_SOFT_EVENT_UPDATE(struct kbase_ioctl_soft_event_update *p) -{ - return 0; -} - -static inline int check_padding_KBASE_IOCTL_KINSTR_JM_FD(union kbase_kinstr_jm_fd *p) -{ - size_t i; - - for (i = 0; i < ARRAY_SIZE(p->in.padding); i++) { - if (p->in.padding[i]) - return -1; - } - - return 0; -} - -#endif /* !MALI_USE_CSF */ - #endif /* _KBASE_IOCTL_HELPERS_H_ */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_jd.c b/drivers/gpu/arm/valhall/mali_kbase_jd.c deleted file mode 100644 index e56bc60ed8c3..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_jd.c +++ /dev/null @@ -1,1492 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include -#if IS_ENABLED(CONFIG_COMPAT) -#include -#endif -#include -#include -#include -#include -#include -#if KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE -#include -#else -#include -#endif - -#include -#include -#include -#include -#include -#include - -/* Return whether katom will run on the GPU or not. Currently only soft jobs and - * dependency-only atoms do not run on the GPU - */ -#define IS_GPU_ATOM(katom) \ - (!((katom->core_req & BASE_JD_REQ_SOFT_JOB) || \ - ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) == BASE_JD_REQ_DEP))) - -/* - * This is the kernel side of the API. Only entry points are: - * - kbase_jd_submit(): Called from userspace to submit a single bag - * - kbase_jd_done(): Called from interrupt context to track the - * completion of a job. - * Callouts: - * - to the job manager (enqueue a job) - * - to the event subsystem (signals the completion/failure of bag/job-chains). - */ - -static void __user *get_compat_pointer(struct kbase_context *kctx, const u64 p) -{ -#if IS_ENABLED(CONFIG_COMPAT) - if (kbase_ctx_flag(kctx, KCTX_COMPAT)) - return compat_ptr(p); -#endif - return u64_to_user_ptr(p); -} - -/* Mark an atom as complete, and trace it in kinstr_jm */ -static void jd_mark_atom_complete(struct kbase_jd_atom *katom) -{ - katom->status = KBASE_JD_ATOM_STATE_COMPLETED; - kbase_kinstr_jm_atom_complete(katom); - dev_dbg(katom->kctx->kbdev->dev, "Atom %pK status to completed\n", (void *)katom); - KBASE_TLSTREAM_TL_JD_ATOM_COMPLETE(katom->kctx->kbdev, katom); -} - -/* Runs an atom, either by handing to the JS or by immediately running it in the case of soft-jobs - * - * Returns whether the JS needs a reschedule. - * - * Note that the caller must also check the atom status and - * if it is KBASE_JD_ATOM_STATE_COMPLETED must call kbase_jd_done_nolock - */ -static bool jd_run_atom(struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx = katom->kctx; - - dev_dbg(kctx->kbdev->dev, "JD run atom %pK in kctx %pK\n", (void *)katom, (void *)kctx); - - KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED); - - if ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) == BASE_JD_REQ_DEP) { - /* Dependency only atom */ - trace_sysgraph(SGR_SUBMIT, kctx->id, kbase_jd_atom_id(katom->kctx, katom)); - jd_mark_atom_complete(katom); - return false; - } else if (katom->core_req & BASE_JD_REQ_SOFT_JOB) { - /* Soft-job */ - if (katom->will_fail_event_code) { - kbase_finish_soft_job(katom); - jd_mark_atom_complete(katom); - return false; - } - if (kbase_process_soft_job(katom) == 0) { - kbase_finish_soft_job(katom); - jd_mark_atom_complete(katom); - } - return false; - } - - katom->status = KBASE_JD_ATOM_STATE_IN_JS; - dev_dbg(kctx->kbdev->dev, "Atom %pK status to in JS\n", (void *)katom); - /* Queue an action about whether we should try scheduling a context */ - return kbasep_js_add_job(kctx, katom); -} - -void kbase_jd_dep_clear_locked(struct kbase_jd_atom *katom) -{ - struct kbase_device *kbdev; - - KBASE_DEBUG_ASSERT(katom); - kbdev = katom->kctx->kbdev; - KBASE_DEBUG_ASSERT(kbdev); - - /* Check whether the atom's other dependencies were already met. If - * katom is a GPU atom then the job scheduler may be able to represent - * the dependencies, hence we may attempt to submit it before they are - * met. Other atoms must have had both dependencies resolved. - */ - if (IS_GPU_ATOM(katom) || (!kbase_jd_katom_dep_atom(&katom->dep[0]) && - !kbase_jd_katom_dep_atom(&katom->dep[1]))) { - /* katom dep complete, attempt to run it */ - bool resched = false; - - KBASE_TLSTREAM_TL_RUN_ATOM_START(katom->kctx->kbdev, katom, - kbase_jd_atom_id(katom->kctx, katom)); - resched = jd_run_atom(katom); - KBASE_TLSTREAM_TL_RUN_ATOM_END(katom->kctx->kbdev, katom, - kbase_jd_atom_id(katom->kctx, katom)); - - if (katom->status == KBASE_JD_ATOM_STATE_COMPLETED) { - /* The atom has already finished */ - resched |= kbase_jd_done_nolock(katom, true); - } - - if (resched) - kbase_js_sched_all(kbdev); - } -} - -void kbase_jd_free_external_resources(struct kbase_jd_atom *katom) -{ - CSTD_UNUSED(katom); -} - -static void kbase_jd_post_external_resources(struct kbase_jd_atom *katom) -{ - KBASE_DEBUG_ASSERT(katom); - KBASE_DEBUG_ASSERT(katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES); - - kbase_gpu_vm_lock(katom->kctx); - /* only roll back if extres is non-NULL */ - if (katom->extres) { - u32 res_no; - - res_no = katom->nr_extres; - while (res_no-- > 0) { - kbase_unmap_external_resource(katom->kctx, katom->extres[res_no]); - } - kfree(katom->extres); - katom->extres = NULL; - } - kbase_gpu_vm_unlock(katom->kctx); -} - -/* - * Set up external resources needed by this job. - * - * jctx.lock must be held when this is called. - */ - -static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, - const struct base_jd_atom *user_atom) -{ - int err = -EINVAL; - u32 res_no; - struct base_external_resource *input_extres; - size_t copy_size; - - KBASE_DEBUG_ASSERT(katom); - KBASE_DEBUG_ASSERT(katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES); - - /* no resources encoded, early out */ - if (!katom->nr_extres) - return -EINVAL; - - katom->extres = kmalloc_array(katom->nr_extres, sizeof(*katom->extres), GFP_KERNEL); - if (!katom->extres) - return -ENOMEM; - - input_extres = kmalloc_array(katom->nr_extres, sizeof(*input_extres), GFP_KERNEL); - if (!input_extres) { - err = -ENOMEM; - goto failed_input_alloc; - } - - if (check_mul_overflow(sizeof(*input_extres), (size_t)katom->nr_extres, ©_size)) { - err = -EINVAL; - goto failed_input_copy; - } - - if (copy_from_user(input_extres, get_compat_pointer(katom->kctx, user_atom->extres_list), - copy_size) != 0) { - err = -EINVAL; - goto failed_input_copy; - } - - /* Take the processes mmap lock */ - down_read(kbase_mem_get_process_mmap_lock()); - - /* need to keep the GPU VM locked while we set up UMM buffers */ - kbase_gpu_vm_lock(katom->kctx); - for (res_no = 0; res_no < katom->nr_extres; res_no++) { - struct base_external_resource *user_res = &input_extres[res_no]; - struct kbase_va_region *reg; - - reg = kbase_region_tracker_find_region_enclosing_address( - katom->kctx, - user_res->ext_resource & ~(__u64)BASE_EXT_RES_ACCESS_EXCLUSIVE); - /* did we find a matching region object? */ - if (unlikely(kbase_is_region_invalid_or_free(reg))) { - /* roll back */ - goto failed_loop; - } - - if (!(katom->core_req & BASE_JD_REQ_SOFT_JOB) && - (reg->flags & KBASE_REG_PROTECTED)) { - katom->atom_flags |= KBASE_KATOM_FLAG_PROTECTED; - } - - err = kbase_map_external_resource(katom->kctx, reg, current->mm); - if (err) - goto failed_loop; - - katom->extres[res_no] = reg; - } - /* successfully parsed the extres array */ - /* drop the vm lock now */ - kbase_gpu_vm_unlock(katom->kctx); - - /* Release the processes mmap lock */ - up_read(kbase_mem_get_process_mmap_lock()); - - /* Free the buffer holding data from userspace */ - kfree(input_extres); - - /* all done OK */ - return 0; - -/* error handling section */ -failed_loop: - /* undo the loop work. We are guaranteed to have access to the VA region - * as we hold a reference to it until it's unmapped - */ - while (res_no-- > 0) { - struct kbase_va_region *reg = katom->extres[res_no]; - - kbase_unmap_external_resource(katom->kctx, reg); - } - kbase_gpu_vm_unlock(katom->kctx); - - /* Release the processes mmap lock */ - up_read(kbase_mem_get_process_mmap_lock()); - -failed_input_copy: - kfree(input_extres); -failed_input_alloc: - kfree(katom->extres); - katom->extres = NULL; - return err; -} - -static inline void jd_resolve_dep(struct list_head *out_list, struct kbase_jd_atom *katom, u8 d, - bool ctx_is_dying) -{ - u8 other_d = !d; - - while (!list_empty(&katom->dep_head[d])) { - struct kbase_jd_atom *dep_atom; - struct kbase_jd_atom *other_dep_atom; - u8 dep_type; - - dep_atom = list_entry(katom->dep_head[d].next, struct kbase_jd_atom, dep_item[d]); - list_del(katom->dep_head[d].next); - - dep_type = kbase_jd_katom_dep_type(&dep_atom->dep[d]); - kbase_jd_katom_dep_clear(&dep_atom->dep[d]); - - if (katom->event_code != BASE_JD_EVENT_DONE && - (dep_type != BASE_JD_DEP_TYPE_ORDER)) { - dep_atom->event_code = katom->event_code; - KBASE_DEBUG_ASSERT(dep_atom->status != KBASE_JD_ATOM_STATE_UNUSED); - - dep_atom->will_fail_event_code = dep_atom->event_code; - } - other_dep_atom = - (struct kbase_jd_atom *)kbase_jd_katom_dep_atom(&dep_atom->dep[other_d]); - - if (!dep_atom->in_jd_list && - (!other_dep_atom || - (IS_GPU_ATOM(dep_atom) && !ctx_is_dying && !dep_atom->will_fail_event_code && - !other_dep_atom->will_fail_event_code))) { - dep_atom->in_jd_list = true; - list_add_tail(&dep_atom->jd_item, out_list); - } - } -} - -/** - * is_dep_valid - Validate that a dependency is valid for early dependency - * submission - * @katom: Dependency atom to validate - * - * A dependency is valid if any of the following are true : - * - It does not exist (a non-existent dependency does not block submission) - * - It is in the job scheduler - * - It has completed, does not have a failure event code, and has not been - * marked to fail in the future - * - * Return: true if valid, false otherwise - */ -static bool is_dep_valid(struct kbase_jd_atom *katom) -{ - /* If there's no dependency then this is 'valid' from the perspective of - * early dependency submission - */ - if (!katom) - return true; - - /* Dependency must have reached the job scheduler */ - if (katom->status < KBASE_JD_ATOM_STATE_IN_JS) - return false; - - /* If dependency has completed and has failed or will fail then it is - * not valid - */ - if (katom->status >= KBASE_JD_ATOM_STATE_HW_COMPLETED && - (katom->event_code != BASE_JD_EVENT_DONE || katom->will_fail_event_code)) - return false; - - return true; -} - -static void jd_try_submitting_deps(struct list_head *out_list, struct kbase_jd_atom *node) -{ - int i; - - for (i = 0; i < 2; i++) { - struct list_head *pos; - - list_for_each(pos, &node->dep_head[i]) { - struct kbase_jd_atom *dep_atom = - list_entry(pos, struct kbase_jd_atom, dep_item[i]); - - if (IS_GPU_ATOM(dep_atom) && !dep_atom->in_jd_list) { - /*Check if atom deps look sane*/ - bool dep0_valid = is_dep_valid(dep_atom->dep[0].atom); - bool dep1_valid = is_dep_valid(dep_atom->dep[1].atom); - - if (dep0_valid && dep1_valid) { - dep_atom->in_jd_list = true; - list_add(&dep_atom->jd_item, out_list); - } - } - } - } -} - -#if MALI_JIT_PRESSURE_LIMIT_BASE -/** - * jd_update_jit_usage - Update just-in-time physical memory usage for an atom. - * - * @katom: An atom that has just finished. - * - * Read back actual just-in-time memory region usage from atoms that provide - * this information, and update the current physical page pressure. - * - * The caller must hold the kbase_jd_context.lock. - */ -static void jd_update_jit_usage(struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx = katom->kctx; - struct kbase_va_region *reg; - struct kbase_vmap_struct mapping; - u64 *ptr; - u64 used_pages; - unsigned int idx; - - lockdep_assert_held(&kctx->jctx.lock); - - /* If this atom wrote to JIT memory, find out how much it has written - * and update the usage information in the region. - */ - for (idx = 0; idx < ARRAY_SIZE(katom->jit_ids) && katom->jit_ids[idx]; idx++) { - enum heap_pointer { LOW = 0, HIGH, COUNT }; - size_t size_to_read; - u64 read_val; - - reg = kctx->jit_alloc[katom->jit_ids[idx]]; - - if (!reg) { - dev_warn(kctx->kbdev->dev, "%s: JIT id[%u]=%u has no region\n", __func__, - idx, katom->jit_ids[idx]); - continue; - } - - if (reg == KBASE_RESERVED_REG_JIT_ALLOC) { - dev_warn(kctx->kbdev->dev, - "%s: JIT id[%u]=%u has failed to allocate a region\n", __func__, - idx, katom->jit_ids[idx]); - continue; - } - - if (!reg->heap_info_gpu_addr) - continue; - - size_to_read = sizeof(*ptr); - if (reg->flags & KBASE_REG_HEAP_INFO_IS_SIZE) - size_to_read = sizeof(u32); - else if (reg->flags & KBASE_REG_TILER_ALIGN_TOP) - size_to_read = sizeof(u64[COUNT]); - - ptr = kbase_vmap_prot(kctx, reg->heap_info_gpu_addr, size_to_read, KBASE_REG_CPU_RD, - &mapping); - - if (!ptr) { - dev_warn(kctx->kbdev->dev, - "%s: JIT id[%u]=%u start=0x%llx unable to map end marker %llx\n", - __func__, idx, katom->jit_ids[idx], reg->start_pfn << PAGE_SHIFT, - reg->heap_info_gpu_addr); - continue; - } - - if (reg->flags & KBASE_REG_HEAP_INFO_IS_SIZE) { - read_val = READ_ONCE(*(u32 *)ptr); - used_pages = PFN_UP(read_val); - } else { - u64 addr_end; - - if (reg->flags & KBASE_REG_TILER_ALIGN_TOP) { - const unsigned long extension_bytes = reg->extension << PAGE_SHIFT; - const u64 low_ptr = ptr[LOW]; - const u64 high_ptr = ptr[HIGH]; - - /* As either the low or high pointer could - * consume their partition and move onto the - * next chunk, we need to account for both. - * In the case where nothing has been allocated - * from the high pointer the whole chunk could - * be backed unnecessarily - but the granularity - * is the chunk size anyway and any non-zero - * offset of low pointer from the start of the - * chunk would result in the whole chunk being - * backed. - */ - read_val = max(high_ptr, low_ptr); - - /* kbase_check_alloc_sizes() already satisfies - * this, but here to avoid future maintenance - * hazards - */ - WARN_ON(!is_power_of_2(extension_bytes)); - addr_end = ALIGN(read_val, extension_bytes); - } else { - addr_end = read_val = READ_ONCE(*ptr); - } - - if (addr_end >= (reg->start_pfn << PAGE_SHIFT)) - used_pages = PFN_UP(addr_end) - reg->start_pfn; - else - used_pages = reg->used_pages; - } - - trace_mali_jit_report(katom, reg, idx, read_val, used_pages); - kbase_trace_jit_report_gpu_mem(kctx, reg, 0u); - - /* We can never have used more pages than the VA size of the - * region - */ - if (used_pages > reg->nr_pages) { - dev_warn( - kctx->kbdev->dev, - "%s: JIT id[%u]=%u start=0x%llx used_pages %llx > %zx (read 0x%llx as %s%s)\n", - __func__, idx, katom->jit_ids[idx], reg->start_pfn << PAGE_SHIFT, - used_pages, reg->nr_pages, read_val, - (reg->flags & KBASE_REG_HEAP_INFO_IS_SIZE) ? "size" : "addr", - (reg->flags & KBASE_REG_TILER_ALIGN_TOP) ? " with align" : ""); - used_pages = reg->nr_pages; - } - /* Note: one real use case has an atom correctly reporting 0 - * pages in use. This happens in normal use-cases but may only - * happen for a few of the application's frames. - */ - - kbase_vunmap(kctx, &mapping); - - kbase_jit_report_update_pressure(kctx, reg, used_pages, 0u); - } - - kbase_jit_retry_pending_alloc(kctx); -} -#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - -bool kbase_jd_done_nolock(struct kbase_jd_atom *katom, bool post_immediately) -{ - struct kbase_context *kctx = katom->kctx; - struct list_head completed_jobs; - struct list_head runnable_jobs; - bool need_to_try_schedule_context = false; - int i; - - lockdep_assert_held(&kctx->jctx.lock); - - KBASE_TLSTREAM_TL_JD_DONE_NO_LOCK_START(kctx->kbdev, katom); - - INIT_LIST_HEAD(&completed_jobs); - INIT_LIST_HEAD(&runnable_jobs); - - KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED); - -#if MALI_JIT_PRESSURE_LIMIT_BASE - if (kbase_ctx_flag(kctx, KCTX_JPL_ENABLED)) - jd_update_jit_usage(katom); -#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - - /* This is needed in case an atom is failed due to being invalid, this - * can happen *before* the jobs that the atom depends on have completed - */ - for (i = 0; i < 2; i++) { - if (kbase_jd_katom_dep_atom(&katom->dep[i])) { - list_del(&katom->dep_item[i]); - kbase_jd_katom_dep_clear(&katom->dep[i]); - } - } - - jd_mark_atom_complete(katom); - - list_add_tail(&katom->jd_item, &completed_jobs); - - while (!list_empty(&completed_jobs)) { - katom = list_entry(completed_jobs.prev, struct kbase_jd_atom, jd_item); - list_del(completed_jobs.prev); - KBASE_DEBUG_ASSERT(katom->status == KBASE_JD_ATOM_STATE_COMPLETED); - - for (i = 0; i < 2; i++) - jd_resolve_dep(&runnable_jobs, katom, i, kbase_ctx_flag(kctx, KCTX_DYING)); - - if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) - kbase_jd_post_external_resources(katom); - - while (!list_empty(&runnable_jobs)) { - struct kbase_jd_atom *node; - - node = list_entry(runnable_jobs.next, struct kbase_jd_atom, jd_item); - list_del(runnable_jobs.next); - node->in_jd_list = false; - - dev_dbg(kctx->kbdev->dev, "List node %pK has status %d\n", node, - node->status); - - KBASE_DEBUG_ASSERT(node->status != KBASE_JD_ATOM_STATE_UNUSED); - if (node->status == KBASE_JD_ATOM_STATE_IN_JS) - continue; - - if (node->status != KBASE_JD_ATOM_STATE_COMPLETED && - !kbase_ctx_flag(kctx, KCTX_DYING)) { - KBASE_TLSTREAM_TL_RUN_ATOM_START(kctx->kbdev, node, - kbase_jd_atom_id(kctx, node)); - need_to_try_schedule_context |= jd_run_atom(node); - KBASE_TLSTREAM_TL_RUN_ATOM_END(kctx->kbdev, node, - kbase_jd_atom_id(kctx, node)); - } else { - node->event_code = katom->event_code; - - if (node->core_req & BASE_JD_REQ_SOFT_JOB) { - WARN_ON(!list_empty(&node->queue)); - kbase_finish_soft_job(node); - } - node->status = KBASE_JD_ATOM_STATE_COMPLETED; - } - - if (node->status == KBASE_JD_ATOM_STATE_COMPLETED) { - list_add_tail(&node->jd_item, &completed_jobs); - } else if (node->status == KBASE_JD_ATOM_STATE_IN_JS && - !node->will_fail_event_code) { - /* Node successfully submitted, try submitting - * dependencies as they may now be representable - * in JS - */ - jd_try_submitting_deps(&runnable_jobs, node); - } - } - - /* Register a completed job as a disjoint event when the GPU - * is in a disjoint state (ie. being reset). - */ - kbase_disjoint_event_potential(kctx->kbdev); - if (post_immediately && list_empty(&kctx->completed_jobs)) - kbase_event_post(kctx, katom); - else - list_add_tail(&katom->jd_item, &kctx->completed_jobs); - - /* Decrement and check the TOTAL number of jobs. This includes - * those not tracked by the scheduler: 'not ready to run' and - * 'dependency-only' jobs. - */ - if (--kctx->jctx.job_nr == 0) - /* All events are safely queued now, and we can signal - * any waiter that we've got no more jobs (so we can be - * safely terminated) - */ - wake_up(&kctx->jctx.zero_jobs_wait); - } - KBASE_TLSTREAM_TL_JD_DONE_NO_LOCK_END(kctx->kbdev, katom); - return need_to_try_schedule_context; -} -KBASE_EXPORT_TEST_API(kbase_jd_done_nolock); - -/* Trace an atom submission. */ -static void jd_trace_atom_submit(struct kbase_context *const kctx, - struct kbase_jd_atom *const katom, int *priority) -{ - struct kbase_device *const kbdev = kctx->kbdev; - - KBASE_TLSTREAM_TL_NEW_ATOM(kbdev, katom, (u32)kbase_jd_atom_id(kctx, katom)); - KBASE_TLSTREAM_TL_RET_ATOM_CTX(kbdev, katom, kctx); - if (priority) - KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY(kbdev, katom, (u32)*priority); - KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(kbdev, katom, TL_ATOM_STATE_IDLE); - kbase_kinstr_jm_atom_queue(katom); -} - -static bool jd_submit_atom(struct kbase_context *const kctx, - const struct base_jd_atom *const user_atom, - struct kbase_jd_atom *const katom) -{ - struct kbase_device *kbdev = kctx->kbdev; - struct kbase_jd_context *jctx = &kctx->jctx; - int queued = 0; - int i; - int sched_prio; - bool will_fail = false; - unsigned long flags; - enum kbase_jd_atom_state status; - - dev_dbg(kbdev->dev, "User did JD submit atom %pK\n", (void *)katom); - - /* Update the TOTAL number of jobs. This includes those not tracked by - * the scheduler: 'not ready to run' and 'dependency-only' jobs. - */ - jctx->job_nr++; - -#if KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE - katom->start_timestamp.tv64 = 0; -#else - katom->start_timestamp = 0; -#endif - katom->udata = user_atom->udata; - katom->kctx = kctx; - katom->nr_extres = user_atom->nr_extres; - katom->extres = NULL; - katom->device_nr = user_atom->device_nr; - katom->jc = user_atom->jc; - katom->core_req = user_atom->core_req; - katom->jobslot = user_atom->jobslot; - katom->seq_nr = user_atom->seq_nr; - katom->atom_flags = 0; - katom->need_cache_flush_cores_retained = 0; - katom->pre_dep = NULL; - katom->post_dep = NULL; - katom->x_pre_dep = NULL; - katom->x_post_dep = NULL; - katom->will_fail_event_code = BASE_JD_EVENT_NOT_STARTED; - katom->softjob_data = NULL; - - trace_sysgraph(SGR_ARRIVE, kctx->id, user_atom->atom_number); - -#if MALI_JIT_PRESSURE_LIMIT_BASE - /* Older API version atoms might have random values where jit_id now - * lives, but we must maintain backwards compatibility - handle the - * issue. - */ - if (!mali_kbase_supports_jit_pressure_limit(kctx->api_version)) { - katom->jit_ids[0] = 0; - katom->jit_ids[1] = 0; - } else { - katom->jit_ids[0] = user_atom->jit_id[0]; - katom->jit_ids[1] = user_atom->jit_id[1]; - } -#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - - /* Implicitly sets katom->protected_state.enter as well. */ - katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; - - katom->age = kctx->age_count++; - - INIT_LIST_HEAD(&katom->queue); - INIT_LIST_HEAD(&katom->jd_item); - - /* Don't do anything if there is a mess up with dependencies. - * This is done in a separate cycle to check both the dependencies at ones, otherwise - * it will be extra complexity to deal with 1st dependency ( just added to the list ) - * if only the 2nd one has invalid config. - */ - for (i = 0; i < 2; i++) { - int dep_atom_number = user_atom->pre_dep[i].atom_id; - base_jd_dep_type dep_atom_type = user_atom->pre_dep[i].dependency_type; - - if (dep_atom_number) { - if (dep_atom_type != BASE_JD_DEP_TYPE_ORDER && - dep_atom_type != BASE_JD_DEP_TYPE_DATA) { - katom->event_code = BASE_JD_EVENT_JOB_CONFIG_FAULT; - katom->status = KBASE_JD_ATOM_STATE_COMPLETED; - dev_dbg(kbdev->dev, "Atom %pK status to completed\n", - (void *)katom); - - /* Wrong dependency setup. Atom will be sent - * back to user space. Do not record any - * dependencies. - */ - jd_trace_atom_submit(kctx, katom, NULL); - return kbase_jd_done_nolock(katom, true); - } - } - } - - /* Add dependencies */ - for (i = 0; i < 2; i++) { - int dep_atom_number = user_atom->pre_dep[i].atom_id; - base_jd_dep_type dep_atom_type; - struct kbase_jd_atom *dep_atom = &jctx->atoms[dep_atom_number]; - - dep_atom_type = user_atom->pre_dep[i].dependency_type; - kbase_jd_katom_dep_clear(&katom->dep[i]); - - if (!dep_atom_number) - continue; - - if (dep_atom->status == KBASE_JD_ATOM_STATE_UNUSED || - dep_atom->status == KBASE_JD_ATOM_STATE_COMPLETED) { - if (dep_atom->event_code == BASE_JD_EVENT_DONE) - continue; - /* don't stop this atom if it has an order dependency - * only to the failed one, try to submit it through - * the normal path - */ - if (dep_atom_type == BASE_JD_DEP_TYPE_ORDER && - dep_atom->event_code > BASE_JD_EVENT_ACTIVE) { - continue; - } - - /* Atom has completed, propagate the error code if any */ - katom->event_code = dep_atom->event_code; - katom->status = KBASE_JD_ATOM_STATE_QUEUED; - dev_dbg(kbdev->dev, "Atom %pK status to queued\n", (void *)katom); - - /* This atom will be sent back to user space. - * Do not record any dependencies. - */ - jd_trace_atom_submit(kctx, katom, NULL); - - will_fail = true; - - } else { - /* Atom is in progress, add this atom to the list */ - list_add_tail(&katom->dep_item[i], &dep_atom->dep_head[i]); - kbase_jd_katom_dep_set(&katom->dep[i], dep_atom, dep_atom_type); - queued = 1; - } - } - - if (will_fail) { - if (!queued) { - if (katom->core_req & BASE_JD_REQ_SOFT_JOB) { - /* This softjob has failed due to a previous - * dependency, however we should still run the - * prepare & finish functions - */ - int err = kbase_prepare_soft_job(katom); - - if (err >= 0) - kbase_finish_soft_job(katom); - } - return kbase_jd_done_nolock(katom, true); - } - - katom->will_fail_event_code = katom->event_code; - } - - /* These must occur after the above loop to ensure that an atom - * that depends on a previous atom with the same number behaves - * as expected - */ - katom->event_code = BASE_JD_EVENT_DONE; - katom->status = KBASE_JD_ATOM_STATE_QUEUED; - dev_dbg(kbdev->dev, "Atom %pK status to queued\n", (void *)katom); - - /* For invalid priority, be most lenient and choose the default */ - sched_prio = kbasep_js_atom_prio_to_sched_prio(user_atom->prio); - if (sched_prio == KBASE_JS_ATOM_SCHED_PRIO_INVALID) - sched_prio = KBASE_JS_ATOM_SCHED_PRIO_DEFAULT; - - /* Cap the priority to jctx.max_priority */ - katom->sched_priority = (sched_prio < kctx->jctx.max_priority) ? kctx->jctx.max_priority : - sched_prio; - - /* Create a new atom. */ - jd_trace_atom_submit(kctx, katom, &katom->sched_priority); - - if (!katom->jc && (katom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) { - /* Reject atoms with job chain = NULL, as these cause issues - * with soft-stop - */ - dev_err(kctx->kbdev->dev, "Rejecting atom with jc = NULL\n"); - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - return kbase_jd_done_nolock(katom, true); - } - - /* Reject atoms with an invalid device_nr */ - if ((katom->core_req & BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) && - (katom->device_nr >= kctx->kbdev->gpu_props.num_core_groups)) { - dev_err(kctx->kbdev->dev, "Rejecting atom with invalid device_nr %d\n", - katom->device_nr); - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - return kbase_jd_done_nolock(katom, true); - } - - /* Reject atoms with invalid core requirements */ - if ((katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) && - (katom->core_req & BASE_JD_REQ_EVENT_COALESCE)) { - dev_err(kctx->kbdev->dev, "Rejecting atom with invalid core requirements\n"); - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - katom->core_req &= ~BASE_JD_REQ_EVENT_COALESCE; - return kbase_jd_done_nolock(katom, true); - } - - /* Reject soft-job atom of certain types from accessing external resources */ - if ((katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) && - (((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == BASE_JD_REQ_SOFT_FENCE_WAIT) || - ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == BASE_JD_REQ_SOFT_JIT_ALLOC) || - ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == BASE_JD_REQ_SOFT_JIT_FREE))) { - dev_err(kctx->kbdev->dev, "Rejecting soft-job atom accessing external resources\n"); - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - return kbase_jd_done_nolock(katom, true); - } - - if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { - /* handle what we need to do to access the external resources */ - if (kbase_jd_pre_external_resources(katom, user_atom) != 0) { - /* setup failed (no access, bad resource, unknown resource types, etc.) */ - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - return kbase_jd_done_nolock(katom, true); - } - } - -#if !MALI_JIT_PRESSURE_LIMIT_BASE - if (mali_kbase_supports_jit_pressure_limit(kctx->api_version) && - (user_atom->jit_id[0] || user_atom->jit_id[1])) { - /* JIT pressure limit is disabled, but we are receiving non-0 - * JIT IDs - atom is invalid. - */ - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - return kbase_jd_done_nolock(katom, true); - } -#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - - /* Validate the atom. Function will return error if the atom is - * malformed. - * - * Soft-jobs never enter the job scheduler but have their own initialize method. - * - * If either fail then we immediately complete the atom with an error. - */ - if ((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0) { - if (!kbase_js_is_atom_valid(kctx->kbdev, katom)) { - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - return kbase_jd_done_nolock(katom, true); - } - } else { - /* Soft-job */ - if (kbase_prepare_soft_job(katom) != 0) { - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - return kbase_jd_done_nolock(katom, true); - } - } - - if (queued && !IS_GPU_ATOM(katom)) - return false; - - if (katom->core_req & BASE_JD_REQ_SOFT_JOB) { - if (kbase_process_soft_job(katom) == 0) { - kbase_finish_soft_job(katom); - return kbase_jd_done_nolock(katom, true); - } - return false; - } - - if ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) { - bool need_to_try_schedule_context; - - katom->status = KBASE_JD_ATOM_STATE_IN_JS; - dev_dbg(kctx->kbdev->dev, "Atom %pK status to in JS\n", (void *)katom); - - need_to_try_schedule_context = kbasep_js_add_job(kctx, katom); - /* If job was cancelled then resolve immediately */ - if (katom->event_code != BASE_JD_EVENT_JOB_CANCELLED) - return need_to_try_schedule_context; - - /* Synchronize with backend reset */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - status = katom->status; - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - if (status == KBASE_JD_ATOM_STATE_HW_COMPLETED) { - dev_dbg(kctx->kbdev->dev, "Atom %d cancelled on HW\n", - kbase_jd_atom_id(katom->kctx, katom)); - return need_to_try_schedule_context; - } - } - - /* This is a pure dependency. Resolve it immediately */ - return kbase_jd_done_nolock(katom, true); -} - -int kbase_jd_submit(struct kbase_context *kctx, void __user *user_addr, u32 nr_atoms, u32 stride, - bool uk6_atom) -{ - struct kbase_jd_context *jctx = &kctx->jctx; - int err = 0; - u32 i; - bool need_to_try_schedule_context = false; - struct kbase_device *kbdev; - u32 latest_flush; - - bool jd_atom_is_v2 = (stride == sizeof(struct base_jd_atom_v2)); - - CSTD_UNUSED(uk6_atom); - - /* - * kbase_jd_submit isn't expected to fail and so all errors with the - * jobs are reported by immediately failing them (through event system) - */ - kbdev = kctx->kbdev; - - if (kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { - dev_err(kbdev->dev, - "Attempt to submit to a context that has SUBMIT_DISABLED set on it\n"); - return -EINVAL; - } - - if (stride != sizeof(struct base_jd_atom_v2) && stride != sizeof(struct base_jd_atom)) { - dev_err(kbdev->dev, - "Stride %u passed to job_submit isn't supported by the kernel\n", stride); - return -EINVAL; - } - - if (nr_atoms > BASE_JD_ATOM_COUNT) { - dev_dbg(kbdev->dev, "Invalid attempt to submit %u atoms at once for kctx %d_%d", - nr_atoms, kctx->tgid, kctx->id); - return -EINVAL; - } - - /* All atoms submitted in this call have the same flush ID */ - latest_flush = kbase_backend_get_current_flush_id(kbdev); - - for (i = 0; i < nr_atoms; i++) { - struct base_jd_atom user_atom = { - .seq_nr = 0, - }; - struct kbase_jd_atom *katom; - - if (unlikely(jd_atom_is_v2)) { - if (copy_from_user(&user_atom.jc, user_addr, - sizeof(struct base_jd_atom_v2)) != 0) { - dev_dbg(kbdev->dev, - "Invalid atom address %pK passed to job_submit\n", - user_addr); - err = -EFAULT; - break; - } - - /* no seq_nr in v2 */ - user_atom.seq_nr = 0; - } else { - if (copy_from_user(&user_atom, user_addr, stride) != 0) { - dev_dbg(kbdev->dev, - "Invalid atom address %pK passed to job_submit\n", - user_addr); - err = -EFAULT; - break; - } - } - - user_addr = (void __user *)((uintptr_t)user_addr + stride); - - mutex_lock(&jctx->lock); -#ifndef compiletime_assert -#define compiletime_assert_defined -#define compiletime_assert(x, msg) \ - do { \ - switch (0) { \ - case 0: \ - case (x):; \ - } \ - } while (false) -#endif - compiletime_assert((1 << (8 * sizeof(user_atom.atom_number))) == BASE_JD_ATOM_COUNT, - "BASE_JD_ATOM_COUNT and base_atom_id type out of sync"); - compiletime_assert(sizeof(user_atom.pre_dep[0].atom_id) == - sizeof(user_atom.atom_number), - "BASE_JD_ATOM_COUNT and base_atom_id type out of sync"); -#ifdef compiletime_assert_defined -#undef compiletime_assert -#undef compiletime_assert_defined -#endif - katom = &jctx->atoms[user_atom.atom_number]; - - /* Record the flush ID for the cache flush optimisation */ - katom->flush_id = latest_flush; - - while (katom->status != KBASE_JD_ATOM_STATE_UNUSED) { - /* Atom number is already in use, wait for the atom to - * complete - */ - mutex_unlock(&jctx->lock); - - /* This thread will wait for the atom to complete. Due - * to thread scheduling we are not sure that the other - * thread that owns the atom will also schedule the - * context, so we force the scheduler to be active and - * hence eventually schedule this context at some point - * later. - */ - kbase_js_sched_all(kbdev); - - if (wait_event_killable(katom->completed, - katom->status == KBASE_JD_ATOM_STATE_UNUSED) != 0) { - /* We're being killed so the result code - * doesn't really matter - */ - return 0; - } - mutex_lock(&jctx->lock); - } - KBASE_TLSTREAM_TL_JD_SUBMIT_ATOM_START(kbdev, katom); - need_to_try_schedule_context |= jd_submit_atom(kctx, &user_atom, katom); - KBASE_TLSTREAM_TL_JD_SUBMIT_ATOM_END(kbdev, katom); - /* Register a completed job as a disjoint event when the GPU is in a disjoint state - * (ie. being reset). - */ - kbase_disjoint_event_potential(kbdev); - - mutex_unlock(&jctx->lock); - if (fatal_signal_pending(current)) { - dev_dbg(kbdev->dev, "Fatal signal pending for kctx %d_%d", kctx->tgid, - kctx->id); - /* We're being killed so the result code doesn't really matter */ - return 0; - } - } - - if (need_to_try_schedule_context) - kbase_js_sched_all(kbdev); - - return err; -} - -KBASE_EXPORT_TEST_API(kbase_jd_submit); - -void kbase_jd_done_worker(struct work_struct *data) -{ - struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work); - struct kbase_jd_context *jctx; - struct kbase_context *kctx; - struct kbasep_js_kctx_info *js_kctx_info; - struct kbase_device *kbdev; - struct kbasep_js_device_data *js_devdata; - u64 cache_jc = katom->jc; - struct kbasep_js_atom_retained_state katom_retained_state; - bool context_idle; - base_jd_core_req core_req = katom->core_req; - - /* Soft jobs should never reach this function */ - KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0); - - kctx = katom->kctx; - jctx = &kctx->jctx; - kbdev = kctx->kbdev; - js_kctx_info = &kctx->jctx.sched_info; - js_devdata = &kbdev->js_data; - - dev_dbg(kbdev->dev, "Enter atom %pK done worker for kctx %pK\n", (void *)katom, - (void *)kctx); - - KBASE_KTRACE_ADD_JM(kbdev, JD_DONE_WORKER, kctx, katom, katom->jc, 0); - - kbase_backend_complete_wq(kbdev, katom); - - /* - * Begin transaction on JD context and JS context - */ - mutex_lock(&jctx->lock); - KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(kbdev, katom, TL_ATOM_STATE_DONE); - mutex_lock(&js_devdata->queue_mutex); - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - - /* This worker only gets called on contexts that are scheduled *in*. This is - * because it only happens in response to an IRQ from a job that was - * running. - */ - KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); - - if (katom->event_code == BASE_JD_EVENT_STOPPED) { - unsigned long flags; - - dev_dbg(kbdev->dev, "Atom %pK has been promoted to stopped\n", (void *)katom); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - mutex_unlock(&js_devdata->queue_mutex); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - katom->status = KBASE_JD_ATOM_STATE_IN_JS; - dev_dbg(kctx->kbdev->dev, "Atom %pK status to in JS\n", (void *)katom); - kbase_js_unpull(kctx, katom); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&jctx->lock); - - return; - } - - if ((katom->event_code != BASE_JD_EVENT_DONE) && !kbase_ctx_flag(katom->kctx, KCTX_DYING) && - !kbase_ctx_flag(katom->kctx, KCTX_PAGE_FAULT_REPORT_SKIP)) - if (!kbase_is_quick_reset_enabled(kbdev)) - dev_err(kbdev->dev, "t6xx: GPU fault 0x%02lx from job slot %d\n", - (unsigned long)katom->event_code, katom->slot_nr); - - /* Retain state before the katom disappears */ - kbasep_js_atom_retained_state_copy(&katom_retained_state, katom); - - context_idle = kbase_js_complete_atom_wq(kctx, katom); - - KBASE_DEBUG_ASSERT(kbasep_js_has_atom_finished(&katom_retained_state)); - - kbasep_js_remove_job(kbdev, kctx, katom); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - mutex_unlock(&js_devdata->queue_mutex); - /* kbase_jd_done_nolock() requires the jsctx_mutex lock to be dropped */ - kbase_jd_done_nolock(katom, false); - - /* katom may have been freed now, do not use! */ - - if (context_idle) { - unsigned long flags; - - context_idle = false; - mutex_lock(&js_devdata->queue_mutex); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - /* If kbase_sched() has scheduled this context back in then - * KCTX_ACTIVE will have been set after we marked it as - * inactive, and another pm reference will have been taken, so - * drop our reference. But do not call kbase_jm_idle_ctx(), as - * the context is active and fast-starting is allowed. - * - * If an atom has been fast-started then - * kbase_jsctx_atoms_pulled(kctx) will return non-zero but - * KCTX_ACTIVE will still be false (as the previous pm - * reference has been inherited). Do NOT drop our reference, as - * it has been re-used, and leave the context as active. - * - * If no new atoms have been started then KCTX_ACTIVE will - * still be false and kbase_jsctx_atoms_pulled(kctx) will - * return zero, so drop the reference and call - * kbase_jm_idle_ctx(). - * - * As the checks are done under both the queue_mutex and - * hwaccess_lock is should be impossible for this to race - * with the scheduler code. - */ - if (kbase_ctx_flag(kctx, KCTX_ACTIVE) || !kbase_jsctx_atoms_pulled(kctx)) { - /* Calling kbase_jm_idle_ctx() here will ensure that - * atoms are not fast-started when we drop the - * hwaccess_lock. This is not performed if - * KCTX_ACTIVE is set as in that case another pm - * reference has been taken and a fast-start would be - * valid. - */ - if (!kbase_ctx_flag(kctx, KCTX_ACTIVE)) - kbase_jm_idle_ctx(kbdev, kctx); - context_idle = true; - } else { - kbase_ctx_flag_set(kctx, KCTX_ACTIVE); - } - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&js_devdata->queue_mutex); - } - - /* - * Transaction complete - */ - mutex_unlock(&jctx->lock); - - /* Job is now no longer running, so can now safely release the context - * reference, and handle any actions that were logged against the - * atom's retained state - */ - - kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, &katom_retained_state); - - kbase_js_sched_all(kbdev); - - if (!atomic_dec_return(&kctx->work_count)) { - /* If worker now idle then post all events that kbase_jd_done_nolock() - * has queued - */ - mutex_lock(&jctx->lock); - while (!list_empty(&kctx->completed_jobs)) { - struct kbase_jd_atom *atom = list_entry(kctx->completed_jobs.next, - struct kbase_jd_atom, jd_item); - list_del(kctx->completed_jobs.next); - - kbase_event_post(kctx, atom); - } - mutex_unlock(&jctx->lock); - } - - kbase_backend_complete_wq_post_sched(kbdev, core_req); - - if (context_idle) - kbase_pm_context_idle(kbdev); - - KBASE_KTRACE_ADD_JM(kbdev, JD_DONE_WORKER_END, kctx, NULL, cache_jc, 0); - - dev_dbg(kbdev->dev, "Leave atom %pK done worker for kctx %pK\n", (void *)katom, - (void *)kctx); -} - -/** - * jd_cancel_worker - Work queue job cancel function. - * @data: a &struct work_struct - * - * Only called as part of 'Zapping' a context (which occurs on termination). - * Operates serially with the kbase_jd_done_worker() on the work queue. - * - * This can only be called on contexts that aren't scheduled. - * - * We don't need to release most of the resources that would occur on - * kbase_jd_done() or kbase_jd_done_worker(), because the atoms here must not be - * running (by virtue of only being called on contexts that aren't - * scheduled). - */ -static void jd_cancel_worker(struct work_struct *data) -{ - struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work); - struct kbase_jd_context *jctx; - struct kbase_context *kctx; - struct kbasep_js_kctx_info *js_kctx_info; - bool need_to_try_schedule_context; - bool attr_state_changed; - struct kbase_device *kbdev; - CSTD_UNUSED(need_to_try_schedule_context); - - /* Soft jobs should never reach this function */ - KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0); - - kctx = katom->kctx; - kbdev = kctx->kbdev; - jctx = &kctx->jctx; - js_kctx_info = &kctx->jctx.sched_info; - - KBASE_KTRACE_ADD_JM(kbdev, JD_CANCEL_WORKER, kctx, katom, katom->jc, 0); - - /* This only gets called on contexts that are scheduled out. Hence, we must - * make sure we don't de-ref the number of running jobs (there aren't - * any), nor must we try to schedule out the context (it's already - * scheduled out). - */ - KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); - - /* Scheduler: Remove the job from the system */ - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - attr_state_changed = kbasep_js_remove_cancelled_job(kbdev, kctx, katom); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - - mutex_lock(&jctx->lock); - - need_to_try_schedule_context = kbase_jd_done_nolock(katom, true); - /* Because we're zapping, we're not adding any more jobs to this ctx, so no need to - * schedule the context. There's also no need for the jsctx_mutex to have been taken - * around this too. - */ - KBASE_DEBUG_ASSERT(!need_to_try_schedule_context); - - /* katom may have been freed now, do not use! */ - mutex_unlock(&jctx->lock); - - if (attr_state_changed) - kbase_js_sched_all(kbdev); -} - -/** - * kbase_jd_done - Complete a job that has been removed from the Hardware - * @katom: atom which has been completed - * @slot_nr: slot the atom was on - * @end_timestamp: completion time - * @done_code: completion code - * - * This must be used whenever a job has been removed from the Hardware, e.g.: - * An IRQ indicates that the job finished (for both error and 'done' codes), or - * the job was evicted from the JS_HEAD_NEXT registers during a Soft/Hard stop. - * - * Some work is carried out immediately, and the rest is deferred onto a - * workqueue - * - * Context: - * This can be called safely from atomic context. - * The caller must hold kbdev->hwaccess_lock - */ -void kbase_jd_done(struct kbase_jd_atom *katom, unsigned int slot_nr, ktime_t *end_timestamp, - kbasep_js_atom_done_code done_code) -{ - struct kbase_context *kctx; - struct kbase_device *kbdev; - - CSTD_UNUSED(end_timestamp); - - KBASE_DEBUG_ASSERT(katom); - kctx = katom->kctx; - KBASE_DEBUG_ASSERT(kctx); - kbdev = kctx->kbdev; - KBASE_DEBUG_ASSERT(kbdev); - - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (done_code & KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT) - katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT; - - KBASE_KTRACE_ADD_JM(kbdev, JD_DONE, kctx, katom, katom->jc, 0); - - kbase_job_check_leave_disjoint(kbdev, katom); - - katom->slot_nr = slot_nr; - - atomic_inc(&kctx->work_count); - -#if IS_ENABLED(CONFIG_DEBUG_FS) - /* a failed job happened and is waiting for dumping*/ - if (!katom->will_fail_event_code && kbase_debug_job_fault_process(katom, katom->event_code)) - return; -#endif - - WARN_ON(work_pending(&katom->work)); - INIT_WORK(&katom->work, kbase_jd_done_worker); - queue_work(kctx->jctx.job_done_wq, &katom->work); -} - -KBASE_EXPORT_TEST_API(kbase_jd_done); - -void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx; - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(katom != NULL); - kctx = katom->kctx; - KBASE_DEBUG_ASSERT(kctx != NULL); - - dev_dbg(kbdev->dev, "JD: cancelling atom %pK\n", (void *)katom); - KBASE_KTRACE_ADD_JM(kbdev, JD_CANCEL, kctx, katom, katom->jc, 0); - - /* This should only be done from a context that is not scheduled */ - KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); - - WARN_ON(work_pending(&katom->work)); - - katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; - - INIT_WORK(&katom->work, jd_cancel_worker); - queue_work(kctx->jctx.job_done_wq, &katom->work); -} - -void kbase_jd_zap_context(struct kbase_context *kctx) -{ - struct kbase_jd_atom *katom; - struct list_head *entry, *tmp; - struct kbase_device *kbdev; - - KBASE_DEBUG_ASSERT(kctx); - - kbdev = kctx->kbdev; - - KBASE_KTRACE_ADD_JM(kbdev, JD_ZAP_CONTEXT, kctx, NULL, 0u, 0u); - - kbase_js_zap_context(kctx); - - mutex_lock(&kctx->jctx.lock); - - /* - * While holding the struct kbase_jd_context lock clean up jobs which are known to kbase but are - * queued outside the job scheduler. - */ - - del_timer_sync(&kctx->soft_job_timeout); - list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) { - katom = list_entry(entry, struct kbase_jd_atom, queue); - kbase_cancel_soft_job(katom); - } - - mutex_unlock(&kctx->jctx.lock); - -#if IS_ENABLED(CONFIG_DEBUG_FS) - kbase_debug_job_fault_kctx_unblock(kctx); -#endif - - kbase_jm_wait_for_zero_jobs(kctx); -} - -KBASE_EXPORT_TEST_API(kbase_jd_zap_context); - -int kbase_jd_init(struct kbase_context *kctx) -{ - int i; - int mali_err = 0; - struct priority_control_manager_device *pcm_device = NULL; - - KBASE_DEBUG_ASSERT(kctx); - pcm_device = kctx->kbdev->pcm_dev; - kctx->jctx.max_priority = KBASE_JS_ATOM_SCHED_PRIO_REALTIME; - - kctx->jctx.job_done_wq = alloc_workqueue("mali_jd", WQ_HIGHPRI | WQ_UNBOUND, 1); - if (kctx->jctx.job_done_wq == NULL) { - mali_err = -ENOMEM; - goto out1; - } - - for (i = 0; i < BASE_JD_ATOM_COUNT; i++) { - init_waitqueue_head(&kctx->jctx.atoms[i].completed); - - INIT_LIST_HEAD(&kctx->jctx.atoms[i].dep_head[0]); - INIT_LIST_HEAD(&kctx->jctx.atoms[i].dep_head[1]); - - /* Catch userspace attempting to use an atom which doesn't exist as a pre-dependency */ - kctx->jctx.atoms[i].event_code = BASE_JD_EVENT_JOB_INVALID; - kctx->jctx.atoms[i].status = KBASE_JD_ATOM_STATE_UNUSED; - -#if IS_ENABLED(CONFIG_SYNC_FILE) - kctx->jctx.atoms[i].dma_fence.context = dma_fence_context_alloc(1); -#endif - } - - mutex_init(&kctx->jctx.lock); - - init_waitqueue_head(&kctx->jctx.zero_jobs_wait); - - spin_lock_init(&kctx->jctx.tb_lock); - - INIT_LIST_HEAD(&kctx->completed_jobs); - - /* Check if there are platform rules for maximum priority */ - if (pcm_device) - kctx->jctx.max_priority = pcm_device->ops.pcm_scheduler_priority_check( - pcm_device, current, KBASE_JS_ATOM_SCHED_PRIO_REALTIME); - - return 0; - -out1: - return mali_err; -} - -KBASE_EXPORT_TEST_API(kbase_jd_init); - -void kbase_jd_exit(struct kbase_context *kctx) -{ - KBASE_DEBUG_ASSERT(kctx); - - /* Work queue is emptied by this */ - destroy_workqueue(kctx->jctx.job_done_wq); -} - -KBASE_EXPORT_TEST_API(kbase_jd_exit); diff --git a/drivers/gpu/arm/valhall/mali_kbase_jd_debugfs.c b/drivers/gpu/arm/valhall/mali_kbase_jd_debugfs.c deleted file mode 100644 index c3b6a63afc3f..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_jd_debugfs.c +++ /dev/null @@ -1,189 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#if IS_ENABLED(CONFIG_DEBUG_FS) - -#include -#include -#include -#if IS_ENABLED(CONFIG_SYNC_FILE) -#include -#endif -#include - -struct kbase_jd_debugfs_depinfo { - u8 id; - char type; -}; - -static void kbase_jd_debugfs_fence_info(struct kbase_jd_atom *atom, struct seq_file *sfile) -{ -#if IS_ENABLED(CONFIG_SYNC_FILE) - struct kbase_sync_fence_info info; - int res; - - switch (atom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { - case BASE_JD_REQ_SOFT_FENCE_TRIGGER: - res = kbase_sync_fence_out_info_get(atom, &info); - if (res == 0) - seq_printf(sfile, "Sa([%pK]%d) ", info.fence, info.status); - break; - case BASE_JD_REQ_SOFT_FENCE_WAIT: - res = kbase_sync_fence_in_info_get(atom, &info); - if (res == 0) - seq_printf(sfile, "Wa([%pK]%d) ", info.fence, info.status); - break; - default: - break; - } -#endif /* CONFIG_SYNC_FILE */ -} - -static void kbasep_jd_debugfs_atom_deps(struct kbase_jd_debugfs_depinfo *deps, - struct kbase_jd_atom *atom) -{ - struct kbase_context *kctx = atom->kctx; - int i; - - for (i = 0; i < 2; i++) { - deps[i].id = (unsigned int)(atom->dep[i].atom ? - kbase_jd_atom_id(kctx, atom->dep[i].atom) : - 0); - - switch (atom->dep[i].dep_type) { - case BASE_JD_DEP_TYPE_INVALID: - deps[i].type = ' '; - break; - case BASE_JD_DEP_TYPE_DATA: - deps[i].type = 'D'; - break; - case BASE_JD_DEP_TYPE_ORDER: - deps[i].type = '>'; - break; - default: - deps[i].type = '?'; - break; - } - } -} -/** - * kbasep_jd_debugfs_atoms_show - Show callback for the JD atoms debugfs file. - * @sfile: The debugfs entry - * @data: Data associated with the entry - * - * This function is called to get the contents of the JD atoms debugfs file. - * This is a report of all atoms managed by kbase_jd_context.atoms - * - * Return: 0 if successfully prints data in debugfs entry file, failure - * otherwise - */ -static int kbasep_jd_debugfs_atoms_show(struct seq_file *sfile, void *data) -{ - struct kbase_context *kctx = sfile->private; - struct kbase_jd_atom *atoms; - unsigned long irq_flags; - int i; - - CSTD_UNUSED(data); - - KBASE_DEBUG_ASSERT(kctx != NULL); - - /* Print version */ - seq_printf(sfile, "v%u\n", MALI_JD_DEBUGFS_VERSION); - - /* Print U/K API version */ - seq_printf(sfile, "ukv%u.%u\n", BASE_UK_VERSION_MAJOR, BASE_UK_VERSION_MINOR); - - /* Print table heading */ - seq_puts(sfile, " ID, Core req, St, Predeps, Start time, Additional info...\n"); - - atoms = kctx->jctx.atoms; - /* General atom states */ - mutex_lock(&kctx->jctx.lock); - /* JS-related states */ - spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, irq_flags); - for (i = 0; i != BASE_JD_ATOM_COUNT; ++i) { - struct kbase_jd_atom *atom = &atoms[i]; - s64 start_timestamp = 0; - struct kbase_jd_debugfs_depinfo deps[2]; - - if (atom->status == KBASE_JD_ATOM_STATE_UNUSED) - continue; - - /* start_timestamp is cleared as soon as the atom leaves UNUSED state - * and set before a job is submitted to the h/w, a non-zero value means - * it is valid - */ - if (ktime_to_ns(atom->start_timestamp)) - start_timestamp = - ktime_to_ns(ktime_sub(ktime_get_raw(), atom->start_timestamp)); - - kbasep_jd_debugfs_atom_deps(deps, atom); - - seq_printf(sfile, "%3u, %8x, %2u, %c%3u %c%3u, %20lld, ", i, atom->core_req, - atom->status, deps[0].type, deps[0].id, deps[1].type, deps[1].id, - start_timestamp); - - kbase_jd_debugfs_fence_info(atom, sfile); - - seq_puts(sfile, "\n"); - } - spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, irq_flags); - mutex_unlock(&kctx->jctx.lock); - - return 0; -} - -/** - * kbasep_jd_debugfs_atoms_open - open operation for atom debugfs file - * @in: &struct inode pointer - * @file: &struct file pointer - * - * Return: file descriptor - */ -static int kbasep_jd_debugfs_atoms_open(struct inode *in, struct file *file) -{ - return single_open(file, kbasep_jd_debugfs_atoms_show, in->i_private); -} - -static const struct file_operations kbasep_jd_debugfs_atoms_fops = { - .owner = THIS_MODULE, - .open = kbasep_jd_debugfs_atoms_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; - -void kbasep_jd_debugfs_ctx_init(struct kbase_context *kctx) -{ - const mode_t mode = 0444; - - /* Caller already ensures this, but we keep the pattern for - * maintenance safety. - */ - if (WARN_ON(!kctx) || WARN_ON(IS_ERR_OR_NULL(kctx->kctx_dentry))) - return; - - /* Expose all atoms */ - debugfs_create_file("atoms", mode, kctx->kctx_dentry, kctx, &kbasep_jd_debugfs_atoms_fops); -} - -#endif /* CONFIG_DEBUG_FS */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_jd_debugfs.h b/drivers/gpu/arm/valhall/mali_kbase_jd_debugfs.h deleted file mode 100644 index 8ea18b36d0d8..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_jd_debugfs.h +++ /dev/null @@ -1,43 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/** - * DOC: Header file for job dispatcher-related entries in debugfs - */ - -#ifndef _KBASE_JD_DEBUGFS_H -#define _KBASE_JD_DEBUGFS_H - -#include - -#define MALI_JD_DEBUGFS_VERSION 3 - -/* Forward declarations */ -struct kbase_context; - -/** - * kbasep_jd_debugfs_ctx_init() - Add debugfs entries for JD system - * - * @kctx: Pointer to kbase_context - */ -void kbasep_jd_debugfs_ctx_init(struct kbase_context *kctx); - -#endif /*_KBASE_JD_DEBUGFS_H*/ diff --git a/drivers/gpu/arm/valhall/mali_kbase_jm.c b/drivers/gpu/arm/valhall/mali_kbase_jm.c deleted file mode 100644 index cac12df225a0..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_jm.c +++ /dev/null @@ -1,153 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2013-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * HW access job manager common APIs - */ - -#include -#include "mali_kbase_hwaccess_jm.h" -#include "mali_kbase_jm.h" - -#if !MALI_USE_CSF -/** - * kbase_jm_next_job() - Attempt to run the next @nr_jobs_to_submit jobs on slot - * @js on the active context. - * @kbdev: Device pointer - * @js: Job slot to run on - * @nr_jobs_to_submit: Number of jobs to attempt to submit - * - * Return: true if slot can still be submitted on, false if slot is now full. - */ -static bool kbase_jm_next_job(struct kbase_device *kbdev, unsigned int js, int nr_jobs_to_submit) -{ - struct kbase_context *kctx; - int i; - - kctx = kbdev->hwaccess.active_kctx[js]; - dev_dbg(kbdev->dev, "Trying to run the next %d jobs in kctx %pK (s:%u)\n", - nr_jobs_to_submit, (void *)kctx, js); - - if (!kctx) - return true; - - for (i = 0; i < nr_jobs_to_submit; i++) { - struct kbase_jd_atom *katom = kbase_js_pull(kctx, js); - - if (!katom) - return true; /* Context has no jobs on this slot */ - - kbase_backend_run_atom(kbdev, katom); - } - - dev_dbg(kbdev->dev, "Slot ringbuffer should now be full (s:%u)\n", js); - return false; -} - -u32 kbase_jm_kick(struct kbase_device *kbdev, u32 js_mask) -{ - u32 ret_mask = 0; - - lockdep_assert_held(&kbdev->hwaccess_lock); - dev_dbg(kbdev->dev, "JM kick slot mask 0x%x\n", js_mask); - - while (js_mask) { - unsigned int js = (unsigned int)ffs((int)js_mask) - 1U; - int nr_jobs_to_submit = kbase_backend_slot_free(kbdev, js); - - if (kbase_jm_next_job(kbdev, js, nr_jobs_to_submit)) - ret_mask |= (1U << js); - - js_mask &= ~(1U << js); - } - - dev_dbg(kbdev->dev, "Can still submit to mask 0x%x\n", ret_mask); - return ret_mask; -} - -void kbase_jm_try_kick(struct kbase_device *kbdev, u32 js_mask) -{ - struct kbasep_js_device_data *js_devdata = &kbdev->js_data; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (!down_trylock(&js_devdata->schedule_sem)) { - kbase_jm_kick(kbdev, js_mask); - up(&js_devdata->schedule_sem); - } -} - -void kbase_jm_try_kick_all(struct kbase_device *kbdev) -{ - struct kbasep_js_device_data *js_devdata = &kbdev->js_data; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (!down_trylock(&js_devdata->schedule_sem)) { - kbase_jm_kick_all(kbdev); - up(&js_devdata->schedule_sem); - } -} - -void kbase_jm_idle_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) -{ - unsigned int js; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { - if (kbdev->hwaccess.active_kctx[js] == kctx) { - dev_dbg(kbdev->dev, "Marking kctx %pK as inactive (s:%u)\n", (void *)kctx, - js); - kbdev->hwaccess.active_kctx[js] = NULL; - } - } -} - -struct kbase_jd_atom *kbase_jm_return_atom_to_js(struct kbase_device *kbdev, - struct kbase_jd_atom *katom) -{ - lockdep_assert_held(&kbdev->hwaccess_lock); - - dev_dbg(kbdev->dev, "Atom %pK is returning with event code 0x%x\n", (void *)katom, - katom->event_code); - - KBASE_KTRACE_ADD_JM(kbdev, JM_RETURN_ATOM_TO_JS, katom->kctx, katom, katom->jc, - katom->event_code); - - if (katom->event_code != BASE_JD_EVENT_STOPPED && - katom->event_code != BASE_JD_EVENT_REMOVED_FROM_NEXT) { - return kbase_js_complete_atom(katom, NULL); - } - - kbase_js_unpull(katom->kctx, katom); - - return NULL; -} - -struct kbase_jd_atom *kbase_jm_complete(struct kbase_device *kbdev, struct kbase_jd_atom *katom, - ktime_t *end_timestamp) -{ - lockdep_assert_held(&kbdev->hwaccess_lock); - - return kbase_js_complete_atom(katom, end_timestamp); -} -#endif /* !MALI_USE_CSF */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_jm.h b/drivers/gpu/arm/valhall/mali_kbase_jm.h deleted file mode 100644 index fb0e69a0aee3..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_jm.h +++ /dev/null @@ -1,117 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2013-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Job manager common APIs - */ - -#ifndef _KBASE_JM_H_ -#define _KBASE_JM_H_ - -#if !MALI_USE_CSF -/** - * kbase_jm_kick() - Indicate that there are jobs ready to run. - * @kbdev: Device pointer - * @js_mask: Mask of the job slots that can be pulled from. - * - * Caller must hold the hwaccess_lock and schedule_sem semaphore - * - * Return: Mask of the job slots that can still be submitted to. - */ -u32 kbase_jm_kick(struct kbase_device *kbdev, u32 js_mask); - -/** - * kbase_jm_kick_all() - Indicate that there are jobs ready to run on all job - * slots. - * @kbdev: Device pointer - * - * Caller must hold the hwaccess_lock and schedule_sem semaphore - * - * Return: Mask of the job slots that can still be submitted to. - */ -static inline u32 kbase_jm_kick_all(struct kbase_device *kbdev) -{ - return kbase_jm_kick(kbdev, (1U << kbdev->gpu_props.num_job_slots) - 1U); -} - -/** - * kbase_jm_try_kick - Attempt to call kbase_jm_kick - * @kbdev: Device pointer - * @js_mask: Mask of the job slots that can be pulled from - * Context: Caller must hold hwaccess_lock - * - * If schedule_sem can be immediately obtained then this function will call - * kbase_jm_kick() otherwise it will do nothing. - */ -void kbase_jm_try_kick(struct kbase_device *kbdev, u32 js_mask); - -/** - * kbase_jm_try_kick_all() - Attempt to call kbase_jm_kick_all - * @kbdev: Device pointer - * Context: Caller must hold hwaccess_lock - * - * If schedule_sem can be immediately obtained then this function will call - * kbase_jm_kick_all() otherwise it will do nothing. - */ -void kbase_jm_try_kick_all(struct kbase_device *kbdev); -#endif /* !MALI_USE_CSF */ - -#if !MALI_USE_CSF -/** - * kbase_jm_idle_ctx() - Mark a context as idle. - * @kbdev: Device pointer - * @kctx: Context to mark as idle - * - * No more atoms will be pulled from this context until it is marked as active - * by kbase_js_use_ctx(). - * - * The context should have no atoms currently pulled from it - * (kbase_jsctx_atoms_pulled(kctx) == 0). - * - * Caller must hold the hwaccess_lock - */ -void kbase_jm_idle_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); - -/** - * kbase_jm_return_atom_to_js() - Return an atom to the job scheduler that has - * been soft-stopped or will fail due to a - * dependency - * @kbdev: Device pointer - * @katom: Atom that has been stopped or will be failed - * - * Return: Atom that has now been unblocked and can now be run, or NULL if none - */ -struct kbase_jd_atom *kbase_jm_return_atom_to_js(struct kbase_device *kbdev, - struct kbase_jd_atom *katom); - -/** - * kbase_jm_complete() - Complete an atom - * @kbdev: Device pointer - * @katom: Atom that has completed - * @end_timestamp: Timestamp of atom completion - * - * Return: Atom that has now been unblocked and can now be run, or NULL if none - */ -struct kbase_jd_atom *kbase_jm_complete(struct kbase_device *kbdev, struct kbase_jd_atom *katom, - ktime_t *end_timestamp); -#endif /* !MALI_USE_CSF */ - -#endif /* _KBASE_JM_H_ */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_js.c b/drivers/gpu/arm/valhall/mali_kbase_js.c deleted file mode 100644 index 8e3dd7023373..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_js.c +++ /dev/null @@ -1,3271 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2011-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Job Scheduler Implementation - */ -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "mali_kbase_jm.h" -#include "mali_kbase_hwaccess_jm.h" -#include -#include -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) -#include -#endif - -/* - * Private types - */ - -/* Bitpattern indicating the result of releasing a context */ -enum { - /* The context was descheduled - caller should try scheduling in a new - * one to keep the runpool full - */ - KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED = (1u << 0), - /* Ctx attributes were changed - caller should try scheduling all - * contexts - */ - KBASEP_JS_RELEASE_RESULT_SCHED_ALL = (1u << 1) -}; - -typedef u32 kbasep_js_release_result; - -const int kbasep_js_atom_priority_to_relative[BASE_JD_NR_PRIO_LEVELS] = { - KBASE_JS_ATOM_SCHED_PRIO_MED, /* BASE_JD_PRIO_MEDIUM */ - KBASE_JS_ATOM_SCHED_PRIO_HIGH, /* BASE_JD_PRIO_HIGH */ - KBASE_JS_ATOM_SCHED_PRIO_LOW, /* BASE_JD_PRIO_LOW */ - KBASE_JS_ATOM_SCHED_PRIO_REALTIME /* BASE_JD_PRIO_REALTIME */ -}; - -const base_jd_prio kbasep_js_relative_priority_to_atom[KBASE_JS_ATOM_SCHED_PRIO_COUNT] = { - BASE_JD_PRIO_REALTIME, /* KBASE_JS_ATOM_SCHED_PRIO_REALTIME */ - BASE_JD_PRIO_HIGH, /* KBASE_JS_ATOM_SCHED_PRIO_HIGH */ - BASE_JD_PRIO_MEDIUM, /* KBASE_JS_ATOM_SCHED_PRIO_MED */ - BASE_JD_PRIO_LOW /* KBASE_JS_ATOM_SCHED_PRIO_LOW */ -}; - -/* - * Private function prototypes - */ -static kbasep_js_release_result -kbasep_js_runpool_release_ctx_internal(struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbasep_js_atom_retained_state *katom_retained_state); - -static unsigned int kbase_js_get_slot(struct kbase_device *kbdev, struct kbase_jd_atom *katom); - -static void kbase_js_foreach_ctx_job(struct kbase_context *kctx, kbasep_js_ctx_job_cb *callback); - -/* Helper for ktrace */ -#if KBASE_KTRACE_ENABLE -static int kbase_ktrace_get_ctx_refcnt(struct kbase_context *kctx) -{ - return atomic_read(&kctx->refcount); -} -#else /* KBASE_KTRACE_ENABLE */ -static int kbase_ktrace_get_ctx_refcnt(struct kbase_context *kctx) -{ - CSTD_UNUSED(kctx); - return 0; -} -#endif /* KBASE_KTRACE_ENABLE */ - -/* - * Private functions - */ - -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) -/** - * gpu_metrics_timer_callback() - Callback function for the GPU metrics hrtimer - * - * @timer: Pointer to the GPU metrics hrtimer - * - * This function will emit power/gpu_work_period tracepoint for all the active - * GPU metrics contexts. The timer will be restarted if needed. - * - * Return: enum value to indicate that timer should not be restarted. - */ -static enum hrtimer_restart gpu_metrics_timer_callback(struct hrtimer *timer) -{ - struct kbasep_js_device_data *js_devdata = - container_of(timer, struct kbasep_js_device_data, gpu_metrics_timer); - struct kbase_device *kbdev = container_of(js_devdata, struct kbase_device, js_data); - unsigned long flags; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_gpu_metrics_emit_tracepoint(kbdev, ktime_get_raw_ns()); - WARN_ON_ONCE(!js_devdata->gpu_metrics_timer_running); - if (js_devdata->gpu_metrics_timer_needed) { - hrtimer_start(&js_devdata->gpu_metrics_timer, - HR_TIMER_DELAY_NSEC(kbase_gpu_metrics_get_tp_emit_interval()), - HRTIMER_MODE_REL); - } else - js_devdata->gpu_metrics_timer_running = false; - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - return HRTIMER_NORESTART; -} - -/** - * gpu_metrics_ctx_init() - Take a reference on GPU metrics context if it exists, - * otherwise allocate and initialise one. - * - * @kctx: Pointer to the Kbase context. - * - * The GPU metrics context represents an "Application" for the purposes of GPU metrics - * reporting. There may be multiple kbase_contexts contributing data to a single GPU - * metrics context. - * This function takes a reference on GPU metrics context if it already exists - * corresponding to the Application that is creating the Kbase context, otherwise - * memory is allocated for it and initialised. - * - * Return: 0 on success, or negative on failure. - */ -static inline int gpu_metrics_ctx_init(struct kbase_context *kctx) -{ - struct kbase_gpu_metrics_ctx *gpu_metrics_ctx; - struct kbase_device *kbdev = kctx->kbdev; - unsigned long flags; - int ret = 0; - - const struct cred *cred = get_current_cred(); - const unsigned int aid = cred->euid.val; - - put_cred(cred); - - /* Return early if this is not a Userspace created context */ - if (unlikely(!kctx->filp)) - return 0; - - /* Serialize against the other threads trying to create/destroy Kbase contexts. */ - mutex_lock(&kbdev->kctx_list_lock); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - gpu_metrics_ctx = kbase_gpu_metrics_ctx_get(kbdev, aid); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - if (!gpu_metrics_ctx) { - gpu_metrics_ctx = kmalloc(sizeof(*gpu_metrics_ctx), GFP_KERNEL); - - if (gpu_metrics_ctx) { - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_gpu_metrics_ctx_init(kbdev, gpu_metrics_ctx, aid); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - } else { - dev_err(kbdev->dev, "Allocation for gpu_metrics_ctx failed"); - ret = -ENOMEM; - } - } - - kctx->gpu_metrics_ctx = gpu_metrics_ctx; - mutex_unlock(&kbdev->kctx_list_lock); - - return ret; -} - -/** - * gpu_metrics_ctx_term() - Drop a reference on a GPU metrics context and free it - * if the refcount becomes 0. - * - * @kctx: Pointer to the Kbase context. - */ -static inline void gpu_metrics_ctx_term(struct kbase_context *kctx) -{ - unsigned long flags; - - /* Return early if this is not a Userspace created context */ - if (unlikely(!kctx->filp)) - return; - - /* Serialize against the other threads trying to create/destroy Kbase contexts. */ - mutex_lock(&kctx->kbdev->kctx_list_lock); - spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags); - kbase_gpu_metrics_ctx_put(kctx->kbdev, kctx->gpu_metrics_ctx); - spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); - mutex_unlock(&kctx->kbdev->kctx_list_lock); -} -#endif - -/** - * core_reqs_from_jsn_features - Convert JSn_FEATURES to core requirements - * @features: parsed JSn_FEATURE register value - * - * Given a JSn_FEATURE register value returns the core requirements that match - * - * Return: Core requirement bit mask - */ -static base_jd_core_req core_reqs_from_jsn_features(struct kbase_js_features_props *features) -{ - base_jd_core_req core_req = 0u; - - if (features->write_value) - core_req |= BASE_JD_REQ_V; - - if (features->cache_flush) - core_req |= BASE_JD_REQ_CF; - - if (features->compute_shader) - core_req |= BASE_JD_REQ_CS; - - if (features->tiler) - core_req |= BASE_JD_REQ_T; - - if (features->fragment_shader) - core_req |= BASE_JD_REQ_FS; - - return core_req; -} - -static void kbase_js_sync_timers(struct kbase_device *kbdev) -{ - mutex_lock(&kbdev->js_data.runpool_mutex); - kbase_backend_ctx_count_changed(kbdev); - mutex_unlock(&kbdev->js_data.runpool_mutex); -} - -/** - * jsctx_rb_none_to_pull_prio(): - Check if there are no pullable atoms - * @kctx: Pointer to kbase context with ring buffer. - * @js: Job slot id to check. - * @prio: Priority to check. - * - * Return true if there are no atoms to pull. There may be running atoms in the - * ring buffer even if there are no atoms to pull. It is also possible for the - * ring buffer to be full (with running atoms) when this functions returns - * true. - * - * Return: true if there are no atoms to pull, false otherwise. - */ -static inline bool jsctx_rb_none_to_pull_prio(struct kbase_context *kctx, unsigned int js, int prio) -{ - bool none_to_pull; - struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - none_to_pull = RB_EMPTY_ROOT(&rb->runnable_tree); - - dev_dbg(kctx->kbdev->dev, "Slot %u (prio %d) is %spullable in kctx %pK\n", js, prio, - none_to_pull ? "not " : "", kctx); - - return none_to_pull; -} - -/** - * jsctx_rb_none_to_pull(): - Check if all priority ring buffers have no - * pullable atoms - * @kctx: Pointer to kbase context with ring buffer. - * @js: Job slot id to check. - * - * Caller must hold hwaccess_lock - * - * Return: true if the ring buffers for all priorities have no pullable atoms, - * false otherwise. - */ -static inline bool jsctx_rb_none_to_pull(struct kbase_context *kctx, unsigned int js) -{ - int prio; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - for (prio = KBASE_JS_ATOM_SCHED_PRIO_FIRST; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { - if (!jsctx_rb_none_to_pull_prio(kctx, js, prio)) - return false; - } - - return true; -} - -/** - * jsctx_queue_foreach_prio(): - Execute callback for each entry in the queue. - * @kctx: Pointer to kbase context with the queue. - * @js: Job slot id to iterate. - * @prio: Priority id to iterate. - * @callback: Function pointer to callback. - * - * Iterate over a queue and invoke @callback for each entry in the queue, and - * remove the entry from the queue. - * - * If entries are added to the queue while this is running those entries may, or - * may not be covered. To ensure that all entries in the buffer have been - * enumerated when this function returns jsctx->lock must be held when calling - * this function. - * - * The HW access lock must always be held when calling this function. - */ -static void jsctx_queue_foreach_prio(struct kbase_context *kctx, unsigned int js, int prio, - kbasep_js_ctx_job_cb *callback) -{ - struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - while (!RB_EMPTY_ROOT(&queue->runnable_tree)) { - struct rb_node *node = rb_first(&queue->runnable_tree); - struct kbase_jd_atom *entry = - rb_entry(node, struct kbase_jd_atom, runnable_tree_node); - - rb_erase(node, &queue->runnable_tree); - callback(kctx->kbdev, entry); - } - - while (!list_empty(&queue->x_dep_head)) { - struct kbase_jd_atom *entry = - list_entry(queue->x_dep_head.next, struct kbase_jd_atom, queue); - - WARN_ON(!(entry->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST)); - dev_dbg(kctx->kbdev->dev, "Del blocked atom %pK from X_DEP list\n", (void *)entry); - - list_del(queue->x_dep_head.next); - entry->atom_flags &= ~KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; - - callback(kctx->kbdev, entry); - } -} - -/** - * jsctx_queue_foreach(): - Execute callback for each entry in every queue - * @kctx: Pointer to kbase context with queue. - * @js: Job slot id to iterate. - * @callback: Function pointer to callback. - * - * Iterate over all the different priorities, and for each call - * jsctx_queue_foreach_prio() to iterate over the queue and invoke @callback - * for each entry, and remove the entry from the queue. - */ -static inline void jsctx_queue_foreach(struct kbase_context *kctx, unsigned int js, - kbasep_js_ctx_job_cb *callback) -{ - int prio; - - for (prio = KBASE_JS_ATOM_SCHED_PRIO_FIRST; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) - jsctx_queue_foreach_prio(kctx, js, prio, callback); -} - -/** - * jsctx_rb_peek_prio(): - Check buffer and get next atom - * @kctx: Pointer to kbase context with ring buffer. - * @js: Job slot id to check. - * @prio: Priority id to check. - * - * Check the ring buffer for the specified @js and @prio and return a pointer to - * the next atom, unless the ring buffer is empty. - * - * Return: Pointer to next atom in buffer, or NULL if there is no atom. - */ -static inline struct kbase_jd_atom *jsctx_rb_peek_prio(struct kbase_context *kctx, unsigned int js, - int prio) -{ - struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; - struct rb_node *node; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - dev_dbg(kctx->kbdev->dev, "Peeking runnable tree of kctx %pK for prio %d (s:%u)\n", - (void *)kctx, prio, js); - - node = rb_first(&rb->runnable_tree); - if (!node) { - dev_dbg(kctx->kbdev->dev, "Tree is empty\n"); - return NULL; - } - - return rb_entry(node, struct kbase_jd_atom, runnable_tree_node); -} - -/** - * jsctx_rb_peek(): - Check all priority buffers and get next atom - * @kctx: Pointer to kbase context with ring buffer. - * @js: Job slot id to check. - * - * Check the ring buffers for all priorities, starting from - * KBASE_JS_ATOM_SCHED_PRIO_REALTIME, for the specified @js and @prio and return a - * pointer to the next atom, unless all the priority's ring buffers are empty. - * - * Caller must hold the hwaccess_lock. - * - * Return: Pointer to next atom in buffer, or NULL if there is no atom. - */ -static inline struct kbase_jd_atom *jsctx_rb_peek(struct kbase_context *kctx, unsigned int js) -{ - int prio; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - for (prio = KBASE_JS_ATOM_SCHED_PRIO_FIRST; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { - struct kbase_jd_atom *katom; - - katom = jsctx_rb_peek_prio(kctx, js, prio); - if (katom) - return katom; - } - - return NULL; -} - -/** - * jsctx_rb_pull(): - Mark atom in list as running - * @kctx: Pointer to kbase context with ring buffer. - * @katom: Pointer to katom to pull. - * - * Mark an atom previously obtained from jsctx_rb_peek() as running. - * - * @katom must currently be at the head of the ring buffer. - */ -static inline void jsctx_rb_pull(struct kbase_context *kctx, struct kbase_jd_atom *katom) -{ - int prio = katom->sched_priority; - unsigned int js = katom->slot_nr; - struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - dev_dbg(kctx->kbdev->dev, "Erasing atom %pK from runnable tree of kctx %pK\n", - (void *)katom, (void *)kctx); - - /* Atoms must be pulled in the correct order. */ - WARN_ON(katom != jsctx_rb_peek_prio(kctx, js, prio)); - - rb_erase(&katom->runnable_tree_node, &rb->runnable_tree); -} - -static void jsctx_tree_add(struct kbase_context *kctx, struct kbase_jd_atom *katom) -{ - struct kbase_device *kbdev = kctx->kbdev; - int prio = katom->sched_priority; - unsigned int js = katom->slot_nr; - struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; - struct rb_node **new = &(queue->runnable_tree.rb_node), *parent = NULL; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - dev_dbg(kbdev->dev, "Adding atom %pK to runnable tree of kctx %pK (s:%u)\n", (void *)katom, - (void *)kctx, js); - - while (*new) { - struct kbase_jd_atom *entry = - container_of(*new, struct kbase_jd_atom, runnable_tree_node); - - parent = *new; - if (kbase_jd_atom_is_younger(katom, entry)) - new = &((*new)->rb_left); - else - new = &((*new)->rb_right); - } - - /* Add new node and rebalance tree. */ - rb_link_node(&katom->runnable_tree_node, parent, new); - rb_insert_color(&katom->runnable_tree_node, &queue->runnable_tree); - - KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(kbdev, katom, TL_ATOM_STATE_READY); -} - -/** - * jsctx_rb_unpull(): - Undo marking of atom in list as running - * @kctx: Pointer to kbase context with ring buffer. - * @katom: Pointer to katom to unpull. - * - * Undo jsctx_rb_pull() and put @katom back in the queue. - * - * jsctx_rb_unpull() must be called on atoms in the same order the atoms were - * pulled. - */ -static inline void jsctx_rb_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom) -{ - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - KBASE_KTRACE_ADD_JM(kctx->kbdev, JS_UNPULL_JOB, kctx, katom, katom->jc, 0u); - - jsctx_tree_add(kctx, katom); -} - -static bool kbase_js_ctx_pullable(struct kbase_context *kctx, unsigned int js, bool is_scheduled); -static bool kbase_js_ctx_list_add_pullable_nolock(struct kbase_device *kbdev, - struct kbase_context *kctx, unsigned int js); -static bool kbase_js_ctx_list_add_unpullable_nolock(struct kbase_device *kbdev, - struct kbase_context *kctx, unsigned int js); - -typedef bool(katom_ordering_func)(const struct kbase_jd_atom *, const struct kbase_jd_atom *); - -bool kbase_js_atom_runs_before(struct kbase_device *kbdev, const struct kbase_jd_atom *katom_a, - const struct kbase_jd_atom *katom_b, - const kbase_atom_ordering_flag_t order_flags) -{ - struct kbase_context *kctx_a = katom_a->kctx; - struct kbase_context *kctx_b = katom_b->kctx; - katom_ordering_func *samectxatomprio_ordering_func = kbase_jd_atom_is_younger; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (order_flags & KBASE_ATOM_ORDERING_FLAG_SEQNR) - samectxatomprio_ordering_func = kbase_jd_atom_is_earlier; - - /* It only makes sense to make this test for atoms on the same slot */ - WARN_ON(katom_a->slot_nr != katom_b->slot_nr); - - if (kbdev->js_ctx_scheduling_mode == KBASE_JS_PROCESS_LOCAL_PRIORITY_MODE) { - /* In local priority mode, querying either way around for "a - * should run before b" and "b should run before a" should - * always be false when they're from different contexts - */ - if (kctx_a != kctx_b) - return false; - } else { - /* In system priority mode, ordering is done first strictly by - * context priority, even when katom_b might be lower priority - * than katom_a. This is due to scheduling of contexts in order - * of highest priority first, regardless of whether the atoms - * for a particular slot from such contexts have the highest - * priority or not. - */ - if (kctx_a != kctx_b) { - if (kctx_a->priority < kctx_b->priority) - return true; - if (kctx_a->priority > kctx_b->priority) - return false; - } - } - - /* For same contexts/contexts with the same context priority (in system - * priority mode), ordering is next done by atom priority - */ - if (katom_a->sched_priority < katom_b->sched_priority) - return true; - if (katom_a->sched_priority > katom_b->sched_priority) - return false; - /* For atoms of same priority on the same kctx, they are - * ordered by seq_nr/age (dependent on caller) - */ - if (kctx_a == kctx_b && samectxatomprio_ordering_func(katom_a, katom_b)) - return true; - - return false; -} - -/* - * Functions private to KBase ('Protected' functions) - */ -int kbasep_js_devdata_init(struct kbase_device *const kbdev) -{ - struct kbasep_js_device_data *jsdd; - int i, j; - - KBASE_DEBUG_ASSERT(kbdev != NULL); - - jsdd = &kbdev->js_data; - - INIT_LIST_HEAD(&jsdd->suspended_soft_jobs_list); - - /* Config attributes */ - jsdd->scheduling_period_ns = DEFAULT_JS_SCHEDULING_PERIOD_NS; - jsdd->soft_stop_ticks = DEFAULT_JS_SOFT_STOP_TICKS; - jsdd->soft_stop_ticks_cl = DEFAULT_JS_SOFT_STOP_TICKS_CL; - jsdd->hard_stop_ticks_ss = DEFAULT_JS_HARD_STOP_TICKS_SS; - jsdd->hard_stop_ticks_cl = DEFAULT_JS_HARD_STOP_TICKS_CL; - jsdd->hard_stop_ticks_dumping = DEFAULT_JS_HARD_STOP_TICKS_DUMPING; - jsdd->gpu_reset_ticks_ss = DEFAULT_JS_RESET_TICKS_SS; - jsdd->gpu_reset_ticks_cl = DEFAULT_JS_RESET_TICKS_CL; - - jsdd->gpu_reset_ticks_dumping = DEFAULT_JS_RESET_TICKS_DUMPING; - jsdd->ctx_timeslice_ns = DEFAULT_JS_CTX_TIMESLICE_NS; - atomic_set(&jsdd->soft_job_timeout_ms, DEFAULT_JS_SOFT_JOB_TIMEOUT); - jsdd->js_free_wait_time_ms = kbase_get_timeout_ms(kbdev, JM_DEFAULT_JS_FREE_TIMEOUT); - - dev_dbg(kbdev->dev, "JS Config Attribs: "); - dev_dbg(kbdev->dev, "\tscheduling_period_ns:%u", jsdd->scheduling_period_ns); - dev_dbg(kbdev->dev, "\tsoft_stop_ticks:%u", jsdd->soft_stop_ticks); - dev_dbg(kbdev->dev, "\tsoft_stop_ticks_cl:%u", jsdd->soft_stop_ticks_cl); - dev_dbg(kbdev->dev, "\thard_stop_ticks_ss:%u", jsdd->hard_stop_ticks_ss); - dev_dbg(kbdev->dev, "\thard_stop_ticks_cl:%u", jsdd->hard_stop_ticks_cl); - dev_dbg(kbdev->dev, "\thard_stop_ticks_dumping:%u", jsdd->hard_stop_ticks_dumping); - dev_dbg(kbdev->dev, "\tgpu_reset_ticks_ss:%u", jsdd->gpu_reset_ticks_ss); - dev_dbg(kbdev->dev, "\tgpu_reset_ticks_cl:%u", jsdd->gpu_reset_ticks_cl); - dev_dbg(kbdev->dev, "\tgpu_reset_ticks_dumping:%u", jsdd->gpu_reset_ticks_dumping); - dev_dbg(kbdev->dev, "\tctx_timeslice_ns:%u", jsdd->ctx_timeslice_ns); - dev_dbg(kbdev->dev, "\tsoft_job_timeout:%i", atomic_read(&jsdd->soft_job_timeout_ms)); - dev_dbg(kbdev->dev, "\tjs_free_wait_time_ms:%u", jsdd->js_free_wait_time_ms); - - if (!(jsdd->soft_stop_ticks < jsdd->hard_stop_ticks_ss && - jsdd->hard_stop_ticks_ss < jsdd->gpu_reset_ticks_ss && - jsdd->soft_stop_ticks < jsdd->hard_stop_ticks_dumping && - jsdd->hard_stop_ticks_dumping < jsdd->gpu_reset_ticks_dumping)) { - dev_err(kbdev->dev, - "Job scheduler timeouts invalid; soft/hard/reset tick counts should be in increasing order\n"); - return -EINVAL; - } - -#if KBASE_DISABLE_SCHEDULING_SOFT_STOPS - dev_dbg(kbdev->dev, - "Job Scheduling Soft-stops disabled, ignoring value for soft_stop_ticks==%u at %uns per tick. Other soft-stops may still occur.", - jsdd->soft_stop_ticks, jsdd->scheduling_period_ns); -#endif -#if KBASE_DISABLE_SCHEDULING_HARD_STOPS - dev_dbg(kbdev->dev, - "Job Scheduling Hard-stops disabled, ignoring values for hard_stop_ticks_ss==%d and hard_stop_ticks_dumping==%u at %uns per tick. Other hard-stops may still occur.", - jsdd->hard_stop_ticks_ss, jsdd->hard_stop_ticks_dumping, - jsdd->scheduling_period_ns); -#endif -#if KBASE_DISABLE_SCHEDULING_SOFT_STOPS && KBASE_DISABLE_SCHEDULING_HARD_STOPS - dev_dbg(kbdev->dev, - "Note: The JS tick timer (if coded) will still be run, but do nothing."); -#endif - - for (i = 0; i < kbdev->gpu_props.num_job_slots; ++i) - jsdd->js_reqs[i] = core_reqs_from_jsn_features(&kbdev->gpu_props.js_features[i]); - - /* On error, we could continue on: providing none of the below resources - * rely on the ones above - */ - - mutex_init(&jsdd->runpool_mutex); - mutex_init(&jsdd->queue_mutex); - sema_init(&jsdd->schedule_sem, 1); - - for (i = 0; i < kbdev->gpu_props.num_job_slots; ++i) { - for (j = KBASE_JS_ATOM_SCHED_PRIO_FIRST; j < KBASE_JS_ATOM_SCHED_PRIO_COUNT; ++j) { - INIT_LIST_HEAD(&jsdd->ctx_list_pullable[i][j]); - INIT_LIST_HEAD(&jsdd->ctx_list_unpullable[i][j]); - } - } - -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) - hrtimer_init(&jsdd->gpu_metrics_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - jsdd->gpu_metrics_timer.function = gpu_metrics_timer_callback; - jsdd->gpu_metrics_timer_needed = false; - jsdd->gpu_metrics_timer_running = false; -#endif - - return 0; -} - -void kbasep_js_devdata_halt(struct kbase_device *kbdev) -{ - CSTD_UNUSED(kbdev); -} - -void kbasep_js_devdata_term(struct kbase_device *kbdev) -{ - struct kbasep_js_device_data *js_devdata; - s8 zero_ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT] = { - 0, - }; - CSTD_UNUSED(js_devdata); - - KBASE_DEBUG_ASSERT(kbdev != NULL); - - js_devdata = &kbdev->js_data; - - /* The caller must de-register all contexts before calling this - */ - KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running == 0); - KBASE_DEBUG_ASSERT(memcmp(js_devdata->runpool_irq.ctx_attr_ref_count, - zero_ctx_attr_ref_count, sizeof(zero_ctx_attr_ref_count)) == 0); - CSTD_UNUSED(zero_ctx_attr_ref_count); - -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) - js_devdata->gpu_metrics_timer_needed = false; - hrtimer_cancel(&js_devdata->gpu_metrics_timer); -#endif -} - -int kbasep_js_kctx_init(struct kbase_context *const kctx) -{ - struct kbasep_js_kctx_info *js_kctx_info; - int i, j; - int ret; - CSTD_UNUSED(js_kctx_info); - - KBASE_DEBUG_ASSERT(kctx != NULL); - - CSTD_UNUSED(ret); -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) - ret = gpu_metrics_ctx_init(kctx); - if (ret) - return ret; -#endif - - for (i = 0; i < BASE_JM_MAX_NR_SLOTS; ++i) - INIT_LIST_HEAD(&kctx->jctx.sched_info.ctx.ctx_list_entry[i]); - - js_kctx_info = &kctx->jctx.sched_info; - - kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); - kbase_ctx_flag_clear(kctx, KCTX_DYING); - - /* Initially, the context is disabled from submission until the create - * flags are set - */ - kbase_ctx_flag_set(kctx, KCTX_SUBMIT_DISABLED); - - /* On error, we could continue on: providing none of the below resources - * rely on the ones above - */ - mutex_init(&js_kctx_info->ctx.jsctx_mutex); - - init_waitqueue_head(&js_kctx_info->ctx.is_scheduled_wait); - - for (i = KBASE_JS_ATOM_SCHED_PRIO_FIRST; i < KBASE_JS_ATOM_SCHED_PRIO_COUNT; i++) { - for (j = 0; j < BASE_JM_MAX_NR_SLOTS; j++) { - INIT_LIST_HEAD(&kctx->jsctx_queue[i][j].x_dep_head); - kctx->jsctx_queue[i][j].runnable_tree = RB_ROOT; - } - } - - return 0; -} - -void kbasep_js_kctx_term(struct kbase_context *kctx) -{ - struct kbase_device *kbdev; - struct kbasep_js_kctx_info *js_kctx_info; - unsigned int js; - bool update_ctx_count = false; - unsigned long flags; - CSTD_UNUSED(js_kctx_info); - - KBASE_DEBUG_ASSERT(kctx != NULL); - - kbdev = kctx->kbdev; - KBASE_DEBUG_ASSERT(kbdev != NULL); - - js_kctx_info = &kctx->jctx.sched_info; - - /* The caller must de-register all jobs before calling this */ - KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); - KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs == 0); - - mutex_lock(&kbdev->js_data.queue_mutex); - mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) - list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - if (kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)) { - WARN_ON(atomic_read(&kbdev->js_data.nr_contexts_runnable) <= 0); - atomic_dec(&kbdev->js_data.nr_contexts_runnable); - update_ctx_count = true; - kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); - } - - mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); - mutex_unlock(&kbdev->js_data.queue_mutex); - - if (update_ctx_count) { - mutex_lock(&kbdev->js_data.runpool_mutex); - kbase_backend_ctx_count_changed(kbdev); - mutex_unlock(&kbdev->js_data.runpool_mutex); - } - - kbase_ctx_sched_remove_ctx(kctx); -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) - gpu_metrics_ctx_term(kctx); -#endif -} - -/* - * Priority blocking management functions - */ - -/* Should not normally use directly - use kbase_jsctx_slot_atom_pulled_dec() instead */ -static void kbase_jsctx_slot_prio_blocked_clear(struct kbase_context *kctx, unsigned int js, - int sched_prio) -{ - struct kbase_jsctx_slot_tracking *slot_tracking = &kctx->slot_tracking[js]; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - slot_tracking->blocked &= ~(((kbase_js_prio_bitmap_t)1) << sched_prio); - KBASE_KTRACE_ADD_JM_SLOT_INFO(kctx->kbdev, JS_SLOT_PRIO_UNBLOCKED, kctx, NULL, 0, js, - (unsigned int)sched_prio); -} - -static int kbase_jsctx_slot_atoms_pulled(struct kbase_context *kctx, unsigned int js) -{ - return atomic_read(&kctx->slot_tracking[js].atoms_pulled); -} - -/* - * A priority level on a slot is blocked when: - * - that priority level is blocked - * - or, any higher priority level is blocked - */ -static bool kbase_jsctx_slot_prio_is_blocked(struct kbase_context *kctx, unsigned int js, - int sched_prio) -{ - struct kbase_jsctx_slot_tracking *slot_tracking = &kctx->slot_tracking[js]; - kbase_js_prio_bitmap_t prio_bit, higher_prios_mask; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - /* done in two separate shifts to prevent future undefined behavior - * should the number of priority levels == (bit width of the type) - */ - prio_bit = (((kbase_js_prio_bitmap_t)1) << sched_prio); - /* all bits of sched_prio or higher, with sched_prio = 0 being the - * highest priority - */ - higher_prios_mask = (prio_bit << 1u) - 1u; - return (slot_tracking->blocked & higher_prios_mask) != 0u; -} - -/** - * kbase_jsctx_slot_atom_pulled_inc - Increase counts of atoms that have being - * pulled for a slot from a ctx, based on - * this atom - * @kctx: kbase context - * @katom: atom pulled - * - * Manages counts of atoms pulled (including per-priority-level counts), for - * later determining when a ctx can become unblocked on a slot. - * - * Once a slot has been blocked at @katom's priority level, it should not be - * pulled from, hence this function should not be called in that case. - * - * The return value is to aid tracking of when @kctx becomes runnable. - * - * Return: new total count of atoms pulled from all slots on @kctx - */ -static int kbase_jsctx_slot_atom_pulled_inc(struct kbase_context *kctx, - const struct kbase_jd_atom *katom) -{ - unsigned int js = katom->slot_nr; - int sched_prio = katom->sched_priority; - struct kbase_jsctx_slot_tracking *slot_tracking = &kctx->slot_tracking[js]; - int nr_atoms_pulled; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - WARN(kbase_jsctx_slot_prio_is_blocked(kctx, js, sched_prio), - "Should not have pulled atoms for slot %u from a context that is blocked at priority %d or higher", - js, sched_prio); - - nr_atoms_pulled = atomic_inc_return(&kctx->atoms_pulled_all_slots); - atomic_inc(&slot_tracking->atoms_pulled); - slot_tracking->atoms_pulled_pri[sched_prio]++; - - return nr_atoms_pulled; -} - -/** - * kbase_jsctx_slot_atom_pulled_dec- Decrease counts of atoms that have being - * pulled for a slot from a ctx, and - * re-evaluate whether a context is blocked - * on this slot - * @kctx: kbase context - * @katom: atom that has just been removed from a job slot - * - * @kctx can become unblocked on a slot for a priority level when it no longer - * has any pulled atoms at that priority level on that slot, and all higher - * (numerically lower) priority levels are also unblocked @kctx on that - * slot. The latter condition is to retain priority ordering within @kctx. - * - * Return: true if the slot was previously blocked but has now become unblocked - * at @katom's priority level, false otherwise. - */ -static bool kbase_jsctx_slot_atom_pulled_dec(struct kbase_context *kctx, - const struct kbase_jd_atom *katom) -{ - unsigned int js = katom->slot_nr; - int sched_prio = katom->sched_priority; - int atoms_pulled_pri; - struct kbase_jsctx_slot_tracking *slot_tracking = &kctx->slot_tracking[js]; - bool slot_prio_became_unblocked = false; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - atomic_dec(&kctx->atoms_pulled_all_slots); - atomic_dec(&slot_tracking->atoms_pulled); - - atoms_pulled_pri = --(slot_tracking->atoms_pulled_pri[sched_prio]); - - /* We can safely clear this priority level's blocked status even if - * higher priority levels are still blocked: a subsequent query to - * kbase_jsctx_slot_prio_is_blocked() will still return true - */ - if (!atoms_pulled_pri && kbase_jsctx_slot_prio_is_blocked(kctx, js, sched_prio)) { - kbase_jsctx_slot_prio_blocked_clear(kctx, js, sched_prio); - - if (!kbase_jsctx_slot_prio_is_blocked(kctx, js, sched_prio)) - slot_prio_became_unblocked = true; - } - - if (slot_prio_became_unblocked) - KBASE_KTRACE_ADD_JM_SLOT_INFO(kctx->kbdev, JS_SLOT_PRIO_AND_HIGHER_UNBLOCKED, kctx, - katom, katom->jc, js, (unsigned int)sched_prio); - - return slot_prio_became_unblocked; -} - -/** - * kbase_js_ctx_list_add_pullable_nolock - Variant of - * kbase_jd_ctx_list_add_pullable() - * where the caller must hold - * hwaccess_lock - * @kbdev: Device pointer - * @kctx: Context to add to queue - * @js: Job slot to use - * - * Caller must hold hwaccess_lock - * - * Return: true if caller should call kbase_backend_ctx_count_changed() - */ -static bool kbase_js_ctx_list_add_pullable_nolock(struct kbase_device *kbdev, - struct kbase_context *kctx, unsigned int js) -{ - bool ret = false; - - lockdep_assert_held(&kbdev->hwaccess_lock); - dev_dbg(kbdev->dev, "Add pullable tail kctx %pK (s:%u)\n", (void *)kctx, js); - - if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])) - list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); - - list_add_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], - &kbdev->js_data.ctx_list_pullable[js][kctx->priority]); - - if (!kctx->slots_pullable) { - kbdev->js_data.nr_contexts_pullable++; - ret = true; - if (!kbase_jsctx_atoms_pulled(kctx)) { - WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); - kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF); - atomic_inc(&kbdev->js_data.nr_contexts_runnable); - } - } - kctx->slots_pullable |= (1UL << js); - - return ret; -} - -/** - * kbase_js_ctx_list_add_pullable_head_nolock - Variant of - * kbase_js_ctx_list_add_pullable_head() - * where the caller must hold - * hwaccess_lock - * @kbdev: Device pointer - * @kctx: Context to add to queue - * @js: Job slot to use - * - * Caller must hold hwaccess_lock - * - * Return: true if caller should call kbase_backend_ctx_count_changed() - */ -static bool kbase_js_ctx_list_add_pullable_head_nolock(struct kbase_device *kbdev, - struct kbase_context *kctx, unsigned int js) -{ - bool ret = false; - - lockdep_assert_held(&kbdev->hwaccess_lock); - dev_dbg(kbdev->dev, "Add pullable head kctx %pK (s:%u)\n", (void *)kctx, js); - - if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])) - list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); - - list_add(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], - &kbdev->js_data.ctx_list_pullable[js][kctx->priority]); - - if (!kctx->slots_pullable) { - kbdev->js_data.nr_contexts_pullable++; - ret = true; - if (!kbase_jsctx_atoms_pulled(kctx)) { - WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); - kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF); - atomic_inc(&kbdev->js_data.nr_contexts_runnable); - } - } - kctx->slots_pullable |= (1UL << js); - - return ret; -} - -/** - * kbase_js_ctx_list_add_pullable_head - Add context to the head of the - * per-slot pullable context queue - * @kbdev: Device pointer - * @kctx: Context to add to queue - * @js: Job slot to use - * - * If the context is on either the pullable or unpullable queues, then it is - * removed before being added to the head. - * - * This function should be used when a context has been scheduled, but no jobs - * can currently be pulled from it. - * - * Return: true if caller should call kbase_backend_ctx_count_changed() - */ -static bool kbase_js_ctx_list_add_pullable_head(struct kbase_device *kbdev, - struct kbase_context *kctx, unsigned int js) -{ - bool ret; - unsigned long flags; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - ret = kbase_js_ctx_list_add_pullable_head_nolock(kbdev, kctx, js); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - return ret; -} - -/** - * kbase_js_ctx_list_add_unpullable_nolock - Add context to the tail of the - * per-slot unpullable context queue - * @kbdev: Device pointer - * @kctx: Context to add to queue - * @js: Job slot to use - * - * The context must already be on the per-slot pullable queue. It will be - * removed from the pullable queue before being added to the unpullable queue. - * - * This function should be used when a context has been pulled from, and there - * are no jobs remaining on the specified slot. - * - * Caller must hold hwaccess_lock - * - * Return: true if caller should call kbase_backend_ctx_count_changed() - */ -static bool kbase_js_ctx_list_add_unpullable_nolock(struct kbase_device *kbdev, - struct kbase_context *kctx, unsigned int js) -{ - bool ret = false; - - lockdep_assert_held(&kbdev->hwaccess_lock); - dev_dbg(kbdev->dev, "Add unpullable tail kctx %pK (s:%u)\n", (void *)kctx, js); - - list_move_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], - &kbdev->js_data.ctx_list_unpullable[js][kctx->priority]); - - if (kctx->slots_pullable == (1UL << js)) { - kbdev->js_data.nr_contexts_pullable--; - ret = true; - if (!kbase_jsctx_atoms_pulled(kctx)) { - WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); - kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); - atomic_dec(&kbdev->js_data.nr_contexts_runnable); - } - } - kctx->slots_pullable &= ~(1UL << js); - - return ret; -} - -/** - * kbase_js_ctx_list_remove_nolock - Remove context from the per-slot pullable - * or unpullable context queues - * @kbdev: Device pointer - * @kctx: Context to remove from queue - * @js: Job slot to use - * - * The context must already be on one of the queues. - * - * This function should be used when a context has no jobs on the GPU, and no - * jobs remaining for the specified slot. - * - * Caller must hold hwaccess_lock - * - * Return: true if caller should call kbase_backend_ctx_count_changed() - */ -static bool kbase_js_ctx_list_remove_nolock(struct kbase_device *kbdev, struct kbase_context *kctx, - unsigned int js) -{ - bool ret = false; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - WARN_ON(list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])); - - list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); - - if (kctx->slots_pullable == (1UL << js)) { - kbdev->js_data.nr_contexts_pullable--; - ret = true; - if (!kbase_jsctx_atoms_pulled(kctx)) { - WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); - kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); - atomic_dec(&kbdev->js_data.nr_contexts_runnable); - } - } - kctx->slots_pullable &= ~(1UL << js); - - return ret; -} - -/** - * kbase_js_ctx_list_pop_head_nolock - Variant of kbase_js_ctx_list_pop_head() - * where the caller must hold - * hwaccess_lock - * @kbdev: Device pointer - * @js: Job slot to use - * - * Caller must hold hwaccess_lock - * - * Return: Context to use for specified slot. - * NULL if no contexts present for specified slot - */ -static struct kbase_context *kbase_js_ctx_list_pop_head_nolock(struct kbase_device *kbdev, - unsigned int js) -{ - struct kbase_context *kctx; - int i; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - for (i = KBASE_JS_ATOM_SCHED_PRIO_FIRST; i < KBASE_JS_ATOM_SCHED_PRIO_COUNT; i++) { - if (list_empty(&kbdev->js_data.ctx_list_pullable[js][i])) - continue; - - kctx = list_entry(kbdev->js_data.ctx_list_pullable[js][i].next, - struct kbase_context, jctx.sched_info.ctx.ctx_list_entry[js]); - - list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); - dev_dbg(kbdev->dev, "Popped %pK from the pullable queue (s:%u)\n", (void *)kctx, - js); - return kctx; - } - return NULL; -} - -/** - * kbase_js_ctx_list_pop_head - Pop the head context off the per-slot pullable - * queue. - * @kbdev: Device pointer - * @js: Job slot to use - * - * Return: Context to use for specified slot. - * NULL if no contexts present for specified slot - */ -static struct kbase_context *kbase_js_ctx_list_pop_head(struct kbase_device *kbdev, unsigned int js) -{ - struct kbase_context *kctx; - unsigned long flags; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kctx = kbase_js_ctx_list_pop_head_nolock(kbdev, js); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - return kctx; -} - -/** - * kbase_js_ctx_pullable - Return if a context can be pulled from on the - * specified slot - * @kctx: Context pointer - * @js: Job slot to use - * @is_scheduled: true if the context is currently scheduled - * - * Caller must hold hwaccess_lock - * - * Return: true if context can be pulled from on specified slot - * false otherwise - */ -static bool kbase_js_ctx_pullable(struct kbase_context *kctx, unsigned int js, bool is_scheduled) -{ - struct kbasep_js_device_data *js_devdata; - struct kbase_jd_atom *katom; - struct kbase_device *kbdev = kctx->kbdev; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - js_devdata = &kbdev->js_data; - - if (is_scheduled) { - if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) { - dev_dbg(kbdev->dev, "JS: No submit allowed for kctx %pK\n", (void *)kctx); - return false; - } - } - katom = jsctx_rb_peek(kctx, js); - if (!katom) { - dev_dbg(kbdev->dev, "JS: No pullable atom in kctx %pK (s:%u)\n", (void *)kctx, js); - return false; /* No pullable atoms */ - } - if (kbase_jsctx_slot_prio_is_blocked(kctx, js, katom->sched_priority)) { - KBASE_KTRACE_ADD_JM_SLOT_INFO(kctx->kbdev, JS_SLOT_PRIO_IS_BLOCKED, kctx, katom, - katom->jc, js, (unsigned int)katom->sched_priority); - dev_dbg(kbdev->dev, - "JS: kctx %pK is blocked from submitting atoms at priority %d and lower (s:%u)\n", - (void *)kctx, katom->sched_priority, js); - return false; - } - if (atomic_read(&katom->blocked)) { - dev_dbg(kbdev->dev, "JS: Atom %pK is blocked in js_ctx_pullable\n", (void *)katom); - return false; /* next atom blocked */ - } - if (katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) { - if (katom->x_pre_dep->gpu_rb_state == KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB || - katom->x_pre_dep->will_fail_event_code) { - dev_dbg(kbdev->dev, - "JS: X pre-dep %pK is not present in slot FIFO or will fail\n", - (void *)katom->x_pre_dep); - return false; - } - if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) && - kbase_backend_nr_atoms_on_slot(kctx->kbdev, js)) { - dev_dbg(kbdev->dev, - "JS: Atom %pK has cross-slot fail dependency and atoms on slot (s:%u)\n", - (void *)katom, js); - return false; - } - } - - dev_dbg(kbdev->dev, "JS: Atom %pK is pullable in kctx %pK (s:%u)\n", (void *)katom, - (void *)kctx, js); - - return true; -} - -static bool kbase_js_dep_validate(struct kbase_context *kctx, struct kbase_jd_atom *katom) -{ - struct kbase_device *kbdev = kctx->kbdev; - bool ret = true; - bool has_dep = false, has_x_dep = false; - unsigned int js = kbase_js_get_slot(kbdev, katom); - int prio = katom->sched_priority; - int i; - - for (i = 0; i < 2; i++) { - struct kbase_jd_atom *dep_atom = katom->dep[i].atom; - - if (dep_atom) { - unsigned int dep_js = kbase_js_get_slot(kbdev, dep_atom); - int dep_prio = dep_atom->sched_priority; - - dev_dbg(kbdev->dev, "Checking dep %d of atom %pK (s:%d) on %pK (s:%d)\n", i, - (void *)katom, js, (void *)dep_atom, dep_js); - - /* Dependent atom must already have been submitted */ - if (!(dep_atom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE)) { - dev_dbg(kbdev->dev, "Blocker not submitted yet\n"); - ret = false; - break; - } - - /* Dependencies with different priorities can't - * be represented in the ringbuffer - */ - if (prio != dep_prio) { - dev_dbg(kbdev->dev, "Different atom priorities\n"); - ret = false; - break; - } - - if (js == dep_js) { - /* Only one same-slot dependency can be - * represented in the ringbuffer - */ - if (has_dep) { - dev_dbg(kbdev->dev, "Too many same-slot deps\n"); - ret = false; - break; - } - /* Each dependee atom can only have one - * same-slot dependency - */ - if (dep_atom->post_dep) { - dev_dbg(kbdev->dev, "Too many same-slot successors\n"); - ret = false; - break; - } - has_dep = true; - } else { - /* Only one cross-slot dependency can be - * represented in the ringbuffer - */ - if (has_x_dep) { - dev_dbg(kbdev->dev, "Too many cross-slot deps\n"); - ret = false; - break; - } - /* Each dependee atom can only have one - * cross-slot dependency - */ - if (dep_atom->x_post_dep) { - dev_dbg(kbdev->dev, "Too many cross-slot successors\n"); - ret = false; - break; - } - /* The dependee atom can not already be in the - * HW access ringbuffer - */ - if (dep_atom->gpu_rb_state != KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { - dev_dbg(kbdev->dev, - "Blocker already in ringbuffer (state:%d)\n", - dep_atom->gpu_rb_state); - ret = false; - break; - } - /* The dependee atom can not already have - * completed - */ - if (dep_atom->status != KBASE_JD_ATOM_STATE_IN_JS) { - dev_dbg(kbdev->dev, - "Blocker already completed (status:%d)\n", - dep_atom->status); - ret = false; - break; - } - - has_x_dep = true; - } - - /* Dependency can be represented in ringbuffers */ - } - } - - /* If dependencies can be represented by ringbuffer then clear them from - * atom structure - */ - if (ret) { - for (i = 0; i < 2; i++) { - struct kbase_jd_atom *dep_atom = katom->dep[i].atom; - - if (dep_atom) { - unsigned int dep_js = kbase_js_get_slot(kbdev, dep_atom); - - dev_dbg(kbdev->dev, - "Clearing dep %d of atom %pK (s:%d) on %pK (s:%d)\n", i, - (void *)katom, js, (void *)dep_atom, dep_js); - - if ((js != dep_js) && - (dep_atom->status != KBASE_JD_ATOM_STATE_COMPLETED) && - (dep_atom->status != KBASE_JD_ATOM_STATE_HW_COMPLETED) && - (dep_atom->status != KBASE_JD_ATOM_STATE_UNUSED)) { - katom->atom_flags |= KBASE_KATOM_FLAG_X_DEP_BLOCKED; - - katom->x_pre_dep = dep_atom; - dep_atom->x_post_dep = katom; - if (kbase_jd_katom_dep_type(&katom->dep[i]) == - BASE_JD_DEP_TYPE_DATA) - katom->atom_flags |= KBASE_KATOM_FLAG_FAIL_BLOCKER; - } - if ((kbase_jd_katom_dep_type(&katom->dep[i]) == - BASE_JD_DEP_TYPE_DATA) && - (js == dep_js)) { - katom->pre_dep = dep_atom; - dep_atom->post_dep = katom; - } - - list_del(&katom->dep_item[i]); - kbase_jd_katom_dep_clear(&katom->dep[i]); - } - } - } else { - dev_dbg(kbdev->dev, "Deps of atom %pK (s:%d) could not be represented\n", - (void *)katom, js); - } - - return ret; -} - -void kbase_js_set_ctx_priority(struct kbase_context *kctx, int new_priority) -{ - struct kbase_device *kbdev = kctx->kbdev; - unsigned int js; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - /* Move kctx to the pullable/upullable list as per the new priority */ - if (new_priority != kctx->priority) { - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { - if (kctx->slots_pullable & (1UL << js)) - list_move_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], - &kbdev->js_data.ctx_list_pullable[js][new_priority]); - else - list_move_tail( - &kctx->jctx.sched_info.ctx.ctx_list_entry[js], - &kbdev->js_data.ctx_list_unpullable[js][new_priority]); - } - - kctx->priority = new_priority; - } -} - -void kbase_js_update_ctx_priority(struct kbase_context *kctx) -{ - struct kbase_device *kbdev = kctx->kbdev; - int new_priority = KBASE_JS_ATOM_SCHED_PRIO_LOW; - int prio; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (kbdev->js_ctx_scheduling_mode == KBASE_JS_SYSTEM_PRIORITY_MODE) { - /* Determine the new priority for context, as per the priority - * of currently in-use atoms. - */ - for (prio = KBASE_JS_ATOM_SCHED_PRIO_FIRST; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; - prio++) { - if (kctx->atoms_count[prio]) { - new_priority = prio; - break; - } - } - } - - kbase_js_set_ctx_priority(kctx, new_priority); -} -KBASE_EXPORT_TEST_API(kbase_js_update_ctx_priority); - -bool kbasep_js_add_job(struct kbase_context *kctx, struct kbase_jd_atom *atom) -{ - unsigned long flags; - struct kbasep_js_kctx_info *js_kctx_info; - struct kbase_device *kbdev; - struct kbasep_js_device_data *js_devdata; - - bool enqueue_required = false; - bool timer_sync = false; - - KBASE_DEBUG_ASSERT(kctx != NULL); - KBASE_DEBUG_ASSERT(atom != NULL); - lockdep_assert_held(&kctx->jctx.lock); - - kbdev = kctx->kbdev; - js_devdata = &kbdev->js_data; - js_kctx_info = &kctx->jctx.sched_info; - - mutex_lock(&js_devdata->queue_mutex); - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - - /* - * Begin Runpool transaction - */ - mutex_lock(&js_devdata->runpool_mutex); - - /* Refcount ctx.nr_jobs */ - KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs < U32_MAX); - ++(js_kctx_info->ctx.nr_jobs); - dev_dbg(kbdev->dev, "Add atom %pK to kctx %pK; now %d in ctx\n", (void *)atom, (void *)kctx, - js_kctx_info->ctx.nr_jobs); - - /* Lock for state available during IRQ */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - if (++kctx->atoms_count[atom->sched_priority] == 1) - kbase_js_update_ctx_priority(kctx); - - if (!kbase_js_dep_validate(kctx, atom)) { - /* Dependencies could not be represented */ - --(js_kctx_info->ctx.nr_jobs); - dev_dbg(kbdev->dev, "Remove atom %pK from kctx %pK; now %d in ctx\n", (void *)atom, - (void *)kctx, js_kctx_info->ctx.nr_jobs); - - /* Setting atom status back to queued as it still has unresolved - * dependencies - */ - atom->status = KBASE_JD_ATOM_STATE_QUEUED; - dev_dbg(kbdev->dev, "Atom %pK status to queued\n", (void *)atom); - - /* Undo the count, as the atom will get added again later but - * leave the context priority adjusted or boosted, in case if - * this was the first higher priority atom received for this - * context. - * This will prevent the scenario of priority inversion, where - * another context having medium priority atoms keeps getting - * scheduled over this context, which is having both lower and - * higher priority atoms, but higher priority atoms are blocked - * due to dependency on lower priority atoms. With priority - * boost the high priority atom will get to run at earliest. - */ - kctx->atoms_count[atom->sched_priority]--; - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&js_devdata->runpool_mutex); - - goto out_unlock; - } - - enqueue_required = kbase_js_dep_resolved_submit(kctx, atom); - - KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, JS_ADD_JOB, kctx, atom, atom->jc, - kbase_ktrace_get_ctx_refcnt(kctx)); - - /* Context Attribute Refcounting */ - kbasep_js_ctx_attr_ctx_retain_atom(kbdev, kctx, atom); - - if (enqueue_required) { - if (kbase_js_ctx_pullable(kctx, atom->slot_nr, false)) - timer_sync = - kbase_js_ctx_list_add_pullable_nolock(kbdev, kctx, atom->slot_nr); - else - timer_sync = - kbase_js_ctx_list_add_unpullable_nolock(kbdev, kctx, atom->slot_nr); - } - /* If this context is active and the atom is the first on its slot, - * kick the job manager to attempt to fast-start the atom - */ - if (enqueue_required && kctx == kbdev->hwaccess.active_kctx[atom->slot_nr]) - kbase_jm_try_kick(kbdev, 1UL << atom->slot_nr); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - if (timer_sync) - kbase_backend_ctx_count_changed(kbdev); - mutex_unlock(&js_devdata->runpool_mutex); - /* End runpool transaction */ - - if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED)) { - if (kbase_ctx_flag(kctx, KCTX_DYING)) { - /* A job got added while/after kbase_job_zap_context() - * was called on a non-scheduled context. Kill that job - * by killing the context. - */ - kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, false); - } else if (js_kctx_info->ctx.nr_jobs == 1) { - /* Handle Refcount going from 0 to 1: schedule the - * context on the Queue - */ - KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); - dev_dbg(kbdev->dev, "JS: Enqueue Context %pK", kctx); - - /* Queue was updated - caller must try to schedule the - * head context - */ - WARN_ON(!enqueue_required); - } - } -out_unlock: - dev_dbg(kbdev->dev, "Enqueue of kctx %pK is %srequired\n", kctx, - enqueue_required ? "" : "not "); - - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - - mutex_unlock(&js_devdata->queue_mutex); - - return enqueue_required; -} - -void kbasep_js_remove_job(struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbase_jd_atom *atom) -{ - struct kbasep_js_kctx_info *js_kctx_info; - unsigned long flags; - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(kctx != NULL); - KBASE_DEBUG_ASSERT(atom != NULL); - - js_kctx_info = &kctx->jctx.sched_info; - - KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, JS_REMOVE_JOB, kctx, atom, atom->jc, - kbase_ktrace_get_ctx_refcnt(kctx)); - - /* De-refcount ctx.nr_jobs */ - KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs > 0); - --(js_kctx_info->ctx.nr_jobs); - dev_dbg(kbdev->dev, "Remove atom %pK from kctx %pK; now %d in ctx\n", (void *)atom, - (void *)kctx, js_kctx_info->ctx.nr_jobs); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - if (--kctx->atoms_count[atom->sched_priority] == 0) - kbase_js_update_ctx_priority(kctx); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -} - -bool kbasep_js_remove_cancelled_job(struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbase_jd_atom *katom) -{ - unsigned long flags; - struct kbasep_js_atom_retained_state katom_retained_state; - bool attr_state_changed; - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(kctx != NULL); - KBASE_DEBUG_ASSERT(katom != NULL); - - kbasep_js_atom_retained_state_copy(&katom_retained_state, katom); - kbasep_js_remove_job(kbdev, kctx, katom); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - /* The atom has 'finished' (will not be re-run), so no need to call - * kbasep_js_has_atom_finished(). - * - * This is because it returns false for soft-stopped atoms, but we - * want to override that, because we're cancelling an atom regardless of - * whether it was soft-stopped or not - */ - attr_state_changed = - kbasep_js_ctx_attr_ctx_release_atom(kbdev, kctx, &katom_retained_state); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - return attr_state_changed; -} - -/** - * kbasep_js_run_jobs_after_ctx_and_atom_release - Try running more jobs after - * releasing a context and/or atom - * @kbdev: The kbase_device to operate on - * @kctx: The kbase_context to operate on - * @katom_retained_state: Retained state from the atom - * @runpool_ctx_attr_change: True if the runpool context attributes have changed - * - * This collates a set of actions that must happen whilst hwaccess_lock is held. - * - * This includes running more jobs when: - * - The previously released kctx caused a ctx attribute change, - * - The released atom caused a ctx attribute change, - * - Slots were previously blocked due to affinity restrictions, - * - Submission during IRQ handling failed. - * - * Return: %KBASEP_JS_RELEASE_RESULT_SCHED_ALL if context attributes were - * changed. The caller should try scheduling all contexts - */ -static kbasep_js_release_result kbasep_js_run_jobs_after_ctx_and_atom_release( - struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbasep_js_atom_retained_state *katom_retained_state, bool runpool_ctx_attr_change) -{ - struct kbasep_js_device_data *js_devdata; - kbasep_js_release_result result = 0; - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(kctx != NULL); - KBASE_DEBUG_ASSERT(katom_retained_state != NULL); - js_devdata = &kbdev->js_data; - - lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex); - lockdep_assert_held(&js_devdata->runpool_mutex); - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (js_devdata->nr_user_contexts_running != 0 && runpool_ctx_attr_change) { - /* A change in runpool ctx attributes might mean we can - * run more jobs than before - */ - result = KBASEP_JS_RELEASE_RESULT_SCHED_ALL; - - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JD_DONE_TRY_RUN_NEXT_JOB, kctx, NULL, 0u, 0); - } - return result; -} - -/** - * kbasep_js_runpool_release_ctx_internal - Internal function to release the reference - * on a ctx and an atom's "retained state", only - * taking the runpool and as transaction mutexes - * @kbdev: The kbase_device to operate on - * @kctx: The kbase_context to operate on - * @katom_retained_state: Retained state from the atom - * - * This also starts more jobs running in the case of an ctx-attribute state change - * - * This does none of the followup actions for scheduling: - * - It does not schedule in a new context - * - It does not requeue or handle dying contexts - * - * For those tasks, just call kbasep_js_runpool_release_ctx() instead - * - * Has following requirements - * - Context is scheduled in, and kctx->as_nr matches kctx_as_nr - * - Context has a non-zero refcount - * - Caller holds js_kctx_info->ctx.jsctx_mutex - * - Caller holds js_devdata->runpool_mutex - * - * Return: A bitpattern, containing KBASEP_JS_RELEASE_RESULT_* flags, indicating - * the result of releasing a context that whether the caller should try - * scheduling a new context or should try scheduling all contexts. - */ -static kbasep_js_release_result -kbasep_js_runpool_release_ctx_internal(struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbasep_js_atom_retained_state *katom_retained_state) -{ - unsigned long flags; - struct kbasep_js_device_data *js_devdata; - struct kbasep_js_kctx_info *js_kctx_info; - - kbasep_js_release_result release_result = 0u; - bool runpool_ctx_attr_change = false; - int kctx_as_nr; - int new_ref_count; - CSTD_UNUSED(kctx_as_nr); - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(kctx != NULL); - js_kctx_info = &kctx->jctx.sched_info; - js_devdata = &kbdev->js_data; - - /* Ensure context really is scheduled in */ - KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); - - kctx_as_nr = kctx->as_nr; - KBASE_DEBUG_ASSERT(kctx_as_nr != KBASEP_AS_NR_INVALID); - KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0); - - /* - * Transaction begins on AS and runpool_irq - * - * Assert about out calling contract - */ - mutex_lock(&kbdev->pm.lock); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - KBASE_DEBUG_ASSERT(kctx_as_nr == kctx->as_nr); - KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0); - - /* Update refcount */ - kbase_ctx_sched_release_ctx(kctx); - new_ref_count = atomic_read(&kctx->refcount); - - /* Release the atom if it finished (i.e. wasn't soft-stopped) */ - if (kbasep_js_has_atom_finished(katom_retained_state)) - runpool_ctx_attr_change |= - kbasep_js_ctx_attr_ctx_release_atom(kbdev, kctx, katom_retained_state); - - if (new_ref_count == 2 && kbase_ctx_flag(kctx, KCTX_PRIVILEGED) && - !kbase_io_is_aw_removed(kbdev) && !kbase_pm_is_suspending(kbdev)) { - /* Context is kept scheduled into an address space even when - * there are no jobs, in this case we have to handle the - * situation where all jobs have been evicted from the GPU and - * submission is disabled. - * - * At this point we re-enable submission to allow further jobs - * to be executed - */ - kbasep_js_set_submit_allowed(js_devdata, kctx); - } - - /* Make a set of checks to see if the context should be scheduled out. - * Note that there'll always be at least 1 reference to the context - * which was previously acquired by kbasep_js_schedule_ctx(). - */ - if (new_ref_count == 1 && - (!kbasep_js_is_submit_allowed(js_devdata, kctx) || kbase_io_is_aw_removed(kbdev) || - kbase_pm_is_suspending(kbdev))) { - int num_slots = kbdev->gpu_props.num_job_slots; - unsigned int slot; - - /* Last reference, and we've been told to remove this context - * from the Run Pool - */ - dev_dbg(kbdev->dev, - "JS: RunPool Remove Context %pK because refcount=%d, jobs=%d, allowed=%d", - kctx, new_ref_count, js_kctx_info->ctx.nr_jobs, - kbasep_js_is_submit_allowed(js_devdata, kctx)); - - KBASE_TLSTREAM_TL_NRET_AS_CTX(kbdev, &kbdev->as[kctx->as_nr], kctx); - - kbase_backend_release_ctx_irq(kbdev, kctx); - - for (slot = 0; slot < num_slots; slot++) { - if (kbdev->hwaccess.active_kctx[slot] == kctx) { - dev_dbg(kbdev->dev, "Marking kctx %pK as inactive (s:%d)\n", - (void *)kctx, slot); - kbdev->hwaccess.active_kctx[slot] = NULL; - } - } - - /* Ctx Attribute handling - * - * Releasing atoms attributes must either happen before this, or - * after the KCTX_SHEDULED flag is changed, otherwise we - * double-decount the attributes - */ - runpool_ctx_attr_change |= kbasep_js_ctx_attr_runpool_release_ctx(kbdev, kctx); - - /* Releasing the context and katom retained state can allow - * more jobs to run - */ - release_result |= kbasep_js_run_jobs_after_ctx_and_atom_release( - kbdev, kctx, katom_retained_state, runpool_ctx_attr_change); - - /* - * Transaction ends on AS and runpool_irq: - * - * By this point, the AS-related data is now clear and ready - * for re-use. - * - * Since releases only occur once for each previous successful - * retain, and no more retains are allowed on this context, no - * other thread will be operating in this - * code whilst we are - */ - - /* Recalculate pullable status for all slots */ - for (slot = 0; slot < num_slots; slot++) { - if (kbase_js_ctx_pullable(kctx, slot, false)) - kbase_js_ctx_list_add_pullable_nolock(kbdev, kctx, slot); - } - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - kbase_backend_release_ctx_noirq(kbdev, kctx); - - mutex_unlock(&kbdev->pm.lock); - - /* Note: Don't reuse kctx_as_nr now */ - - /* Synchronize with any timers */ - kbase_backend_ctx_count_changed(kbdev); - - /* update book-keeping info */ - kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); - /* Signal any waiter that the context is not scheduled, so is - * safe for termination - once the jsctx_mutex is also dropped, - * and jobs have finished. - */ - wake_up(&js_kctx_info->ctx.is_scheduled_wait); - - /* Queue an action to occur after we've dropped the lock */ - release_result |= KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED | - KBASEP_JS_RELEASE_RESULT_SCHED_ALL; - } else { - kbasep_js_run_jobs_after_ctx_and_atom_release(kbdev, kctx, katom_retained_state, - runpool_ctx_attr_change); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kbdev->pm.lock); - } - - return release_result; -} - -void kbasep_js_runpool_release_ctx_nolock(struct kbase_device *kbdev, struct kbase_context *kctx) -{ - struct kbasep_js_atom_retained_state katom_retained_state; - - /* Setup a dummy katom_retained_state */ - kbasep_js_atom_retained_state_init_invalid(&katom_retained_state); - - kbasep_js_runpool_release_ctx_internal(kbdev, kctx, &katom_retained_state); -} - -void kbasep_js_runpool_requeue_or_kill_ctx(struct kbase_device *kbdev, struct kbase_context *kctx, - bool has_pm_ref) -{ - CSTD_UNUSED(has_pm_ref); - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(kctx != NULL); - - /* This is called if and only if you've you've detached the context from - * the Runpool Queue, and not added it back to the Runpool - */ - KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); - - if (kbase_ctx_flag(kctx, KCTX_DYING)) { - /* Dying: don't requeue, but kill all jobs on the context. This - * happens asynchronously - */ - dev_dbg(kbdev->dev, "JS: ** Killing Context %pK on RunPool Remove **", kctx); - kbase_js_foreach_ctx_job(kctx, &kbase_jd_cancel); - } -} - -void kbasep_js_runpool_release_ctx_and_katom_retained_state( - struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbasep_js_atom_retained_state *katom_retained_state) -{ - struct kbasep_js_device_data *js_devdata; - struct kbasep_js_kctx_info *js_kctx_info; - kbasep_js_release_result release_result; - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(kctx != NULL); - js_kctx_info = &kctx->jctx.sched_info; - js_devdata = &kbdev->js_data; - - mutex_lock(&js_devdata->queue_mutex); - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - mutex_lock(&js_devdata->runpool_mutex); - - release_result = kbasep_js_runpool_release_ctx_internal(kbdev, kctx, katom_retained_state); - - /* Drop the runpool mutex to allow requeing kctx */ - mutex_unlock(&js_devdata->runpool_mutex); - - if ((release_result & KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED) != 0u) - kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, true); - - /* Drop the jsctx_mutex to allow scheduling in a new context */ - - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - mutex_unlock(&js_devdata->queue_mutex); - - if (release_result & KBASEP_JS_RELEASE_RESULT_SCHED_ALL) - kbase_js_sched_all(kbdev); -} - -void kbasep_js_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) -{ - struct kbasep_js_atom_retained_state katom_retained_state; - - kbasep_js_atom_retained_state_init_invalid(&katom_retained_state); - - kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, &katom_retained_state); -} - -/* Variant of kbasep_js_runpool_release_ctx() that doesn't call into - * kbase_js_sched_all() - */ -static void kbasep_js_runpool_release_ctx_no_schedule(struct kbase_device *kbdev, - struct kbase_context *kctx) -{ - struct kbasep_js_device_data *js_devdata; - struct kbasep_js_kctx_info *js_kctx_info; - kbasep_js_release_result release_result; - struct kbasep_js_atom_retained_state katom_retained_state_struct; - struct kbasep_js_atom_retained_state *katom_retained_state = &katom_retained_state_struct; - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(kctx != NULL); - js_kctx_info = &kctx->jctx.sched_info; - js_devdata = &kbdev->js_data; - kbasep_js_atom_retained_state_init_invalid(katom_retained_state); - - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - mutex_lock(&js_devdata->runpool_mutex); - - release_result = kbasep_js_runpool_release_ctx_internal(kbdev, kctx, katom_retained_state); - - /* Drop the runpool mutex to allow requeing kctx */ - mutex_unlock(&js_devdata->runpool_mutex); - if ((release_result & KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED) != 0u) - kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, true); - - /* Drop the jsctx_mutex to allow scheduling in a new context */ - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - - /* NOTE: could return release_result if the caller would like to know - * whether it should schedule a new context, but currently no callers do - */ -} - -void kbase_js_set_timeouts(struct kbase_device *kbdev) -{ - lockdep_assert_held(&kbdev->hwaccess_lock); - - kbase_backend_timeouts_changed(kbdev); -} - -static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, struct kbase_context *kctx, - unsigned int js) -{ - struct kbasep_js_device_data *js_devdata; - struct kbasep_js_kctx_info *js_kctx_info; - unsigned long flags; - bool kctx_suspended = false; - int as_nr; - - dev_dbg(kbdev->dev, "Scheduling kctx %pK (s:%u)\n", kctx, js); - - js_devdata = &kbdev->js_data; - js_kctx_info = &kctx->jctx.sched_info; - - /* Pick available address space for this context */ - mutex_lock(&kbdev->mmu_hw_mutex); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - as_nr = kbase_ctx_sched_retain_ctx(kctx); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kbdev->mmu_hw_mutex); - if (as_nr == KBASEP_AS_NR_INVALID) { - as_nr = kbase_backend_find_and_release_free_address_space(kbdev, kctx); - if (as_nr != KBASEP_AS_NR_INVALID) { - /* Attempt to retain the context again, this should - * succeed - */ - mutex_lock(&kbdev->mmu_hw_mutex); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - as_nr = kbase_ctx_sched_retain_ctx(kctx); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kbdev->mmu_hw_mutex); - - WARN_ON(as_nr == KBASEP_AS_NR_INVALID); - } - } - if ((as_nr < 0) || (as_nr >= BASE_MAX_NR_AS)) - return false; /* No address space currently available */ - - /* - * Atomic transaction on the Context and Run Pool begins - */ - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - mutex_lock(&js_devdata->runpool_mutex); - mutex_lock(&kbdev->mmu_hw_mutex); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - /* Check to see if context is dying due to kbase_job_zap_context() */ - if (kbase_ctx_flag(kctx, KCTX_DYING)) { - /* Roll back the transaction so far and return */ - kbase_ctx_sched_release_ctx(kctx); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kbdev->mmu_hw_mutex); - mutex_unlock(&js_devdata->runpool_mutex); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - - return false; - } - - KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, JS_TRY_SCHEDULE_HEAD_CTX, kctx, NULL, 0u, - kbase_ktrace_get_ctx_refcnt(kctx)); - - kbase_ctx_flag_set(kctx, KCTX_SCHEDULED); - - /* Assign context to previously chosen address space */ - if (!kbase_backend_use_ctx(kbdev, kctx, as_nr)) { - /* Roll back the transaction so far and return */ - kbase_ctx_sched_release_ctx(kctx); - kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kbdev->mmu_hw_mutex); - mutex_unlock(&js_devdata->runpool_mutex); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - - return false; - } - - kbdev->hwaccess.active_kctx[js] = kctx; - - KBASE_TLSTREAM_TL_RET_AS_CTX(kbdev, &kbdev->as[kctx->as_nr], kctx); - - /* Cause any future waiter-on-termination to wait until the context is - * descheduled - */ - wake_up(&js_kctx_info->ctx.is_scheduled_wait); - - /* Re-check for suspending: a suspend could've occurred, and all the - * contexts could've been removed from the runpool before we took this - * lock. In this case, we don't want to allow this context to run jobs, - * we just want it out immediately. - * - * The DMB required to read the suspend flag was issued recently as part - * of the hwaccess_lock locking. If a suspend occurs *after* that lock - * was taken (i.e. this condition doesn't execute), then the - * kbasep_js_suspend() code will cleanup this context instead (by virtue - * of it being called strictly after the suspend flag is set, and will - * wait for this lock to drop) - */ - if (kbase_pm_is_suspending(kbdev) || kbase_io_is_aw_removed(kbdev)) { - /* Cause it to leave at some later point */ - bool retained; - CSTD_UNUSED(retained); - - retained = kbase_ctx_sched_inc_refcount_nolock(kctx); - KBASE_DEBUG_ASSERT(retained); - - kbasep_js_clear_submit_allowed(js_devdata, kctx); - kctx_suspended = true; - } - - kbase_ctx_flag_clear(kctx, KCTX_PULLED_SINCE_ACTIVE_JS0 << js); - - /* Transaction complete */ - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kbdev->mmu_hw_mutex); - - /* Synchronize with any timers */ - kbase_backend_ctx_count_changed(kbdev); - - mutex_unlock(&js_devdata->runpool_mutex); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - /* Note: after this point, the context could potentially get scheduled - * out immediately - */ - - if (kctx_suspended) { - /* Finishing forcing out the context due to a suspend. Use a - * variant of kbasep_js_runpool_release_ctx() that doesn't - * schedule a new context, to prevent a risk of recursion back - * into this function - */ - kbasep_js_runpool_release_ctx_no_schedule(kbdev, kctx); - return false; - } - return true; -} - -static bool kbase_js_use_ctx(struct kbase_device *kbdev, struct kbase_context *kctx, - unsigned int js) -{ - unsigned long flags; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && kbase_backend_use_ctx_sched(kbdev, kctx, js)) { - dev_dbg(kbdev->dev, "kctx %pK already has ASID - mark as active (s:%u)\n", - (void *)kctx, js); - - if (kbdev->hwaccess.active_kctx[js] != kctx) { - kbdev->hwaccess.active_kctx[js] = kctx; - kbase_ctx_flag_clear(kctx, KCTX_PULLED_SINCE_ACTIVE_JS0 << js); - } - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - return true; /* Context already scheduled */ - } - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - return kbasep_js_schedule_ctx(kbdev, kctx, js); -} - -void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) -{ - struct kbasep_js_kctx_info *js_kctx_info; - struct kbasep_js_device_data *js_devdata; - bool is_scheduled; - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(kctx != NULL); - - js_devdata = &kbdev->js_data; - js_kctx_info = &kctx->jctx.sched_info; - - /* This should only happen in response to a system call - * from a user-space thread. - * In a non-arbitrated environment this can never happen - * whilst suspending. - * - * In an arbitrated environment, user-space threads can run - * while we are suspended (for example GPU not available - * to this VM), however in that case we will block on - * the wait event for KCTX_SCHEDULED, since no context - * can be scheduled until we have the GPU again. - */ - if (!kbase_has_arbiter(kbdev)) { - if (WARN_ON(kbase_pm_is_suspending(kbdev))) - return; - } - - mutex_lock(&js_devdata->queue_mutex); - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - - /* Mark the context as privileged */ - kbase_ctx_flag_set(kctx, KCTX_PRIVILEGED); - - is_scheduled = kbase_ctx_flag(kctx, KCTX_SCHEDULED); - if (!is_scheduled) { - /* Add the context to the pullable list */ - if (kbase_js_ctx_list_add_pullable_head(kbdev, kctx, 0)) - kbase_js_sync_timers(kbdev); - - /* Fast-starting requires the jsctx_mutex to be dropped, - * because it works on multiple ctxs - */ - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - mutex_unlock(&js_devdata->queue_mutex); - - /* Try to schedule the context in */ - kbase_js_sched_all(kbdev); - - /* Wait for the context to be scheduled in */ - wait_event(kctx->jctx.sched_info.ctx.is_scheduled_wait, - kbase_ctx_flag(kctx, KCTX_SCHEDULED)); - } else { - /* Already scheduled in - We need to retain it to keep the - * corresponding address space - */ - WARN_ON(!kbase_ctx_sched_inc_refcount(kctx)); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - mutex_unlock(&js_devdata->queue_mutex); - } -} -KBASE_EXPORT_TEST_API(kbasep_js_schedule_privileged_ctx); - -void kbasep_js_release_privileged_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) -{ - struct kbasep_js_kctx_info *js_kctx_info; - - KBASE_DEBUG_ASSERT(kctx != NULL); - js_kctx_info = &kctx->jctx.sched_info; - - /* We don't need to use the address space anymore */ - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - kbase_ctx_flag_clear(kctx, KCTX_PRIVILEGED); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - - /* Release the context - it will be scheduled out */ - kbasep_js_runpool_release_ctx(kbdev, kctx); - - kbase_js_sched_all(kbdev); -} -KBASE_EXPORT_TEST_API(kbasep_js_release_privileged_ctx); - -void kbasep_js_suspend(struct kbase_device *kbdev) -{ - unsigned long flags; - struct kbasep_js_device_data *js_devdata; - int i; - u16 retained = 0u; - - KBASE_DEBUG_ASSERT(kbdev); - KBASE_DEBUG_ASSERT(kbase_pm_is_suspending(kbdev)); - js_devdata = &kbdev->js_data; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - /* Prevent all contexts from submitting */ - js_devdata->runpool_irq.submit_allowed = 0; - - /* Retain each of the contexts, so we can cause it to leave even if it - * had no refcount to begin with - */ - for (i = BASE_MAX_NR_AS - 1; i >= 0; --i) { - struct kbase_context *kctx = kbdev->as_to_kctx[i]; - - retained = retained << 1; - - if (kctx && !(kbdev->as_free & (1u << i))) { - kbase_ctx_sched_retain_ctx_refcount(kctx); - retained |= 1u; - /* This loop will not have an effect on the privileged - * contexts as they would have an extra ref count - * compared to the normal contexts, so they will hold - * on to their address spaces. MMU will re-enabled for - * them on resume. - */ - } - } - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - /* De-ref the previous retain to ensure each context gets pulled out - * sometime later. - */ - for (i = 0; i < BASE_MAX_NR_AS; ++i, retained = retained >> 1) { - struct kbase_context *kctx = kbdev->as_to_kctx[i]; - - if (retained & 1u) - kbasep_js_runpool_release_ctx(kbdev, kctx); - } - - /* Caller must wait for all Power Manager active references to be - * dropped - */ -} - -void kbasep_js_resume(struct kbase_device *kbdev) -{ - struct kbasep_js_device_data *js_devdata; - unsigned int js; - int prio; - - KBASE_DEBUG_ASSERT(kbdev); - js_devdata = &kbdev->js_data; - KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev)); - - mutex_lock(&js_devdata->queue_mutex); - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { - for (prio = KBASE_JS_ATOM_SCHED_PRIO_FIRST; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; - prio++) { - struct kbase_context *kctx, *n; - unsigned long flags; - - if (kbase_has_arbiter(kbdev)) { - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - list_for_each_entry_safe( - kctx, n, &kbdev->js_data.ctx_list_unpullable[js][prio], - jctx.sched_info.ctx.ctx_list_entry[js]) { - struct kbasep_js_kctx_info *js_kctx_info; - bool timer_sync = false; - - /* Drop lock so we can take kctx mutexes */ - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - js_kctx_info = &kctx->jctx.sched_info; - - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - mutex_lock(&js_devdata->runpool_mutex); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED) && - kbase_js_ctx_pullable(kctx, js, false)) - timer_sync = kbase_js_ctx_list_add_pullable_nolock( - kbdev, kctx, js); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - if (timer_sync) - kbase_backend_ctx_count_changed(kbdev); - - mutex_unlock(&js_devdata->runpool_mutex); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - - /* Take lock before accessing list again */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - } - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - } else { - bool timer_sync = false; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - list_for_each_entry_safe( - kctx, n, &kbdev->js_data.ctx_list_unpullable[js][prio], - jctx.sched_info.ctx.ctx_list_entry[js]) { - if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED) && - kbase_js_ctx_pullable(kctx, js, false)) - timer_sync |= kbase_js_ctx_list_add_pullable_nolock( - kbdev, kctx, js); - } - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - if (timer_sync) { - mutex_lock(&js_devdata->runpool_mutex); - kbase_backend_ctx_count_changed(kbdev); - mutex_unlock(&js_devdata->runpool_mutex); - } - } - } - } - mutex_unlock(&js_devdata->queue_mutex); - - /* Restart atom processing */ - kbase_js_sched_all(kbdev); - - /* JS Resume complete */ -} - -bool kbase_js_is_atom_valid(struct kbase_device *kbdev, struct kbase_jd_atom *katom) -{ - CSTD_UNUSED(kbdev); - - if ((katom->core_req & BASE_JD_REQ_FS) && - (katom->core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T))) - return false; - - if ((katom->core_req & BASE_JD_REQ_JOB_SLOT) && (katom->jobslot >= BASE_JM_MAX_NR_SLOTS)) - return false; - - return true; -} - -static unsigned int kbase_js_get_slot(struct kbase_device *kbdev, struct kbase_jd_atom *katom) -{ - if (katom->core_req & BASE_JD_REQ_JOB_SLOT) - return katom->jobslot; - - if (katom->core_req & BASE_JD_REQ_FS) - return 0; - - if (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) { - if (katom->device_nr == 1 && kbdev->gpu_props.num_core_groups == 2) - return 2; - } - - return 1; -} - -bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, struct kbase_jd_atom *katom) -{ - bool enqueue_required; - - katom->slot_nr = kbase_js_get_slot(kctx->kbdev, katom); - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - lockdep_assert_held(&kctx->jctx.lock); - - /* If slot will transition from unpullable to pullable then add to - * pullable list - */ - enqueue_required = jsctx_rb_none_to_pull(kctx, katom->slot_nr); - - if ((katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) || - (katom->pre_dep && - (katom->pre_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST))) { - int prio = katom->sched_priority; - unsigned int js = katom->slot_nr; - struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; - - dev_dbg(kctx->kbdev->dev, "Add atom %pK to X_DEP list (s:%u)\n", (void *)katom, js); - - list_add_tail(&katom->queue, &queue->x_dep_head); - katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; - enqueue_required = false; - } else { - dev_dbg(kctx->kbdev->dev, "Atom %pK not added to X_DEP list\n", (void *)katom); - /* Check if there are lower priority jobs to soft stop */ - kbase_job_slot_ctx_priority_check_locked(kctx, katom); - - /* Add atom to ring buffer. */ - jsctx_tree_add(kctx, katom); - katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_TREE; - } - - dev_dbg(kctx->kbdev->dev, "Enqueue of kctx %pK is %srequired to submit atom %pK\n", kctx, - enqueue_required ? "" : "not ", katom); - - return enqueue_required; -} - -/** - * kbase_js_move_to_tree - Move atom (and any dependent atoms) to the - * runnable_tree, ready for execution - * @katom: Atom to submit - * - * It is assumed that @katom does not have KBASE_KATOM_FLAG_X_DEP_BLOCKED set, - * but is still present in the x_dep list. If @katom has a same-slot dependent - * atom then that atom (and any dependents) will also be moved. - */ -static void kbase_js_move_to_tree(struct kbase_jd_atom *katom) -{ - lockdep_assert_held(&katom->kctx->kbdev->hwaccess_lock); - - while (katom) { - WARN_ON(!(katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST)); - - if (!(katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED)) { - dev_dbg(katom->kctx->kbdev->dev, - "Del atom %pK from X_DEP list in js_move_to_tree\n", (void *)katom); - - list_del(&katom->queue); - katom->atom_flags &= ~KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; - jsctx_tree_add(katom->kctx, katom); - katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_TREE; - } else { - dev_dbg(katom->kctx->kbdev->dev, - "Atom %pK blocked on x-dep in js_move_to_tree\n", (void *)katom); - break; - } - - katom = katom->post_dep; - } -} - -/** - * kbase_js_evict_deps - Evict dependencies of a failed atom. - * @kctx: Context pointer - * @katom: Pointer to the atom that has failed. - * @js: The job slot the katom was run on. - * @prio: Priority of the katom. - * - * Remove all post dependencies of an atom from the context ringbuffers. - * - * The original atom's event_code will be propagated to all dependent atoms. - * - * Context: Caller must hold the HW access lock - */ -static void kbase_js_evict_deps(struct kbase_context *kctx, struct kbase_jd_atom *katom, - unsigned int js, int prio) -{ - struct kbase_jd_atom *x_dep = katom->x_post_dep; - struct kbase_jd_atom *next_katom = katom->post_dep; - - CSTD_UNUSED(js); - CSTD_UNUSED(prio); - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - if (next_katom) { - KBASE_DEBUG_ASSERT(next_katom->status != KBASE_JD_ATOM_STATE_HW_COMPLETED); - next_katom->will_fail_event_code = katom->event_code; - } - - /* Has cross slot depenency. */ - if (x_dep && (x_dep->atom_flags & - (KBASE_KATOM_FLAG_JSCTX_IN_TREE | KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST))) { - /* Remove dependency.*/ - x_dep->atom_flags &= ~KBASE_KATOM_FLAG_X_DEP_BLOCKED; - - dev_dbg(kctx->kbdev->dev, "Cleared X_DEP flag on atom %pK\n", (void *)x_dep); - - /* Fail if it had a data dependency. */ - if (x_dep->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) - x_dep->will_fail_event_code = katom->event_code; - - if (x_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST) - kbase_js_move_to_tree(x_dep); - } -} - -struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, unsigned int js) -{ - struct kbase_jd_atom *katom; - struct kbasep_js_device_data *js_devdata; - struct kbase_device *kbdev; - int pulled; - - KBASE_DEBUG_ASSERT(kctx); - - kbdev = kctx->kbdev; - dev_dbg(kbdev->dev, "JS: pulling an atom from kctx %pK (s:%u)\n", (void *)kctx, js); - - js_devdata = &kbdev->js_data; - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) { - dev_dbg(kbdev->dev, "JS: No submit allowed for kctx %pK\n", (void *)kctx); - return NULL; - } - if (kbase_pm_is_suspending(kbdev) || kbase_io_is_aw_removed(kbdev)) - return NULL; - - katom = jsctx_rb_peek(kctx, js); - if (!katom) { - dev_dbg(kbdev->dev, "JS: No pullable atom in kctx %pK (s:%u)\n", (void *)kctx, js); - return NULL; - } - if (kbase_jsctx_slot_prio_is_blocked(kctx, js, katom->sched_priority)) { - dev_dbg(kbdev->dev, - "JS: kctx %pK is blocked from submitting atoms at priority %d and lower (s:%u)\n", - (void *)kctx, katom->sched_priority, js); - return NULL; - } - if (atomic_read(&katom->blocked)) { - dev_dbg(kbdev->dev, "JS: Atom %pK is blocked in js_pull\n", (void *)katom); - return NULL; - } - - /* Due to ordering restrictions when unpulling atoms on failure, we do - * not allow multiple runs of fail-dep atoms from the same context to be - * present on the same slot - */ - if (katom->pre_dep && kbase_jsctx_slot_atoms_pulled(kctx, js)) { - struct kbase_jd_atom *prev_atom = kbase_backend_inspect_tail(kbdev, js); - - if (prev_atom && prev_atom->kctx != kctx) - return NULL; - } - - if (katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) { - if (katom->x_pre_dep->gpu_rb_state == KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB || - katom->x_pre_dep->will_fail_event_code) { - dev_dbg(kbdev->dev, - "JS: X pre-dep %pK is not present in slot FIFO or will fail\n", - (void *)katom->x_pre_dep); - return NULL; - } - if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) && - kbase_backend_nr_atoms_on_slot(kbdev, js)) { - dev_dbg(kbdev->dev, - "JS: Atom %pK has cross-slot fail dependency and atoms on slot (s:%u)\n", - (void *)katom, js); - return NULL; - } - } - - KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, JS_PULL_JOB, kctx, katom, katom->jc, js, - (u64)katom->sched_priority); - kbase_ctx_flag_set(kctx, KCTX_PULLED); - kbase_ctx_flag_set(kctx, (KCTX_PULLED_SINCE_ACTIVE_JS0 << js)); - - pulled = kbase_jsctx_slot_atom_pulled_inc(kctx, katom); - if (pulled == 1 && !kctx->slots_pullable) { - WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); - kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF); - atomic_inc(&kbdev->js_data.nr_contexts_runnable); - } - jsctx_rb_pull(kctx, katom); - - kbase_ctx_sched_retain_ctx_refcount(kctx); - - katom->ticks = 0; - - dev_dbg(kbdev->dev, "JS: successfully pulled atom %pK from kctx %pK (s:%u)\n", - (void *)katom, (void *)kctx, js); - - return katom; -} - -static void js_return_worker(struct work_struct *data) -{ - struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work); - struct kbase_context *kctx = katom->kctx; - struct kbase_device *kbdev = kctx->kbdev; - struct kbasep_js_device_data *js_devdata = &kbdev->js_data; - struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info; - struct kbasep_js_atom_retained_state retained_state; - unsigned int js = katom->slot_nr; - bool slot_became_unblocked; - bool timer_sync = false; - bool context_idle = false; - unsigned long flags; - base_jd_core_req core_req = katom->core_req; - u64 cache_jc = katom->jc; - - dev_dbg(kbdev->dev, "%s for atom %pK with event code 0x%x\n", __func__, (void *)katom, - katom->event_code); - - KBASE_KTRACE_ADD_JM(kbdev, JS_RETURN_WORKER, kctx, katom, katom->jc, 0); - KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_EX(kbdev, katom); - - kbase_backend_complete_wq(kbdev, katom); - - kbasep_js_atom_retained_state_copy(&retained_state, katom); - - mutex_lock(&js_devdata->queue_mutex); - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - - atomic_dec(&katom->blocked); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - slot_became_unblocked = kbase_jsctx_slot_atom_pulled_dec(kctx, katom); - - if (!kbase_jsctx_slot_atoms_pulled(kctx, js) && jsctx_rb_none_to_pull(kctx, js)) - timer_sync |= kbase_js_ctx_list_remove_nolock(kbdev, kctx, js); - - /* If the context is now unblocked on this slot after soft-stopped - * atoms, then only mark it as pullable on this slot if it is not - * idle - */ - if (slot_became_unblocked && kbase_jsctx_atoms_pulled(kctx) && - kbase_js_ctx_pullable(kctx, js, true)) - timer_sync |= kbase_js_ctx_list_add_pullable_nolock(kbdev, kctx, js); - - if (!kbase_jsctx_atoms_pulled(kctx)) { - dev_dbg(kbdev->dev, "No atoms currently pulled from context %pK\n", (void *)kctx); - - if (!kctx->slots_pullable) { - dev_dbg(kbdev->dev, "Context %pK %s counted as runnable\n", (void *)kctx, - kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF) ? "is" : "isn't"); - - WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); - kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); - atomic_dec(&kbdev->js_data.nr_contexts_runnable); - timer_sync = true; - } - - if (kctx->as_nr != KBASEP_AS_NR_INVALID && !kbase_ctx_flag(kctx, KCTX_DYING)) { - unsigned int num_slots = kbdev->gpu_props.num_job_slots; - unsigned int slot; - - if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) - kbasep_js_set_submit_allowed(js_devdata, kctx); - - for (slot = 0; slot < num_slots; slot++) { - if (kbase_js_ctx_pullable(kctx, slot, true)) - timer_sync |= kbase_js_ctx_list_add_pullable_nolock( - kbdev, kctx, slot); - } - } - - kbase_jm_idle_ctx(kbdev, kctx); - - context_idle = true; - } - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - if (context_idle) { - dev_dbg(kbdev->dev, "Context %pK %s counted as active\n", (void *)kctx, - kbase_ctx_flag(kctx, KCTX_ACTIVE) ? "is" : "isn't"); - WARN_ON(!kbase_ctx_flag(kctx, KCTX_ACTIVE)); - kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); - kbase_pm_context_idle(kbdev); - } - - if (timer_sync) - kbase_js_sync_timers(kbdev); - - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - mutex_unlock(&js_devdata->queue_mutex); - - dev_dbg(kbdev->dev, "JS: retained state %s finished", - kbasep_js_has_atom_finished(&retained_state) ? "has" : "hasn't"); - - WARN_ON(kbasep_js_has_atom_finished(&retained_state)); - - kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, &retained_state); - - kbase_js_sched_all(kbdev); - - kbase_backend_complete_wq_post_sched(kbdev, core_req); - - KBASE_KTRACE_ADD_JM(kbdev, JS_RETURN_WORKER_END, kctx, NULL, cache_jc, 0); - - dev_dbg(kbdev->dev, "Leaving %s for atom %pK\n", __func__, (void *)katom); -} - -void kbase_js_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom) -{ - dev_dbg(kctx->kbdev->dev, "Unpulling atom %pK in kctx %pK\n", (void *)katom, (void *)kctx); - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - jsctx_rb_unpull(kctx, katom); - - WARN_ON(work_pending(&katom->work)); - - /* Block re-submission until workqueue has run */ - atomic_inc(&katom->blocked); - - kbase_job_check_leave_disjoint(kctx->kbdev, katom); - - INIT_WORK(&katom->work, js_return_worker); - queue_work(kctx->jctx.job_done_wq, &katom->work); -} - -bool kbase_js_complete_atom_wq(struct kbase_context *kctx, struct kbase_jd_atom *katom) -{ - struct kbasep_js_kctx_info *js_kctx_info; - struct kbasep_js_device_data *js_devdata; - struct kbase_device *kbdev; - unsigned long flags; - bool timer_sync = false; - unsigned int atom_slot; - bool context_idle = false; - int prio = katom->sched_priority; - - kbdev = kctx->kbdev; - atom_slot = katom->slot_nr; - - dev_dbg(kbdev->dev, "%s for atom %pK (s:%u)\n", __func__, (void *)katom, atom_slot); - - js_kctx_info = &kctx->jctx.sched_info; - js_devdata = &kbdev->js_data; - - lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); - - mutex_lock(&js_devdata->runpool_mutex); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - if (katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE) { - bool slot_became_unblocked; - - dev_dbg(kbdev->dev, "Atom %pK is in runnable_tree\n", (void *)katom); - - slot_became_unblocked = kbase_jsctx_slot_atom_pulled_dec(kctx, katom); - context_idle = !kbase_jsctx_atoms_pulled(kctx); - - if (!kbase_jsctx_atoms_pulled(kctx) && !kctx->slots_pullable) { - WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); - kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); - atomic_dec(&kbdev->js_data.nr_contexts_runnable); - timer_sync = true; - } - - /* If this slot has been blocked due to soft-stopped atoms, and - * all atoms have now been processed at this priority level and - * higher, then unblock the slot - */ - if (slot_became_unblocked) { - dev_dbg(kbdev->dev, - "kctx %pK is no longer blocked from submitting on slot %u at priority %d or higher\n", - (void *)kctx, atom_slot, prio); - - if (kbase_js_ctx_pullable(kctx, atom_slot, true)) - timer_sync |= kbase_js_ctx_list_add_pullable_nolock(kbdev, kctx, - atom_slot); - } - } - WARN_ON(!(katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE)); - - if (!kbase_jsctx_slot_atoms_pulled(kctx, atom_slot) && - jsctx_rb_none_to_pull(kctx, atom_slot)) { - if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[atom_slot])) - timer_sync |= kbase_js_ctx_list_remove_nolock(kctx->kbdev, kctx, atom_slot); - } - - /* - * If submission is disabled on this context (most likely due to an - * atom failure) and there are now no atoms left in the system then - * re-enable submission so that context can be scheduled again. - */ - if (!kbasep_js_is_submit_allowed(js_devdata, kctx) && !kbase_jsctx_atoms_pulled(kctx) && - !kbase_ctx_flag(kctx, KCTX_DYING)) { - unsigned int js; - - kbasep_js_set_submit_allowed(js_devdata, kctx); - - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { - if (kbase_js_ctx_pullable(kctx, js, true)) - timer_sync |= - kbase_js_ctx_list_add_pullable_nolock(kbdev, kctx, js); - } - } else if (katom->x_post_dep && kbasep_js_is_submit_allowed(js_devdata, kctx)) { - unsigned int js; - - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { - if (kbase_js_ctx_pullable(kctx, js, true)) - timer_sync |= - kbase_js_ctx_list_add_pullable_nolock(kbdev, kctx, js); - } - } - - /* Mark context as inactive. The pm reference will be dropped later in - * jd_done_worker(). - */ - if (context_idle) { - dev_dbg(kbdev->dev, "kctx %pK is no longer active\n", (void *)kctx); - kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); - } - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - if (timer_sync) - kbase_backend_ctx_count_changed(kbdev); - mutex_unlock(&js_devdata->runpool_mutex); - - dev_dbg(kbdev->dev, "Leaving %s\n", __func__); - return context_idle; -} - -struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, ktime_t *end_timestamp) -{ - struct kbase_device *kbdev; - struct kbase_context *kctx = katom->kctx; - struct kbase_jd_atom *x_dep = katom->x_post_dep; - - kbdev = kctx->kbdev; - dev_dbg(kbdev->dev, "Atom %pK complete in kctx %pK (post-dep %pK)\n", (void *)katom, - (void *)kctx, (void *)x_dep); - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - if (katom->will_fail_event_code) - katom->event_code = katom->will_fail_event_code; - - katom->status = KBASE_JD_ATOM_STATE_HW_COMPLETED; - dev_dbg(kbdev->dev, "Atom %pK status to HW completed\n", (void *)katom); - if (kbase_is_quick_reset_enabled(kbdev)) { - kbdev->num_of_atoms_hw_completed++; - if (kbdev->num_of_atoms_hw_completed >= 20) - kbase_disable_quick_reset(kbdev); - } - - if (katom->event_code != BASE_JD_EVENT_DONE) { - kbase_js_evict_deps(kctx, katom, katom->slot_nr, katom->sched_priority); - } - - KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT(kbdev, NULL, katom->slot_nr, 0, TL_JS_EVENT_STOP); - - trace_sysgraph_gpu(SGR_COMPLETE, kctx->id, kbase_jd_atom_id(katom->kctx, katom), - katom->slot_nr); - - KBASE_TLSTREAM_TL_JD_DONE_START(kbdev, katom); - kbase_jd_done(katom, katom->slot_nr, end_timestamp, 0); - KBASE_TLSTREAM_TL_JD_DONE_END(kbdev, katom); - - /* Unblock cross dependency if present */ - if (x_dep && - (katom->event_code == BASE_JD_EVENT_DONE || - !(x_dep->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER)) && - (x_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST)) { - bool was_pullable = kbase_js_ctx_pullable(kctx, x_dep->slot_nr, false); - x_dep->atom_flags &= ~KBASE_KATOM_FLAG_X_DEP_BLOCKED; - dev_dbg(kbdev->dev, "Cleared X_DEP flag on atom %pK\n", (void *)x_dep); - - kbase_js_move_to_tree(x_dep); - - if (!was_pullable && kbase_js_ctx_pullable(kctx, x_dep->slot_nr, false)) - kbase_js_ctx_list_add_pullable_nolock(kbdev, kctx, x_dep->slot_nr); - - if (x_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE) { - dev_dbg(kbdev->dev, "Atom %pK is in runnable tree\n", (void *)x_dep); - return x_dep; - } - } else { - dev_dbg(kbdev->dev, "No cross-slot dep to unblock for atom %pK\n", (void *)katom); - } - - return NULL; -} - -void kbase_js_sched(struct kbase_device *kbdev, unsigned int js_mask) -{ - struct kbasep_js_device_data *js_devdata; - struct kbase_context *last_active_kctx[BASE_JM_MAX_NR_SLOTS]; - bool timer_sync = false; - bool ctx_waiting[BASE_JM_MAX_NR_SLOTS]; - unsigned int js; - - KBASE_TLSTREAM_TL_JS_SCHED_START(kbdev, 0); - - dev_dbg(kbdev->dev, "%s kbdev %pK mask 0x%x\n", __func__, (void *)kbdev, - (unsigned int)js_mask); - - js_devdata = &kbdev->js_data; - - down(&js_devdata->schedule_sem); - mutex_lock(&js_devdata->queue_mutex); - - for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { - last_active_kctx[js] = kbdev->hwaccess.active_kctx[js]; - ctx_waiting[js] = false; - } - - while (js_mask) { - js = (unsigned int)ffs((int)js_mask) - 1; - - while (1) { - struct kbase_context *kctx; - unsigned long flags; - bool context_idle = false; - - kctx = kbase_js_ctx_list_pop_head(kbdev, js); - - if (!kctx) { - js_mask &= ~(1UL << js); - dev_dbg(kbdev->dev, "No kctx on pullable list (s:%u)\n", js); - break; - } - - if (!kbase_ctx_flag(kctx, KCTX_ACTIVE)) { - context_idle = true; - - dev_dbg(kbdev->dev, "kctx %pK is not active (s:%u)\n", (void *)kctx, - js); - - if (kbase_pm_context_active_handle_suspend( - kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE)) { - dev_dbg(kbdev->dev, "Suspend pending (s:%u)\n", js); - /* Suspend pending - return context to - * queue and stop scheduling - */ - mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); - if (kbase_js_ctx_list_add_pullable_head(kctx->kbdev, kctx, - js)) - kbase_js_sync_timers(kbdev); - mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); - mutex_unlock(&js_devdata->queue_mutex); - up(&js_devdata->schedule_sem); - KBASE_TLSTREAM_TL_JS_SCHED_END(kbdev, 0); - return; - } - kbase_ctx_flag_set(kctx, KCTX_ACTIVE); - } - - if (!kbase_js_use_ctx(kbdev, kctx, js)) { - mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); - - dev_dbg(kbdev->dev, "kctx %pK cannot be used at this time\n", kctx); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - if (kbase_js_ctx_pullable(kctx, js, false) || - kbase_ctx_flag(kctx, KCTX_PRIVILEGED)) - timer_sync |= kbase_js_ctx_list_add_pullable_head_nolock( - kctx->kbdev, kctx, js); - else - timer_sync |= kbase_js_ctx_list_add_unpullable_nolock( - kctx->kbdev, kctx, js); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); - if (context_idle) { - WARN_ON(!kbase_ctx_flag(kctx, KCTX_ACTIVE)); - kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); - kbase_pm_context_idle(kbdev); - } - - /* No more jobs can be submitted on this slot */ - js_mask &= ~(1UL << js); - break; - } - mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - kbase_ctx_flag_clear(kctx, KCTX_PULLED); - - if (!kbase_jm_kick(kbdev, 1UL << js)) { - dev_dbg(kbdev->dev, "No more jobs can be submitted (s:%u)\n", js); - js_mask &= ~(1UL << js); - } - if (!kbase_ctx_flag(kctx, KCTX_PULLED)) { - bool pullable; - - dev_dbg(kbdev->dev, "No atoms pulled from kctx %pK (s:%u)\n", - (void *)kctx, js); - - pullable = kbase_js_ctx_pullable(kctx, js, true); - - /* Failed to pull jobs - push to head of list. - * Unless this context is already 'active', in - * which case it's effectively already scheduled - * so push it to the back of the list. - */ - if (pullable && kctx == last_active_kctx[js] && - kbase_ctx_flag(kctx, (KCTX_PULLED_SINCE_ACTIVE_JS0 << js))) - timer_sync |= kbase_js_ctx_list_add_pullable_nolock( - kctx->kbdev, kctx, js); - else if (pullable) - timer_sync |= kbase_js_ctx_list_add_pullable_head_nolock( - kctx->kbdev, kctx, js); - else - timer_sync |= kbase_js_ctx_list_add_unpullable_nolock( - kctx->kbdev, kctx, js); - - /* If this context is not the active context, - * but the active context is pullable on this - * slot, then we need to remove the active - * marker to prevent it from submitting atoms in - * the IRQ handler, which would prevent this - * context from making progress. - */ - if (last_active_kctx[js] && kctx != last_active_kctx[js] && - kbase_js_ctx_pullable(last_active_kctx[js], js, true)) - ctx_waiting[js] = true; - - if (context_idle) { - kbase_jm_idle_ctx(kbdev, kctx); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - WARN_ON(!kbase_ctx_flag(kctx, KCTX_ACTIVE)); - kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); - kbase_pm_context_idle(kbdev); - } else { - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - } - mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); - - js_mask &= ~(1UL << js); - break; /* Could not run atoms on this slot */ - } - - dev_dbg(kbdev->dev, "Push kctx %pK to back of list\n", (void *)kctx); - if (kbase_js_ctx_pullable(kctx, js, true)) - timer_sync |= kbase_js_ctx_list_add_pullable_nolock(kctx->kbdev, - kctx, js); - else - timer_sync |= kbase_js_ctx_list_add_unpullable_nolock(kctx->kbdev, - kctx, js); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); - } - } - - if (timer_sync) - kbase_js_sync_timers(kbdev); - - for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { - if (kbdev->hwaccess.active_kctx[js] == last_active_kctx[js] && ctx_waiting[js]) { - dev_dbg(kbdev->dev, "Marking kctx %pK as inactive (s:%u)\n", - (void *)last_active_kctx[js], js); - kbdev->hwaccess.active_kctx[js] = NULL; - } - } - - mutex_unlock(&js_devdata->queue_mutex); - up(&js_devdata->schedule_sem); - KBASE_TLSTREAM_TL_JS_SCHED_END(kbdev, 0); -} - -void kbase_js_zap_context(struct kbase_context *kctx) -{ - struct kbase_device *kbdev = kctx->kbdev; - struct kbasep_js_device_data *js_devdata = &kbdev->js_data; - struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info; - - /* - * Critical assumption: No more submission is possible outside of the - * workqueue. This is because the OS *must* prevent U/K calls (IOCTLs) - * whilst the struct kbase_context is terminating. - */ - - /* First, atomically do the following: - * - mark the context as dying - * - try to evict it from the queue - */ - mutex_lock(&kctx->jctx.lock); - mutex_lock(&js_devdata->queue_mutex); - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - kbase_ctx_flag_set(kctx, KCTX_DYING); - - dev_dbg(kbdev->dev, "Zap: Try Evict Ctx %pK", kctx); - - /* - * At this point we know: - * - If eviction succeeded, it was in the queue, but now no - * longer is - * - We must cancel the jobs here. No Power Manager active reference to - * release. - * - This happens asynchronously - kbase_jd_zap_context() will wait for - * those jobs to be killed. - * - If eviction failed, then it wasn't in the queue. It is one - * of the following: - * - a. it didn't have any jobs, and so is not in the Queue or - * the Run Pool (not scheduled) - * - Hence, no more work required to cancel jobs. No Power Manager - * active reference to release. - * - b. it was in the middle of a scheduling transaction (and thus must - * have at least 1 job). This can happen from a syscall or a - * kernel thread. We still hold the jsctx_mutex, and so the thread - * must be waiting inside kbasep_js_try_schedule_head_ctx(), - * before checking whether the runpool is full. That thread will - * continue after we drop the mutex, and will notice the context - * is dying. It will rollback the transaction, killing all jobs at - * the same time. kbase_jd_zap_context() will wait for those jobs - * to be killed. - * - Hence, no more work required to cancel jobs, or to release the - * Power Manager active reference. - * - c. it is scheduled, and may or may not be running jobs - * - We must cause it to leave the runpool by stopping it from - * submitting any more jobs. When it finally does leave, - * kbasep_js_runpool_requeue_or_kill_ctx() will kill all remaining jobs - * (because it is dying), release the Power Manager active reference, - * and will not requeue the context in the queue. - * kbase_jd_zap_context() will wait for those jobs to be killed. - * - Hence, work required just to make it leave the runpool. Cancelling - * jobs and releasing the Power manager active reference will be - * handled when it leaves the runpool. - */ - if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED)) { - unsigned long flags; - unsigned int js; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { - if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])) - list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); - } - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - /* The following events require us to kill off remaining jobs - * and update PM book-keeping: - * - we evicted it correctly (it must have jobs to be in the - * Queue) - * - * These events need no action, but take this path anyway: - * - Case a: it didn't have any jobs, and was never in the Queue - * - Case b: scheduling transaction will be partially rolled- - * back (this already cancels the jobs) - */ - - KBASE_KTRACE_ADD_JM(kbdev, JM_ZAP_NON_SCHEDULED, kctx, NULL, 0u, - kbase_ctx_flag(kctx, KCTX_SCHEDULED)); - - dev_dbg(kbdev->dev, "Zap: Ctx %pK scheduled=0", kctx); - - /* Only cancel jobs when we evicted from the - * queue. No Power Manager active reference was held. - * - * Having is_dying set ensures that this kills, and doesn't - * requeue - */ - kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, false); - - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - mutex_unlock(&js_devdata->queue_mutex); - mutex_unlock(&kctx->jctx.lock); - } else { - unsigned long flags; - bool was_retained; - CSTD_UNUSED(was_retained); - - /* Case c: didn't evict, but it is scheduled - it's in the Run - * Pool - */ - KBASE_KTRACE_ADD_JM(kbdev, JM_ZAP_SCHEDULED, kctx, NULL, 0u, - kbase_ctx_flag(kctx, KCTX_SCHEDULED)); - dev_dbg(kbdev->dev, "Zap: Ctx %pK is in RunPool", kctx); - - /* Disable the ctx from submitting any more jobs */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - kbasep_js_clear_submit_allowed(js_devdata, kctx); - - /* Retain and (later) release the context whilst it is now - * disallowed from submitting jobs - ensures that someone - * somewhere will be removing the context later on - */ - was_retained = kbase_ctx_sched_inc_refcount_nolock(kctx); - - /* Since it's scheduled and we have the jsctx_mutex, it must be - * retained successfully - */ - KBASE_DEBUG_ASSERT(was_retained); - - dev_dbg(kbdev->dev, "Zap: Ctx %pK Kill Any Running jobs", kctx); - - /* Cancel any remaining running jobs for this kctx - if any. - * Submit is disallowed which takes effect immediately, so no - * more new jobs will appear after we do this. - */ - kbase_backend_jm_kill_running_jobs_from_kctx(kctx); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - mutex_unlock(&js_devdata->queue_mutex); - mutex_unlock(&kctx->jctx.lock); - - dev_dbg(kbdev->dev, - "Zap: Ctx %pK Release (may or may not schedule out immediately)", kctx); - - kbasep_js_runpool_release_ctx(kbdev, kctx); - } - - KBASE_KTRACE_ADD_JM(kbdev, JM_ZAP_DONE, kctx, NULL, 0u, 0u); - - /* After this, you must wait on both the - * kbase_jd_context::zero_jobs_wait and the - * kbasep_js_kctx_info::ctx::is_scheduled_waitq - to wait for the jobs - * to be destroyed, and the context to be de-scheduled (if it was on the - * runpool). - * - * kbase_jd_zap_context() will do this. - */ -} - -static inline int trace_get_refcnt(struct kbase_device *kbdev, struct kbase_context *kctx) -{ - CSTD_UNUSED(kbdev); - return atomic_read(&kctx->refcount); -} - -/** - * kbase_js_foreach_ctx_job(): - Call a function on all jobs in context - * @kctx: Pointer to context. - * @callback: Pointer to function to call for each job. - * - * Call a function on all jobs belonging to a non-queued, non-running - * context, and detach the jobs from the context as it goes. - * - * Due to the locks that might be held at the time of the call, the callback - * may need to defer work on a workqueue to complete its actions (e.g. when - * cancelling jobs) - * - * Atoms will be removed from the queue, so this must only be called when - * cancelling jobs (which occurs as part of context destruction). - * - * The locking conditions on the caller are as follows: - * - it will be holding kbasep_js_kctx_info::ctx::jsctx_mutex. - */ -static void kbase_js_foreach_ctx_job(struct kbase_context *kctx, kbasep_js_ctx_job_cb *callback) -{ - struct kbase_device *kbdev; - unsigned long flags; - unsigned int js; - - kbdev = kctx->kbdev; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, JS_POLICY_FOREACH_CTX_JOBS, kctx, NULL, 0u, - trace_get_refcnt(kbdev, kctx)); - - /* Invoke callback on jobs on each slot in turn */ - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) - jsctx_queue_foreach(kctx, js, callback); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -} - -base_jd_prio kbase_js_priority_check(struct kbase_device *kbdev, base_jd_prio priority) -{ - struct priority_control_manager_device *pcm_device = kbdev->pcm_dev; - int req_priority, out_priority; - - req_priority = kbasep_js_atom_prio_to_sched_prio(priority); - out_priority = req_priority; - /* Does not use pcm defined priority check if PCM not defined or if - * kbasep_js_atom_prio_to_sched_prio returns an error - * (KBASE_JS_ATOM_SCHED_PRIO_INVALID). - */ - if (pcm_device && (req_priority != KBASE_JS_ATOM_SCHED_PRIO_INVALID)) - out_priority = pcm_device->ops.pcm_scheduler_priority_check(pcm_device, current, - req_priority); - return kbasep_js_sched_prio_to_atom_prio(kbdev, out_priority); -} diff --git a/drivers/gpu/arm/valhall/mali_kbase_js.h b/drivers/gpu/arm/valhall/mali_kbase_js.h deleted file mode 100644 index 89c3b45c735b..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_js.h +++ /dev/null @@ -1,36 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2011-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/** - * DOC: Job Scheduler APIs. - */ - -#ifndef _KBASE_JS_H_ -#define _KBASE_JS_H_ - -#include "context/mali_kbase_context.h" -#include "mali_kbase_defs.h" -#include "mali_kbase_debug.h" -#include -#include "jm/mali_kbase_jm_js.h" -#include "jm/mali_kbase_js_defs.h" - -#endif /* _KBASE_JS_H_ */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_js_ctx_attr.c b/drivers/gpu/arm/valhall/mali_kbase_js_ctx_attr.c deleted file mode 100644 index 6fc6b8a93027..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_js_ctx_attr.c +++ /dev/null @@ -1,324 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2012-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include -#include - -/* - * Private functions follow - */ - -/** - * kbasep_js_ctx_attr_runpool_retain_attr - Check whether a ctx has a certain attribute - * and if so, retain that attribute on the runpool. - * - * @kbdev: Device pointer - * @kctx: KBase context - * @attribute: Atribute to check/retain - * - * Requires: - * - jsctx mutex - * - runpool_irq spinlock - * - ctx is scheduled on the runpool - * - * Return: true indicates a change in ctx attributes state of the runpool. - * In this state, the scheduler might be able to submit more jobs than - * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() - * or similar is called sometime later. - * false indicates no change in ctx attributes state of the runpool. - */ -static bool kbasep_js_ctx_attr_runpool_retain_attr(struct kbase_device *kbdev, - struct kbase_context *kctx, - enum kbasep_js_ctx_attr attribute) -{ - struct kbasep_js_device_data *js_devdata; - struct kbasep_js_kctx_info *js_kctx_info; - bool runpool_state_changed = false; - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(kctx != NULL); - KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); - js_devdata = &kbdev->js_data; - js_kctx_info = &kctx->jctx.sched_info; - - lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); - lockdep_assert_held(&kbdev->hwaccess_lock); - - KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); - - if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, attribute) != false) { - KBASE_DEBUG_ASSERT(js_devdata->runpool_irq.ctx_attr_ref_count[attribute] < S8_MAX); - ++(js_devdata->runpool_irq.ctx_attr_ref_count[attribute]); - - if (js_devdata->runpool_irq.ctx_attr_ref_count[attribute] == 1) { - /* First refcount indicates a state change */ - runpool_state_changed = true; - KBASE_KTRACE_ADD_JM(kbdev, JS_CTX_ATTR_NOW_ON_RUNPOOL, kctx, NULL, 0u, - attribute); - } - } - - return runpool_state_changed; -} - -/** - * kbasep_js_ctx_attr_runpool_release_attr - Check whether a ctx has a certain attribute, - * and if so, release that attribute on the runpool. - * - * @kbdev: Device pointer - * @kctx: KBase context - * @attribute: Atribute to release - * - * Requires: - * - jsctx mutex - * - runpool_irq spinlock - * - ctx is scheduled on the runpool - * - * Return: true indicates a change in ctx attributes state of the runpool. - * In this state, the scheduler might be able to submit more jobs than - * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() - * or similar is called sometime later. - * false indicates no change in ctx attributes state of the runpool. - */ -static bool kbasep_js_ctx_attr_runpool_release_attr(struct kbase_device *kbdev, - struct kbase_context *kctx, - enum kbasep_js_ctx_attr attribute) -{ - struct kbasep_js_device_data *js_devdata; - struct kbasep_js_kctx_info *js_kctx_info; - bool runpool_state_changed = false; - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(kctx != NULL); - KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); - js_devdata = &kbdev->js_data; - js_kctx_info = &kctx->jctx.sched_info; - - lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); - lockdep_assert_held(&kbdev->hwaccess_lock); - KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); - - if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, attribute) != false) { - KBASE_DEBUG_ASSERT(js_devdata->runpool_irq.ctx_attr_ref_count[attribute] > 0); - --(js_devdata->runpool_irq.ctx_attr_ref_count[attribute]); - - if (js_devdata->runpool_irq.ctx_attr_ref_count[attribute] == 0) { - /* Last de-refcount indicates a state change */ - runpool_state_changed = true; - KBASE_KTRACE_ADD_JM(kbdev, JS_CTX_ATTR_NOW_OFF_RUNPOOL, kctx, NULL, 0u, - attribute); - } - } - - return runpool_state_changed; -} - -/** - * kbasep_js_ctx_attr_ctx_retain_attr - Retain a certain attribute on a ctx, - * also retaining it on the runpool if the context is scheduled. - * - * @kbdev: Device pointer - * @kctx: KBase context - * @attribute: Atribute to retain - * - * Requires: - * - jsctx mutex - * - If the context is scheduled, then runpool_irq spinlock must also be held - * - * Return: true indicates a change in ctx attributes state of the runpool. - * This may allow the scheduler to submit more jobs than previously. - * false indicates no change in ctx attributes state of the runpool. - */ -static bool kbasep_js_ctx_attr_ctx_retain_attr(struct kbase_device *kbdev, - struct kbase_context *kctx, - enum kbasep_js_ctx_attr attribute) -{ - struct kbasep_js_kctx_info *js_kctx_info; - bool runpool_state_changed = false; - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(kctx != NULL); - KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); - js_kctx_info = &kctx->jctx.sched_info; - - lockdep_assert_held(&kbdev->hwaccess_lock); - lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); - KBASE_DEBUG_ASSERT(js_kctx_info->ctx.ctx_attr_ref_count[attribute] < U32_MAX); - - ++(js_kctx_info->ctx.ctx_attr_ref_count[attribute]); - - if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && - js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) { - /* Only ref-count the attribute on the runpool for the first time this contexts sees this attribute */ - KBASE_KTRACE_ADD_JM(kbdev, JS_CTX_ATTR_NOW_ON_CTX, kctx, NULL, 0u, attribute); - runpool_state_changed = - kbasep_js_ctx_attr_runpool_retain_attr(kbdev, kctx, attribute); - } - - return runpool_state_changed; -} - -/** - * kbasep_js_ctx_attr_ctx_release_attr - Release a certain attribute on a ctx, - * also releasing it from the runpool if the context is scheduled. - * - * @kbdev: Device pointer - * @kctx: KBase context - * @attribute: Atribute to release - * - * Requires: - * - jsctx mutex - * - If the context is scheduled, then runpool_irq spinlock must also be held - * - * Return: true indicates a change in ctx attributes state of the runpool. - * This may allow the scheduler to submit more jobs than previously. - * false indicates no change in ctx attributes state of the runpool. - */ -static bool kbasep_js_ctx_attr_ctx_release_attr(struct kbase_device *kbdev, - struct kbase_context *kctx, - enum kbasep_js_ctx_attr attribute) -{ - struct kbasep_js_kctx_info *js_kctx_info; - bool runpool_state_changed = false; - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(kctx != NULL); - KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); - js_kctx_info = &kctx->jctx.sched_info; - - lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); - KBASE_DEBUG_ASSERT(js_kctx_info->ctx.ctx_attr_ref_count[attribute] > 0); - - if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && - js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) { - lockdep_assert_held(&kbdev->hwaccess_lock); - /* Only de-ref-count the attribute on the runpool when this is the last ctx-reference to it */ - runpool_state_changed = - kbasep_js_ctx_attr_runpool_release_attr(kbdev, kctx, attribute); - KBASE_KTRACE_ADD_JM(kbdev, JS_CTX_ATTR_NOW_OFF_CTX, kctx, NULL, 0u, attribute); - } - - /* De-ref must happen afterwards, because kbasep_js_ctx_attr_runpool_release() needs to check it too */ - --(js_kctx_info->ctx.ctx_attr_ref_count[attribute]); - - return runpool_state_changed; -} - -/* - * More commonly used public functions - */ - -void kbasep_js_ctx_attr_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) -{ - bool runpool_state_changed; - int i; - - /* Retain any existing attributes */ - for (i = 0; i < KBASEP_JS_CTX_ATTR_COUNT; ++i) { - if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, (enum kbasep_js_ctx_attr)i) != false) { - /* The context is being scheduled in, so update the runpool with the new attributes */ - runpool_state_changed = kbasep_js_ctx_attr_runpool_retain_attr( - kbdev, kctx, (enum kbasep_js_ctx_attr)i); - - /* We don't need to know about state changed, because retaining a - * context occurs on scheduling it, and that itself will also try - * to run new atoms - */ - CSTD_UNUSED(runpool_state_changed); - } - } -} - -bool kbasep_js_ctx_attr_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) -{ - bool runpool_state_changed = false; - int i; - - /* Release any existing attributes */ - for (i = 0; i < KBASEP_JS_CTX_ATTR_COUNT; ++i) { - if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, (enum kbasep_js_ctx_attr)i) != false) { - /* The context is being scheduled out, so update the runpool on the removed attributes */ - runpool_state_changed |= kbasep_js_ctx_attr_runpool_release_attr( - kbdev, kctx, (enum kbasep_js_ctx_attr)i); - } - } - - return runpool_state_changed; -} - -void kbasep_js_ctx_attr_ctx_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbase_jd_atom *katom) -{ - bool runpool_state_changed = false; - base_jd_core_req core_req; - - KBASE_DEBUG_ASSERT(katom); - core_req = katom->core_req; - - if (core_req & BASE_JD_REQ_ONLY_COMPUTE) - runpool_state_changed |= - kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE); - else - runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr( - kbdev, kctx, KBASEP_JS_CTX_ATTR_NON_COMPUTE); - - if ((core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T)) != 0 && - (core_req & (BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP)) == 0) { - /* Atom that can run on slot1 or slot2, and can use all cores */ - runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr( - kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES); - } - - /* We don't need to know about state changed, because retaining an atom - * occurs on adding it, and that itself will also try to run new atoms - */ - CSTD_UNUSED(runpool_state_changed); -} - -bool kbasep_js_ctx_attr_ctx_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbasep_js_atom_retained_state *katom_retained_state) -{ - bool runpool_state_changed = false; - base_jd_core_req core_req; - - KBASE_DEBUG_ASSERT(katom_retained_state); - core_req = katom_retained_state->core_req; - - /* No-op for invalid atoms */ - if (kbasep_js_atom_retained_state_is_valid(katom_retained_state) == false) - return false; - - if (core_req & BASE_JD_REQ_ONLY_COMPUTE) - runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr( - kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE); - else - runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr( - kbdev, kctx, KBASEP_JS_CTX_ATTR_NON_COMPUTE); - - if ((core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T)) != 0 && - (core_req & (BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP)) == 0) { - /* Atom that can run on slot1 or slot2, and can use all cores */ - runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr( - kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES); - } - - return runpool_state_changed; -} diff --git a/drivers/gpu/arm/valhall/mali_kbase_js_ctx_attr.h b/drivers/gpu/arm/valhall/mali_kbase_js_ctx_attr.h deleted file mode 100644 index 471f54081d75..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_js_ctx_attr.h +++ /dev/null @@ -1,152 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2012-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/** - * DOC: Job Scheduler Context Attribute APIs - */ - -#ifndef _KBASE_JS_CTX_ATTR_H_ -#define _KBASE_JS_CTX_ATTR_H_ - -/** - * kbasep_js_ctx_attr_runpool_retain_ctx - Retain all attributes of a context - * - * @kbdev: KBase device - * @kctx: KBase context - * - * This occurs on scheduling in the context on the runpool (but after - * is_scheduled is set) - * - * Requires: - * - jsctx mutex - * - runpool_irq spinlock - * - ctx->is_scheduled is true - */ -void kbasep_js_ctx_attr_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); - -/** - * kbasep_js_ctx_attr_runpool_release_ctx - Release all attributes of a context - * - * @kbdev: KBase device - * @kctx: KBase context - * - * This occurs on scheduling out the context from the runpool (but before - * is_scheduled is cleared) - * - * Requires: - * - jsctx mutex - * - runpool_irq spinlock - * - ctx->is_scheduled is true - * - * Return: true indicates a change in ctx attributes state of the runpool. - * In this state, the scheduler might be able to submit more jobs than - * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() - * or similar is called sometime later. - * false indicates no change in ctx attributes state of the runpool. - */ -bool kbasep_js_ctx_attr_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); - -/** - * kbasep_js_ctx_attr_ctx_retain_atom - Retain all attributes of an atom - * - * @kbdev: KBase device - * @kctx: KBase context - * @katom: Atom - * - * This occurs on adding an atom to a context - * - * Requires: - * - jsctx mutex - * - If the context is scheduled, then runpool_irq spinlock must also be held - */ -void kbasep_js_ctx_attr_ctx_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbase_jd_atom *katom); - -/** - * kbasep_js_ctx_attr_ctx_release_atom - Release all attributes of an atom, - * given its retained state. - * - * @kbdev: KBase device - * @kctx: KBase context - * @katom_retained_state: Retained state - * - * This occurs after (permanently) removing an atom from a context - * - * Requires: - * - jsctx mutex - * - If the context is scheduled, then runpool_irq spinlock must also be held - * - * This is a no-op when \a katom_retained_state is invalid. - * - * Return: true indicates a change in ctx attributes state of the runpool. - * In this state, the scheduler might be able to submit more jobs than - * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() - * or similar is called sometime later. - * false indicates no change in ctx attributes state of the runpool. - */ -bool kbasep_js_ctx_attr_ctx_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbasep_js_atom_retained_state *katom_retained_state); - -/* - * Requires: - * - runpool_irq spinlock - */ -static inline s8 kbasep_js_ctx_attr_count_on_runpool(struct kbase_device *kbdev, - enum kbasep_js_ctx_attr attribute) -{ - struct kbasep_js_device_data *js_devdata; - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); - js_devdata = &kbdev->js_data; - - return js_devdata->runpool_irq.ctx_attr_ref_count[attribute]; -} - -/* - * Requires: - * - runpool_irq spinlock - */ -static inline bool kbasep_js_ctx_attr_is_attr_on_runpool(struct kbase_device *kbdev, - enum kbasep_js_ctx_attr attribute) -{ - /* In general, attributes are 'on' when they have a non-zero refcount (note: the refcount will never be < 0) */ - return (bool)kbasep_js_ctx_attr_count_on_runpool(kbdev, attribute); -} - -/* - * Requires: - * - jsctx mutex - */ -static inline bool kbasep_js_ctx_attr_is_attr_on_ctx(struct kbase_context *kctx, - enum kbasep_js_ctx_attr attribute) -{ - struct kbasep_js_kctx_info *js_kctx_info; - - KBASE_DEBUG_ASSERT(kctx != NULL); - KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); - js_kctx_info = &kctx->jctx.sched_info; - - /* In general, attributes are 'on' when they have a refcount (which should never be < 0) */ - return (bool)(js_kctx_info->ctx.ctx_attr_ref_count[attribute]); -} - -#endif /* _KBASE_JS_DEFS_H_ */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_kinstr_jm.c b/drivers/gpu/arm/valhall/mali_kbase_kinstr_jm.c deleted file mode 100644 index 13e225507017..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_kinstr_jm.c +++ /dev/null @@ -1,880 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * mali_kbase_kinstr_jm.c - * Kernel driver public interface to job manager atom tracing - */ - -#include "mali_kbase_kinstr_jm.h" -#include - -#include "mali_kbase.h" -#include "mali_kbase_linux.h" - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* Explicitly include epoll header for old kernels. Not required from 4.16. */ -#if KERNEL_VERSION(4, 16, 0) > LINUX_VERSION_CODE -#include -#endif - -/* Define static_assert(). - * - * The macro was introduced in kernel 5.1. But older vendor kernels may define - * it too. - */ -#if KERNEL_VERSION(5, 1, 0) <= LINUX_VERSION_CODE -#include -#elif !defined(static_assert) -// Stringify the expression if no message is given. -#define static_assert(e, ...) __static_assert(e, #__VA_ARGS__, #e) -#define __static_assert(e, msg, ...) _Static_assert(e, msg) -#endif - -/* The module printing prefix */ -#define PR_ "mali_kbase_kinstr_jm: " - -/* Allows us to perform ASM goto for the tracing - * https://www.kernel.org/doc/Documentation/static-keys.txt - */ -DEFINE_STATIC_KEY_FALSE(basep_kinstr_jm_reader_static_key); - -#define KBASE_KINSTR_JM_VERSION 2 - -/** - * struct kbase_kinstr_jm - The context for the kernel job manager atom tracing - * @readers: a bitlocked list of opened readers. Readers are attached to the - * private data of a file descriptor that the user opens with the - * KBASE_IOCTL_KINSTR_JM_FD IO control call. - * @refcount: reference count for the context. Any reader will have a link - * back to the context so that they can remove themselves from the - * list. - * - * This is opaque outside this compilation unit - */ -struct kbase_kinstr_jm { - struct hlist_bl_head readers; - struct kref refcount; -}; - -/** - * struct kbase_kinstr_jm_atom_state_change - Represents an atom changing to a - * new state - * @timestamp: Raw monotonic nanoseconds of the state change - * @state: The state that the atom has moved to - * @atom: The atom number that has changed state - * @flags: Flags associated with the state change. See - * KBASE_KINSTR_JM_ATOM_STATE_FLAG_* defines. - * @reserved: Reserved for future use. - * @data: Extra data for the state change. Active member depends on state. - * @data.start: Extra data for the state change. Active member depends on - * state. - * @data.start.slot: Extra data for the state change. Active member depends on - * state. - * @data.padding: Padding - * - * We can add new fields to the structure and old user code will gracefully - * ignore the new fields. - * - * We can change the size of the structure and old user code will gracefully - * skip over the new size via `struct kbase_kinstr_jm_fd_out->size`. - * - * If we remove fields, the version field in `struct - * kbase_kinstr_jm_fd_out->version` will be incremented and old user code will - * gracefully fail and tell the user that the kernel API is too new and has - * backwards-incompatible changes. Note that one userspace can opt to handle - * multiple kernel major versions of the structure. - * - * If we need to change the _meaning_ of one of the fields, i.e. the state - * machine has had a incompatible change, we can keep the same members in the - * structure and update the version as above. User code will no longer - * recognise that it has the supported field and can gracefully explain to the - * user that the kernel API is no longer supported. - * - * When making changes to this structure, make sure they are either: - * - additions to the end (for minor version bumps (i.e. only a size increase)) - * such that the layout of existing fields doesn't change, or; - * - update the version reported to userspace so that it can fail explicitly. - */ -struct kbase_kinstr_jm_atom_state_change { - u64 timestamp; - s8 state; /* enum kbase_kinstr_jm_reader_atom_state */ - u8 atom; - u8 flags; - u8 reserved[1]; - /* Tagged union based on state. Ensure members are aligned correctly! */ - union { - struct { - u8 slot; - } start; - u8 padding[4]; - } data; -}; -static_assert(((1 << 8 * sizeof(((struct kbase_kinstr_jm_atom_state_change *)0)->state)) - 1) >= - KBASE_KINSTR_JM_READER_ATOM_STATE_COUNT); - -#define KBASE_KINSTR_JM_ATOM_STATE_FLAG_OVERFLOW BIT(0) - -/** - * struct reader_changes - The circular buffer of kernel atom state changes - * @data: The allocated buffer. This is allocated when the user requests - * the reader file descriptor. It is released when the user calls - * close() on the fd. When accessing this, lock the producer spin - * lock to prevent races on the allocated memory. The consume lock - * does not need to be held because newly-inserted data will always - * be outside the currenly-read range. - * @producer: The producing spinlock which allows us to push changes into the - * buffer at the same time as a user read occurring. This needs to - * be locked when saving/restoring the IRQ because we can receive an - * interrupt from the GPU when an atom completes. The CPU could have - * a task preempted that is holding this lock. - * @consumer: The consuming mutex which locks around the user read(). - * Must be held when updating the tail of the circular buffer. - * @head: The head of the circular buffer. Can be used with Linux @c CIRC_ - * helpers. The producer should lock and update this with an SMP - * store when a new change lands. The consumer can read with an - * SMP load. This allows the producer to safely insert new changes - * into the circular buffer. - * @tail: The tail of the circular buffer. Can be used with Linux @c CIRC_ - * helpers. The producer should do a READ_ONCE load and the consumer - * should SMP store. - * @size: The number of changes that are allowed in @c data. Can be used - * with Linux @c CIRC_ helpers. Will always be a power of two. The - * producer lock should be held when updating this and stored with - * an SMP release memory barrier. This means that the consumer can - * do an SMP load. - * @threshold: The number of changes above which threads polling on the reader - * file descriptor will be woken up. - */ -struct reader_changes { - struct kbase_kinstr_jm_atom_state_change *data; - spinlock_t producer; - struct mutex consumer; - u32 head; - u32 tail; - u32 size; - u32 threshold; -}; - -/** - * reader_changes_is_valid_size() - Determines if requested changes buffer size - * is valid. - * @size: The requested memory size - * - * We have a constraint that the underlying physical buffer must be a - * power of two so that we can use the efficient circular buffer helpers that - * the kernel provides. It also needs to be representable within a u32. - * - * Return: - * * true - the size is valid - * * false - the size is invalid - */ -static inline bool reader_changes_is_valid_size(const size_t size) -{ - const size_t elem_size = sizeof(*((struct reader_changes *)0)->data); - const size_t size_size = sizeof(((struct reader_changes *)0)->size); - const size_t size_max = (1ull << (size_size * 8)) - 1; - - return is_power_of_2(size) && /* Is a power of two */ - ((size / elem_size) <= size_max); /* Small enough */ -} - -/** - * reader_changes_init() - Initializes the reader changes and allocates the - * changes buffer - * @changes: The context pointer, must point to a zero-inited allocated reader - * changes structure. We may support allocating the structure in the - * future. - * @size: The requested changes buffer size - * - * Return: - * (0, U16_MAX] - the number of data elements allocated - * -ERANGE - the requested memory size was invalid - * -ENOMEM - could not allocate the memory - */ -static int reader_changes_init(struct reader_changes *const changes, const size_t size) -{ - BUILD_BUG_ON((PAGE_SIZE % sizeof(*changes->data)) != 0); - - if (!reader_changes_is_valid_size(size)) { - pr_warn(PR_ "invalid size %zu\n", size); - return -ERANGE; - } - - changes->data = vmalloc(size); - if (!changes->data) - return -ENOMEM; - - spin_lock_init(&changes->producer); - mutex_init(&changes->consumer); - - changes->size = size / sizeof(*changes->data); - changes->threshold = - min(((size_t)(changes->size)) / 4, ((size_t)(PAGE_SIZE)) / sizeof(*changes->data)); - - return (int)changes->size; -} - -/** - * reader_changes_term() - Cleans up a reader changes structure - * @changes: The context to clean up - * - * Releases the allocated state changes memory - */ -static void reader_changes_term(struct reader_changes *const changes) -{ - struct kbase_kinstr_jm_atom_state_change *data = NULL; - unsigned long irq; - - /* - * Although changes->data is used on the consumer side, too, no active - * consumer is possible by the time we clean up the reader changes, so - * no need to take the consumer lock. However, we do need the producer - * lock because the list removal can race with list traversal. - */ - spin_lock_irqsave(&changes->producer, irq); - swap(changes->data, data); - spin_unlock_irqrestore(&changes->producer, irq); - - mutex_destroy(&changes->consumer); - vfree(data); -} - -/** - * reader_changes_count_locked() - Retrieves the count of state changes from the - * tail to the physical end of the buffer - * @changes: The state changes context - * - * The consumer mutex must be held. Uses the CIRC_CNT_TO_END macro to - * determine the count, so there may be more items. However, that's the maximum - * number that can be read in one contiguous read. - * - * Return: the number of changes in the circular buffer until the end of the - * allocation - */ -static u32 reader_changes_count_locked(struct reader_changes *const changes) -{ - u32 head; - - lockdep_assert_held_once(&changes->consumer); - - head = smp_load_acquire(&changes->head); - - return CIRC_CNT_TO_END(head, changes->tail, changes->size); -} - -/** - * reader_changes_count() - Retrieves the count of state changes from the - * tail to the physical end of the buffer - * @changes: The state changes context - * - * Return: the number of changes in the circular buffer until the end of the - * allocation - */ -static u32 reader_changes_count(struct reader_changes *const changes) -{ - u32 ret; - - mutex_lock(&changes->consumer); - ret = reader_changes_count_locked(changes); - mutex_unlock(&changes->consumer); - return ret; -} - -/** - * reader_changes_push() - Pushes a change into the reader circular buffer. - * @changes: The buffer to insert the change into - * @change: Kernel atom change to insert - * @wait_queue: The queue to be kicked when changes should be read from - * userspace. Kicked when a threshold is reached or there is - * overflow. - */ -static void reader_changes_push(struct reader_changes *const changes, - const struct kbase_kinstr_jm_atom_state_change *const change, - wait_queue_head_t *const wait_queue) -{ - u32 head, tail, size, space; - unsigned long irq; - struct kbase_kinstr_jm_atom_state_change *data; - - spin_lock_irqsave(&changes->producer, irq); - - /* We may be called for a reader_changes that's awaiting cleanup. */ - data = changes->data; - if (!data) - goto unlock; - - size = changes->size; - head = changes->head; - tail = smp_load_acquire(&changes->tail); - - space = CIRC_SPACE(head, tail, size); - if (space >= 1) { - data[head] = *change; - if (space == 1) { - data[head].flags |= KBASE_KINSTR_JM_ATOM_STATE_FLAG_OVERFLOW; - pr_warn(PR_ "overflow of circular buffer\n"); - } - smp_store_release(&changes->head, (head + 1) & (size - 1)); - } - - /* Wake for either overflow or over-threshold cases. */ - if (CIRC_CNT(head + 1, tail, size) >= changes->threshold) - wake_up_interruptible(wait_queue); - -unlock: - spin_unlock_irqrestore(&changes->producer, irq); -} - -/** - * struct reader - Allows the kernel state changes to be read by user space. - * @node: The node in the @c readers locked list - * @rcu_head: storage for the RCU callback to free this reader (see kfree_rcu) - * @changes: The circular buffer of user changes - * @wait_queue: A wait queue for poll - * @context: a pointer to the parent context that created this reader. Can be - * used to remove the reader from the list of readers. Reference - * counted. - * - * The reader is a circular buffer in kernel space. State changes are pushed - * into the buffer. The flow from user space is: - * - * * Request file descriptor with KBASE_IOCTL_KINSTR_JM_FD. This will - * allocate the kernel side circular buffer with a size specified in the - * ioctl argument. - * * The user will then poll the file descriptor for data - * * Upon receiving POLLIN, perform a read() on the file descriptor to get - * the data out. - * * The buffer memory will be freed when the file descriptor is closed - */ -struct reader { - struct hlist_bl_node node; - struct rcu_head rcu_head; - struct reader_changes changes; - wait_queue_head_t wait_queue; - struct kbase_kinstr_jm *context; -}; - -static struct kbase_kinstr_jm *kbase_kinstr_jm_ref_get(struct kbase_kinstr_jm *const ctx); -static void kbase_kinstr_jm_ref_put(struct kbase_kinstr_jm *const ctx); -static int kbase_kinstr_jm_readers_add(struct kbase_kinstr_jm *const ctx, - struct reader *const reader); -static void kbase_kinstr_jm_readers_del(struct kbase_kinstr_jm *const ctx, - struct reader *const reader); - -/** - * reader_term() - Terminate a instrumentation job manager reader context. - * @reader: Pointer to context to be terminated. - */ -static void reader_term(struct reader *const reader) -{ - if (!reader) - return; - - kbase_kinstr_jm_readers_del(reader->context, reader); - reader_changes_term(&reader->changes); - kbase_kinstr_jm_ref_put(reader->context); - - kfree_rcu(reader, rcu_head); -} - -/** - * reader_init() - Initialise a instrumentation job manager reader context. - * @out_reader: Non-NULL pointer to where the pointer to the created context - * will be stored on success. - * @ctx: the pointer to the parent context. Reference count will be - * increased if initialization is successful - * @num_changes: The number of changes to allocate a buffer for - * - * Return: 0 on success, else error code. - */ -static int reader_init(struct reader **const out_reader, struct kbase_kinstr_jm *const ctx, - size_t const num_changes) -{ - struct reader *reader = NULL; - const size_t change_size = sizeof(struct kbase_kinstr_jm_atom_state_change); - int status; - - if (!out_reader || !ctx || !num_changes) - return -EINVAL; - - reader = kzalloc(sizeof(*reader), GFP_KERNEL); - if (!reader) - return -ENOMEM; - - INIT_HLIST_BL_NODE(&reader->node); - init_waitqueue_head(&reader->wait_queue); - - reader->context = kbase_kinstr_jm_ref_get(ctx); - - status = reader_changes_init(&reader->changes, num_changes * change_size); - if (status < 0) - goto fail; - - status = kbase_kinstr_jm_readers_add(ctx, reader); - if (status < 0) - goto fail; - - *out_reader = reader; - - return 0; - -fail: - kbase_kinstr_jm_ref_put(reader->context); - kfree(reader); - return status; -} - -/** - * reader_release() - Invoked when the reader file descriptor is released - * @node: The inode that the file descriptor that the file corresponds to. In - * our case our reader file descriptor is backed by an anonymous node so - * not much is in this. - * @file: the file data. Our reader context is held in the private data - * Return: zero on success - */ -static int reader_release(struct inode *const node, struct file *const file) -{ - struct reader *const reader = file->private_data; - - CSTD_UNUSED(node); - - reader_term(reader); - file->private_data = NULL; - - return 0; -} - -/** - * reader_changes_copy_to_user() - Copy any changes from a changes structure to - * the user-provided buffer. - * @changes: The changes structure from which to copy. - * @buffer: The user buffer to copy the data to. - * @buffer_size: The number of bytes in the buffer. - * Return: The number of bytes copied or negative errno on failure. - */ -static ssize_t reader_changes_copy_to_user(struct reader_changes *const changes, - char __user *buffer, size_t buffer_size) -{ - ssize_t ret = 0; - struct kbase_kinstr_jm_atom_state_change const *src_buf = READ_ONCE(changes->data); - size_t const entry_size = sizeof(*src_buf); - size_t changes_tail, changes_count, read_size; - size_t copy_size; - - /* Needed for the quick buffer capacity calculation below. - * Note that we can't use is_power_of_2() since old compilers don't - * understand it's a constant expression. - */ -#define is_power_of_two(x) ((x) && !((x) & ((x)-1))) - static_assert(is_power_of_two(sizeof(struct kbase_kinstr_jm_atom_state_change))); -#undef is_power_of_two - - lockdep_assert_held_once(&changes->consumer); - - /* Read continuously until either: - * - we've filled the output buffer, or - * - there are no changes when we check. - * - * If more changes arrive while we're copying to the user, we can copy - * those as well, space permitting. - */ - do { - changes_tail = changes->tail; - changes_count = reader_changes_count_locked(changes); - - if (check_mul_overflow(changes_count, entry_size, ©_size)) { - ret = -EINVAL; - goto exit; - } - - read_size = min(copy_size, buffer_size & ~(entry_size - 1)); - - if (!read_size) - break; - - if (copy_to_user(buffer, &(src_buf[changes_tail]), read_size)) { - ret = -EFAULT; - goto exit; - } - - buffer += read_size; - buffer_size -= read_size; - ret += (ssize_t)read_size; - changes_tail = (changes_tail + read_size / entry_size) & (changes->size - 1); - smp_store_release(&changes->tail, changes_tail); - } while (read_size); -exit: - return ret; -} - -/** - * reader_read() - Handles a read call on the reader file descriptor - * - * @filp: The file that the read was performed on - * @buffer: The destination buffer - * @buffer_size: The maximum number of bytes to read - * @offset: The offset into the 'file' to read from. - * - * Note the destination buffer needs to be fully mapped in userspace or the read - * will fault. - * - * Return: - * * The number of bytes read or: - * * -EBADF - the file descriptor did not have an attached reader - * * -EFAULT - memory access fault - * * -EAGAIN - if the file is set to nonblocking reads with O_NONBLOCK and there - * is no data available - * - * Note: The number of bytes read will always be a multiple of the size of an - * entry. - */ -static ssize_t reader_read(struct file *const filp, char __user *const buffer, - size_t const buffer_size, loff_t *const offset) -{ - struct reader *const reader = filp->private_data; - struct reader_changes *changes; - ssize_t ret; - - CSTD_UNUSED(offset); - - if (!reader) - return -EBADF; - - if (buffer_size < sizeof(struct kbase_kinstr_jm_atom_state_change)) - return -ENOBUFS; - -#if KERNEL_VERSION(5, 0, 0) <= LINUX_VERSION_CODE - if (!access_ok(buffer, buffer_size)) - return -EIO; -#else - if (!access_ok(VERIFY_WRITE, buffer, buffer_size)) - return -EIO; -#endif - - changes = &reader->changes; - - mutex_lock(&changes->consumer); - if (!reader_changes_count_locked(changes)) { - if (filp->f_flags & O_NONBLOCK) { - ret = -EAGAIN; - goto exit; - } - - if (wait_event_interruptible(reader->wait_queue, - !!reader_changes_count_locked(changes))) { - ret = -EINTR; - goto exit; - } - } - - ret = reader_changes_copy_to_user(changes, buffer, buffer_size); - -exit: - mutex_unlock(&changes->consumer); - return ret; -} - -/** - * reader_poll() - Handles a poll call on the reader file descriptor - * @file: The file that the poll was performed on - * @wait: The poll table - * - * The results of the poll will be unreliable if there is no mapped memory as - * there is no circular buffer to push atom state changes into. - * - * Return: - * * 0 - no data ready - * * EPOLLIN | EPOLLRDNORM - state changes have been buffered - * * EPOLLHUP | EPOLLERR - IO control arguments were invalid or the file - * descriptor did not have an attached reader. - */ -static __poll_t reader_poll(struct file *const file, struct poll_table_struct *const wait) -{ - struct reader *reader; - struct reader_changes *changes; - __poll_t mask = 0; - - if (unlikely(!file || !wait)) - return EPOLLHUP | EPOLLERR; - - reader = file->private_data; - if (unlikely(!reader)) - return EPOLLHUP | EPOLLERR; - - changes = &reader->changes; - if (reader_changes_count(changes) >= changes->threshold) - return EPOLLIN | EPOLLRDNORM; - - poll_wait(file, &reader->wait_queue, wait); - - if (reader_changes_count(changes) > 0) - mask |= EPOLLIN | EPOLLRDNORM; - - return mask; -} - -/* The file operations virtual function table */ -static const struct file_operations file_operations = { .owner = THIS_MODULE, - .llseek = no_llseek, - .read = reader_read, - .poll = reader_poll, - .release = reader_release }; - -/* The maximum amount of readers that can be created on a context. */ -static const size_t kbase_kinstr_jm_readers_max = 16; - -/** - * kbase_kinstr_jm_release() - Invoked when the reference count is dropped - * @ref: the context reference count - */ -static void kbase_kinstr_jm_release(struct kref *const ref) -{ - struct kbase_kinstr_jm *const ctx = container_of(ref, struct kbase_kinstr_jm, refcount); - - kfree(ctx); -} - -/** - * kbase_kinstr_jm_ref_get() - Reference counts the instrumentation context - * @ctx: the context to reference count - * Return: the reference counted context - */ -static struct kbase_kinstr_jm *kbase_kinstr_jm_ref_get(struct kbase_kinstr_jm *const ctx) -{ - if (likely(ctx)) - kref_get(&ctx->refcount); - return ctx; -} - -/** - * kbase_kinstr_jm_ref_put() - Dereferences the instrumentation context - * @ctx: the context to lower the reference count on - */ -static void kbase_kinstr_jm_ref_put(struct kbase_kinstr_jm *const ctx) -{ - if (likely(ctx)) - kref_put(&ctx->refcount, kbase_kinstr_jm_release); -} - -/** - * kbase_kinstr_jm_readers_add() - Adds a reader to the list of readers - * @ctx: the instrumentation context - * @reader: the reader to add - * - * Return: - * 0 - success - * -ENOMEM - too many readers already added. - */ -static int kbase_kinstr_jm_readers_add(struct kbase_kinstr_jm *const ctx, - struct reader *const reader) -{ - struct hlist_bl_head *const readers = &ctx->readers; - struct hlist_bl_node *node; - struct reader *temp; - size_t count = 0; - - hlist_bl_lock(readers); - - hlist_bl_for_each_entry_rcu(temp, node, readers, node) - ++count; - - if (kbase_kinstr_jm_readers_max < count) { - hlist_bl_unlock(readers); - return -ENOMEM; - } - - hlist_bl_add_head_rcu(&reader->node, readers); - - hlist_bl_unlock(readers); - - static_branch_inc(&basep_kinstr_jm_reader_static_key); - - return 0; -} - -/** - * kbase_kinstr_jm_readers_del() - Deletes a reader from the list of readers - * @ctx: the instrumentation context - * @reader: the reader to delete - */ -static void kbase_kinstr_jm_readers_del(struct kbase_kinstr_jm *const ctx, - struct reader *const reader) -{ - struct hlist_bl_head *const readers = &ctx->readers; - - hlist_bl_lock(readers); - hlist_bl_del_rcu(&reader->node); - hlist_bl_unlock(readers); - - static_branch_dec(&basep_kinstr_jm_reader_static_key); -} - -int kbase_kinstr_jm_get_fd(struct kbase_kinstr_jm *const ctx, union kbase_kinstr_jm_fd *jm_fd_arg) -{ - struct kbase_kinstr_jm_fd_in const *in; - struct reader *reader; - size_t const change_size = sizeof(struct kbase_kinstr_jm_atom_state_change); - int status; - int fd; - - if (!ctx || !jm_fd_arg) - return -EINVAL; - - in = &jm_fd_arg->in; - - if (!is_power_of_2(in->count)) - return -EINVAL; - - status = reader_init(&reader, ctx, in->count); - if (status < 0) - return status; - - jm_fd_arg->out.version = KBASE_KINSTR_JM_VERSION; - jm_fd_arg->out.size = change_size; - memset(&jm_fd_arg->out.padding, 0, sizeof(jm_fd_arg->out.padding)); - - fd = anon_inode_getfd("[mali_kinstr_jm]", &file_operations, reader, O_CLOEXEC); - if (fd < 0) - reader_term(reader); - - return fd; -} - -int kbase_kinstr_jm_init(struct kbase_kinstr_jm **const out_ctx) -{ - struct kbase_kinstr_jm *ctx = NULL; - - if (!out_ctx) - return -EINVAL; - - ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); - if (!ctx) - return -ENOMEM; - - INIT_HLIST_BL_HEAD(&ctx->readers); - kref_init(&ctx->refcount); - - *out_ctx = ctx; - - return 0; -} - -void kbase_kinstr_jm_term(struct kbase_kinstr_jm *const ctx) -{ - kbase_kinstr_jm_ref_put(ctx); -} - -void kbasep_kinstr_jm_atom_state(struct kbase_jd_atom *const katom, - const enum kbase_kinstr_jm_reader_atom_state state) -{ - struct kbase_context *const kctx = katom->kctx; - struct kbase_kinstr_jm *const ctx = kctx->kinstr_jm; - const u8 id = kbase_jd_atom_id(kctx, katom); - struct kbase_kinstr_jm_atom_state_change change = { .timestamp = ktime_get_raw_ns(), - .atom = id, - .state = state }; - struct reader *reader; - struct hlist_bl_node *node; - - WARN(KBASE_KINSTR_JM_READER_ATOM_STATE_COUNT < state || 0 > state, - PR_ "unsupported katom (%u) state (%i)", id, state); - - switch (state) { - case KBASE_KINSTR_JM_READER_ATOM_STATE_START: - change.data.start.slot = katom->slot_nr; - break; - default: - break; - } - - rcu_read_lock(); - hlist_bl_for_each_entry_rcu(reader, node, &ctx->readers, node) - reader_changes_push(&reader->changes, &change, &reader->wait_queue); - rcu_read_unlock(); -} - -KBASE_EXPORT_TEST_API(kbasep_kinstr_jm_atom_state); - -void kbasep_kinstr_jm_atom_hw_submit(struct kbase_jd_atom *const katom) -{ - struct kbase_context *const kctx = katom->kctx; - struct kbase_device *const kbdev = kctx->kbdev; - const unsigned int slot = katom->slot_nr; - struct kbase_jd_atom *const submitted = kbase_gpu_inspect(kbdev, slot, 0); - - BUILD_BUG_ON(SLOT_RB_SIZE != 2); - - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (WARN_ON(slot >= GPU_MAX_JOB_SLOTS)) - return; - if (WARN_ON(!submitted)) - return; - - if (submitted == katom) - kbase_kinstr_jm_atom_state_start(katom); -} - -void kbasep_kinstr_jm_atom_hw_release(struct kbase_jd_atom *const katom) -{ - struct kbase_context *const kctx = katom->kctx; - struct kbase_device *const kbdev = kctx->kbdev; - const unsigned int slot = katom->slot_nr; - struct kbase_jd_atom *const submitted = kbase_gpu_inspect(kbdev, slot, 0); - struct kbase_jd_atom *const queued = kbase_gpu_inspect(kbdev, slot, 1); - - BUILD_BUG_ON(SLOT_RB_SIZE != 2); - - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (WARN_ON(slot >= GPU_MAX_JOB_SLOTS)) - return; - if (WARN_ON(!submitted)) - return; - if (WARN_ON((submitted != katom) && (queued != katom))) - return; - - if (queued == katom) - return; - - if (katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) - kbase_kinstr_jm_atom_state_stop(katom); - if (queued && queued->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) - kbase_kinstr_jm_atom_state_start(queued); -} diff --git a/drivers/gpu/arm/valhall/mali_kbase_kinstr_jm.h b/drivers/gpu/arm/valhall/mali_kbase_kinstr_jm.h deleted file mode 100644 index ae5b63f0372f..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_kinstr_jm.h +++ /dev/null @@ -1,257 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * mali_kbase_kinstr_jm.h - * Kernel driver public interface to job manager atom tracing. This API provides - * a method to get the atom state changes into user space. - * - * The flow of operation is: - * - * | kernel | user | - * | ----------------------------------- | ----------------------------------- | - * | Initialize API with | | - * | kbase_kinstr_jm_init() | | - * | | | - * | Kernel code injects states with | | - * | kbase_kinstr_jm_atom_state_*() APIs | | - * | | Call ioctl() to get file descriptor | - * | | via KBASE_IOCTL_KINSTR_JM_FD | - * | Allocates a reader attached to FD | | - * | Allocates circular buffer and | | - * | patches, via ASM goto, the | | - * | kbase_kinstr_jm_atom_state_*() | | - * | | loop: | - * | | Call poll() on FD for POLLIN | - * | When threshold of changes is hit, | | - * | the poll is interrupted with | | - * | POLLIN. If circular buffer is | | - * | full then store the missed count | | - * | and interrupt poll | Call read() to get data from | - * | | circular buffer via the fd | - * | Kernel advances tail of circular | | - * | buffer | | - * | | Close file descriptor | - * | Deallocates circular buffer | | - * | | | - * | Terminate API with | | - * | kbase_kinstr_jm_term() | | - * - * All tracepoints are guarded on a static key. The static key is activated when - * a user space reader gets created. This means that there is negligible cost - * inserting the tracepoints into code when there are no readers. - */ - -#ifndef _KBASE_KINSTR_JM_H_ -#define _KBASE_KINSTR_JM_H_ - -#include - -#ifdef __KERNEL__ -#include -#include -#else -/* empty wrapper macros for userspace */ -#define static_branch_unlikely(key) (1) -#endif /* __KERNEL__ */ - -/* Forward declarations */ -struct kbase_context; -struct kbase_kinstr_jm; -struct kbase_jd_atom; -union kbase_kinstr_jm_fd; - -/** - * kbase_kinstr_jm_init() - Initialise an instrumentation job manager context. - * @ctx: Non-NULL pointer to where the pointer to the created context will - * be stored on success. - * - * Return: 0 on success, else error code. - */ -int kbase_kinstr_jm_init(struct kbase_kinstr_jm **ctx); - -/** - * kbase_kinstr_jm_term() - Terminate an instrumentation job manager context. - * @ctx: Pointer to context to be terminated. - */ -void kbase_kinstr_jm_term(struct kbase_kinstr_jm *ctx); - -/** - * kbase_kinstr_jm_get_fd() - Retrieves a file descriptor that can be used to - * read the atom state changes from userspace - * - * @ctx: Pointer to the initialized context - * @jm_fd_arg: Pointer to the union containing the in/out params - * Return: -1 on failure, valid file descriptor on success - */ -int kbase_kinstr_jm_get_fd(struct kbase_kinstr_jm *const ctx, union kbase_kinstr_jm_fd *jm_fd_arg); - -/** - * kbasep_kinstr_jm_atom_state() - Signifies that an atom has changed state - * @atom: The atom that has changed state - * @state: The new state of the atom - * - * This performs the actual storage of the state ready for user space to - * read the data. It is only called when the static key is enabled from - * kbase_kinstr_jm_atom_state(). There is almost never a need to invoke this - * function directly. - */ -void kbasep_kinstr_jm_atom_state(struct kbase_jd_atom *const atom, - const enum kbase_kinstr_jm_reader_atom_state state); - -/* Allows ASM goto patching to reduce tracing overhead. This is - * incremented/decremented when readers are created and terminated. This really - * shouldn't be changed externally, but if you do, make sure you use - * a static_key_inc()/static_key_dec() pair. - */ -extern struct static_key_false basep_kinstr_jm_reader_static_key; - -/** - * kbase_kinstr_jm_atom_state() - Signifies that an atom has changed state - * @atom: The atom that has changed state - * @state: The new state of the atom - * - * This uses a static key to reduce overhead when tracing is disabled - */ -static inline void kbase_kinstr_jm_atom_state(struct kbase_jd_atom *const atom, - const enum kbase_kinstr_jm_reader_atom_state state) -{ - if (static_branch_unlikely(&basep_kinstr_jm_reader_static_key)) - kbasep_kinstr_jm_atom_state(atom, state); -} - -/** - * kbase_kinstr_jm_atom_state_queue() - Signifies that an atom has entered a - * hardware or software queue. - * @atom: The atom that has changed state - */ -static inline void kbase_kinstr_jm_atom_state_queue(struct kbase_jd_atom *const atom) -{ - kbase_kinstr_jm_atom_state(atom, KBASE_KINSTR_JM_READER_ATOM_STATE_QUEUE); -} - -/** - * kbase_kinstr_jm_atom_state_start() - Signifies that work has started on an - * atom - * @atom: The atom that has changed state - */ -static inline void kbase_kinstr_jm_atom_state_start(struct kbase_jd_atom *const atom) -{ - kbase_kinstr_jm_atom_state(atom, KBASE_KINSTR_JM_READER_ATOM_STATE_START); -} - -/** - * kbase_kinstr_jm_atom_state_stop() - Signifies that work has stopped on an - * atom - * @atom: The atom that has changed state - */ -static inline void kbase_kinstr_jm_atom_state_stop(struct kbase_jd_atom *const atom) -{ - kbase_kinstr_jm_atom_state(atom, KBASE_KINSTR_JM_READER_ATOM_STATE_STOP); -} - -/** - * kbase_kinstr_jm_atom_state_complete() - Signifies that all work has completed - * on an atom - * @atom: The atom that has changed state - */ -static inline void kbase_kinstr_jm_atom_state_complete(struct kbase_jd_atom *const atom) -{ - kbase_kinstr_jm_atom_state(atom, KBASE_KINSTR_JM_READER_ATOM_STATE_COMPLETE); -} - -/** - * kbase_kinstr_jm_atom_queue() - A software *or* hardware atom is queued for - * execution - * @atom: The atom that has changed state - */ -static inline void kbase_kinstr_jm_atom_queue(struct kbase_jd_atom *const atom) -{ - kbase_kinstr_jm_atom_state_queue(atom); -} - -/** - * kbase_kinstr_jm_atom_complete() - A software *or* hardware atom is fully - * completed - * @atom: The atom that has changed state - */ -static inline void kbase_kinstr_jm_atom_complete(struct kbase_jd_atom *const atom) -{ - kbase_kinstr_jm_atom_state_complete(atom); -} - -/** - * kbase_kinstr_jm_atom_sw_start() - A software atom has started work - * @atom: The atom that has changed state - */ -static inline void kbase_kinstr_jm_atom_sw_start(struct kbase_jd_atom *const atom) -{ - kbase_kinstr_jm_atom_state_start(atom); -} - -/** - * kbase_kinstr_jm_atom_sw_stop() - A software atom has stopped work - * @atom: The atom that has changed state - */ -static inline void kbase_kinstr_jm_atom_sw_stop(struct kbase_jd_atom *const atom) -{ - kbase_kinstr_jm_atom_state_stop(atom); -} - -/** - * kbasep_kinstr_jm_atom_hw_submit() - A hardware atom has been submitted - * @atom: The atom that has been submitted - * - * This private implementation should not be called directly, it is protected - * by a static key in kbase_kinstr_jm_atom_hw_submit(). Use that instead. - */ -void kbasep_kinstr_jm_atom_hw_submit(struct kbase_jd_atom *const atom); - -/** - * kbase_kinstr_jm_atom_hw_submit() - A hardware atom has been submitted - * @atom: The atom that has been submitted - */ -static inline void kbase_kinstr_jm_atom_hw_submit(struct kbase_jd_atom *const atom) -{ - if (static_branch_unlikely(&basep_kinstr_jm_reader_static_key)) - kbasep_kinstr_jm_atom_hw_submit(atom); -} - -/** - * kbasep_kinstr_jm_atom_hw_release() - A hardware atom has been released - * @atom: The atom that has been released - * - * This private implementation should not be called directly, it is protected - * by a static key in kbase_kinstr_jm_atom_hw_release(). Use that instead. - */ -void kbasep_kinstr_jm_atom_hw_release(struct kbase_jd_atom *const atom); - -/** - * kbase_kinstr_jm_atom_hw_release() - A hardware atom has been released - * @atom: The atom that has been released - */ -static inline void kbase_kinstr_jm_atom_hw_release(struct kbase_jd_atom *const atom) -{ - if (static_branch_unlikely(&basep_kinstr_jm_reader_static_key)) - kbasep_kinstr_jm_atom_hw_release(atom); -} - -#endif /* _KBASE_KINSTR_JM_H_ */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_mem.c b/drivers/gpu/arm/valhall/mali_kbase_mem.c index cb2e230fbc91..4f5e1980d772 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_mem.c +++ b/drivers/gpu/arm/valhall/mali_kbase_mem.c @@ -246,9 +246,6 @@ int kbase_mem_init(struct kbase_device *kbdev) } } - kbase_mem_pool_group_config_set_max_size(&kbdev->mem_pool_defaults, - KBASE_MEM_POOL_MAX_SIZE_KCTX); - spin_lock_init(&kbdev->gpu_mem_usage_lock); kbdev->process_root = RB_ROOT; kbdev->dma_buf_root = RB_ROOT; @@ -282,20 +279,37 @@ int kbase_mem_init(struct kbase_device *kbdev) } } of_node_put(mgm_node); + if (unlikely(err)) + goto init_fail; } #endif - if (likely(!err)) { - struct kbase_mem_pool_group_config mem_pool_defaults; + err = kbase_mem_pool_init_no_reclaim(&kbdev->pgd_mem_pool, BASE_PGD_MEM_POOL_MAX_SIZE_KBDEV, + KBASE_MEM_POOL_SMALL_PAGE_TABLE_ORDER, 0, kbdev); - kbase_mem_pool_group_config_set_max_size(&mem_pool_defaults, - KBASE_MEM_POOL_MAX_SIZE_KBDEV); + if (unlikely(err)) + goto init_fail; - err = kbase_mem_pool_group_init(&kbdev->mem_pools, kbdev, &mem_pool_defaults, NULL); - if (likely(!err)) - return err; - } + err = kbase_mem_pool_init_no_reclaim(&kbdev->fw_mem_pools.small, + KBASE_FW_MEM_POOL_MAX_SIZE_KBDEV, + KBASE_MEM_POOL_SMALL_PAGE_TABLE_ORDER, 0, kbdev); + if (unlikely(err)) + goto term_kbase_pgd_mem_pool; + + err = kbase_mem_pool_init_no_reclaim(&kbdev->fw_mem_pools.large, + KBASE_FW_MEM_POOL_MAX_SIZE_KBDEV >> + (KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER - + KBASE_MEM_POOL_SMALL_PAGE_TABLE_ORDER), + KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER, 0, kbdev); + + if (likely(!err)) + return err; + + kbase_mem_pool_term(&kbdev->fw_mem_pools.small); +term_kbase_pgd_mem_pool: + kbase_mem_pool_term(&kbdev->pgd_mem_pool); +init_fail: kmem_cache_destroy(kbdev->page_metadata_slab); kbdev->page_metadata_slab = NULL; page_metadata_slab_fail: @@ -324,7 +338,9 @@ void kbase_mem_term(struct kbase_device *kbdev) if (pages != 0) dev_warn(kbdev->dev, "%s: %d pages in use!\n", __func__, pages); - kbase_mem_pool_group_term(&kbdev->mem_pools); + kbase_mem_pool_term(&kbdev->pgd_mem_pool); + kbase_mem_pool_term(&kbdev->fw_mem_pools.small); + kbase_mem_pool_term(&kbdev->fw_mem_pools.large); kbase_mem_migrate_term(kbdev); @@ -848,11 +864,10 @@ static int kbase_do_syncset(struct kbase_context *kctx, struct basep_syncset *ss kbase_gpu_vm_lock(kctx); /* find the region where the virtual address is contained */ - reg = kbase_region_tracker_find_region_enclosing_address(kctx, - sset->mem_handle.basep.handle); + reg = kbase_region_tracker_find_region_enclosing_address(kctx, sset->gpu_va); if (kbase_is_region_invalid_or_free(reg)) { dev_warn(kctx->kbdev->dev, "Can't find a valid region at VA 0x%016llX", - sset->mem_handle.basep.handle); + sset->gpu_va); err = -EINVAL; goto out_unlock; } @@ -938,8 +953,8 @@ int kbase_sync_now(struct kbase_context *kctx, struct basep_syncset *sset) KBASE_DEBUG_ASSERT(kctx != NULL); KBASE_DEBUG_ASSERT(sset != NULL); - if (sset->mem_handle.basep.handle & ~PAGE_MASK) { - dev_warn(kctx->kbdev->dev, "mem_handle: passed parameter is invalid"); + if (sset->gpu_va & ~PAGE_MASK) { + dev_warn(kctx->kbdev->dev, "gpu_va 0x%016llX parameter is invalid", sset->gpu_va); return -EINVAL; } @@ -1003,7 +1018,6 @@ int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *re goto out; } -#if MALI_USE_CSF if (((kbase_bits_to_zone(reg->flags)) == FIXED_VA_ZONE) || ((kbase_bits_to_zone(reg->flags)) == EXEC_FIXED_VA_ZONE)) { if (reg->flags & KBASE_REG_FIXED_ADDRESS) @@ -1011,7 +1025,6 @@ int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *re else atomic64_dec(&kctx->num_fixable_allocs); } -#endif /* This will also free the physical pages */ kbase_free_alloced_region(reg); @@ -1137,12 +1150,6 @@ int kbase_update_region_flags(struct kbase_context *kctx, struct kbase_va_region reg->flags |= KBASE_REG_SHARE_IN; } -#if !MALI_USE_CSF - if (flags & BASE_MEM_TILER_ALIGN_TOP) - reg->flags |= KBASE_REG_TILER_ALIGN_TOP; -#endif /* !MALI_USE_CSF */ - -#if MALI_USE_CSF if (flags & BASE_MEM_CSF_EVENT) { reg->flags |= KBASE_REG_CSF_EVENT; reg->flags |= KBASE_REG_PERMANENT_KERNEL_MAPPING; @@ -1155,7 +1162,6 @@ int kbase_update_region_flags(struct kbase_context *kctx, struct kbase_va_region reg->flags &= ~KBASE_REG_GPU_CACHED; } } -#endif /* Set up default MEMATTR usage */ if (!(reg->flags & KBASE_REG_GPU_CACHED)) { @@ -1168,12 +1174,10 @@ int kbase_update_region_flags(struct kbase_context *kctx, struct kbase_va_region "Can't allocate GPU uncached memory due to MMU in Legacy Mode\n"); return -EINVAL; } -#if MALI_USE_CSF } else if (reg->flags & KBASE_REG_CSF_EVENT) { WARN_ON(!(reg->flags & KBASE_REG_SHARE_BOTH)); reg->flags |= KBASE_REG_MEMATTR_INDEX(KBASE_MEMATTR_INDEX_SHARED); -#endif } else if (kctx->kbdev->system_coherency == COHERENCY_ACE && (reg->flags & KBASE_REG_SHARE_BOTH)) { reg->flags |= KBASE_REG_MEMATTR_INDEX(KBASE_MEMATTR_INDEX_DEFAULT_ACE); @@ -1193,10 +1197,8 @@ int kbase_update_region_flags(struct kbase_context *kctx, struct kbase_va_region if (flags & BASE_MEM_GPU_VA_SAME_4GB_PAGE) reg->flags |= KBASE_REG_GPU_VA_SAME_4GB_PAGE; -#if MALI_USE_CSF if (flags & BASE_MEM_FIXED) reg->flags |= KBASE_REG_FIXED_ADDRESS; -#endif if (flags & BASEP_MEM_ACTIVE_JIT_ALLOC) reg->flags |= BASEP_MEM_ACTIVE_JIT_ALLOC; @@ -2085,26 +2087,12 @@ bool kbase_check_alloc_flags(struct kbase_context *kctx, unsigned long flags) (flags & (BASE_MEM_PROT_GPU_WR | BASE_MEM_GROW_ON_GPF))) return false; -#if !MALI_USE_CSF - /* GPU executable memory also cannot have the top of its initial - * commit aligned to 'extension' - */ - if ((flags & BASE_MEM_PROT_GPU_EX) && (flags & BASE_MEM_TILER_ALIGN_TOP)) - return false; -#endif /* !MALI_USE_CSF */ - /* To have an allocation lie within a 4GB chunk is required only for * TLS memory, which will never be used to contain executable code. */ if ((flags & BASE_MEM_GPU_VA_SAME_4GB_PAGE) && (flags & BASE_MEM_PROT_GPU_EX)) return false; -#if !MALI_USE_CSF - /* TLS memory should also not be used for tiler heap */ - if ((flags & BASE_MEM_GPU_VA_SAME_4GB_PAGE) && (flags & BASE_MEM_TILER_ALIGN_TOP)) - return false; -#endif /* !MALI_USE_CSF */ - /* GPU should have at least read or write access otherwise there is no * reason for allocating. */ @@ -2127,13 +2115,11 @@ bool kbase_check_alloc_flags(struct kbase_context *kctx, unsigned long flags) (BASE_MEM_COHERENT_LOCAL | BASE_MEM_COHERENT_SYSTEM)) return false; -#if MALI_USE_CSF if ((flags & BASE_MEM_SAME_VA) && (flags & (BASE_MEM_FIXABLE | BASE_MEM_FIXED))) return false; if ((flags & BASE_MEM_FIXABLE) && (flags & BASE_MEM_FIXED)) return false; -#endif /* Cannot be set only allocation, only with base_mem_set */ if ((flags & BASE_MEM_DONT_NEED) && @@ -2155,30 +2141,13 @@ bool kbase_check_alloc_flags(struct kbase_context *kctx, unsigned long flags) (mali_kbase_supports_reject_alloc_mem_unused_bit_7(kctx->api_version))) return false; -#if !MALI_USE_CSF - if ((flags & BASE_MEM_UNUSED_BIT_8) && - (mali_kbase_supports_reject_alloc_mem_unused_bit_8(kctx->api_version))) - return false; - - if ((flags & BASE_MEM_UNUSED_BIT_19) && - (mali_kbase_supports_reject_alloc_mem_unused_bit_19(kctx->api_version))) - return false; - -#else if ((flags & BASE_MEM_UNUSED_BIT_20) && (mali_kbase_supports_reject_alloc_mem_unused_bit_20(kctx->api_version))) return false; -#endif /* !MALI_USE_CSF */ if ((flags & BASE_MEM_UNUSED_BIT_27) && (mali_kbase_supports_reject_alloc_mem_unused_bit_27(kctx->api_version))) return false; -#if !MALI_USE_CSF - - if ((flags & BASE_MEM_UNUSED_BIT_29) && - (mali_kbase_supports_reject_alloc_mem_unused_bit_29(kctx->api_version))) - return false; -#endif /* !MALI_USE_CSF */ return true; } @@ -2201,15 +2170,9 @@ bool kbase_check_import_flags(unsigned long flags) if (flags & BASE_MEM_GROW_ON_GPF) return false; -#if MALI_USE_CSF /* Imported memory cannot be fixed */ if ((flags & (BASE_MEM_FIXED | BASE_MEM_FIXABLE))) return false; -#else - /* Imported memory cannot be aligned to the end of its initial commit */ - if (flags & BASE_MEM_TILER_ALIGN_TOP) - return false; -#endif /* !MALI_USE_CSF */ /* GPU should have at least read or write access otherwise there is no * reason for importing. @@ -2267,59 +2230,12 @@ int kbase_check_alloc_sizes(struct kbase_context *kctx, unsigned long flags, u64 return -EINVAL; } -#if !MALI_USE_CSF - if ((flags & BASE_MEM_TILER_ALIGN_TOP) && (test_reg.extension == 0)) { - dev_warn(dev, KBASE_MSG_PRE "BASE_MEM_TILER_ALIGN_TOP but extension == 0\n"); - return -EINVAL; - } - - if (!(flags & (BASE_MEM_GROW_ON_GPF | BASE_MEM_TILER_ALIGN_TOP)) && - test_reg.extension != 0) { - dev_warn( - dev, KBASE_MSG_PRE - "neither BASE_MEM_GROW_ON_GPF nor BASE_MEM_TILER_ALIGN_TOP set but extension != 0\n"); - return -EINVAL; - } -#else if (!(flags & BASE_MEM_GROW_ON_GPF) && test_reg.extension != 0) { dev_warn(dev, KBASE_MSG_PRE "BASE_MEM_GROW_ON_GPF not set but extension != 0\n"); return -EINVAL; } -#endif /* !MALI_USE_CSF */ -#if !MALI_USE_CSF - /* BASE_MEM_TILER_ALIGN_TOP memory has a number of restrictions */ - if (flags & BASE_MEM_TILER_ALIGN_TOP) { -#define KBASE_MSG_PRE_FLAG KBASE_MSG_PRE "BASE_MEM_TILER_ALIGN_TOP and " - unsigned long small_extension; - - if (large_extension > BASE_MEM_TILER_ALIGN_TOP_EXTENSION_MAX_PAGES) { - dev_warn(dev, KBASE_MSG_PRE_FLAG "extension==%lld pages exceeds limit %lld", - (unsigned long long)large_extension, - BASE_MEM_TILER_ALIGN_TOP_EXTENSION_MAX_PAGES); - return -EINVAL; - } - /* For use with is_power_of_2, which takes unsigned long, so - * must ensure e.g. on 32-bit kernel it'll fit in that type - */ - small_extension = (unsigned long)large_extension; - - if (!is_power_of_2(small_extension)) { - dev_warn(dev, KBASE_MSG_PRE_FLAG "extension==%ld not a non-zero power of 2", - small_extension); - return -EINVAL; - } - - if (commit_pages > large_extension) { - dev_warn(dev, KBASE_MSG_PRE_FLAG "commit_pages==%ld exceeds extension==%ld", - (unsigned long)commit_pages, (unsigned long)large_extension); - return -EINVAL; - } -#undef KBASE_MSG_PRE_FLAG - } -#else CSTD_UNUSED(commit_pages); -#endif /* !MALI_USE_CSF */ if ((flags & BASE_MEM_GPU_VA_SAME_4GB_PAGE) && (va_pages > (BASE_MEM_PFN_MASK_4GB + 1))) { dev_warn( @@ -2343,9 +2259,7 @@ KBASE_EXPORT_TEST_API(kbase_gpu_vm_lock); void kbase_gpu_vm_lock_with_pmode_sync(struct kbase_context *kctx) { -#if MALI_USE_CSF down_read(&kctx->kbdev->csf.mmu_sync_sem); -#endif kbase_gpu_vm_lock(kctx); } @@ -2359,9 +2273,7 @@ KBASE_EXPORT_TEST_API(kbase_gpu_vm_unlock); void kbase_gpu_vm_unlock_with_pmode_sync(struct kbase_context *kctx) { kbase_gpu_vm_unlock(kctx); -#if MALI_USE_CSF up_read(&kctx->kbdev->csf.mmu_sync_sem); -#endif } #if IS_ENABLED(CONFIG_DEBUG_FS) @@ -2524,17 +2436,11 @@ static int kbase_jit_debugfs_used_get(struct kbase_jit_debugfs_data *data) struct kbase_context *kctx = data->kctx; struct kbase_va_region *reg; -#if !MALI_USE_CSF - mutex_lock(&kctx->jctx.lock); -#endif /* !MALI_USE_CSF */ mutex_lock(&kctx->jit_evict_lock); list_for_each_entry(reg, &kctx->jit_active_head, jit_node) { data->active_value += reg->used_pages; } mutex_unlock(&kctx->jit_evict_lock); -#if !MALI_USE_CSF - mutex_unlock(&kctx->jctx.lock); -#endif /* !MALI_USE_CSF */ return 0; } @@ -2550,9 +2456,6 @@ static int kbase_jit_debugfs_trim_get(struct kbase_jit_debugfs_data *data) struct kbase_context *kctx = data->kctx; struct kbase_va_region *reg; -#if !MALI_USE_CSF - mutex_lock(&kctx->jctx.lock); -#endif /* !MALI_USE_CSF */ kbase_gpu_vm_lock(kctx); mutex_lock(&kctx->jit_evict_lock); list_for_each_entry(reg, &kctx->jit_active_head, jit_node) { @@ -2570,9 +2473,6 @@ static int kbase_jit_debugfs_trim_get(struct kbase_jit_debugfs_data *data) } mutex_unlock(&kctx->jit_evict_lock); kbase_gpu_vm_unlock(kctx); -#if !MALI_USE_CSF - mutex_unlock(&kctx->jctx.lock); -#endif /* !MALI_USE_CSF */ return 0; } @@ -2677,14 +2577,9 @@ int kbase_jit_init(struct kbase_context *kctx) INIT_LIST_HEAD(&kctx->jit_destroy_head); INIT_WORK(&kctx->jit_work, kbase_jit_destroy_worker); -#if MALI_USE_CSF mutex_init(&kctx->csf.kcpu_queues.jit_lock); INIT_LIST_HEAD(&kctx->csf.kcpu_queues.jit_cmds_head); INIT_LIST_HEAD(&kctx->csf.kcpu_queues.jit_blocked_queues); -#else /* !MALI_USE_CSF */ - INIT_LIST_HEAD(&kctx->jctx.jit_atoms_head); - INIT_LIST_HEAD(&kctx->jctx.jit_pending_alloc); -#endif /* MALI_USE_CSF */ mutex_unlock(&kctx->jit_evict_lock); return 0; @@ -2702,16 +2597,6 @@ static bool meet_size_and_tiler_align_top_requirements(const struct kbase_va_reg if (walker->nr_pages != info->va_pages) meet_reqs = false; -#if !MALI_USE_CSF - if (meet_reqs && (info->flags & BASE_JIT_ALLOC_MEM_TILER_ALIGN_TOP)) { - size_t align = info->extension; - size_t align_mask = align - 1; - - if ((walker->start_pfn + info->commit_pages) & align_mask) - meet_reqs = false; - } -#endif /* !MALI_USE_CSF */ - return meet_reqs; } @@ -2729,9 +2614,6 @@ static int kbase_mem_jit_trim_pages_from_region(struct kbase_context *kctx, size_t to_free = 0u; size_t max_allowed_pages = old_pages; -#if !MALI_USE_CSF - lockdep_assert_held(&kctx->jctx.lock); -#endif /* !MALI_USE_CSF */ lockdep_assert_held(&kctx->reg_lock); /* Is this a JIT allocation that has been reported on? */ @@ -2846,9 +2728,6 @@ static size_t kbase_mem_jit_trim_pages(struct kbase_context *kctx, size_t pages_ struct kbase_va_region *reg, *tmp; size_t total_freed = 0; -#if !MALI_USE_CSF - lockdep_assert_held(&kctx->jctx.lock); -#endif /* !MALI_USE_CSF */ lockdep_assert_held(&kctx->reg_lock); lockdep_assert_held(&kctx->jit_evict_lock); @@ -3048,9 +2927,6 @@ void kbase_jit_trim_necessary_pages(struct kbase_context *kctx, size_t needed_pa size_t jit_backing = 0; size_t pages_to_trim = 0; -#if !MALI_USE_CSF - lockdep_assert_held(&kctx->jctx.lock); -#endif /* !MALI_USE_CSF */ lockdep_assert_held(&kctx->reg_lock); lockdep_assert_held(&kctx->jit_evict_lock); @@ -3096,11 +2972,7 @@ void kbase_jit_trim_necessary_pages(struct kbase_context *kctx, size_t needed_pa static bool jit_allow_allocate(struct kbase_context *kctx, const struct base_jit_alloc_info *info, bool ignore_pressure_limit) { -#if !MALI_USE_CSF - lockdep_assert_held(&kctx->jctx.lock); -#else /* MALI_USE_CSF */ lockdep_assert_held(&kctx->csf.kcpu_queues.jit_lock); -#endif /* !MALI_USE_CSF */ #if MALI_JIT_PRESSURE_LIMIT_BASE if (!ignore_pressure_limit && @@ -3187,11 +3059,7 @@ struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, */ const enum kbase_caller_mmu_sync_info mmu_sync_info = CALLER_MMU_SYNC; -#if !MALI_USE_CSF - lockdep_assert_held(&kctx->jctx.lock); -#else /* MALI_USE_CSF */ lockdep_assert_held(&kctx->csf.kcpu_queues.jit_lock); -#endif /* !MALI_USE_CSF */ if (!jit_allow_allocate(kctx, info, ignore_pressure_limit)) return NULL; @@ -3330,11 +3198,6 @@ struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, BASEP_MEM_ACTIVE_JIT_ALLOC; u64 gpu_addr; -#if !MALI_USE_CSF - if (info->flags & BASE_JIT_ALLOC_MEM_TILER_ALIGN_TOP) - flags |= BASE_MEM_TILER_ALIGN_TOP; -#endif /* !MALI_USE_CSF */ - flags |= kbase_mem_group_id_set(kctx->jit_group_id); #if MALI_JIT_PRESSURE_LIMIT_BASE if (!ignore_pressure_limit) { @@ -3431,11 +3294,7 @@ void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg) { u64 old_pages; -#if !MALI_USE_CSF - lockdep_assert_held(&kctx->jctx.lock); -#else /* MALI_USE_CSF */ lockdep_assert_held(&kctx->csf.kcpu_queues.jit_lock); -#endif /* !MALI_USE_CSF */ /* JIT id not immediately available here, so use 0u */ trace_mali_jit_free(reg, 0u); @@ -3669,10 +3528,6 @@ void kbase_jit_report_update_pressure(struct kbase_context *kctx, struct kbase_v { u64 diff; -#if !MALI_USE_CSF - lockdep_assert_held(&kctx->jctx.lock); -#endif /* !MALI_USE_CSF */ - trace_mali_jit_report_pressure( reg, new_used_pages, kctx->jit_current_phys_pressure + new_used_pages - reg->used_pages, flags); @@ -4533,9 +4388,6 @@ void kbase_user_buf_from_dma_mapped_to_pinned(struct kbase_context *kctx, dev_dbg(kctx->kbdev->dev, "%s %pK in kctx %pK\n", __func__, (void *)reg, (void *)kctx); if (WARN_ON(reg->gpu_alloc->imported.user_buf.state != KBASE_USER_BUF_STATE_DMA_MAPPED)) return; -#if !MALI_USE_CSF - kbase_mem_shrink_cpu_mapping(kctx, reg, 0, reg->gpu_alloc->nents); -#endif kbase_user_buf_dma_unmap_pages(kctx, reg); reg->gpu_alloc->imported.user_buf.state = KBASE_USER_BUF_STATE_PINNED; diff --git a/drivers/gpu/arm/valhall/mali_kbase_mem.h b/drivers/gpu/arm/valhall/mali_kbase_mem.h index 455c3efc3bd2..3a92ce9ec8b6 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_mem.h +++ b/drivers/gpu/arm/valhall/mali_kbase_mem.h @@ -174,7 +174,6 @@ static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, int /* Normal memory, inner non-cacheable, outer non-cacheable (ARMv8 mode only) */ #define KBASE_MEMATTR_INDEX_NON_CACHEABLE 5 -#if MALI_USE_CSF /* Set to shared memory, that is inner cacheable on ACE and inner or outer * shared, otherwise inner non-cacheable. * Outer cacheable if inner or outer shared, otherwise outer non-cacheable. @@ -187,14 +186,12 @@ static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, int /* Normal memory, shared between MCU and Host */ #define KBASE_MEMATTR_INDEX_SHARED 6 -#endif #define KBASE_REG_PROTECTED (1ul << 19) /* Imported buffer is padded? */ #define KBASE_REG_IMPORT_PAD (1ul << 21) -#if MALI_USE_CSF /* CSF event memory */ #define KBASE_REG_CSF_EVENT (1ul << 22) /* Bit 23 is reserved. @@ -202,17 +199,6 @@ static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, int * Do not remove, use the next unreserved bit for new flags */ #define KBASE_REG_RESERVED_BIT_23 (1ul << 23) -#else -/* Bit 22 is reserved. - * - * Do not remove, use the next unreserved bit for new flags - */ -#define KBASE_REG_RESERVED_BIT_22 (1ul << 22) -/* The top of the initial commit is aligned to extension pages. - * Extent must be a power of 2 - */ -#define KBASE_REG_TILER_ALIGN_TOP (1ul << 23) -#endif /* MALI_USE_CSF */ /* Bit 24 is currently unused and is available for use for a new flag */ @@ -241,15 +227,11 @@ static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, int */ #define KBASE_REG_HEAP_INFO_IS_SIZE (1ul << 27) -#if MALI_USE_CSF /* This flag only applies to allocations in the EXEC_FIXED_VA and FIXED_VA * memory zones, and it determines whether they were created with a fixed * GPU VA address requested by the user. */ #define KBASE_REG_FIXED_ADDRESS (1ul << 29) -#else -#define KBASE_REG_RESERVED_BIT_29 (1ul << 29) -#endif /* * A CPU mapping */ @@ -975,11 +957,25 @@ static inline int kbase_reg_prepare_native(struct kbase_va_region *reg, struct k */ #define KBASE_MEM_POOL_MAX_SIZE_KBDEV (SZ_64M >> PAGE_SHIFT) +/* + * Max size for kbdev pgd memory pool (in pages) + */ +#define BASE_PGD_MEM_POOL_MAX_SIZE_KBDEV (SZ_2M >> PAGE_SHIFT) + +/* + * Max size for kbdev fw memory pool (in pages) + */ +#define KBASE_FW_MEM_POOL_MAX_SIZE_KBDEV ((4 * SZ_2M) >> PAGE_SHIFT) /* * Max size for kctx memory pool (in pages) */ #define KBASE_MEM_POOL_MAX_SIZE_KCTX (SZ_64M >> PAGE_SHIFT) +/* + * Max size of kctx pgd memory pool (in pages) + */ +#define KBASE_PGD_MM_POOLMAX_SIZE_KCTX (SZ_2M >> PAGE_SHIFT) + /* * The order required for a 2MB page allocation (2^order * PAGE_SIZE = 2MB) */ @@ -1022,13 +1018,12 @@ kbase_mem_pool_config_get_max_size(const struct kbase_mem_pool_config *const con /** * kbase_mem_pool_init - Create a memory pool for a kbase device * @pool: Memory pool to initialize - * @config: Initial configuration for the memory pool + * @max_size: Maximum size for the memory pool * @order: Page order for physical page size (order=0 => small page, order != 0 => 2MB) * @group_id: A memory group ID to be passed to a platform-specific * memory group manager, if present. * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). * @kbdev: Kbase device where memory is used - * @next_pool: Pointer to the next pool or NULL. * * Allocations from @pool are in whole pages. Each @pool has a free list where * pages can be quickly allocated from. The free list is initially empty and @@ -1046,9 +1041,26 @@ kbase_mem_pool_config_get_max_size(const struct kbase_mem_pool_config *const con * * Return: 0 on success, negative -errno on error */ -int kbase_mem_pool_init(struct kbase_mem_pool *pool, const struct kbase_mem_pool_config *config, - unsigned int order, int group_id, struct kbase_device *kbdev, - struct kbase_mem_pool *next_pool); +int kbase_mem_pool_init(struct kbase_mem_pool *pool, size_t max_size, unsigned int order, + int group_id, struct kbase_device *kbdev); + +/** + * kbase_mem_pool_init_no_reclaim - Create a memory pool for a kbase device, with no reclaim. + * @pool: Memory pool to initialize + * @max_size: Maximum size for the memory pool + * @order: Page order for physical page size (order=0 => small page, order != 0 => 2MB) + * @group_id: A memory group ID to be passed to a platform-specific + * memory group manager, if present. + * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). + * @kbdev: Kbase device where memory is used + * + * Identical to kbase_mem_pool_init, except the pool does not support reclaiming of pages. + * For that reason it also omits support of the next_pool feature. + * + * Return: 0 on success, negative -errno on error + */ +int kbase_mem_pool_init_no_reclaim(struct kbase_mem_pool *pool, size_t max_size, unsigned int order, + int group_id, struct kbase_device *kbdev); /** * kbase_mem_pool_term - Destroy a memory pool @@ -2000,9 +2012,6 @@ void kbase_jit_trim_necessary_pages(struct kbase_context *kctx, size_t needed_pa static inline void kbase_jit_request_phys_increase_locked(struct kbase_context *kctx, size_t needed_pages) { -#if !MALI_USE_CSF - lockdep_assert_held(&kctx->jctx.lock); -#endif /* !MALI_USE_CSF */ lockdep_assert_held(&kctx->reg_lock); lockdep_assert_held(&kctx->jit_evict_lock); @@ -2038,9 +2047,6 @@ static inline void kbase_jit_request_phys_increase_locked(struct kbase_context * */ static inline void kbase_jit_request_phys_increase(struct kbase_context *kctx, size_t needed_pages) { -#if !MALI_USE_CSF - lockdep_assert_held(&kctx->jctx.lock); -#endif /* !MALI_USE_CSF */ lockdep_assert_held(&kctx->reg_lock); mutex_lock(&kctx->jit_evict_lock); @@ -2395,7 +2401,6 @@ static inline void kbase_mem_pool_unlock(struct kbase_mem_pool *pool) */ void kbase_mem_evictable_mark_reclaim(struct kbase_mem_phy_alloc *alloc); -#if MALI_USE_CSF /** * kbase_link_event_mem_page - Add the new event memory region to the per * context list of event pages. @@ -2451,7 +2456,6 @@ int kbase_mcu_shared_interface_region_tracker_init(struct kbase_device *kbdev); * @kbdev: Pointer to the kbase device */ void kbase_mcu_shared_interface_region_tracker_term(struct kbase_device *kbdev); -#endif /** * kbase_mem_umm_map - Map dma-buf diff --git a/drivers/gpu/arm/valhall/mali_kbase_mem_flags.h b/drivers/gpu/arm/valhall/mali_kbase_mem_flags.h index 84948f9fa445..3ffd7701bd0b 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_mem_flags.h +++ b/drivers/gpu/arm/valhall/mali_kbase_mem_flags.h @@ -22,11 +22,7 @@ #ifndef _KBASE_MEM_FLAGS_H_ #define _KBASE_MEM_FLAGS_H_ -#if MALI_USE_CSF #include -#else -#include -#endif /* Will be permanently mapped in kernel space. * Flag is only allowed on allocations originating from kbase. diff --git a/drivers/gpu/arm/valhall/mali_kbase_mem_linux.c b/drivers/gpu/arm/valhall/mali_kbase_mem_linux.c index 321df3afd279..c8e36707aae9 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_mem_linux.c +++ b/drivers/gpu/arm/valhall/mali_kbase_mem_linux.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2025 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -97,10 +97,8 @@ MODULE_PARM_DESC( "Note that this should only be enabled for testing worst case memory usage " "and should not be enabled in production"); -#if MALI_USE_CSF static int kbase_csf_cpu_mmap_user_reg_page(struct kbase_context *kctx, struct vm_area_struct *vma); static int kbase_csf_cpu_mmap_user_io_pages(struct kbase_context *kctx, struct vm_area_struct *vma); -#endif static int kbase_vmap_phy_pages(struct kbase_context *kctx, struct kbase_va_region *reg, u64 offset_bytes, size_t size, struct kbase_vmap_struct *map, @@ -132,7 +130,6 @@ static bool is_process_exiting(struct vm_area_struct *vma) */ static struct kbase_va_region *kbase_find_event_mem_region(struct kbase_context *kctx, u64 gpu_addr) { -#if MALI_USE_CSF u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; struct kbase_va_region *reg; @@ -149,10 +146,6 @@ static struct kbase_va_region *kbase_find_event_mem_region(struct kbase_context return reg; } } -#else - CSTD_UNUSED(kctx); - CSTD_UNUSED(gpu_addr); -#endif return NULL; } @@ -310,13 +303,8 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages dev_dbg(dev, "Allocating %lld va_pages, %lld commit_pages, %lld extension, 0x%llX flags\n", va_pages, commit_pages, extension, *flags); -#if MALI_USE_CSF if (!(*flags & BASE_MEM_FIXED)) *gpu_va = 0; /* return 0 on failure */ -#else - if (!(*flags & BASEP_MEM_FLAG_MAP_FIXED)) - *gpu_va = 0; /* return 0 on failure */ -#endif else dev_dbg(dev, "Keeping requested GPU VA of 0x%llx\n", (unsigned long long)*gpu_va); @@ -370,7 +358,6 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages /* find out which VA zone to use */ if (*flags & BASE_MEM_SAME_VA) zone = SAME_VA_ZONE; -#if MALI_USE_CSF /* fixed va_zone always exists */ else if (*flags & (BASE_MEM_FIXED | BASE_MEM_FIXABLE)) { if (*flags & BASE_MEM_PROT_GPU_EX) { @@ -378,9 +365,7 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages } else { zone = FIXED_VA_ZONE; } - } -#endif - else if ((*flags & BASE_MEM_PROT_GPU_EX) && kbase_has_exec_va_zone(kctx)) { + } else if ((*flags & BASE_MEM_PROT_GPU_EX) && kbase_has_exec_va_zone(kctx)) { zone = EXEC_VA_ZONE; } else { zone = CUSTOM_VA_ZONE; @@ -415,10 +400,6 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages * reg->extension. */ reg->extension = extension; -#if !MALI_USE_CSF - } else if (*flags & BASE_MEM_TILER_ALIGN_TOP) { - reg->extension = extension; -#endif /* !MALI_USE_CSF */ } else { reg->extension = 0; } @@ -480,10 +461,6 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages } if (*gpu_va) align = 1; -#if !MALI_USE_CSF - if (reg->flags & KBASE_REG_TILER_ALIGN_TOP) - align = 1; -#endif /* !MALI_USE_CSF */ } if (kbase_gpu_mmap(kctx, reg, *gpu_va, va_pages, align, mmu_sync_info) != 0) { dev_warn(dev, "Failed to map memory on GPU"); @@ -507,12 +484,10 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages kbase_gpu_vm_unlock_with_pmode_sync(kctx); -#if MALI_USE_CSF if (*flags & BASE_MEM_FIXABLE) atomic64_inc(&kctx->num_fixable_allocs); else if (*flags & BASE_MEM_FIXED) atomic64_inc(&kctx->num_fixed_allocs); -#endif return reg; @@ -617,13 +592,8 @@ int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, u64 query, u64 *co if (KBASE_REG_PROTECTED & reg->flags) *out |= BASE_MEM_PROTECTED; } -#if !MALI_USE_CSF - if (KBASE_REG_TILER_ALIGN_TOP & reg->flags) - *out |= BASE_MEM_TILER_ALIGN_TOP; -#endif /* !MALI_USE_CSF */ if (!(KBASE_REG_GPU_CACHED & reg->flags)) *out |= BASE_MEM_UNCACHED_GPU; -#if MALI_USE_CSF if (KBASE_REG_CSF_EVENT & reg->flags) *out |= BASE_MEM_CSF_EVENT; if ((kbase_bits_to_zone(reg->flags) == FIXED_VA_ZONE) || @@ -633,7 +603,6 @@ int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, u64 query, u64 *co else *out |= BASE_MEM_FIXABLE; } -#endif /* MALI_USE_CSF */ if (KBASE_REG_GPU_VA_SAME_4GB_PAGE & reg->flags) *out |= BASE_MEM_GPU_VA_SAME_4GB_PAGE; if (mali_kbase_supports_query_mem_import_sync_on_map_unmap(kctx->api_version)) { @@ -741,13 +710,11 @@ static unsigned long kbase_mem_evictable_reclaim_scan_objects(struct shrinker *s kctx = KBASE_GET_KBASE_DATA_FROM_SHRINKER(s, struct kbase_context, reclaim); -#if MALI_USE_CSF if (!down_read_trylock(&kctx->kbdev->csf.mmu_sync_sem)) { dev_warn(kctx->kbdev->dev, "Can't shrink GPU memory when P.Mode entrance is in progress"); return 0; } -#endif mutex_lock(&kctx->jit_evict_lock); list_for_each_entry_safe(alloc, tmp, &kctx->evict_list, evict_node) { @@ -786,9 +753,7 @@ static unsigned long kbase_mem_evictable_reclaim_scan_objects(struct shrinker *s } mutex_unlock(&kctx->jit_evict_lock); -#if MALI_USE_CSF up_read(&kctx->kbdev->csf.mmu_sync_sem); -#endif return freed; } @@ -1494,15 +1459,33 @@ static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx, in *flags &= ~BASE_MEM_SAME_VA; /* - * Force CPU cached flag. - * - * We can't query the dma-buf exporter to get details about the CPU - * cache attributes of CPU mappings, so we have to assume that the - * buffer may be cached, and call into the exporter for cache - * maintenance, and rely on the exporter to do the right thing when - * handling our calls. + * mgm_get_import_memory_cached_access_permitted() returns true if externally visible + * accesses by the GPU for this dma_buf are allowed to be cached. Set CPU cacheabilty + * flag to match. */ - *flags |= BASE_MEM_CACHED_CPU; + if (kctx->kbdev->mgm_dev->ops.mgm_get_import_memory_cached_access_permitted) { + struct memory_group_manager_import_data mgm_import_data; + + mgm_import_data.type = MEMORY_GROUP_MANAGER_IMPORT_TYPE_DMA_BUF; + mgm_import_data.u.dma_buf = dma_buf; + + if (kctx->kbdev->mgm_dev->ops.mgm_get_import_memory_cached_access_permitted( + kctx->kbdev->mgm_dev, &mgm_import_data)) + *flags |= BASE_MEM_CACHED_CPU; + else + *flags &= ~BASE_MEM_CACHED_CPU; + } else { + /* + * Force CPU cached flag if we do not have the API to query MGM + * + * We can't query the dma-buf exporter to get details about the CPU + * cache attributes of CPU mappings, so we have to assume that the + * buffer may be cached, and call into the exporter for cache + * maintenance, and rely on the exporter to do the right thing when + * handling our calls. + */ + *flags |= BASE_MEM_CACHED_CPU; + } if (*flags & BASE_MEM_IMPORT_SHARED) shared_zone = true; @@ -1825,8 +1808,8 @@ u64 kbase_mem_alias(struct kbase_context *kctx, base_mem_alloc_flags *flags, u64 /* validate and add src handles */ for (i = 0; i < nents; i++) { - if (ai[i].handle.basep.handle < BASE_MEM_FIRST_FREE_ADDRESS) { - if (ai[i].handle.basep.handle != BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE) + if (ai[i].gpu_va < BASE_MEM_FIRST_FREE_ADDRESS) { + if (ai[i].gpu_va != BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE) goto bad_handle; /* unsupported magic handle */ if (!ai[i].length) goto bad_handle; /* must be > 0 */ @@ -1840,7 +1823,7 @@ u64 kbase_mem_alias(struct kbase_context *kctx, base_mem_alloc_flags *flags, u64 struct kbase_mem_phy_alloc *alloc; aliasing_reg = kbase_region_tracker_find_region_base_address( - kctx, (ai[i].handle.basep.handle >> PAGE_SHIFT) << PAGE_SHIFT); + kctx, (ai[i].gpu_va >> PAGE_SHIFT) << PAGE_SHIFT); /* validate found region */ if (kbase_is_region_invalid_or_free(aliasing_reg)) @@ -2734,10 +2717,8 @@ static int kbasep_reg_mmap(struct kbase_context *kctx, struct vm_area_struct *vm kctx->pending_regions[cookie] = NULL; bitmap_set(kctx->cookies, cookie, 1); -#if MALI_USE_CSF if (reg->flags & KBASE_REG_CSF_EVENT) kbase_link_event_mem_page(kctx, reg); -#endif /* * Overwrite the offset with the region start_pfn, so we effectively @@ -2813,7 +2794,6 @@ int kbase_context_mmap(struct kbase_context *const kctx, struct vm_area_struct * err = -EINVAL; goto out_unlock; #endif /* defined(CONFIG_MALI_VECTOR_DUMP) */ -#if MALI_USE_CSF case PFN_DOWN(BASEP_MEM_CSF_USER_REG_PAGE_HANDLE): kbase_gpu_vm_unlock_with_pmode_sync(kctx); err = kbase_csf_cpu_mmap_user_reg_page(kctx, vma); @@ -2825,7 +2805,6 @@ int kbase_context_mmap(struct kbase_context *const kctx, struct vm_area_struct * mutex_unlock(&kctx->csf.lock); goto out; } -#endif case PFN_DOWN(BASE_MEM_COOKIE_BASE)... PFN_DOWN(BASE_MEM_FIRST_FREE_ADDRESS) - 1: { err = kbasep_reg_mmap(kctx, vma, ®, &nr_pages, &aligned_offset); if (err != 0) @@ -3319,7 +3298,6 @@ static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_ return 0; } -#if MALI_USE_CSF static unsigned long get_queue_doorbell_pfn(struct kbase_device *kbdev, struct kbase_queue *queue) { lockdep_assert_held(&kbdev->csf.reg_lock); @@ -3462,13 +3440,19 @@ static vm_fault_t kbase_csf_user_io_pages_vm_fault(struct vm_fault *vmf) output_cpu_addr = input_cpu_addr + PAGE_SIZE; if (mali_kbase_supports_csg_cs_user_page_allocation(queue->kctx->api_version)) { - if (!queue->group) { - ret = VM_FAULT_SIGBUS; - goto exit; + if (likely(queue->group)) { + input_page_pfn = PFN_DOWN(as_phys_addr_t(queue->group->phys[0])); + output_page_pfn = PFN_DOWN(as_phys_addr_t(queue->group->phys[1])); + } else { + /* This could happen if userspace tries to access this memory + * after the group has already been terminated due to a fault. + * Re-map to the dummy page to render the access harmless. + */ + input_page_pfn = + PFN_DOWN(as_phys_addr_t(kbdev->csf.user_reg.dummy_page)); + output_page_pfn = + PFN_DOWN(as_phys_addr_t(kbdev->csf.user_reg.dummy_page)); } - - input_page_pfn = PFN_DOWN(as_phys_addr_t(queue->group->phys[0])); - output_page_pfn = PFN_DOWN(as_phys_addr_t(queue->group->phys[1])); } else { input_page_pfn = PFN_DOWN(as_phys_addr_t(queue->phys[0])); output_page_pfn = PFN_DOWN(as_phys_addr_t(queue->phys[1])); @@ -3757,5 +3741,3 @@ static int kbase_csf_cpu_mmap_user_reg_page(struct kbase_context *kctx, struct v return 0; } - -#endif /* MALI_USE_CSF */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_mem_pool.c b/drivers/gpu/arm/valhall/mali_kbase_mem_pool.c index 150236e05e37..3ebbe32ae74c 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_mem_pool.c +++ b/drivers/gpu/arm/valhall/mali_kbase_mem_pool.c @@ -34,9 +34,8 @@ #include #endif -#define pool_dbg(pool, format, ...) \ - dev_dbg(pool->kbdev->dev, "%s-pool [%zu/%zu]: " format, \ - (pool->next_pool) ? "kctx" : "kbdev", kbase_mem_pool_size(pool), \ +#define pool_dbg(pool, format, ...) \ + dev_dbg(pool->kbdev->dev, "%s-pool [%zu/%zu]: " format, "kctx", kbase_mem_pool_size(pool), \ kbase_mem_pool_max_size(pool), ##__VA_ARGS__) #define NOT_DIRTY false @@ -257,14 +256,6 @@ static void kbase_mem_pool_zero_page(struct kbase_mem_pool *pool, struct page *p kbase_mem_pool_sync_page(pool, p); } -static void kbase_mem_pool_spill(struct kbase_mem_pool *next_pool, struct page *p) -{ - /* Zero page before spilling */ - kbase_mem_pool_zero_page(next_pool, p); - - kbase_mem_pool_add(next_pool, p); -} - struct page *kbase_mem_alloc_page(struct kbase_mem_pool *pool) { struct page *p; @@ -308,13 +299,15 @@ struct page *kbase_mem_alloc_page(struct kbase_mem_pool *pool) return p; } +KBASE_EXPORT_TEST_API(kbase_mem_alloc_page); static void enqueue_free_pool_pages_work(struct kbase_mem_pool *pool) { - struct kbase_mem_migrate *mem_migrate = &pool->kbdev->mem_migrate; + if (!pool->order && kbase_is_page_migration_enabled()) { + struct kbase_mem_migrate *mem_migrate = &pool->kbdev->mem_migrate; - if (!pool->order && kbase_is_page_migration_enabled()) queue_work(mem_migrate->free_pages_workq, &mem_migrate->free_pages_work); + } } void kbase_mem_pool_free_page(struct kbase_mem_pool *pool, struct page *p) @@ -345,6 +338,7 @@ void kbase_mem_pool_free_page(struct kbase_mem_pool *pool, struct page *p) pool_dbg(pool, "freed page to kernel\n"); } } +KBASE_EXPORT_TEST_API(kbase_mem_pool_free_page); static size_t kbase_mem_pool_shrink_locked(struct kbase_mem_pool *pool, size_t nr_to_shrink) { @@ -383,10 +377,13 @@ int kbase_mem_pool_grow(struct kbase_mem_pool *pool, size_t nr_to_grow, kbase_mem_pool_lock(pool); - pool->dont_reclaim = true; + pool->reclaim_allowed = false; + for (i = 0; i < nr_to_grow; i++) { if (pool->dying) { - pool->dont_reclaim = false; + if (pool->pool_supports_reclaim) + pool->reclaim_allowed = true; + kbase_mem_pool_shrink_locked(pool, nr_to_grow); kbase_mem_pool_unlock(pool); if (page_owner) @@ -402,9 +399,11 @@ int kbase_mem_pool_grow(struct kbase_mem_pool *pool, size_t nr_to_grow, p = kbase_mem_alloc_page(pool); if (!p) { - kbase_mem_pool_lock(pool); - pool->dont_reclaim = false; - kbase_mem_pool_unlock(pool); + if (pool->pool_supports_reclaim) { + kbase_mem_pool_lock(pool); + pool->reclaim_allowed = true; + kbase_mem_pool_unlock(pool); + } return -ENOMEM; } @@ -412,7 +411,10 @@ int kbase_mem_pool_grow(struct kbase_mem_pool *pool, size_t nr_to_grow, kbase_mem_pool_lock(pool); kbase_mem_pool_add_locked(pool, p); } - pool->dont_reclaim = false; + + if (pool->pool_supports_reclaim) + pool->reclaim_allowed = true; + kbase_mem_pool_unlock(pool); return 0; @@ -443,6 +445,7 @@ void kbase_mem_pool_trim(struct kbase_mem_pool *pool, size_t new_size) (new_size - cur_size), (grown_size - cur_size)); } } +KBASE_EXPORT_TEST_API(kbase_mem_pool_trim); void kbase_mem_pool_set_max_size(struct kbase_mem_pool *pool, size_t max_size) { @@ -473,8 +476,12 @@ static unsigned long kbase_mem_pool_reclaim_count_objects(struct shrinker *s, pool = KBASE_GET_KBASE_DATA_FROM_SHRINKER(s, struct kbase_mem_pool, reclaim); + /* Pools not supporting reclaims are not assumed to register reclaim callbacks */ + if (WARN_ON(!pool->pool_supports_reclaim)) + return 0; + kbase_mem_pool_lock(pool); - if (pool->dont_reclaim && !pool->dying) { + if (!pool->reclaim_allowed && !pool->dying) { kbase_mem_pool_unlock(pool); /* Tell shrinker to skip reclaim * even though freeable pages are available @@ -495,10 +502,13 @@ static unsigned long kbase_mem_pool_reclaim_scan_objects(struct shrinker *s, pool = KBASE_GET_KBASE_DATA_FROM_SHRINKER(s, struct kbase_mem_pool, reclaim); + if (WARN_ON(!pool->pool_supports_reclaim)) + return SHRINK_STOP; + kbase_mem_pool_lock(pool); - if (pool->dont_reclaim && !pool->dying) { + if (!pool->reclaim_allowed && !pool->dying) { kbase_mem_pool_unlock(pool); - /* Tell shrinker that reclaim can't be made and + /* Tell shrinker that reclaim can't be done, and * do not attempt again for this reclaim context. */ return SHRINK_STOP; @@ -515,9 +525,8 @@ static unsigned long kbase_mem_pool_reclaim_scan_objects(struct shrinker *s, return freed; } -int kbase_mem_pool_init(struct kbase_mem_pool *pool, const struct kbase_mem_pool_config *config, - unsigned int order, int group_id, struct kbase_device *kbdev, - struct kbase_mem_pool *next_pool) +static int kbasep_mem_pool_init(struct kbase_mem_pool *pool, size_t max_size, unsigned int order, + int group_id, struct kbase_device *kbdev, bool support_reclaim) { struct shrinker *reclaim; @@ -526,71 +535,73 @@ int kbase_mem_pool_init(struct kbase_mem_pool *pool, const struct kbase_mem_pool } pool->cur_size = 0; - pool->max_size = kbase_mem_pool_config_get_max_size(config); + pool->max_size = max_size; pool->order = order; pool->group_id = group_id; pool->kbdev = kbdev; - pool->next_pool = next_pool; pool->dying = false; + pool->pool_supports_reclaim = support_reclaim; + pool->reclaim_allowed = false; atomic_set(&pool->isolation_in_progress_cnt, 0); spin_lock_init(&pool->pool_lock); INIT_LIST_HEAD(&pool->page_list); - reclaim = KBASE_INIT_RECLAIM(pool, reclaim, "mali-mem-pool"); - if (!reclaim) - return -ENOMEM; - KBASE_SET_RECLAIM(pool, reclaim, reclaim); + if (support_reclaim) { + reclaim = KBASE_INIT_RECLAIM(pool, reclaim, "mali-mem-pool"); + if (!reclaim) + return -ENOMEM; + KBASE_SET_RECLAIM(pool, reclaim, reclaim); - reclaim->count_objects = kbase_mem_pool_reclaim_count_objects; - reclaim->scan_objects = kbase_mem_pool_reclaim_scan_objects; - reclaim->seeks = DEFAULT_SEEKS; - reclaim->batch = 0; + reclaim->count_objects = kbase_mem_pool_reclaim_count_objects; + reclaim->scan_objects = kbase_mem_pool_reclaim_scan_objects; + reclaim->seeks = DEFAULT_SEEKS; + reclaim->batch = 0; - KBASE_REGISTER_SHRINKER(reclaim, "mali-mem-pool", pool); + KBASE_REGISTER_SHRINKER(reclaim, "mali-mem-pool", pool); + } pool_dbg(pool, "initialized\n"); return 0; } + +int kbase_mem_pool_init(struct kbase_mem_pool *pool, size_t max_size, unsigned int order, + int group_id, struct kbase_device *kbdev) +{ + return kbasep_mem_pool_init(pool, max_size, order, group_id, kbdev, true); +} KBASE_EXPORT_TEST_API(kbase_mem_pool_init); +int kbase_mem_pool_init_no_reclaim(struct kbase_mem_pool *pool, size_t max_size, unsigned int order, + int group_id, struct kbase_device *kbdev) +{ + return kbasep_mem_pool_init(pool, max_size, order, group_id, kbdev, false); +} +KBASE_EXPORT_TEST_API(kbase_mem_pool_init_no_reclaim); + void kbase_mem_pool_mark_dying(struct kbase_mem_pool *pool) { kbase_mem_pool_lock(pool); pool->dying = true; kbase_mem_pool_unlock(pool); } +KBASE_EXPORT_TEST_API(kbase_mem_pool_mark_dying); void kbase_mem_pool_term(struct kbase_mem_pool *pool) { - struct kbase_mem_pool *next_pool = pool->next_pool; struct page *p, *tmp; - size_t nr_to_spill = 0; LIST_HEAD(spill_list); LIST_HEAD(free_list); - size_t i; pool_dbg(pool, "terminate()\n"); - KBASE_UNREGISTER_SHRINKER(pool->reclaim); + if (pool->pool_supports_reclaim) + KBASE_UNREGISTER_SHRINKER(pool->reclaim); kbase_mem_pool_lock(pool); pool->max_size = 0; - if (next_pool && !kbase_mem_pool_is_full(next_pool)) { - /* Spill to next pool (may overspill) */ - nr_to_spill = kbase_mem_pool_capacity(next_pool); - nr_to_spill = min(kbase_mem_pool_size(pool), nr_to_spill); - - /* Zero pages first without holding the next_pool lock */ - for (i = 0; i < nr_to_spill; i++) { - p = kbase_mem_pool_remove_locked(pool, SPILL_IN_PROGRESS); - if (p) - list_add(&p->lru, &spill_list); - } - } - while (!kbase_mem_pool_is_empty(pool)) { /* Free remaining pages to kernel */ p = kbase_mem_pool_remove_locked(pool, FREE_IN_PROGRESS); @@ -600,16 +611,6 @@ void kbase_mem_pool_term(struct kbase_mem_pool *pool) kbase_mem_pool_unlock(pool); - if (next_pool && nr_to_spill) { - list_for_each_entry(p, &spill_list, lru) - kbase_mem_pool_zero_page(pool, p); - - /* Add new page list to next_pool */ - kbase_mem_pool_add_list(next_pool, &spill_list, nr_to_spill); - - pool_dbg(pool, "terminate() spilled %zu pages\n", nr_to_spill); - } - list_for_each_entry_safe(p, tmp, &free_list, lru) { list_del_init(&p->lru); kbase_mem_pool_free_page(pool, p); @@ -633,18 +634,12 @@ struct page *kbase_mem_pool_alloc(struct kbase_mem_pool *pool) { struct page *p; - do { - pool_dbg(pool, "alloc()\n"); - p = kbase_mem_pool_remove(pool, ALLOCATE_IN_PROGRESS); + pool_dbg(pool, "alloc()\n"); + p = kbase_mem_pool_remove(pool, ALLOCATE_IN_PROGRESS); - if (p) - return p; - - pool = pool->next_pool; - } while (pool); - - return NULL; + return p; } +KBASE_EXPORT_TEST_API(kbase_mem_pool_alloc); struct page *kbase_mem_pool_alloc_locked(struct kbase_mem_pool *pool) { @@ -656,8 +651,6 @@ struct page *kbase_mem_pool_alloc_locked(struct kbase_mem_pool *pool) void kbase_mem_pool_free(struct kbase_mem_pool *pool, struct page *p, bool dirty) { - struct kbase_mem_pool *next_pool = pool->next_pool; - pool_dbg(pool, "free()\n"); if (!kbase_mem_pool_is_full(pool)) { @@ -666,9 +659,6 @@ void kbase_mem_pool_free(struct kbase_mem_pool *pool, struct page *p, bool dirty kbase_mem_pool_sync_page(pool, p); kbase_mem_pool_add(pool, p); - } else if (next_pool && !kbase_mem_pool_is_full(next_pool)) { - /* Spill to next pool */ - kbase_mem_pool_spill(next_pool, p); } else { /* Free page */ kbase_mem_pool_free_page(pool, p); @@ -676,6 +666,7 @@ void kbase_mem_pool_free(struct kbase_mem_pool *pool, struct page *p, bool dirty enqueue_free_pool_pages_work(pool); } } +KBASE_EXPORT_TEST_API(kbase_mem_pool_free); void kbase_mem_pool_free_locked(struct kbase_mem_pool *pool, struct page *p, bool dirty) { @@ -696,6 +687,7 @@ void kbase_mem_pool_free_locked(struct kbase_mem_pool *pool, struct page *p, boo enqueue_free_pool_pages_work(pool); } } +KBASE_EXPORT_TEST_API(kbase_mem_pool_free_locked); int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_small_pages, struct tagged_addr *pages, bool partial_allowed, @@ -735,42 +727,31 @@ int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_small_page } kbase_mem_pool_unlock(pool); - if (i != nr_small_pages && pool->next_pool) { - /* Allocate via next pool */ - err = kbase_mem_pool_alloc_pages(pool->next_pool, nr_small_pages - i, pages + i, - partial_allowed, page_owner); - - if (err < 0) + /* Get any remaining pages from kernel */ + while (i != nr_small_pages) { + if (unlikely(!can_alloc_page(pool, page_owner))) goto err_rollback; - i += (size_t)err; - } else { - /* Get any remaining pages from kernel */ - while (i != nr_small_pages) { - if (unlikely(!can_alloc_page(pool, page_owner))) + p = kbase_mem_alloc_page(pool); + if (!p) { + if (partial_allowed) + goto done; + else goto err_rollback; + } - p = kbase_mem_alloc_page(pool); - if (!p) { - if (partial_allowed) - goto done; - else - goto err_rollback; - } - - if (pool->order) { - uint j; - - pages[i++] = as_tagged_tag(page_to_phys(p), HUGE_PAGE | HUGE_HEAD); - for (j = 1; j < (1u << pool->order); j++) { - phys_addr_t phys; - - phys = page_to_phys(p) + PAGE_SIZE * j; - pages[i++] = as_tagged_tag(phys, HUGE_PAGE); - } - } else { - pages[i++] = as_tagged(page_to_phys(p)); + if (pool->order) { + uint j; + + pages[i++] = as_tagged_tag(page_to_phys(p), HUGE_PAGE | HUGE_HEAD); + for (j = 1; j < (1u << pool->order); j++) { + phys_addr_t phys; + + phys = page_to_phys(p) + PAGE_SIZE * j; + pages[i++] = as_tagged_tag(phys, HUGE_PAGE); } + } else { + pages[i++] = as_tagged(page_to_phys(p)); } } @@ -782,6 +763,7 @@ err_rollback: kbase_mem_pool_free_pages(pool, i, pages, NOT_DIRTY, NOT_RECLAIMED); return err; } +KBASE_EXPORT_TEST_API(kbase_mem_pool_alloc_pages); int kbase_mem_pool_alloc_pages_locked(struct kbase_mem_pool *pool, size_t nr_small_pages, struct tagged_addr *pages) @@ -822,6 +804,7 @@ int kbase_mem_pool_alloc_pages_locked(struct kbase_mem_pool *pool, size_t nr_sma return nr_small_pages; } +KBASE_EXPORT_TEST_API(kbase_mem_pool_alloc_pages_locked); static void kbase_mem_pool_add_array(struct kbase_mem_pool *pool, size_t nr_pages, struct tagged_addr *pages, bool zero, bool sync) @@ -902,7 +885,6 @@ static void kbase_mem_pool_add_array_locked(struct kbase_mem_pool *pool, size_t void kbase_mem_pool_free_pages(struct kbase_mem_pool *pool, size_t nr_pages, struct tagged_addr *pages, bool dirty, bool reclaimed) { - struct kbase_mem_pool *next_pool = pool->next_pool; struct page *p; size_t nr_to_pool; LIST_HEAD(to_pool_list); @@ -919,15 +901,6 @@ void kbase_mem_pool_free_pages(struct kbase_mem_pool *pool, size_t nr_pages, kbase_mem_pool_add_array(pool, nr_to_pool, pages, false, dirty); i += nr_to_pool; - - if (i != nr_pages && next_pool) { - /* Spill to next pool (may overspill) */ - nr_to_pool = kbase_mem_pool_capacity(next_pool); - nr_to_pool = min(nr_pages - i, nr_to_pool); - - kbase_mem_pool_add_array(next_pool, nr_to_pool, pages + i, true, dirty); - i += nr_to_pool; - } } /* Free any remaining pages to kernel */ @@ -952,6 +925,7 @@ void kbase_mem_pool_free_pages(struct kbase_mem_pool *pool, size_t nr_pages, pool_dbg(pool, "free_pages(%zu) done\n", nr_pages); } +KBASE_EXPORT_TEST_API(kbase_mem_pool_free_pages); void kbase_mem_pool_free_pages_locked(struct kbase_mem_pool *pool, size_t nr_pages, struct tagged_addr *pages, bool dirty, bool reclaimed) @@ -999,3 +973,4 @@ void kbase_mem_pool_free_pages_locked(struct kbase_mem_pool *pool, size_t nr_pag pool_dbg(pool, "free_pages_locked(%zu) done\n", nr_pages); } +KBASE_EXPORT_TEST_API(kbase_mem_pool_free_pages_locked); diff --git a/drivers/gpu/arm/valhall/mali_kbase_mem_pool_debugfs.c b/drivers/gpu/arm/valhall/mali_kbase_mem_pool_debugfs.c index b757a1941124..3b89574790ba 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_mem_pool_debugfs.c +++ b/drivers/gpu/arm/valhall/mali_kbase_mem_pool_debugfs.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -165,4 +165,10 @@ void kbase_mem_pool_debugfs_init(struct dentry *parent, struct kbase_context *kc debugfs_create_file("lp_mem_pool_max_size", mode, parent, &kctx->mem_pools.large, &kbase_mem_pool_debugfs_max_size_fops); + + debugfs_create_file("pdg_pool_size", mode, parent, &kctx->pgd_mem_pool, + &kbase_mem_pool_debugfs_fops); + + debugfs_create_file("pgd_mem_pool_max_size", mode, parent, &kctx->pgd_mem_pool, + &kbase_mem_pool_debugfs_max_size_fops); } diff --git a/drivers/gpu/arm/valhall/mali_kbase_mem_pool_group.c b/drivers/gpu/arm/valhall/mali_kbase_mem_pool_group.c index 1b224f7d0cc0..0c3c90d62995 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_mem_pool_group.c +++ b/drivers/gpu/arm/valhall/mali_kbase_mem_pool_group.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -25,36 +25,19 @@ #include -void kbase_mem_pool_group_config_set_max_size(struct kbase_mem_pool_group_config *const configs, - size_t const max_size) -{ - size_t const large_max_size = max_size >> (KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER - - KBASE_MEM_POOL_SMALL_PAGE_TABLE_ORDER); - int gid; - - for (gid = 0; gid < MEMORY_GROUP_MANAGER_NR_GROUPS; ++gid) { - kbase_mem_pool_config_set_max_size(&configs->small[gid], max_size); - - kbase_mem_pool_config_set_max_size(&configs->large[gid], large_max_size); - } -} - int kbase_mem_pool_group_init(struct kbase_mem_pool_group *const mem_pools, - struct kbase_device *const kbdev, - const struct kbase_mem_pool_group_config *const configs, - struct kbase_mem_pool_group *next_pools) + struct kbase_device *const kbdev, size_t small_max_size, + size_t large_max_size, struct kbase_mem_pool_group *next_pools) { int gid, err = 0; for (gid = 0; gid < MEMORY_GROUP_MANAGER_NR_GROUPS; ++gid) { - err = kbase_mem_pool_init(&mem_pools->small[gid], &configs->small[gid], - KBASE_MEM_POOL_SMALL_PAGE_TABLE_ORDER, gid, kbdev, - next_pools ? &next_pools->small[gid] : NULL); + err = kbase_mem_pool_init(&mem_pools->small[gid], small_max_size, + KBASE_MEM_POOL_SMALL_PAGE_TABLE_ORDER, gid, kbdev); if (!err) { - err = kbase_mem_pool_init(&mem_pools->large[gid], &configs->large[gid], - KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER, gid, kbdev, - next_pools ? &next_pools->large[gid] : NULL); + err = kbase_mem_pool_init(&mem_pools->large[gid], large_max_size, + KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER, gid, kbdev); if (err) kbase_mem_pool_term(&mem_pools->small[gid]); } diff --git a/drivers/gpu/arm/valhall/mali_kbase_mem_pool_group.h b/drivers/gpu/arm/valhall/mali_kbase_mem_pool_group.h index cb69eab061ac..b32c71e3db4a 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_mem_pool_group.h +++ b/drivers/gpu/arm/valhall/mali_kbase_mem_pool_group.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -24,53 +24,15 @@ #include -/** - * kbase_mem_pool_group_select() - Select the memory pool to use. - * - * @kbdev: Device pointer. - * @mem_group_id: Physical memory group ID to use. - * @is_small_page: Flag used to select between the small and - * large memory pool. - * - * Return: A pointer to the selected memory pool. - */ -static inline struct kbase_mem_pool * -kbase_mem_pool_group_select(struct kbase_device *kbdev, u32 mem_group_id, bool is_small_page) -{ - if (WARN_ON(unlikely(kbdev == NULL))) - return NULL; - - WARN_ON(mem_group_id > BASE_MEM_GROUP_COUNT); - - if (is_small_page) - return &kbdev->mem_pools.small[mem_group_id]; - - return &kbdev->mem_pools.large[mem_group_id]; -} - -/** - * kbase_mem_pool_group_config_set_max_size - Set the initial configuration for - * a set of memory pools - * - * @configs: Initial configuration for the set of memory pools - * @max_size: Maximum number of free small pages each pool can hold - * - * This function sets the initial configuration for every memory pool so that - * the maximum amount of free memory that each pool can hold is identical. - * The equivalent number of 2 MiB pages is calculated automatically for the - * purpose of configuring the large page pools. - */ -void kbase_mem_pool_group_config_set_max_size(struct kbase_mem_pool_group_config *configs, - size_t max_size); - /** * kbase_mem_pool_group_init - Initialize a set of memory pools * - * @mem_pools: Set of memory pools to initialize - * @kbdev: Kbase device where memory is used - * @configs: Initial configuration for the set of memory pools - * @next_pools: Set of memory pools from which to allocate memory if there - * is no free memory in one of the @mem_pools + * @mem_pools: Set of memory pools to initialize + * @kbdev: Kbase device where memory is used + * @small_max_size: Maximum size for small memory pools + * @large_max_size: Maximum size for large memory pools + * @next_pools: Set of memory pools from which to allocate memory if there + * is no free memory in one of the @mem_pools * * Initializes a complete set of physical memory pools. Memory pools are used to * allow efficient reallocation of previously-freed physical pages. A pair of @@ -87,7 +49,7 @@ void kbase_mem_pool_group_config_set_max_size(struct kbase_mem_pool_group_config * Return: 0 on success, otherwise a negative error code */ int kbase_mem_pool_group_init(struct kbase_mem_pool_group *mem_pools, struct kbase_device *kbdev, - const struct kbase_mem_pool_group_config *configs, + size_t small_max_size, size_t large_max_size, struct kbase_mem_pool_group *next_pools); /** diff --git a/drivers/gpu/arm/valhall/mali_kbase_pbha.c b/drivers/gpu/arm/valhall/mali_kbase_pbha.c index c201163b5424..b79d419e24de 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_pbha.c +++ b/drivers/gpu/arm/valhall/mali_kbase_pbha.c @@ -24,9 +24,7 @@ #include #include -#if MALI_USE_CSF #define DTB_SET_SIZE 2 -#endif static bool read_setting_valid(unsigned int prod_model, unsigned int id, unsigned int read_setting) { @@ -164,7 +162,6 @@ int kbase_pbha_record_settings(struct kbase_device *kbdev, bool runtime, unsigne unsigned int const sysc_alloc_num = id / sizeof(u32); u32 modified_reg; -#if MALI_USE_CSF if (runtime) { uint i; @@ -175,9 +172,6 @@ int kbase_pbha_record_settings(struct kbase_device *kbdev, bool runtime, unsigne kbase_reg_read32(kbdev, GPU_SYSC_ALLOC_OFFSET(i)); kbase_pm_context_idle(kbdev); } -#else - CSTD_UNUSED(runtime); -#endif /* MALI_USE_CSF */ modified_reg = kbdev->sysc_alloc[sysc_alloc_num]; @@ -224,7 +218,6 @@ int kbase_pbha_record_settings(struct kbase_device *kbdev, bool runtime, unsigne void kbase_pbha_write_settings(struct kbase_device *kbdev) { -#if MALI_USE_CSF if (kbasep_pbha_supported(kbdev)) { uint i; @@ -249,12 +242,8 @@ void kbase_pbha_write_settings(struct kbase_device *kbdev) kbase_reg_write32(kbdev, GPU_SYSC_PBHA_OVERRIDE_OFFSET(reg_index), pbha_override_val); } -#else - CSTD_UNUSED(kbdev); -#endif /* MALI_USE_CSF */ } -#if MALI_USE_CSF static int kbase_pbha_read_int_id_override_property(struct kbase_device *kbdev, const struct device_node *pbha_node) { @@ -376,11 +365,9 @@ static int kbase_pbha_read_mma_wa_id_property(struct kbase_device *kbdev, kbdev->mma_wa_id = mma_wa_id; return 0; } -#endif /* MALI_USE_CSF */ int kbase_pbha_read_dtb(struct kbase_device *kbdev) { -#if MALI_USE_CSF const struct device_node *pbha_node; int err; @@ -404,7 +391,4 @@ int kbase_pbha_read_dtb(struct kbase_device *kbdev) err = kbase_pbha_read_mma_wa_id_property(kbdev, pbha_node); return err; -#else - return 0; -#endif } diff --git a/drivers/gpu/arm/valhall/mali_kbase_pbha_debugfs.c b/drivers/gpu/arm/valhall/mali_kbase_pbha_debugfs.c index 81f2df5ea977..8341d664dd34 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_pbha_debugfs.c +++ b/drivers/gpu/arm/valhall/mali_kbase_pbha_debugfs.c @@ -25,9 +25,7 @@ #include #include -#if MALI_USE_CSF #include "backend/gpu/mali_kbase_pm_internal.h" -#endif static int int_id_overrides_show(struct seq_file *sfile, void *data) { @@ -43,11 +41,7 @@ static int int_id_overrides_show(struct seq_file *sfile, void *data) for (i = 0; i < GPU_SYSC_ALLOC_COUNT; ++i) { uint j; -#if MALI_USE_CSF u32 reg = kbase_reg_read32(kbdev, GPU_SYSC_ALLOC_OFFSET(i)); -#else /* MALI_USE_CSF */ - u32 reg = 0; -#endif /* MALI_USE_CSF */ for (j = 0; j < sizeof(u32); ++j) { u8 r_val = 0; @@ -120,7 +114,6 @@ static int int_id_overrides_open(struct inode *in, struct file *file) return single_open(file, int_id_overrides_show, in->i_private); } -#if MALI_USE_CSF /** * propagate_bits_show - Read PBHA bits from L2_CONFIG out to debugfs. * @@ -207,7 +200,6 @@ static const struct file_operations pbha_propagate_bits_fops = { .llseek = seq_lseek, .release = single_release, }; -#endif /* MALI_USE_CSF */ static const struct file_operations pbha_int_id_overrides_fops = { .owner = THIS_MODULE, @@ -233,10 +225,8 @@ void kbase_pbha_debugfs_init(struct kbase_device *kbdev) debugfs_create_file("int_id_overrides", mode, debugfs_pbha_dir, kbdev, &pbha_int_id_overrides_fops); -#if MALI_USE_CSF if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_PBHA_HWU)) debugfs_create_file("propagate_bits", mode, debugfs_pbha_dir, kbdev, &pbha_propagate_bits_fops); -#endif /* MALI_USE_CSF */ } } diff --git a/drivers/gpu/arm/valhall/mali_kbase_pm.c b/drivers/gpu/arm/valhall/mali_kbase_pm.c index e7e70b5389d8..d13cbbe48458 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_pm.c +++ b/drivers/gpu/arm/valhall/mali_kbase_pm.c @@ -35,16 +35,6 @@ #include -int kbase_pm_powerup(struct kbase_device *kbdev, unsigned int flags) -{ - return kbase_hwaccess_pm_powerup(kbdev, flags); -} - -void kbase_pm_halt(struct kbase_device *kbdev) -{ - kbase_hwaccess_pm_halt(kbdev); -} - void kbase_pm_context_active(struct kbase_device *kbdev) { (void)kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE); @@ -62,13 +52,11 @@ kbasep_pm_context_active_handle_suspend_locked(struct kbase_device *kbdev, current->pid); lockdep_assert_held(&kbdev->pm.lock); -#if MALI_USE_CSF /* Check scheduler lock */ if (sched_lock_held) lockdep_assert_held(&kbdev->csf.scheduler.lock); else kbase_lockdep_assert_not_held(&kbdev->csf.scheduler.lock); -#endif /* If there is an Arbiter, wait for Arbiter to grant GPU back to KBase * so suspend request can be handled. @@ -170,40 +158,20 @@ static void reenable_hwcnt_on_resume(struct kbase_device *kbdev) unsigned long flags; /* Re-enable GPU hardware counters */ -#if MALI_USE_CSF - kbase_csf_scheduler_spin_lock(kbdev, &flags); - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); - kbase_csf_scheduler_spin_unlock(kbdev, flags); -#else - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#endif + if (kbdev->csf.firmware_hctl_core_pwr) { + kbase_csf_scheduler_spin_lock(kbdev, &flags); + kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); + kbase_csf_scheduler_spin_unlock(kbdev, flags); + } /* Resume HW counters intermediaries. */ -#if MALI_USE_CSF if (kbdev->csf.firmware_inited) -#endif - { kbase_kinstr_prfcnt_resume(kbdev->kinstr_prfcnt_ctx); - } } static void resume_job_scheduling(struct kbase_device *kbdev) { -#if !MALI_USE_CSF - /* Resume any blocked atoms (which may cause contexts to be scheduled in - * and dependent atoms to run) - */ - kbase_resume_suspended_soft_jobs(kbdev); - - /* Resume the Job Scheduler and associated components, and start running - * atoms - */ - kbasep_js_resume(kbdev); -#else kbase_csf_scheduler_pm_resume(kbdev); -#endif } int kbase_pm_driver_suspend(struct kbase_device *kbdev) @@ -214,17 +182,14 @@ int kbase_pm_driver_suspend(struct kbase_device *kbdev) /* Suspend HW counter intermediaries. This blocks until workers and timers * are no longer running. */ -#if MALI_USE_CSF if (kbdev->csf.firmware_inited) -#endif - { kbase_kinstr_prfcnt_suspend(kbdev->kinstr_prfcnt_ctx); - } /* Disable GPU hardware counters. * This call will block until counters are disabled. */ - kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); + if (kbdev->csf.firmware_hctl_core_pwr) + kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); mutex_lock(&kbdev->pm.lock); if (WARN_ON(kbase_pm_is_suspending(kbdev))) { @@ -238,20 +203,9 @@ int kbase_pm_driver_suspend(struct kbase_device *kbdev) if (kbase_has_arbiter(kbdev)) { unsigned long flags; -#if MALI_USE_CSF spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbase_disjoint_state_up(kbdev); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#else - unsigned int i; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbdev->js_data.runpool_irq.submit_allowed = 0; - kbase_disjoint_state_up(kbdev); - for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) - kbase_job_slot_softstop(kbdev, i, NULL); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#endif } /* From now on, the active count will drop towards zero. Sometimes, @@ -260,15 +214,8 @@ int kbase_pm_driver_suspend(struct kbase_device *kbdev) * all pm references */ -#if !MALI_USE_CSF - /* Suspend job scheduler and associated components, so that it releases all - * the PM active count references - */ - kbasep_js_suspend(kbdev); -#else if (kbase_csf_scheduler_pm_suspend(kbdev)) goto exit; -#endif scheduling_suspended = true; @@ -281,7 +228,6 @@ int kbase_pm_driver_suspend(struct kbase_device *kbdev) wait_event(kbdev->pm.zero_active_count_wait, kbdev->pm.active_count == 0); dev_dbg(kbdev->dev, ">wait_event - waiting done\n"); -#if MALI_USE_CSF /* At this point, any kbase context termination should either have run to * completion and any further context termination can only begin after * the system resumes. Therefore, it is now safe to skip taking the context @@ -289,7 +235,6 @@ int kbase_pm_driver_suspend(struct kbase_device *kbdev) */ if (kbase_csf_kcpu_queue_halt_timers(kbdev)) goto exit; -#endif timers_halted = true; @@ -310,21 +255,17 @@ int kbase_pm_driver_suspend(struct kbase_device *kbdev) mutex_unlock(&kbdev->pm.arb_vm_state->vm_state_lock); } -#if MALI_USE_CSF kbase_backend_invalidate_gpu_timestamp_offset(kbdev); -#endif return 0; exit: if (timers_halted) { -#if MALI_USE_CSF /* Resume the timers in case of suspend failure. But that needs to * be done before clearing the 'pm.suspending' flag so as to keep the * context termination blocked. */ kbase_csf_kcpu_queue_resume_timers(kbdev); -#endif } mutex_lock(&kbdev->pm.lock); @@ -358,9 +299,7 @@ void kbase_pm_driver_resume(struct kbase_device *kbdev, bool arb_gpu_start) resume_job_scheduling(kbdev); -#if MALI_USE_CSF kbase_csf_kcpu_queue_resume_timers(kbdev); -#endif /* Matching idle call, to power off the GPU/cores if we didn't actually * need it and the policy doesn't want it on diff --git a/drivers/gpu/arm/valhall/mali_kbase_pm.h b/drivers/gpu/arm/valhall/mali_kbase_pm.h index f176f8cdfc31..e1c5388a7d54 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_pm.h +++ b/drivers/gpu/arm/valhall/mali_kbase_pm.h @@ -52,29 +52,6 @@ struct kbase_device; */ int kbase_pm_init(struct kbase_device *kbdev); -/** - * kbase_pm_powerup - Power up GPU after all modules have been initialized - * and interrupt handlers installed. - * - * @kbdev: The kbase device structure for the device (must be a valid pointer) - * @flags: Flags to pass on to kbase_pm_init_hw - * - * Return: 0 if powerup was successful. - */ -int kbase_pm_powerup(struct kbase_device *kbdev, unsigned int flags); - -/** - * kbase_pm_halt - Halt the power management framework. - * - * @kbdev: The kbase device structure for the device (must be a valid pointer) - * - * Should ensure that no new interrupts are generated, - * but allow any currently running interrupt handlers to complete successfully. - * The GPU is forced off by the time this function returns, regardless of - * whether or not the active power policy asks for the GPU to be powered off. - */ -void kbase_pm_halt(struct kbase_device *kbdev); - /** * kbase_pm_term - Terminate the power management framework. * diff --git a/drivers/gpu/arm/valhall/mali_kbase_reg_track.c b/drivers/gpu/arm/valhall/mali_kbase_reg_track.c index 2d9ae789f7f4..9c26e33be18d 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_reg_track.c +++ b/drivers/gpu/arm/valhall/mali_kbase_reg_track.c @@ -42,14 +42,12 @@ char *kbase_reg_zone_get_name(enum kbase_memory_zone zone) return "CUSTOM_VA"; case EXEC_VA_ZONE: return "EXEC_VA"; -#if MALI_USE_CSF case MCU_SHARED_ZONE: return "MCU_SHARED"; case EXEC_FIXED_VA_ZONE: return "EXEC_FIXED_VA"; case FIXED_VA_ZONE: return "FIXED_VA"; -#endif default: return NULL; } @@ -345,7 +343,7 @@ kbase_region_tracker_find_region_meeting_reqs(struct kbase_va_region *reg_reqs, void kbase_remove_va_region(struct kbase_device *kbdev, struct kbase_va_region *reg) { struct rb_node *rbprev; - struct kbase_reg_zone *zone = container_of(reg->rbtree, struct kbase_reg_zone, reg_rbtree); + struct kbase_reg_zone *zone; struct kbase_va_region *prev = NULL; struct rb_node *rbnext; struct kbase_va_region *next = NULL; @@ -355,6 +353,10 @@ void kbase_remove_va_region(struct kbase_device *kbdev, struct kbase_va_region * int merged_front = 0; int merged_back = 0; + if (WARN_ON(reg == NULL)) + return; + + zone = container_of(reg->rbtree, struct kbase_reg_zone, reg_rbtree); reg_rbtree = reg->rbtree; if (WARN_ON(RB_EMPTY_ROOT(reg_rbtree))) @@ -566,14 +568,10 @@ int kbase_add_va_region(struct kbase_context *kctx, struct kbase_va_region *reg, * Also, executable allocations from EXEC_VA don't need the special * alignment. */ -#if MALI_USE_CSF /* The same is also true for the EXEC_FIXED_VA zone. */ -#endif if (!(reg->flags & KBASE_REG_GPU_NX) && !addr && -#if MALI_USE_CSF ((kbase_bits_to_zone(reg->flags)) != EXEC_FIXED_VA_ZONE) && -#endif ((kbase_bits_to_zone(reg->flags)) != EXEC_VA_ZONE)) { if (cpu_va_bits > gpu_pc_bits) { align = max(align, (size_t)((1ULL << gpu_pc_bits) >> PAGE_SHIFT)); @@ -665,16 +663,6 @@ int kbase_add_va_region_rbtree(struct kbase_device *kbdev, struct kbase_va_regio size_t align_offset = align; size_t align_mask = align - 1; -#if !MALI_USE_CSF - if ((reg->flags & KBASE_REG_TILER_ALIGN_TOP)) { - WARN(align > 1, - "%s with align %lx might not be honored for KBASE_REG_TILER_ALIGN_TOP memory", - __func__, (unsigned long)align); - align_mask = reg->extension - 1; - align_offset = reg->extension - reg->initial_commit; - } -#endif /* !MALI_USE_CSF */ - tmp = kbase_region_tracker_find_region_meeting_reqs(reg, nr_pages, align_offset, align_mask, &start_pfn); if (tmp) { @@ -768,7 +756,6 @@ static int kbase_reg_zone_same_va_init(struct kbase_context *kctx, u64 gpu_va_li lockdep_assert_held(&kctx->reg_lock); -#if MALI_USE_CSF if ((base_pfn + nr_pages) > KBASE_REG_ZONE_EXEC_VA_BASE_64) { /* Depending on how the kernel is configured, it's possible (eg on aarch64) for * same_va_bits to reach 48 bits. Cap same_va_pages so that the same_va zone @@ -776,7 +763,6 @@ static int kbase_reg_zone_same_va_init(struct kbase_context *kctx, u64 gpu_va_li */ nr_pages = KBASE_REG_ZONE_EXEC_VA_BASE_64 - base_pfn; } -#endif err = kbase_reg_zone_init(kctx->kbdev, zone, SAME_VA_ZONE, base_pfn, nr_pages); if (err) return -ENOMEM; @@ -841,17 +827,9 @@ static inline u64 kbase_get_exec_va_zone_base(struct kbase_context *kctx) { u64 base_pfn; -#if MALI_USE_CSF base_pfn = KBASE_REG_ZONE_EXEC_VA_BASE_64; if (kbase_ctx_compat_mode(kctx)) base_pfn = KBASE_REG_ZONE_EXEC_VA_BASE_32; -#else - CSTD_UNUSED(kctx); - /* EXEC_VA zone's codepaths are slightly easier when its base_pfn is - * initially U64_MAX - */ - base_pfn = U64_MAX; -#endif return base_pfn; } @@ -864,10 +842,6 @@ static inline int kbase_reg_zone_exec_va_init(struct kbase_context *kctx, u64 gp CSTD_UNUSED(gpu_va_limit); -#if !MALI_USE_CSF - nr_pages = 0; -#endif - return kbase_reg_zone_init(kctx->kbdev, zone, EXEC_VA_ZONE, base_pfn, nr_pages); } @@ -878,7 +852,6 @@ static void kbase_reg_zone_exec_va_term(struct kbase_context *kctx) kbase_reg_zone_term(zone); } -#if MALI_USE_CSF static inline u64 kbase_get_exec_fixed_va_zone_base(struct kbase_context *kctx) { return kbase_get_exec_va_zone_base(kctx) + KBASE_REG_ZONE_EXEC_VA_SIZE; @@ -931,7 +904,6 @@ static void kbase_reg_zone_fixed_va_term(struct kbase_context *kctx) kbase_reg_zone_term(zone); } -#endif typedef int kbase_memory_zone_init(struct kbase_context *kctx, u64 gpu_va_limit); typedef void kbase_memory_zone_term(struct kbase_context *kctx); @@ -949,13 +921,11 @@ static const struct kbase_memory_zone_init_meta zones_init[] = { "Could not initialize CUSTOM_VA zone" }, [EXEC_VA_ZONE] = { kbase_reg_zone_exec_va_init, kbase_reg_zone_exec_va_term, "Could not initialize EXEC_VA zone" }, -#if MALI_USE_CSF [EXEC_FIXED_VA_ZONE] = { kbase_reg_zone_exec_fixed_va_init, kbase_reg_zone_exec_fixed_va_term, "Could not initialize EXEC_FIXED_VA zone" }, [FIXED_VA_ZONE] = { kbase_reg_zone_fixed_va_init, kbase_reg_zone_fixed_va_term, "Could not initialize FIXED_VA zone" }, -#endif }; int kbase_region_tracker_init(struct kbase_context *kctx) @@ -975,9 +945,7 @@ int kbase_region_tracker_init(struct kbase_context *kctx) goto term; } } -#if MALI_USE_CSF INIT_LIST_HEAD(&kctx->csf.event_pages_head); -#endif kctx->jit_va = false; kbase_gpu_vm_unlock(kctx); @@ -1217,16 +1185,6 @@ KBASE_EXPORT_TEST_API(kbase_region_tracker_init_jit); int kbase_region_tracker_init_exec(struct kbase_context *kctx, u64 exec_va_pages) { -#if !MALI_USE_CSF - struct kbase_reg_zone *exec_va_zone; - struct kbase_reg_zone *target_zone; - struct kbase_va_region *target_reg; - u64 target_zone_base_addr; - enum kbase_memory_zone target_zone_id; - u64 exec_va_start; - int err; -#endif - /* The EXEC_VA zone shall be created by making space either: * - for 64-bit clients, at the end of the process's address space * - for 32-bit clients, in the CUSTOM zone @@ -1239,85 +1197,14 @@ int kbase_region_tracker_init_exec(struct kbase_context *kctx, u64 exec_va_pages if (exec_va_pages == 0 || exec_va_pages > KBASE_REG_ZONE_EXEC_VA_MAX_PAGES) return -EINVAL; -#if MALI_USE_CSF /* For CSF GPUs we now setup the EXEC_VA zone during initialization, * so this request is a null-op. */ CSTD_UNUSED(kctx); return 0; -#else - kbase_gpu_vm_lock(kctx); - - /* Verify that we've not already created a EXEC_VA zone, and that the - * EXEC_VA zone must come before JIT's CUSTOM_VA. - */ - if (kbase_has_exec_va_zone_locked(kctx) || kctx->jit_va) { - err = -EPERM; - goto exit_unlock; - } - - if (exec_va_pages > kctx->gpu_va_end) { - err = -ENOMEM; - goto exit_unlock; - } - - /* Verify no allocations have already been made */ - if (kbase_region_tracker_has_allocs(kctx)) { - err = -ENOMEM; - goto exit_unlock; - } - - if (kbase_ctx_compat_mode(kctx)) { - /* 32-bit client: take from CUSTOM_VA zone */ - target_zone_id = CUSTOM_VA_ZONE; - } else { - /* 64-bit client: take from SAME_VA zone */ - target_zone_id = SAME_VA_ZONE; - } - - target_zone = kbase_ctx_reg_zone_get(kctx, target_zone_id); - target_zone_base_addr = target_zone->base_pfn << PAGE_SHIFT; - - target_reg = kbase_region_tracker_find_region_base_address(kctx, target_zone_base_addr); - if (WARN(!target_reg, - "Already found a free region at the start of every zone, but now cannot find any region for zone base 0x%.16llx zone %s", - (unsigned long long)target_zone_base_addr, - kbase_reg_zone_get_name(target_zone_id))) { - err = -ENOMEM; - goto exit_unlock; - } - /* kbase_region_tracker_has_allocs() above has already ensured that all - * of the zones have no allocs, so no need to check that again on - * target_reg - */ - WARN_ON((!(target_reg->flags & KBASE_REG_FREE)) || - target_reg->nr_pages != target_zone->va_size_pages); - - if (target_reg->nr_pages <= exec_va_pages || target_zone->va_size_pages <= exec_va_pages) { - err = -ENOMEM; - goto exit_unlock; - } - - /* Taken from the end of the target zone */ - exec_va_start = kbase_reg_zone_end_pfn(target_zone) - exec_va_pages; - exec_va_zone = kbase_ctx_reg_zone_get(kctx, EXEC_VA_ZONE); - if (kbase_reg_zone_init(kctx->kbdev, exec_va_zone, EXEC_VA_ZONE, exec_va_start, - exec_va_pages)) - return -ENOMEM; - - /* Update target zone and corresponding region */ - target_reg->nr_pages -= exec_va_pages; - target_zone->va_size_pages -= exec_va_pages; - err = 0; - -exit_unlock: - kbase_gpu_vm_unlock(kctx); - return err; -#endif /* MALI_USE_CSF */ } KBASE_EXPORT_TEST_API(kbase_region_tracker_init_exec); -#if MALI_USE_CSF void kbase_mcu_shared_interface_region_tracker_term(struct kbase_device *kbdev) { kbase_reg_zone_term(&kbdev->csf.mcu_shared_zone); @@ -1328,7 +1215,6 @@ int kbase_mcu_shared_interface_region_tracker_init(struct kbase_device *kbdev) return kbase_reg_zone_init(kbdev, &kbdev->csf.mcu_shared_zone, MCU_SHARED_ZONE, KBASE_REG_ZONE_MCU_SHARED_BASE, MCU_SHARED_ZONE_SIZE); } -#endif /** * kbase_alloc_free_region - Allocate a free region object. @@ -1410,12 +1296,10 @@ KBASE_EXPORT_TEST_API(kbase_ctx_alloc_free_region); */ void kbase_free_alloced_region(struct kbase_va_region *reg) { -#if MALI_USE_CSF if (kbase_bits_to_zone(reg->flags) == MCU_SHARED_ZONE) { kfree(reg); return; } -#endif if (!(reg->flags & KBASE_REG_FREE)) { struct kbase_context *kctx = kbase_reg_to_kctx(reg); @@ -1427,7 +1311,6 @@ void kbase_free_alloced_region(struct kbase_va_region *reg) dev_dbg(kctx->kbdev->dev, "Freeing memory region %pK\n of zone %s", (void *)reg, kbase_reg_zone_get_name(kbase_bits_to_zone(reg->flags))); -#if MALI_USE_CSF if (reg->flags & KBASE_REG_CSF_EVENT) /* * This should not be reachable if called from 'mcu_shared' functions @@ -1437,7 +1320,6 @@ void kbase_free_alloced_region(struct kbase_va_region *reg) */ kbase_unlink_event_mem_page(kctx, reg); -#endif mutex_lock(&kctx->jit_evict_lock); diff --git a/drivers/gpu/arm/valhall/mali_kbase_reg_track.h b/drivers/gpu/arm/valhall/mali_kbase_reg_track.h index d29999fdcb9a..376951a9d569 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_reg_track.h +++ b/drivers/gpu/arm/valhall/mali_kbase_reg_track.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -33,22 +33,13 @@ struct kbase_va_region; struct kbase_device; struct kmem_cache; -#if MALI_USE_CSF /* Space for 8 different zones */ #define KBASE_REG_ZONE_BITS 3 -#else -/* Space for 4 different zones */ -#define KBASE_REG_ZONE_BITS 2 -#endif /** * KBASE_REG_ZONE_MAX - Maximum number of GPU memory region zones */ -#if MALI_USE_CSF #define KBASE_REG_ZONE_MAX 6ul -#else -#define KBASE_REG_ZONE_MAX 4ul -#endif /* The bits 11-13 (inclusive) of the kbase_va_region flag are reserved * for information about the zone in which it was allocated. @@ -62,22 +53,10 @@ struct kmem_cache; #define KBASE_REG_ZONE_CUSTOM_VA_BASE (0x100000000ULL >> PAGE_SHIFT) -#if MALI_USE_CSF /* only used with 32-bit clients */ /* On a 32bit platform, custom VA should be wired from 4GB to 2^(43). */ #define KBASE_REG_ZONE_CUSTOM_VA_SIZE (((1ULL << 43) >> PAGE_SHIFT) - KBASE_REG_ZONE_CUSTOM_VA_BASE) -#else -/* only used with 32-bit clients */ -/* On a 32bit platform, custom VA should be wired from 4GB to the VA limit of the - * GPU. Unfortunately, the Linux mmap() interface limits us to 2^32 pages (2^44 - * bytes, see mmap64 man page for reference). So we put the default limit to the - * maximum possible on Linux and shrink it down, if required by the GPU, during - * initialization. - */ -#define KBASE_REG_ZONE_CUSTOM_VA_SIZE (((1ULL << 44) >> PAGE_SHIFT) - KBASE_REG_ZONE_CUSTOM_VA_BASE) -/* end 32-bit clients only */ -#endif /* The starting address and size of the GPU-executable zone are dynamic * and depend on the platform and the number of pages requested by the @@ -86,7 +65,6 @@ struct kmem_cache; #define KBASE_REG_ZONE_EXEC_VA_MAX_PAGES ((1ULL << 32) >> PAGE_SHIFT) /* 4 GB */ #define KBASE_REG_ZONE_EXEC_VA_SIZE KBASE_REG_ZONE_EXEC_VA_MAX_PAGES -#if MALI_USE_CSF #define KBASE_REG_ZONE_MCU_SHARED_BASE (0x04000000ULL >> PAGE_SHIFT) #define MCU_SHARED_ZONE_SIZE (((0x08000000ULL) >> PAGE_SHIFT) - KBASE_REG_ZONE_MCU_SHARED_BASE) @@ -109,8 +87,6 @@ struct kmem_cache; #define KBASE_REG_ZONE_FIXED_VA_END_64 ((1ULL << 48) >> PAGE_SHIFT) #define KBASE_REG_ZONE_FIXED_VA_END_32 ((1ULL << 44) >> PAGE_SHIFT) -#endif - /** * enum kbase_memory_zone - Kbase memory zone identifier * @SAME_VA_ZONE: Memory zone for allocations where the GPU and CPU VA coincide. @@ -142,17 +118,11 @@ enum kbase_memory_zone { SAME_VA_ZONE, CUSTOM_VA_ZONE, EXEC_VA_ZONE, -#if IS_ENABLED(MALI_USE_CSF) EXEC_FIXED_VA_ZONE, FIXED_VA_ZONE, MCU_SHARED_ZONE, -#endif MEMORY_ZONE_MAX, -#if IS_ENABLED(MALI_USE_CSF) CONTEXT_ZONE_MAX = FIXED_VA_ZONE + 1 -#else - CONTEXT_ZONE_MAX = EXEC_VA_ZONE + 1 -#endif }; /** @@ -196,7 +166,7 @@ unsigned long kbase_zone_to_bits(enum kbase_memory_zone zone); enum kbase_memory_zone kbase_bits_to_zone(unsigned long zone_bits); /** - * kbase_mem_zone_get_name - Get the string name for a given memory zone + * kbase_reg_zone_get_name - Get the string name for a given memory zone * @zone: Memory zone identifier * * Return: string for valid memory zone, NULL otherwise @@ -212,11 +182,7 @@ char *kbase_reg_zone_get_name(enum kbase_memory_zone zone); */ static inline bool kbase_is_ctx_reg_zone(enum kbase_memory_zone zone) { -#if MALI_USE_CSF return !(zone == MCU_SHARED_ZONE); -#else - return true; -#endif } /** diff --git a/drivers/gpu/arm/valhall/mali_kbase_softjobs.c b/drivers/gpu/arm/valhall/mali_kbase_softjobs.c index 9f060f42d8dd..df66b3b81609 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_softjobs.c +++ b/drivers/gpu/arm/valhall/mali_kbase_softjobs.c @@ -29,7 +29,6 @@ #include #include #include -#include #include #include #include @@ -41,830 +40,6 @@ #include #include -#if !MALI_USE_CSF - -/** - * struct kbase_external_resource_list - Structure which describes a list of external - * resources. This structure is used for the processing of - * EXT_RES_MAP and EXT_RES_UNMAP softjobs, instead of ioctl - * structure 'base_external_resource_list'. This is done to - * avoid UBSAN falsely detecting that an out of bound access - * is going to be made for ext_res[1] array, defined inside - * 'base_external_resource_list', when number of external - * resources to be processed are more than 1. - * - * @count: The number of external resources. - * @ext_res: Pointer to an array of external resources. - */ -struct kbase_external_resource_list { - u64 count; - struct base_external_resource *ext_res; -}; - -/** - * DOC: This file implements the logic behind software only jobs that are - * executed within the driver rather than being handed over to the GPU. - */ - -static void kbasep_add_waiting_soft_job(struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx = katom->kctx; - unsigned long lflags; - - spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); - list_add_tail(&katom->queue, &kctx->waiting_soft_jobs); - spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); -} - -void kbasep_remove_waiting_soft_job(struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx = katom->kctx; - unsigned long lflags; - - spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); - list_del(&katom->queue); - spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); -} - -static void kbasep_add_waiting_with_timeout(struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx = katom->kctx; - - /* Record the start time of this atom so we could cancel it at - * the right time. - */ - katom->start_timestamp = ktime_get_raw(); - - /* Add the atom to the waiting list before the timer is - * (re)started to make sure that it gets processed. - */ - kbasep_add_waiting_soft_job(katom); - - /* Schedule timeout of this atom after a period if it is not active */ - if (!timer_pending(&kctx->soft_job_timeout)) { - unsigned int timeout_ms = - (unsigned int)atomic_read(&kctx->kbdev->js_data.soft_job_timeout_ms); - mod_timer(&kctx->soft_job_timeout, jiffies + msecs_to_jiffies(timeout_ms)); - } -} - -static int kbasep_read_soft_event_status(struct kbase_context *kctx, u64 evt, unsigned char *status) -{ - unsigned char *mapped_evt; - struct kbase_vmap_struct map; - - mapped_evt = kbase_vmap_prot(kctx, evt, sizeof(*mapped_evt), KBASE_REG_CPU_RD, &map); - if (!mapped_evt) - return -EFAULT; - - *status = *mapped_evt; - - kbase_vunmap(kctx, &map); - - return 0; -} - -static int kbasep_write_soft_event_status(struct kbase_context *kctx, u64 evt, - unsigned char new_status) -{ - unsigned char *mapped_evt; - struct kbase_vmap_struct map; - - if ((new_status != BASE_JD_SOFT_EVENT_SET) && (new_status != BASE_JD_SOFT_EVENT_RESET)) - return -EINVAL; - - mapped_evt = kbase_vmap_prot(kctx, evt, sizeof(*mapped_evt), KBASE_REG_CPU_WR, &map); - if (!mapped_evt) - return -EFAULT; - - *mapped_evt = new_status; - - kbase_vunmap(kctx, &map); - - return 0; -} - -static int kbase_dump_cpu_gpu_time(struct kbase_jd_atom *katom) -{ - struct kbase_vmap_struct map; - void *user_result; - struct timespec64 ts; - struct base_dump_cpu_gpu_counters data; - u64 system_time = 0ULL; - u64 cycle_counter; - u64 jc = katom->jc; - struct kbase_context *kctx = katom->kctx; - int pm_active_err; - - memset(&data, 0, sizeof(data)); - - /* Take the PM active reference as late as possible - otherwise, it could - * delay suspend until we process the atom (which may be at the end of a - * long chain of dependencies - */ - if (kbase_has_arbiter(kctx->kbdev)) - atomic_inc(&kctx->kbdev->pm.gpu_users_waiting); - pm_active_err = kbase_pm_context_active_handle_suspend( - kctx->kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE); - if (pm_active_err) { - struct kbasep_js_device_data *js_devdata = &kctx->kbdev->js_data; - - /* We're suspended - queue this on the list of suspended jobs - * Use dep_item[1], because dep_item[0] was previously in use - * for 'waiting_soft_jobs'. - */ - mutex_lock(&js_devdata->runpool_mutex); - list_add_tail(&katom->dep_item[1], &js_devdata->suspended_soft_jobs_list); - mutex_unlock(&js_devdata->runpool_mutex); - - /* Also adding this to the list of waiting soft job */ - kbasep_add_waiting_soft_job(katom); - - return pm_active_err; - } else if (kbase_has_arbiter(kctx->kbdev)) - atomic_dec(&kctx->kbdev->pm.gpu_users_waiting); - - kbase_backend_get_gpu_time(kctx->kbdev, &cycle_counter, &system_time, &ts); - - kbase_pm_context_idle(kctx->kbdev); - - data.sec = (__u64)ts.tv_sec; - data.usec = ts.tv_nsec / 1000; - data.system_time = system_time; - data.cycle_counter = cycle_counter; - - /* Assume this atom will be cancelled until we know otherwise */ - katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; - - /* GPU_WR access is checked on the range for returning the result to - * userspace for the following reasons: - * - security, this is currently how imported user bufs are checked. - * - userspace ddk guaranteed to assume region was mapped as GPU_WR - */ - user_result = kbase_vmap_prot(kctx, jc, sizeof(data), KBASE_REG_GPU_WR, &map); - if (!user_result) - return 0; - - memcpy(user_result, &data, sizeof(data)); - - kbase_vunmap(kctx, &map); - - /* Atom was fine - mark it as done */ - katom->event_code = BASE_JD_EVENT_DONE; - - return 0; -} - -#if IS_ENABLED(CONFIG_SYNC_FILE) -/* Called by the explicit fence mechanism when a fence wait has completed */ -void kbase_soft_event_wait_callback(struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx = katom->kctx; - - mutex_lock(&kctx->jctx.lock); - kbasep_remove_waiting_soft_job(katom); - kbase_finish_soft_job(katom); - if (kbase_jd_done_nolock(katom, true)) - kbase_js_sched_all(kctx->kbdev); - mutex_unlock(&kctx->jctx.lock); -} -#endif - -static void kbasep_soft_event_complete_job(struct work_struct *work) -{ - struct kbase_jd_atom *katom = container_of(work, struct kbase_jd_atom, work); - struct kbase_context *kctx = katom->kctx; - int resched; - - mutex_lock(&kctx->jctx.lock); - resched = kbase_jd_done_nolock(katom, true); - mutex_unlock(&kctx->jctx.lock); - - if (resched) - kbase_js_sched_all(kctx->kbdev); -} - -void kbasep_complete_triggered_soft_events(struct kbase_context *kctx, u64 evt) -{ - int cancel_timer = 1; - struct list_head *entry, *tmp; - unsigned long lflags; - - spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); - list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) { - struct kbase_jd_atom *katom = list_entry(entry, struct kbase_jd_atom, queue); - - switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { - case BASE_JD_REQ_SOFT_EVENT_WAIT: - if (katom->jc == evt) { - list_del(&katom->queue); - - katom->event_code = BASE_JD_EVENT_DONE; - INIT_WORK(&katom->work, kbasep_soft_event_complete_job); - queue_work(kctx->jctx.job_done_wq, &katom->work); - } else { - /* There are still other waiting jobs, we cannot - * cancel the timer yet. - */ - cancel_timer = 0; - } - break; -#ifdef CONFIG_MALI_VALHALL_FENCE_DEBUG - case BASE_JD_REQ_SOFT_FENCE_WAIT: - /* Keep the timer running if fence debug is enabled and - * there are waiting fence jobs. - */ - cancel_timer = 0; - break; -#endif - } - } - - if (cancel_timer) - del_timer(&kctx->soft_job_timeout); - spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); -} - -#ifdef CONFIG_MALI_VALHALL_FENCE_DEBUG -static void kbase_fence_debug_check_atom(struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx = katom->kctx; - struct device *dev = kctx->kbdev->dev; - int i; - - for (i = 0; i < 2; i++) { - struct kbase_jd_atom *dep; - - list_for_each_entry(dep, &katom->dep_head[i], dep_item[i]) { - if (dep->status == KBASE_JD_ATOM_STATE_UNUSED || - dep->status == KBASE_JD_ATOM_STATE_COMPLETED) - continue; - - if ((dep->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == - BASE_JD_REQ_SOFT_FENCE_TRIGGER) { - /* Found blocked trigger fence. */ - struct kbase_sync_fence_info info; - - if (!kbase_sync_fence_in_info_get(dep, &info)) { - dev_warn(dev, - "\tVictim trigger atom %d fence [%pK] %s: %s\n", - kbase_jd_atom_id(kctx, dep), info.fence, info.name, - kbase_sync_status_string(info.status)); - } - } - - kbase_fence_debug_check_atom(dep); - } - } -} - -static void kbase_fence_debug_wait_timeout(struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx = katom->kctx; - struct device *dev = katom->kctx->kbdev->dev; - int timeout_ms = atomic_read(&kctx->kbdev->js_data.soft_job_timeout_ms); - unsigned long lflags; - struct kbase_sync_fence_info info; - - spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); - - if (kbase_sync_fence_in_info_get(katom, &info)) { - /* Fence must have signaled just after timeout. */ - spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); - return; - } - - dev_warn(dev, "ctx %d_%d: Atom %d still waiting for fence [%pK] after %dms\n", kctx->tgid, - kctx->id, kbase_jd_atom_id(kctx, katom), info.fence, timeout_ms); - dev_warn(dev, "\tGuilty fence [%pK] %s: %s\n", info.fence, info.name, - kbase_sync_status_string(info.status)); - - /* Search for blocked trigger atoms */ - kbase_fence_debug_check_atom(katom); - - spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); - - kbase_sync_fence_in_dump(katom); -} - -struct kbase_fence_debug_work { - struct kbase_jd_atom *katom; - struct work_struct work; -}; - -static void kbase_fence_debug_wait_timeout_worker(struct work_struct *work) -{ - struct kbase_fence_debug_work *w = container_of(work, struct kbase_fence_debug_work, work); - struct kbase_jd_atom *katom = w->katom; - struct kbase_context *kctx = katom->kctx; - - mutex_lock(&kctx->jctx.lock); - kbase_fence_debug_wait_timeout(katom); - mutex_unlock(&kctx->jctx.lock); - - kfree(w); -} - -static void kbase_fence_debug_timeout(struct kbase_jd_atom *katom) -{ - struct kbase_fence_debug_work *work; - struct kbase_context *kctx = katom->kctx; - - /* Enqueue fence debug worker. Use job_done_wq to get - * debug print ordered with job completion. - */ - work = kzalloc(sizeof(struct kbase_fence_debug_work), GFP_ATOMIC); - /* Ignore allocation failure. */ - if (work) { - work->katom = katom; - INIT_WORK(&work->work, kbase_fence_debug_wait_timeout_worker); - queue_work(kctx->jctx.job_done_wq, &work->work); - } -} -#endif /* CONFIG_MALI_VALHALL_FENCE_DEBUG */ - -void kbasep_soft_job_timeout_worker(struct timer_list *timer) -{ - struct kbase_context *kctx = container_of(timer, struct kbase_context, soft_job_timeout); - u32 timeout_ms = (u32)atomic_read(&kctx->kbdev->js_data.soft_job_timeout_ms); - ktime_t cur_time = ktime_get_raw(); - bool restarting = false; - unsigned long lflags; - struct list_head *entry, *tmp; - - spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); - list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) { - struct kbase_jd_atom *katom = list_entry(entry, struct kbase_jd_atom, queue); - s64 elapsed_time = ktime_to_ms(ktime_sub(cur_time, katom->start_timestamp)); - - if (elapsed_time < (s64)timeout_ms) { - restarting = true; - continue; - } - - switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { - case BASE_JD_REQ_SOFT_EVENT_WAIT: - /* Take it out of the list to ensure that it - * will be cancelled in all cases - */ - list_del(&katom->queue); - - katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; - INIT_WORK(&katom->work, kbasep_soft_event_complete_job); - queue_work(kctx->jctx.job_done_wq, &katom->work); - break; -#ifdef CONFIG_MALI_VALHALL_FENCE_DEBUG - case BASE_JD_REQ_SOFT_FENCE_WAIT: - kbase_fence_debug_timeout(katom); - break; -#endif - } - } - - if (restarting) - mod_timer(timer, jiffies + msecs_to_jiffies(timeout_ms)); - spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); -} - -static int kbasep_soft_event_wait(struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx = katom->kctx; - unsigned char status; - - /* The status of this soft-job is stored in jc */ - if (kbasep_read_soft_event_status(kctx, katom->jc, &status)) { - katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; - return 0; - } - - if (status == BASE_JD_SOFT_EVENT_SET) - return 0; /* Event already set, nothing to do */ - - kbasep_add_waiting_with_timeout(katom); - - return 1; -} - -static void kbasep_soft_event_update_locked(struct kbase_jd_atom *katom, unsigned char new_status) -{ - /* Complete jobs waiting on the same event */ - struct kbase_context *kctx = katom->kctx; - - if (kbasep_write_soft_event_status(kctx, katom->jc, new_status) != 0) { - katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; - return; - } - - if (new_status == BASE_JD_SOFT_EVENT_SET) - kbasep_complete_triggered_soft_events(kctx, katom->jc); -} - -/** - * kbase_soft_event_update() - Update soft event state - * @kctx: Pointer to context - * @event: Event to update - * @new_status: New status value of event - * - * Update the event, and wake up any atoms waiting for the event. - * - * Return: 0 on success, a negative error code on failure. - */ -int kbase_soft_event_update(struct kbase_context *kctx, u64 event, unsigned char new_status) -{ - int err = 0; - - mutex_lock(&kctx->jctx.lock); - - if (kbasep_write_soft_event_status(kctx, event, new_status)) { - err = -ENOENT; - goto out; - } - - if (new_status == BASE_JD_SOFT_EVENT_SET) - kbasep_complete_triggered_soft_events(kctx, event); - -out: - mutex_unlock(&kctx->jctx.lock); - - return err; -} - -static void kbasep_soft_event_cancel_job(struct kbase_jd_atom *katom) -{ - katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; - if (kbase_jd_done_nolock(katom, true)) - kbase_js_sched_all(katom->kctx->kbdev); -} - -#if IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST -static void kbase_debug_copy_finish(struct kbase_jd_atom *katom) -{ - struct kbase_debug_copy_buffer *buffers = katom->softjob_data; - unsigned int i; - unsigned int nr = katom->nr_extres; - - if (!buffers) - return; - - kbase_gpu_vm_lock(katom->kctx); - for (i = 0; i < nr; i++) { - uint p; - struct kbase_mem_phy_alloc *gpu_alloc = buffers[i].gpu_alloc; - - if (!buffers[i].pages) - break; - for (p = 0; p < buffers[i].nr_pages; p++) { - struct page *pg = buffers[i].pages[p]; - - if (pg) - put_page(pg); - } - if (buffers[i].is_vmalloc) - vfree(buffers[i].pages); - else - kfree(buffers[i].pages); - if (gpu_alloc) { - switch (gpu_alloc->type) { - case KBASE_MEM_TYPE_IMPORTED_USER_BUF: { - kbase_free_user_buffer(&buffers[i]); - break; - } - default: - /* Nothing to be done. */ - break; - } - kbase_mem_phy_alloc_put(gpu_alloc); - } - } - kbase_gpu_vm_unlock(katom->kctx); - kfree(buffers); - - katom->softjob_data = NULL; -} - -static int kbase_debug_copy_prepare(struct kbase_jd_atom *katom) -{ - struct kbase_debug_copy_buffer *buffers; - struct base_jd_debug_copy_buffer *user_buffers = NULL; - unsigned int i; - unsigned int nr = katom->nr_extres; - int ret = 0; - void __user *user_structs = (void __user *)(uintptr_t)katom->jc; - size_t copy_size; - - if (!user_structs) - return -EINVAL; - - buffers = kcalloc(nr, sizeof(*buffers), GFP_KERNEL); - if (!buffers) { - ret = -ENOMEM; - goto out_cleanup; - } - katom->softjob_data = buffers; - - user_buffers = kmalloc_array(nr, sizeof(*user_buffers), GFP_KERNEL); - - if (!user_buffers) { - ret = -ENOMEM; - goto out_cleanup; - } - - if (check_mul_overflow(sizeof(*user_buffers), (size_t)nr, ©_size)) { - ret = -EINVAL; - goto out_cleanup; - } - - ret = copy_from_user(user_buffers, user_structs, copy_size); - if (ret) { - ret = -EFAULT; - goto out_cleanup; - } - - for (i = 0; i < nr; i++) { - u64 addr = user_buffers[i].address; - u64 page_addr = addr & PAGE_MASK; - u64 end_page_addr = addr + user_buffers[i].size - 1; - u64 last_page_addr = end_page_addr & PAGE_MASK; - size_t nr_pages = (last_page_addr - page_addr) / PAGE_SIZE + 1; - int pinned_pages; - struct kbase_va_region *reg; - struct base_external_resource user_extres; - - if (!addr) - continue; - - if (last_page_addr < page_addr) { - ret = -EINVAL; - goto out_cleanup; - } - - buffers[i].nr_pages = nr_pages; - buffers[i].offset = addr & ~PAGE_MASK; - if (buffers[i].offset >= PAGE_SIZE) { - ret = -EINVAL; - goto out_cleanup; - } - buffers[i].size = user_buffers[i].size; - - if (nr_pages > (KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD / sizeof(struct page *))) { - buffers[i].is_vmalloc = true; - buffers[i].pages = vzalloc(nr_pages * sizeof(struct page *)); - } else { - buffers[i].is_vmalloc = false; - buffers[i].pages = kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL); - } - - if (!buffers[i].pages) { - ret = -ENOMEM; - goto out_cleanup; - } - - pinned_pages = get_user_pages_fast(page_addr, nr_pages, 1, /* Write */ - buffers[i].pages); - if (pinned_pages < 0) { - /* get_user_pages_fast has failed - page array is not - * valid. Don't try to release any pages. - */ - buffers[i].nr_pages = 0; - - ret = pinned_pages; - goto out_cleanup; - } - if ((size_t)pinned_pages != nr_pages) { - /* Adjust number of pages, so that we only attempt to - * release pages in the array that we know are valid. - */ - buffers[i].nr_pages = (unsigned int)pinned_pages; - - ret = -EINVAL; - goto out_cleanup; - } - - user_extres = user_buffers[i].extres; - if (user_extres.ext_resource == 0ULL) { - ret = -EINVAL; - goto out_cleanup; - } - - kbase_gpu_vm_lock(katom->kctx); - reg = kbase_region_tracker_find_region_enclosing_address( - katom->kctx, - user_extres.ext_resource & ~(__u64)BASE_EXT_RES_ACCESS_EXCLUSIVE); - - if (kbase_is_region_invalid_or_free(reg) || reg->gpu_alloc == NULL) { - ret = -EINVAL; - goto out_unlock; - } - - buffers[i].gpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); - buffers[i].nr_extres_pages = reg->nr_pages; - - if (reg->nr_pages * PAGE_SIZE != buffers[i].size) - dev_warn( - katom->kctx->kbdev->dev, - "Copy buffer is not of same size as the external resource to copy.\n"); - - switch (reg->gpu_alloc->type) { - case KBASE_MEM_TYPE_IMPORTED_USER_BUF: { - struct kbase_mem_phy_alloc *alloc = reg->gpu_alloc; - const unsigned long nr_pages = alloc->imported.user_buf.nr_pages; - const unsigned long start = alloc->imported.user_buf.address; - - if (alloc->imported.user_buf.mm != current->mm) { - ret = -EINVAL; - goto out_unlock; - } - buffers[i].extres_pages = - kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL); - if (!buffers[i].extres_pages) { - ret = -ENOMEM; - goto out_unlock; - } - kbase_gpu_vm_unlock(katom->kctx); - ret = get_user_pages_fast(start, nr_pages, 0, buffers[i].extres_pages); - kbase_gpu_vm_lock(katom->kctx); - if (ret != (int)nr_pages) { - /* Adjust number of pages, so that we only - * attempt to release pages in the array that we - * know are valid. - */ - if (ret < 0) - buffers[i].nr_extres_pages = 0; - else - buffers[i].nr_extres_pages = (unsigned int)ret; - - goto out_unlock; - } - ret = 0; - break; - } - default: - /* Nothing to be done. */ - break; - } - kbase_gpu_vm_unlock(katom->kctx); - } - kfree(user_buffers); - - return ret; - -out_unlock: - kbase_gpu_vm_unlock(katom->kctx); - -out_cleanup: - /* Frees allocated memory for kbase_debug_copy_job struct, including - * members, and sets jc to 0 - */ - kbase_debug_copy_finish(katom); - kfree(user_buffers); - - return ret; -} - -#if KERNEL_VERSION(5, 6, 0) <= LINUX_VERSION_CODE -static void *dma_buf_kmap_page(struct kbase_mem_phy_alloc *gpu_alloc, unsigned long page_num, - struct page **page) -{ - struct sg_table *sgt = gpu_alloc->imported.umm.sgt; - struct sg_page_iter sg_iter; - unsigned long page_index = 0; - - if (WARN_ON(gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM)) - return NULL; - - if (!sgt) - return NULL; - - if (WARN_ON(page_num >= gpu_alloc->nents)) - return NULL; - - for_each_sg_page(sgt->sgl, &sg_iter, sgt->nents, 0) { - if (page_index == page_num) { - *page = sg_page_iter_page(&sg_iter); - - return kbase_kmap(*page); - } - page_index++; - } - - return NULL; -} -#endif - -/** - * kbase_mem_copy_from_extres() - Copy from external resources. - * - * @kctx: kbase context within which the copying is to take place. - * @buf_data: Pointer to the information about external resources: - * pages pertaining to the external resource, number of - * pages to copy. - * - * Return: 0 on success, error code otherwise. - */ -static int kbase_mem_copy_from_extres(struct kbase_context *kctx, - struct kbase_debug_copy_buffer *buf_data) -{ - unsigned int i; - unsigned int target_page_nr = 0; - struct page **pages = buf_data->pages; - u64 offset = buf_data->offset; - size_t extres_size = buf_data->nr_extres_pages * PAGE_SIZE; - size_t to_copy = min(extres_size, buf_data->size); - struct kbase_mem_phy_alloc *gpu_alloc = buf_data->gpu_alloc; - int ret = 0; - size_t dma_to_copy; - - KBASE_DEBUG_ASSERT(pages != NULL); - - kbase_gpu_vm_lock(kctx); - if (!gpu_alloc) { - ret = -EINVAL; - goto out_unlock; - } - - switch (gpu_alloc->type) { - case KBASE_MEM_TYPE_IMPORTED_USER_BUF: { - for (i = 0; i < buf_data->nr_extres_pages && target_page_nr < buf_data->nr_pages; - i++) { - struct page *pg = buf_data->extres_pages[i]; - void *extres_page = kbase_kmap(pg); - if (extres_page) { - ret = kbase_mem_copy_to_pinned_user_pages(pages, extres_page, - &to_copy, - buf_data->nr_pages, - &target_page_nr, offset); - kbase_kunmap(pg, extres_page); - if (ret) - goto out_unlock; - } - } - } break; - case KBASE_MEM_TYPE_IMPORTED_UMM: { - struct dma_buf *dma_buf = gpu_alloc->imported.umm.dma_buf; - - KBASE_DEBUG_ASSERT(dma_buf != NULL); - if (dma_buf->size > buf_data->nr_extres_pages * PAGE_SIZE) - dev_warn(kctx->kbdev->dev, "External resources buffer size mismatch"); - - dma_to_copy = min(dma_buf->size, (size_t)(buf_data->nr_extres_pages * PAGE_SIZE)); - ret = dma_buf_begin_cpu_access(dma_buf, DMA_FROM_DEVICE); - if (ret) - goto out_unlock; - - for (i = 0; i < dma_to_copy / PAGE_SIZE && target_page_nr < buf_data->nr_pages; - i++) { -#if KERNEL_VERSION(5, 6, 0) <= LINUX_VERSION_CODE - struct page *pg; - void *extres_page = dma_buf_kmap_page(gpu_alloc, i, &pg); -#else - void *extres_page = dma_buf_kmap(dma_buf, i); -#endif - if (extres_page) { - ret = kbase_mem_copy_to_pinned_user_pages(pages, extres_page, - &to_copy, - buf_data->nr_pages, - &target_page_nr, offset); - -#if KERNEL_VERSION(5, 6, 0) <= LINUX_VERSION_CODE - kbase_kunmap(pg, extres_page); -#else - dma_buf_kunmap(dma_buf, i, extres_page); -#endif - if (ret) - break; - } - } - dma_buf_end_cpu_access(dma_buf, DMA_FROM_DEVICE); - break; - } - default: - ret = -EINVAL; - } -out_unlock: - kbase_gpu_vm_unlock(kctx); - return ret; -} - -static int kbase_debug_copy(struct kbase_jd_atom *katom) -{ - struct kbase_debug_copy_buffer *buffers = katom->softjob_data; - unsigned int i; - - if (WARN_ON(!buffers)) - return -EINVAL; - - for (i = 0; i < katom->nr_extres; i++) { - int res = kbase_mem_copy_from_extres(katom->kctx, &buffers[i]); - - if (res) - return res; - } - - return 0; -} -#endif /* IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST */ -#endif /* !MALI_USE_CSF */ - #define KBASEP_JIT_ALLOC_GPU_ADDR_ALIGNMENT ((u32)0x7) int kbasep_jit_alloc_validate(struct kbase_context *kctx, struct base_jit_alloc_info *info) @@ -910,817 +85,5 @@ int kbasep_jit_alloc_validate(struct kbase_context *kctx, struct base_jit_alloc_ return -EINVAL; #endif -#if !MALI_USE_CSF - /* If BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE is set, heap_info_gpu_addr - * cannot be 0 - */ - if ((info->flags & BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE) && !info->heap_info_gpu_addr) - return -EINVAL; -#endif /* !MALI_USE_CSF */ - return 0; } - -#if !MALI_USE_CSF - -static int kbase_jit_allocate_prepare(struct kbase_jd_atom *katom) -{ - __user u8 *data = (__user u8 *)(uintptr_t)katom->jc; - struct base_jit_alloc_info *info; - struct kbase_context *kctx = katom->kctx; - struct kbase_device *kbdev = kctx->kbdev; - u32 count; - int ret; - u32 i; - - if (!kbase_mem_allow_alloc(kctx)) { - dev_dbg(kbdev->dev, "Invalid attempt to allocate JIT memory by %s/%d for ctx %d_%d", - current->comm, current->pid, kctx->tgid, kctx->id); - ret = -EINVAL; - goto fail; - } - - /* For backwards compatibility, and to prevent reading more than 1 jit - * info struct on jit version 1 - */ - if (katom->nr_extres == 0) - katom->nr_extres = 1; - count = katom->nr_extres; - - /* Sanity checks */ - if (!data || count > kctx->jit_max_allocations || count > ARRAY_SIZE(kctx->jit_alloc)) { - ret = -EINVAL; - goto fail; - } - - /* Copy the information for safe access and future storage */ - info = kmalloc_array(count, sizeof(*info), GFP_KERNEL); - if (!info) { - ret = -ENOMEM; - goto fail; - } - - katom->softjob_data = info; - - for (i = 0; i < count; i++, info++, data += sizeof(*info)) { - if (copy_from_user(info, data, sizeof(*info)) != 0) { - ret = -EINVAL; - goto free_info; - } - - ret = kbasep_jit_alloc_validate(kctx, info); - if (ret) - goto free_info; - KBASE_TLSTREAM_TL_ATTRIB_ATOM_JITALLOCINFO( - kbdev, katom, info->va_pages, info->commit_pages, info->extension, info->id, - info->bin_id, info->max_allocations, info->flags, info->usage_id); - } - - katom->jit_blocked = false; - - lockdep_assert_held(&kctx->jctx.lock); - list_add_tail(&katom->jit_node, &kctx->jctx.jit_atoms_head); - - /* - * Note: - * The provided info->gpu_alloc_addr isn't validated here as - * userland can cache allocations which means that even - * though the region is valid it doesn't represent the - * same thing it used to. - * - * Complete validation of va_pages, commit_pages and extension - * isn't done here as it will be done during the call to - * kbase_mem_alloc. - */ - return 0; - -free_info: - kfree(katom->softjob_data); - katom->softjob_data = NULL; -fail: - return ret; -} - -static u8 *kbase_jit_free_get_ids(struct kbase_jd_atom *katom) -{ - if (WARN_ON((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) != BASE_JD_REQ_SOFT_JIT_FREE)) - return NULL; - - return (u8 *)katom->softjob_data; -} - -static void kbase_jit_add_to_pending_alloc_list(struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx = katom->kctx; - struct list_head *target_list_head = NULL; - struct kbase_jd_atom *entry; - - list_for_each_entry(entry, &kctx->jctx.jit_pending_alloc, queue) { - if (katom->age < entry->age) { - target_list_head = &entry->queue; - break; - } - } - - if (target_list_head == NULL) - target_list_head = &kctx->jctx.jit_pending_alloc; - - list_add_tail(&katom->queue, target_list_head); -} - -static int kbase_jit_allocate_process(struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx = katom->kctx; - struct kbase_device *kbdev = kctx->kbdev; - struct base_jit_alloc_info *info; - struct kbase_va_region *reg; - struct kbase_vmap_struct mapping; - u64 *ptr, new_addr; - u32 count = katom->nr_extres; - u32 i; - bool ignore_pressure_limit = false; - - trace_sysgraph(SGR_SUBMIT, kctx->id, kbase_jd_atom_id(kctx, katom)); - - if (katom->jit_blocked) { - list_del(&katom->queue); - katom->jit_blocked = false; - } - - info = katom->softjob_data; - if (WARN_ON(!info)) { - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - return 0; - } - - for (i = 0; i < count; i++, info++) { - /* The JIT ID is still in use so fail the allocation */ - if (kctx->jit_alloc[info->id]) { - katom->event_code = BASE_JD_EVENT_MEM_GROWTH_FAILED; - return 0; - } - } - -#if MALI_JIT_PRESSURE_LIMIT_BASE - /* - * If this is the only JIT_ALLOC atom in-flight or if JIT pressure limit - * is disabled at the context scope, then bypass JIT pressure limit - * logic in kbase_jit_allocate(). - */ - if (!kbase_ctx_flag(kctx, KCTX_JPL_ENABLED) || (kctx->jit_current_allocations == 0)) { - ignore_pressure_limit = true; - } -#else - ignore_pressure_limit = true; -#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - - for (i = 0, info = katom->softjob_data; i < count; i++, info++) { - if (kctx->jit_alloc[info->id]) { - /* The JIT ID is duplicated in this atom. Roll back - * previous allocations and fail. - */ - u32 j; - - info = katom->softjob_data; - for (j = 0; j < i; j++, info++) { - kbase_jit_free(kctx, kctx->jit_alloc[info->id]); - kctx->jit_alloc[info->id] = KBASE_RESERVED_REG_JIT_ALLOC; - } - - katom->event_code = BASE_JD_EVENT_MEM_GROWTH_FAILED; - return 0; - } - - /* Create a JIT allocation */ - reg = kbase_jit_allocate(kctx, info, ignore_pressure_limit); - if (!reg) { - struct kbase_jd_atom *jit_atom; - bool can_block = false; - - lockdep_assert_held(&kctx->jctx.lock); - - list_for_each_entry(jit_atom, &kctx->jctx.jit_atoms_head, jit_node) { - if (jit_atom == katom) - break; - - if ((jit_atom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == - BASE_JD_REQ_SOFT_JIT_FREE) { - u8 *free_ids = kbase_jit_free_get_ids(jit_atom); - - if (free_ids && *free_ids && kctx->jit_alloc[*free_ids]) { - /* A JIT free which is active and - * submitted before this atom - */ - can_block = true; - break; - } - } - } - - if (!can_block) { - /* Mark the failed allocation as well as the - * other un-attempted allocations in the set, - * so we know they are in use even if the - * allocation itself failed. - */ - for (; i < count; i++, info++) { - kctx->jit_alloc[info->id] = KBASE_RESERVED_REG_JIT_ALLOC; - } - - katom->event_code = BASE_JD_EVENT_MEM_GROWTH_FAILED; - dev_warn_ratelimited(kbdev->dev, - "JIT alloc softjob failed: atom id %d\n", - kbase_jd_atom_id(kctx, katom)); - return 0; - } - - /* There are pending frees for an active allocation - * so we should wait to see whether they free the - * memory. Add to the list of atoms for which JIT - * allocation is pending. - */ - kbase_jit_add_to_pending_alloc_list(katom); - katom->jit_blocked = true; - - /* Rollback, the whole set will be re-attempted */ - while (i-- > 0) { - info--; - kbase_jit_free(kctx, kctx->jit_alloc[info->id]); - kctx->jit_alloc[info->id] = NULL; - } - - return 1; - } - - /* Bind it to the user provided ID. */ - kctx->jit_alloc[info->id] = reg; - } - - for (i = 0, info = katom->softjob_data; i < count; i++, info++) { - u64 entry_mmu_flags = 0; - /* - * Write the address of the JIT allocation to the user provided - * GPU allocation. - */ - ptr = kbase_vmap_prot(kctx, info->gpu_alloc_addr, sizeof(*ptr), KBASE_REG_CPU_WR, - &mapping); - if (!ptr) { - /* - * Leave the allocations "live" as the JIT free atom - * will be submitted anyway. - */ - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - return 0; - } - - reg = kctx->jit_alloc[info->id]; - new_addr = reg->start_pfn << PAGE_SHIFT; - *ptr = new_addr; - -#if defined(CONFIG_MALI_VECTOR_DUMP) - /* - * Retrieve the mmu flags for JIT allocation - * only if dumping is enabled - */ - entry_mmu_flags = kbase_mmu_create_ate(kbdev, (struct tagged_addr){ 0 }, reg->flags, - MIDGARD_MMU_BOTTOMLEVEL, kctx->jit_group_id); -#endif - - KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT(kbdev, katom, info->gpu_alloc_addr, new_addr, - info->flags, entry_mmu_flags, info->id, - info->commit_pages, info->extension, - info->va_pages); - kbase_vunmap(kctx, &mapping); - - kbase_trace_jit_report_gpu_mem(kctx, reg, KBASE_JIT_REPORT_ON_ALLOC_OR_FREE); - } - - katom->event_code = BASE_JD_EVENT_DONE; - - return 0; -} - -static void kbase_jit_allocate_finish(struct kbase_jd_atom *katom) -{ - struct base_jit_alloc_info *info; - - lockdep_assert_held(&katom->kctx->jctx.lock); - - if (WARN_ON(!katom->softjob_data)) - return; - - /* Remove atom from jit_atoms_head list */ - list_del(&katom->jit_node); - - if (katom->jit_blocked) { - list_del(&katom->queue); - katom->jit_blocked = false; - } - - info = katom->softjob_data; - /* Free the info structure */ - kfree(info); -} - -static int kbase_jit_free_prepare(struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx = katom->kctx; - struct kbase_device *kbdev = kctx->kbdev; - __user void *data = (__user void *)(uintptr_t)katom->jc; - u8 *ids; - u32 count = MAX(katom->nr_extres, 1); - u32 i; - int ret; - size_t copy_size; - - /* Sanity checks */ - if (count > ARRAY_SIZE(kctx->jit_alloc)) { - ret = -EINVAL; - goto fail; - } - - /* Copy the information for safe access and future storage */ - ids = kmalloc_array(count, sizeof(*ids), GFP_KERNEL); - if (!ids) { - ret = -ENOMEM; - goto fail; - } - - lockdep_assert_held(&kctx->jctx.lock); - katom->softjob_data = ids; - - /* For backwards compatibility */ - if (katom->nr_extres) { - /* Fail the job if there is no list of ids */ - if (!data) { - ret = -EINVAL; - goto free_info; - } - if (check_mul_overflow(sizeof(*ids), (size_t)count, ©_size)) { - ret = -EINVAL; - goto free_info; - } - - if (copy_from_user(ids, data, copy_size) != 0) { - ret = -EINVAL; - goto free_info; - } - } else { - katom->nr_extres = 1; - *ids = (u8)katom->jc; - } - for (i = 0; i < count; i++) - KBASE_TLSTREAM_TL_ATTRIB_ATOM_JITFREEINFO(kbdev, katom, ids[i]); - - list_add_tail(&katom->jit_node, &kctx->jctx.jit_atoms_head); - - return 0; - -free_info: - kfree(katom->softjob_data); - katom->softjob_data = NULL; -fail: - return ret; -} - -static void kbase_jit_free_process(struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx = katom->kctx; - u8 *ids = kbase_jit_free_get_ids(katom); - u32 count = katom->nr_extres; - u32 i; - - if (ids == NULL) { - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - return; - } - - for (i = 0; i < count; i++, ids++) { - /* - * If the ID is zero or it is not in use yet then fail the job. - */ - if ((*ids == 0) || (kctx->jit_alloc[*ids] == NULL)) { - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - return; - } - } -} - -static void kbasep_jit_finish_worker(struct work_struct *work) -{ - struct kbase_jd_atom *katom = container_of(work, struct kbase_jd_atom, work); - struct kbase_context *kctx = katom->kctx; - int resched; - - mutex_lock(&kctx->jctx.lock); - kbase_finish_soft_job(katom); - resched = kbase_jd_done_nolock(katom, true); - mutex_unlock(&kctx->jctx.lock); - - if (resched) - kbase_js_sched_all(kctx->kbdev); -} - -void kbase_jit_retry_pending_alloc(struct kbase_context *kctx) -{ - LIST_HEAD(jit_pending_alloc_list); - struct list_head *i, *tmp; - - list_splice_tail_init(&kctx->jctx.jit_pending_alloc, &jit_pending_alloc_list); - - list_for_each_safe(i, tmp, &jit_pending_alloc_list) { - struct kbase_jd_atom *pending_atom = list_entry(i, struct kbase_jd_atom, queue); - KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_START(kctx->kbdev, pending_atom); - kbase_kinstr_jm_atom_sw_start(pending_atom); - if (kbase_jit_allocate_process(pending_atom) == 0) { - /* Atom has completed */ - INIT_WORK(&pending_atom->work, kbasep_jit_finish_worker); - queue_work(kctx->jctx.job_done_wq, &pending_atom->work); - } - KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_END(kctx->kbdev, pending_atom); - kbase_kinstr_jm_atom_sw_stop(pending_atom); - } -} - -static void kbase_jit_free_finish(struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx = katom->kctx; - u8 *ids; - size_t j; - - lockdep_assert_held(&kctx->jctx.lock); - - ids = kbase_jit_free_get_ids(katom); - if (WARN_ON(ids == NULL)) - return; - - /* Remove this atom from the jit_atoms_head list */ - list_del(&katom->jit_node); - - for (j = 0; j != katom->nr_extres; ++j) { - if ((ids[j] != 0) && (kctx->jit_alloc[ids[j]] != NULL)) { - /* - * If the ID is valid but the allocation request failed - * still succeed this soft job but don't try and free - * the allocation. - */ - if (kctx->jit_alloc[ids[j]] != KBASE_RESERVED_REG_JIT_ALLOC) { - KBASE_TLSTREAM_TL_JIT_USEDPAGES( - kctx->kbdev, kctx->jit_alloc[ids[j]]->gpu_alloc->nents, - ids[j]); - kbase_jit_free(kctx, kctx->jit_alloc[ids[j]]); - } - kctx->jit_alloc[ids[j]] = NULL; - } - } - /* Free the list of ids */ - kfree(ids); - - kbase_jit_retry_pending_alloc(kctx); -} - -static int kbase_ext_res_prepare(struct kbase_jd_atom *katom) -{ - __user struct base_external_resource_list *user_ext_res; - struct kbase_external_resource_list *ext_res; - u64 count = 0; - size_t copy_size; - - user_ext_res = (__user struct base_external_resource_list *)(uintptr_t)katom->jc; - - /* Fail the job if there is no info structure */ - if (!user_ext_res) - return -EINVAL; - - if (copy_from_user(&count, &user_ext_res->count, sizeof(u64)) != 0) - return -EINVAL; - - /* Is the number of external resources in range? */ - if (!count || count > BASE_EXT_RES_COUNT_MAX) - return -EINVAL; - - /* Copy the information for safe access and future storage */ - copy_size = sizeof(struct base_external_resource) * count; - ext_res = kmalloc(sizeof(*ext_res) + copy_size, GFP_KERNEL); - if (!ext_res) - return -ENOMEM; - - ext_res->count = count; - ext_res->ext_res = (struct base_external_resource *)(ext_res + 1); - if (copy_from_user(ext_res->ext_res, user_ext_res->ext_res, copy_size) != 0) { - kfree(ext_res); - return -EINVAL; - } - - katom->softjob_data = ext_res; - - return 0; -} - -static void kbase_ext_res_process(struct kbase_jd_atom *katom, bool map) -{ - struct kbase_external_resource_list *ext_res; - uint64_t i; - bool failed = false; - - ext_res = katom->softjob_data; - if (!ext_res) - goto failed_jc; - - kbase_gpu_vm_lock(katom->kctx); - - for (i = 0; i < ext_res->count; i++) { - u64 gpu_addr; - - gpu_addr = ext_res->ext_res[i].ext_resource & ~(__u64)BASE_EXT_RES_ACCESS_EXCLUSIVE; - if (map) { - if (!kbase_sticky_resource_acquire(katom->kctx, gpu_addr, NULL)) - goto failed_loop; - } else { - if (!kbase_sticky_resource_release_force(katom->kctx, NULL, gpu_addr)) - failed = true; - } - } - - /* - * In the case of unmap we continue unmapping other resources in the - * case of failure but will always report failure if _any_ unmap - * request fails. - */ - if (failed) - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - else - katom->event_code = BASE_JD_EVENT_DONE; - - kbase_gpu_vm_unlock(katom->kctx); - - return; - -failed_loop: - while (i > 0) { - u64 const gpu_addr = ext_res->ext_res[i - 1].ext_resource & - ~(__u64)BASE_EXT_RES_ACCESS_EXCLUSIVE; - - kbase_sticky_resource_release_force(katom->kctx, NULL, gpu_addr); - - --i; - } - - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - kbase_gpu_vm_unlock(katom->kctx); - -failed_jc: - return; -} - -static void kbase_ext_res_finish(struct kbase_jd_atom *katom) -{ - struct kbase_external_resource_list *ext_res; - - ext_res = katom->softjob_data; - /* Free the info structure */ - kfree(ext_res); -} - -int kbase_process_soft_job(struct kbase_jd_atom *katom) -{ - int ret = 0; - struct kbase_context *kctx = katom->kctx; - struct kbase_device *kbdev = kctx->kbdev; - - KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_START(kbdev, katom); - kbase_kinstr_jm_atom_sw_start(katom); - - trace_sysgraph(SGR_SUBMIT, kctx->id, kbase_jd_atom_id(kctx, katom)); - - switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { - case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: - ret = kbase_dump_cpu_gpu_time(katom); - break; - -#if IS_ENABLED(CONFIG_SYNC_FILE) - case BASE_JD_REQ_SOFT_FENCE_TRIGGER: - katom->event_code = kbase_sync_fence_out_trigger( - katom, katom->event_code == BASE_JD_EVENT_DONE ? 0 : -EFAULT); - break; - case BASE_JD_REQ_SOFT_FENCE_WAIT: { - ret = kbase_sync_fence_in_wait(katom); - - if (ret == 1) { -#ifdef CONFIG_MALI_VALHALL_FENCE_DEBUG - kbasep_add_waiting_with_timeout(katom); -#else - kbasep_add_waiting_soft_job(katom); -#endif - } - break; - } -#endif - case BASE_JD_REQ_SOFT_EVENT_WAIT: - ret = kbasep_soft_event_wait(katom); - break; - case BASE_JD_REQ_SOFT_EVENT_SET: - kbasep_soft_event_update_locked(katom, BASE_JD_SOFT_EVENT_SET); - break; - case BASE_JD_REQ_SOFT_EVENT_RESET: - kbasep_soft_event_update_locked(katom, BASE_JD_SOFT_EVENT_RESET); - break; -#if IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST - case BASE_JD_REQ_SOFT_DEBUG_COPY: { - int res = kbase_debug_copy(katom); - - if (res) - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - break; - } -#endif /* IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST */ - case BASE_JD_REQ_SOFT_JIT_ALLOC: - ret = kbase_jit_allocate_process(katom); - break; - case BASE_JD_REQ_SOFT_JIT_FREE: - kbase_jit_free_process(katom); - break; - case BASE_JD_REQ_SOFT_EXT_RES_MAP: - kbase_ext_res_process(katom, true); - break; - case BASE_JD_REQ_SOFT_EXT_RES_UNMAP: - kbase_ext_res_process(katom, false); - break; - } - - /* Atom is complete */ - KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_END(kbdev, katom); - kbase_kinstr_jm_atom_sw_stop(katom); - return ret; -} - -void kbase_cancel_soft_job(struct kbase_jd_atom *katom) -{ - switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { -#if IS_ENABLED(CONFIG_SYNC_FILE) - case BASE_JD_REQ_SOFT_FENCE_WAIT: - kbase_sync_fence_in_cancel_wait(katom); - break; -#endif - case BASE_JD_REQ_SOFT_EVENT_WAIT: - kbasep_soft_event_cancel_job(katom); - break; - default: - /* This soft-job doesn't support cancellation! */ - KBASE_DEBUG_ASSERT(0); - } -} - -int kbase_prepare_soft_job(struct kbase_jd_atom *katom) -{ - switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { - case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: { - if (!IS_ALIGNED(katom->jc, cache_line_size())) - return -EINVAL; - } break; -#if IS_ENABLED(CONFIG_SYNC_FILE) - case BASE_JD_REQ_SOFT_FENCE_TRIGGER: { - struct base_fence fence; - int fd; - - if (copy_from_user(&fence, (__user void *)(uintptr_t)katom->jc, sizeof(fence)) != 0) - return -EINVAL; - - fd = kbase_sync_fence_out_create(katom, fence.basep.stream_fd); - if (fd < 0) - return -EINVAL; - - fence.basep.fd = fd; - if (copy_to_user((__user void *)(uintptr_t)katom->jc, &fence, sizeof(fence)) != 0) { - kbase_sync_fence_out_remove(katom); - /* fd should have been closed here, but there's - * no good way of doing that. Since - * copy_to_user() very rarely fails, and the fd - * will get closed on process termination this - * won't be a problem. - */ - fence.basep.fd = -EINVAL; - return -EINVAL; - } - } break; - case BASE_JD_REQ_SOFT_FENCE_WAIT: { - struct base_fence fence; - int ret; - - if (copy_from_user(&fence, (__user void *)(uintptr_t)katom->jc, sizeof(fence)) != 0) - return -EINVAL; - - /* Get a reference to the fence object */ - ret = kbase_sync_fence_in_from_fd(katom, fence.basep.fd); - if (ret < 0) - return ret; - } break; -#endif /* CONFIG_SYNC_FILE */ - case BASE_JD_REQ_SOFT_JIT_ALLOC: - return kbase_jit_allocate_prepare(katom); - case BASE_JD_REQ_SOFT_JIT_FREE: - return kbase_jit_free_prepare(katom); - case BASE_JD_REQ_SOFT_EVENT_WAIT: - case BASE_JD_REQ_SOFT_EVENT_SET: - case BASE_JD_REQ_SOFT_EVENT_RESET: - if (katom->jc == 0) - return -EINVAL; - break; -#if IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST - case BASE_JD_REQ_SOFT_DEBUG_COPY: - return kbase_debug_copy_prepare(katom); -#endif /* IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST */ - case BASE_JD_REQ_SOFT_EXT_RES_MAP: - return kbase_ext_res_prepare(katom); - case BASE_JD_REQ_SOFT_EXT_RES_UNMAP: - return kbase_ext_res_prepare(katom); - default: - /* Unsupported soft-job */ - return -EINVAL; - } - return 0; -} - -void kbase_finish_soft_job(struct kbase_jd_atom *katom) -{ - trace_sysgraph(SGR_COMPLETE, katom->kctx->id, kbase_jd_atom_id(katom->kctx, katom)); - - switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { - case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: - /* Nothing to do */ - break; -#if IS_ENABLED(CONFIG_SYNC_FILE) - case BASE_JD_REQ_SOFT_FENCE_TRIGGER: - /* If fence has not yet been signaled, do it now */ - kbase_sync_fence_out_trigger(katom, - katom->event_code == BASE_JD_EVENT_DONE ? 0 : -EFAULT); - break; - case BASE_JD_REQ_SOFT_FENCE_WAIT: - /* Release katom's reference to fence object */ - kbase_sync_fence_in_remove(katom); - break; -#endif /* CONFIG_SYNC_FILE */ -#if IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST - case BASE_JD_REQ_SOFT_DEBUG_COPY: - kbase_debug_copy_finish(katom); - break; -#endif /* IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST */ - case BASE_JD_REQ_SOFT_JIT_ALLOC: - kbase_jit_allocate_finish(katom); - break; - case BASE_JD_REQ_SOFT_EXT_RES_MAP: - kbase_ext_res_finish(katom); - break; - case BASE_JD_REQ_SOFT_EXT_RES_UNMAP: - kbase_ext_res_finish(katom); - break; - case BASE_JD_REQ_SOFT_JIT_FREE: - kbase_jit_free_finish(katom); - break; - } -} - -void kbase_resume_suspended_soft_jobs(struct kbase_device *kbdev) -{ - LIST_HEAD(local_suspended_soft_jobs); - struct kbase_jd_atom *tmp_iter; - struct kbase_jd_atom *katom_iter; - struct kbasep_js_device_data *js_devdata; - bool resched = false; - - KBASE_DEBUG_ASSERT(kbdev); - - js_devdata = &kbdev->js_data; - - /* Move out the entire list */ - mutex_lock(&js_devdata->runpool_mutex); - list_splice_init(&js_devdata->suspended_soft_jobs_list, &local_suspended_soft_jobs); - mutex_unlock(&js_devdata->runpool_mutex); - - /* - * Each atom must be detached from the list and ran separately - - * it could be re-added to the old list, but this is unlikely - */ - list_for_each_entry_safe(katom_iter, tmp_iter, &local_suspended_soft_jobs, dep_item[1]) { - struct kbase_context *kctx = katom_iter->kctx; - - mutex_lock(&kctx->jctx.lock); - - /* Remove from the global list */ - list_del(&katom_iter->dep_item[1]); - /* Remove from the context's list of waiting soft jobs */ - kbasep_remove_waiting_soft_job(katom_iter); - - if (kbase_process_soft_job(katom_iter) == 0) { - kbase_finish_soft_job(katom_iter); - resched |= kbase_jd_done_nolock(katom_iter, true); - if (kbase_has_arbiter(kctx->kbdev)) - atomic_dec(&kbdev->pm.gpu_users_waiting); - } - mutex_unlock(&kctx->jctx.lock); - } - - if (resched) - kbase_js_sched_all(kbdev); -} -#endif /* !MALI_USE_CSF */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_sync.h b/drivers/gpu/arm/valhall/mali_kbase_sync.h index 350296c5da12..7a91a36471be 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_sync.h +++ b/drivers/gpu/arm/valhall/mali_kbase_sync.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2012-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2012-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -36,11 +36,6 @@ #include -#if !MALI_USE_CSF -struct kbase_jd_atom; -struct work_struct; -#endif - /** * struct kbase_sync_fence_info - Information about a fence * @fence: Pointer to fence (type is void*, as underlaying struct can differ) @@ -71,30 +66,6 @@ struct kbase_sync_fence_info { */ int kbase_sync_fence_stream_create(const char *name, int *const out_fd); -#if !MALI_USE_CSF -/** - * kbase_sync_fence_out_create - Create an explicit output fence to specified atom - * - * @katom: Atom to assign the new explicit fence to - * @stream_fd: File descriptor for stream object to create fence on - * - * Return: Valid file descriptor to fence or < 0 on error - */ -int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int stream_fd); - -/** - * kbase_sync_fence_in_from_fd() - Assigns an existing fence to specified atom - * @katom: Atom to assign the existing explicit fence to - * @fd: File descriptor to an existing fence - * - * Assigns an explicit input fence to atom. - * This can later be waited for by calling @kbase_sync_fence_in_wait - * - * Return: 0 on success, < 0 on error - */ -int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd); -#endif /* !MALI_USE_CSF */ - /** * kbase_sync_fence_validate() - Validate a fd to be a valid fence * @@ -107,78 +78,6 @@ int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd); */ int kbase_sync_fence_validate(int fd); -#if !MALI_USE_CSF -/** - * kbase_sync_fence_out_trigger - Signal explicit output fence attached on katom - * @katom: Atom with an explicit fence to signal - * @result: < 0 means signal with error, 0 >= indicates success - * - * Signal output fence attached on katom and remove the fence from the atom. - * - * Return: The "next" event code for atom, typically JOB_CANCELLED or EVENT_DONE - */ -enum base_jd_event_code kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result); - -/** - * kbase_sync_fence_in_wait() - Wait for explicit input fence to be signaled - * @katom: Atom with explicit fence to wait for - * - * If the fence is already signaled, then 0 is returned, and the caller must - * continue processing of the katom. - * - * If the fence isn't already signaled, then this kbase_sync framework will - * take responsibility to continue the processing once the fence is signaled. - * - * Return: 0 if already signaled, otherwise 1 - */ -int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom); - -/** - * kbase_sync_fence_in_cancel_wait() - Cancel explicit input fence waits - * @katom: Atom to cancel wait for - * - * This function is fully responsible for continuing processing of this atom - * (remove_waiting_soft_job + finish_soft_job + jd_done + js_sched_all) - */ -void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom); - -/** - * kbase_sync_fence_in_remove() - Remove the input fence from the katom - * @katom: Atom to remove explicit input fence for - * - * This will also release the corresponding reference. - */ -void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom); - -/** - * kbase_sync_fence_out_remove() - Remove the output fence from the katom - * @katom: Atom to remove explicit output fence for - * - * This will also release the corresponding reference. - */ -void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom); -#endif /* !MALI_USE_CSF */ - -#if !MALI_USE_CSF -/** - * kbase_sync_fence_in_info_get() - Retrieves information about input fence - * @katom: Atom to get fence information from - * @info: Struct to be filled with fence information - * - * Return: 0 on success, < 0 on error - */ -int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, struct kbase_sync_fence_info *info); - -/** - * kbase_sync_fence_out_info_get() - Retrieves information about output fence - * @katom: Atom to get fence information from - * @info: Struct to be filled with fence information - * - * Return: 0 on success, < 0 on error - */ -int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, struct kbase_sync_fence_info *info); -#endif /* !MALI_USE_CSF */ - #if IS_ENABLED(CONFIG_SYNC_FILE) void kbase_sync_fence_info_get(struct dma_fence *fence, struct kbase_sync_fence_info *info); #endif @@ -191,20 +90,4 @@ void kbase_sync_fence_info_get(struct dma_fence *fence, struct kbase_sync_fence_ */ const char *kbase_sync_status_string(int status); -#if !MALI_USE_CSF -/* - * Internal worker used to continue processing of atom. - */ -void kbase_sync_fence_wait_worker(struct work_struct *data); - -#ifdef CONFIG_MALI_VALHALL_FENCE_DEBUG -/** - * kbase_sync_fence_in_dump() - Trigger a debug dump of atoms input fence state - * - * @katom: Atom to trigger fence debug dump for - */ -void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom); -#endif -#endif /* !MALI_USE_CSF */ - #endif /* MALI_KBASE_SYNC_H */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_sync_common.c b/drivers/gpu/arm/valhall/mali_kbase_sync_common.c index 5ee7fc3ce72b..05b896f4616f 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_sync_common.c +++ b/drivers/gpu/arm/valhall/mali_kbase_sync_common.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2012-2016, 2018-2021 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2012-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -29,16 +29,6 @@ #include "mali_kbase.h" #include "mali_kbase_sync.h" -#if !MALI_USE_CSF -void kbase_sync_fence_wait_worker(struct work_struct *data) -{ - struct kbase_jd_atom *katom; - - katom = container_of(data, struct kbase_jd_atom, work); - kbase_soft_event_wait_callback(katom); -} -#endif /* !MALI_USE_CSF */ - const char *kbase_sync_status_string(int status) { if (status == 0) diff --git a/drivers/gpu/arm/valhall/mali_kbase_sync_file.c b/drivers/gpu/arm/valhall/mali_kbase_sync_file.c index 38cd63d7a15b..940bcd034f58 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_sync_file.c +++ b/drivers/gpu/arm/valhall/mali_kbase_sync_file.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2012-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2012-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -52,66 +52,6 @@ int kbase_sync_fence_stream_create(const char *name, int *const out_fd) return 0; } -#if !MALI_USE_CSF -int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int stream_fd) -{ - struct dma_fence *fence; - struct sync_file *sync_file; - int fd; - - CSTD_UNUSED(stream_fd); - - fence = kbase_fence_out_new(katom); - if (!fence) - return -ENOMEM; - -#if (KERNEL_VERSION(4, 9, 67) >= LINUX_VERSION_CODE) - /* Take an extra reference to the fence on behalf of the sync_file. - * This is only needed on older kernels where sync_file_create() - * does not take its own reference. This was changed in v4.9.68, - * where sync_file_create() now takes its own reference. - */ - dma_fence_get(fence); -#endif - - /* create a sync_file fd representing the fence */ - sync_file = sync_file_create(fence); - if (!sync_file) { -#if (KERNEL_VERSION(4, 9, 67) >= LINUX_VERSION_CODE) - dma_fence_put(fence); -#endif - kbase_fence_out_remove(katom); - return -ENOMEM; - } - - fd = get_unused_fd_flags(O_CLOEXEC); - if (fd < 0) { - fput(sync_file->file); - kbase_fence_out_remove(katom); - return fd; - } - - fd_install((unsigned int)fd, sync_file->file); - - return fd; -} - -int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd) -{ - struct dma_fence *fence = sync_file_get_fence(fd); - - lockdep_assert_held(&katom->kctx->jctx.lock); - - if (!fence) - return -ENOENT; - - kbase_fence_fence_in_set(katom, fence); - katom->dma_fence.fence_cb_added = false; - - return 0; -} -#endif /* !MALI_USE_CSF */ - int kbase_sync_fence_validate(int fd) { struct dma_fence *fence = sync_file_get_fence(fd); @@ -124,175 +64,6 @@ int kbase_sync_fence_validate(int fd) return 0; } -#if !MALI_USE_CSF -enum base_jd_event_code kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result) -{ - int res; - - if (!kbase_fence_out_is_ours(katom)) { - /* Not our fence */ - return BASE_JD_EVENT_JOB_CANCELLED; - } - - res = kbase_fence_out_signal(katom, result); - if (unlikely(res < 0)) { - dev_warn(katom->kctx->kbdev->dev, "fence_signal() failed with %d\n", res); - } - - kbase_sync_fence_out_remove(katom); - - return (result != 0) ? BASE_JD_EVENT_JOB_CANCELLED : BASE_JD_EVENT_DONE; -} - -static void kbase_fence_wait_callback(struct dma_fence *fence, struct dma_fence_cb *cb) -{ - struct kbase_jd_atom *katom = container_of(cb, struct kbase_jd_atom, dma_fence.fence_cb); - struct kbase_context *kctx = katom->kctx; - - CSTD_UNUSED(fence); - - /* Cancel atom if fence is erroneous */ - if (dma_fence_is_signaled(katom->dma_fence.fence_in) && -#if (KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE || \ - (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE && \ - KERNEL_VERSION(4, 9, 68) <= LINUX_VERSION_CODE)) - katom->dma_fence.fence_in->error < 0) -#else - katom->dma_fence.fence_in->status < 0) -#endif - katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; - - /* To prevent a potential deadlock we schedule the work onto the - * job_done_wq workqueue - * - * The issue is that we may signal the timeline while holding - * kctx->jctx.lock and the callbacks are run synchronously from - * sync_timeline_signal. So we simply defer the work. - */ - INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); - queue_work(kctx->jctx.job_done_wq, &katom->work); -} - -int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom) -{ - int err; - struct dma_fence *fence; - - lockdep_assert_held(&katom->kctx->jctx.lock); - - fence = katom->dma_fence.fence_in; - if (!fence) - return 0; /* no input fence to wait for, good to go! */ - - err = dma_fence_add_callback(fence, &katom->dma_fence.fence_cb, kbase_fence_wait_callback); - if (err == -ENOENT) { - int fence_status = dma_fence_get_status(fence); - - if (fence_status == 1) { - /* Fence is already signaled with no error. The completion - * for FENCE_WAIT softjob can be done right away. - */ - return 0; - } - - /* Fence shouldn't be in not signaled state */ - if (!fence_status) { - struct kbase_sync_fence_info info; - - kbase_sync_fence_in_info_get(katom, &info); - - dev_warn(katom->kctx->kbdev->dev, - "Unexpected status for fence %s of ctx:%d_%d atom:%d", info.name, - katom->kctx->tgid, katom->kctx->id, - kbase_jd_atom_id(katom->kctx, katom)); - } - - /* If fence is signaled with an error, then the FENCE_WAIT softjob is - * considered to be failed. - */ - } - - if (unlikely(err)) { - /* We should cause the dependent jobs in the bag to be failed. */ - katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; - - /* The completion for FENCE_WAIT softjob can be done right away. */ - return 0; - } - - /* Callback was successfully installed */ - katom->dma_fence.fence_cb_added = true; - - /* Completion to be done later by callback/worker */ - return 1; -} - -void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom) -{ - lockdep_assert_held(&katom->kctx->jctx.lock); - - if (katom->dma_fence.fence_cb_added) { - if (!dma_fence_remove_callback(katom->dma_fence.fence_in, - &katom->dma_fence.fence_cb)) { - /* The callback is already removed so leave the cleanup - * for kbase_fence_wait_callback. - */ - return; - } - } else { - struct kbase_sync_fence_info info; - - kbase_sync_fence_in_info_get(katom, &info); - dev_warn(katom->kctx->kbdev->dev, - "Callback was not added earlier for fence %s of ctx:%d_%d atom:%d", - info.name, katom->kctx->tgid, katom->kctx->id, - kbase_jd_atom_id(katom->kctx, katom)); - } - - /* Wait was cancelled - zap the atoms */ - katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; - - kbasep_remove_waiting_soft_job(katom); - kbase_finish_soft_job(katom); - - if (kbase_jd_done_nolock(katom, true)) - kbase_js_sched_all(katom->kctx->kbdev); -} - -void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom) -{ - kbase_fence_out_remove(katom); -} - -void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom) -{ - lockdep_assert_held(&katom->kctx->jctx.lock); - - if (katom->dma_fence.fence_cb_added) { - bool removed = dma_fence_remove_callback(katom->dma_fence.fence_in, - &katom->dma_fence.fence_cb); - - /* Here it is expected that the callback should have already been removed - * previously either by kbase_sync_fence_in_cancel_wait() or when the fence - * was signaled and kbase_sync_fence_wait_worker() was called. - */ - if (removed) { - struct kbase_sync_fence_info info; - - kbase_sync_fence_in_info_get(katom, &info); - dev_warn( - katom->kctx->kbdev->dev, - "Callback was not removed earlier for fence %s of ctx:%d_%d atom:%d", - info.name, katom->kctx->tgid, katom->kctx->id, - kbase_jd_atom_id(katom->kctx, katom)); - } - } - - kbase_fence_in_remove(katom); - katom->dma_fence.fence_cb_added = false; -} -#endif /* !MALI_USE_CSF */ - void kbase_sync_fence_info_get(struct dma_fence *fence, struct kbase_sync_fence_info *info) { int status; @@ -318,43 +89,3 @@ void kbase_sync_fence_info_get(struct dma_fence *fence, struct kbase_sync_fence_ scnprintf(info->name, sizeof(info->name), "%llu#%llu", fence->context, fence->seqno); #endif } - -#if !MALI_USE_CSF -int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, struct kbase_sync_fence_info *info) -{ - struct dma_fence *fence; - - fence = kbase_fence_in_get(katom); - if (!fence) - return -ENOENT; - - kbase_sync_fence_info_get(fence, info); - - kbase_fence_put(fence); - - return 0; -} - -int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, struct kbase_sync_fence_info *info) -{ - struct dma_fence *fence; - - fence = kbase_fence_out_get(katom); - if (!fence) - return -ENOENT; - - kbase_sync_fence_info_get(fence, info); - - kbase_fence_put(fence); - - return 0; -} - -#ifdef CONFIG_MALI_VALHALL_FENCE_DEBUG -void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom) -{ - /* Not implemented */ - CSTD_UNUSED(katom); -} -#endif -#endif /* !MALI_USE_CSF*/ diff --git a/drivers/gpu/arm/valhall/mali_linux_trace.h b/drivers/gpu/arm/valhall/mali_linux_trace.h index 4dd5f437dad2..1b73153f6c9d 100644 --- a/drivers/gpu/arm/valhall/mali_linux_trace.h +++ b/drivers/gpu/arm/valhall/mali_linux_trace.h @@ -137,7 +137,6 @@ TRACE_EVENT(mali_total_alloc_pages_change, TP_PROTO(u32 gpu_id, s64 event_id), __print_symbolic(KBASE_MMU_FAULT_STATUS_ACCESS(status), \ KBASE_MMU_FAULT_ACCESS_SYMBOLIC_STRINGS) -#if MALI_USE_CSF #define KBASE_MMU_FAULT_CODE_VALID(code) \ ((code >= 0xC0 && code <= 0xEB) && (!(code >= 0xC5 && code <= 0xC7)) && \ (!(code >= 0xCC && code <= 0xD8)) && (!(code >= 0xDC && code <= 0xDF)) && \ @@ -148,20 +147,6 @@ TRACE_EVENT(mali_total_alloc_pages_change, TP_PROTO(u32 gpu_id, s64 event_id), { 0xD8, "ACCESS_FLAG_" }, { 0xE0, "ADDRESS_SIZE_FAULT_IN" }, \ { 0xE4, "ADDRESS_SIZE_FAULT_OUT" }, \ { 0xE8, "MEMORY_ATTRIBUTES_FAULT_" }) -#else /* MALI_USE_CSF */ -#define KBASE_MMU_FAULT_CODE_VALID(code) \ - ((code >= 0xC0 && code <= 0xEF) && (!(code >= 0xC5 && code <= 0xC6)) && \ - (!(code >= 0xCC && code <= 0xCF)) && (!(code >= 0xD4 && code <= 0xD7)) && \ - (!(code >= 0xDC && code <= 0xDF))) -#define KBASE_MMU_FAULT_CODE_SYMBOLIC_STRINGS \ - _ENSURE_PARENTHESIS({ 0xC0, "TRANSLATION_FAULT_" }, \ - { 0xC4, "TRANSLATION_FAULT(_7==_IDENTITY)_" }, \ - { 0xC8, "PERMISSION_FAULT_" }, { 0xD0, "TRANSTAB_BUS_FAULT_" }, \ - { 0xD8, "ACCESS_FLAG_" }, { 0xE0, "ADDRESS_SIZE_FAULT_IN" }, \ - { 0xE4, "ADDRESS_SIZE_FAULT_OUT" }, \ - { 0xE8, "MEMORY_ATTRIBUTES_FAULT_" }, \ - { 0xEC, "MEMORY_ATTRIBUTES_NONCACHEABLE_" }) -#endif /* MALI_USE_CSF */ #endif /* __TRACE_MALI_MMU_HELPERS */ /* trace_mali_mmu_page_fault_extra_grow @@ -255,35 +240,6 @@ DEFINE_EVENT_PRINT(mali_jit_softjob_template, mali_jit_free, TP_printk("start=0x%llx va_pages=0x%zx backed_size=0x%zx", __entry->start_addr, __entry->nr_pages, __entry->backed_pages)); -#if !MALI_USE_CSF -#if MALI_JIT_PRESSURE_LIMIT_BASE -/* trace_mali_jit_report - * - * Tracepoint about the GPU data structure read to form a just-in-time memory - * allocation report, and its calculated physical page usage - */ -TRACE_EVENT(mali_jit_report, - TP_PROTO(struct kbase_jd_atom *katom, struct kbase_va_region *reg, unsigned int id_idx, - u64 read_val, u64 used_pages), - TP_ARGS(katom, reg, id_idx, read_val, used_pages), - TP_STRUCT__entry(__field(u64, start_addr) __field(u64, read_val) - __field(u64, used_pages) __field(unsigned long, flags) - __field(u8, id_idx) __field(u8, jit_id)), - TP_fast_assign(__entry->start_addr = ((u64)reg->start_pfn) << PAGE_SHIFT; - __entry->read_val = read_val; __entry->used_pages = used_pages; - __entry->flags = reg->flags; __entry->id_idx = id_idx; - __entry->jit_id = katom->jit_ids[id_idx];), - TP_printk("start=0x%llx jit_ids[%u]=%u read_type='%s' read_val=0x%llx used_pages=%llu", - __entry->start_addr, __entry->id_idx, __entry->jit_id, - __print_symbolic(__entry->flags, { 0, "address" }, - { KBASE_REG_TILER_ALIGN_TOP, "address with align" }, - { KBASE_REG_HEAP_INFO_IS_SIZE, "size" }, - { KBASE_REG_HEAP_INFO_IS_SIZE | KBASE_REG_TILER_ALIGN_TOP, - "size with align (invalid)" }), - __entry->read_val, __entry->used_pages)); -#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ -#endif /* !MALI_USE_CSF */ - TRACE_DEFINE_ENUM(KBASE_JIT_REPORT_ON_ALLOC_OR_FREE); #if MALI_JIT_PRESSURE_LIMIT_BASE /* trace_mali_jit_report_pressure diff --git a/drivers/gpu/arm/valhall/mmu/Kbuild b/drivers/gpu/arm/valhall/mmu/Kbuild index bedae1d67275..ac726539f4b1 100644 --- a/drivers/gpu/arm/valhall/mmu/Kbuild +++ b/drivers/gpu/arm/valhall/mmu/Kbuild @@ -23,13 +23,6 @@ valhall_kbase-y += \ mmu/mali_kbase_mmu_hw_direct.o \ mmu/mali_kbase_mmu_faults_decoder_luts.o \ mmu/mali_kbase_mmu_faults_decoder.o \ - mmu/mali_kbase_mmu_mode_aarch64.o - -ifeq ($(CONFIG_MALI_VALHALL_CSF_SUPPORT),y) - valhall_kbase-y += mmu/backend/mali_kbase_mmu_csf.o \ + mmu/mali_kbase_mmu_mode_aarch64.o \ + mmu/backend/mali_kbase_mmu_csf.o \ mmu/backend/mali_kbase_mmu_faults_decoder_luts_csf.o -else - valhall_kbase-y += mmu/backend/mali_kbase_mmu_jm.o \ - mmu/backend/mali_kbase_mmu_faults_decoder_luts_jm.o - -endif diff --git a/drivers/gpu/arm/valhall/mmu/backend/mali_kbase_mmu_faults_decoder_luts_jm.c b/drivers/gpu/arm/valhall/mmu/backend/mali_kbase_mmu_faults_decoder_luts_jm.c deleted file mode 100644 index a053a93978b5..000000000000 --- a/drivers/gpu/arm/valhall/mmu/backend/mali_kbase_mmu_faults_decoder_luts_jm.c +++ /dev/null @@ -1,74 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/** - * DOC: Base kernel MMU faults decoder for Job Manager GPUs. - */ - -#include - -#define GPU_ID_ARCH_ID_MAJOR_GET(gpu_id) ((gpu_id >> 16) & 0xFF) -#define GPU_ID_ARCH_ID_MINOR_AND_REV_GET(gpu_id) (gpu_id & 0xFFFF) -#define NELEMS(s) (sizeof(s) / sizeof((s)[0])) - -struct decode_lut_element { - u16 arch_minor_rev; - u16 key; - const char *text; -}; - -static const char *decode_lut_element_lookup(u16 arch_minor_rev, u16 key, - struct decode_lut_element *decode_element_lut, - unsigned int lut_len) -{ - struct decode_lut_element *p; - - for (p = decode_element_lut; p < decode_element_lut + lut_len; p++) { - if (p->key == key && - (p->arch_minor_rev == 0xffff || p->arch_minor_rev == arch_minor_rev)) - break; - } - if (p < decode_element_lut + lut_len) - return p->text; - else - return "unknown"; -} - -/* Auto-generated code: DO NOT MODIFY! */ - -static struct decode_lut_element lut_fault_source_jm_t_major_9[] = { - { 0xFFFF, 0, "js" }, - { 0xFFFF, 1, "pcm" }, -}; - -const char *decode_fault_source_jm_t(u16 idx, u32 gpu_id) -{ - u16 min_rev = GPU_ID_ARCH_ID_MINOR_AND_REV_GET(gpu_id); - const char *ret = "unknown"; - - switch (GPU_ID_ARCH_ID_MAJOR_GET(gpu_id)) { - case 9: - ret = decode_lut_element_lookup(min_rev, idx, lut_fault_source_jm_t_major_9, - NELEMS(lut_fault_source_jm_t_major_9)); - break; - } - return ret; -} diff --git a/drivers/gpu/arm/valhall/mmu/backend/mali_kbase_mmu_faults_decoder_luts_jm.h b/drivers/gpu/arm/valhall/mmu/backend/mali_kbase_mmu_faults_decoder_luts_jm.h deleted file mode 100644 index f686e555d86a..000000000000 --- a/drivers/gpu/arm/valhall/mmu/backend/mali_kbase_mmu_faults_decoder_luts_jm.h +++ /dev/null @@ -1,37 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#ifndef _MALI_KBASE_MMU_FAULTS_DECODER_LUTS_JM_H_ -#define _MALI_KBASE_MMU_FAULTS_DECODER_LUTS_JM_H_ -#include - -/** - * decode_fault_source_jm_t() - Get internal requester of a - * fault in a human readable format. - * - * @idx: Internal requester part of SOURCE_ID field of the fault. - * @gpu_id: GPU id composed of arch_major << 16 | arch_minor << 8 | arch_rev. - * - * Return: Internal requester of a fault in a human readable format for a JM core. - */ -const char *decode_fault_source_jm_t(u16 idx, u32 gpu_id); - -#endif /* _MALI_KBASE_MMU_FAULTS_DECODER_LUTS_JM_H_ */ diff --git a/drivers/gpu/arm/valhall/mmu/backend/mali_kbase_mmu_jm.c b/drivers/gpu/arm/valhall/mmu/backend/mali_kbase_mmu_jm.c deleted file mode 100644 index 2ff0c1c0eae8..000000000000 --- a/drivers/gpu/arm/valhall/mmu/backend/mali_kbase_mmu_jm.c +++ /dev/null @@ -1,418 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/** - * DOC: Base kernel MMU management specific for Job Manager GPU. - */ - -#include -#include -#include -#include -#include -#include -#include - -void kbase_mmu_get_as_setup(struct kbase_mmu_table *mmut, struct kbase_mmu_setup *const setup) -{ - /* Set up the required caching policies at the correct indices - * in the memattr register. - */ - setup->memattr = - (KBASE_MEMATTR_IMPL_DEF_CACHE_POLICY - << (KBASE_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY * 8)) | - (KBASE_MEMATTR_FORCE_TO_CACHE_ALL << (KBASE_MEMATTR_INDEX_FORCE_TO_CACHE_ALL * 8)) | - (KBASE_MEMATTR_WRITE_ALLOC << (KBASE_MEMATTR_INDEX_WRITE_ALLOC * 8)) | - (KBASE_MEMATTR_AARCH64_OUTER_IMPL_DEF << (KBASE_MEMATTR_INDEX_OUTER_IMPL_DEF * 8)) | - (KBASE_MEMATTR_AARCH64_OUTER_WA << (KBASE_MEMATTR_INDEX_OUTER_WA * 8)) | - (KBASE_MEMATTR_AARCH64_NON_CACHEABLE << (KBASE_MEMATTR_INDEX_NON_CACHEABLE * 8)); - - setup->transtab = (u64)mmut->pgd & AS_TRANSTAB_BASE_MASK; - setup->transcfg = AS_TRANSCFG_MODE_SET(0ULL, AS_TRANSCFG_MODE_AARCH64_4K); -} - -void kbase_gpu_report_bus_fault_and_kill(struct kbase_context *kctx, struct kbase_as *as, - struct kbase_fault *fault) -{ - struct kbase_device *const kbdev = kctx->kbdev; - const u32 status = fault->status; - const u32 exception_type = AS_FAULTSTATUS_EXCEPTION_TYPE_GET(status); - const u32 access_type = AS_FAULTSTATUS_ACCESS_TYPE_GET(status); - const u32 source_id = AS_FAULTSTATUS_SOURCE_ID_GET(status); - unsigned int const as_no = as->number; - unsigned long flags; - const uintptr_t fault_addr = fault->addr; - - /* terminal fault, print info about the fault */ - dev_err(kbdev->dev, - "GPU bus fault in AS%u at PA %pK\n" - "raw fault status: 0x%X\n" - "exception type 0x%X: %s\n" - "access type 0x%X: %s\n" - "source id 0x%X (core_id:utlb:IR 0x%X:0x%X:0x%X): %s, %s\n" - "pid: %d\n", - as_no, (void *)fault_addr, status, exception_type, - kbase_gpu_exception_name(exception_type), access_type, - kbase_gpu_access_type_name(kbdev, access_type), source_id, - FAULT_SOURCE_ID_CORE_ID_GET(source_id), FAULT_SOURCE_ID_UTLB_ID_GET(source_id), - fault_source_id_internal_requester_get(kbdev, source_id), - fault_source_id_core_type_description_get(kbdev, source_id), - fault_source_id_internal_requester_get_str(kbdev, source_id, access_type), - kctx->pid); - - /* switch to UNMAPPED mode, will abort all jobs and stop any hw counter - * dumping AS transaction begin - */ - mutex_lock(&kbdev->mmu_hw_mutex); - - /* Set the MMU into unmapped mode */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_mmu_disable(kctx); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - mutex_unlock(&kbdev->mmu_hw_mutex); - /* AS transaction end */ - - kbase_mmu_hw_clear_fault(kbdev, as, KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); - kbase_mmu_hw_enable_fault(kbdev, as, KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); - -} - -/* - * The caller must ensure it's retained the ctx to prevent it from being - * scheduled out whilst it's being worked on. - */ -void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, struct kbase_as *as, - const char *reason_str, struct kbase_fault *fault) -{ - unsigned long flags; - struct kbase_device *kbdev = kctx->kbdev; - struct kbasep_js_device_data *js_devdata = &kbdev->js_data; - unsigned int as_no = as->number; - - /* Make sure the context was active */ - if (WARN_ON(atomic_read(&kctx->refcount) <= 0)) - return; - - if (!kbase_ctx_flag(kctx, KCTX_PAGE_FAULT_REPORT_SKIP)) { - /* decode the fault status */ - const u32 status = fault->status; - const u32 exception_type = AS_FAULTSTATUS_EXCEPTION_TYPE_GET(status); - const u32 access_type = AS_FAULTSTATUS_ACCESS_TYPE_GET(status); - const u32 source_id = AS_FAULTSTATUS_SOURCE_ID_GET(status); - /* terminal fault, print info about the fault */ - if (kbdev->gpu_props.gpu_id.product_model < GPU_ID_MODEL_MAKE(9, 0)) { - dev_err(kbdev->dev, - "Unhandled Page fault in AS%u at VA 0x%016llX\n" - "Reason: %s\n" - "raw fault status: 0x%X\n" - "exception type 0x%X: %s\n" - "access type 0x%X: %s\n" - "pid: %d\n", - as_no, fault->addr, reason_str, status, exception_type, - kbase_gpu_exception_name(exception_type), access_type, - kbase_gpu_access_type_name(kbdev, status), kctx->pid); - } else { - dev_err(kbdev->dev, - "Unhandled Page fault in AS%u at VA 0x%016llX\n" - "Reason: %s\n" - "raw fault status: 0x%X\n" - "exception type 0x%X: %s\n" - "access type 0x%X: %s\n" - "source id 0x%X (core_id:utlb:IR 0x%X:0x%X:0x%X): %s, %s\n" - "pid: %d\n", - as_no, fault->addr, reason_str, status, exception_type, - kbase_gpu_exception_name(exception_type), access_type, - kbase_gpu_access_type_name(kbdev, status), source_id, - FAULT_SOURCE_ID_CORE_ID_GET(source_id), - FAULT_SOURCE_ID_UTLB_ID_GET(source_id), - fault_source_id_internal_requester_get(kbdev, source_id), - fault_source_id_core_type_description_get(kbdev, source_id), - fault_source_id_internal_requester_get_str(kbdev, source_id, - access_type), - kctx->pid); - } - } - - /* hardware counters dump fault handling */ - spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - if ((kbdev->hwcnt.kctx) && (kbdev->hwcnt.kctx->as_nr == as_no) && - (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DUMPING)) { - if ((fault->addr >= kbdev->hwcnt.addr) && - (fault->addr < (kbdev->hwcnt.addr + kbdev->hwcnt.addr_bytes))) - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_FAULT; - } - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - - /* Stop the kctx from submitting more jobs and cause it to be scheduled - * out/rescheduled - this will occur on releasing the context's refcount - */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbasep_js_clear_submit_allowed(js_devdata, kctx); - - /* Kill any running jobs from the context. Submit is disallowed, so no - * more jobs from this context can appear in the job slots from this - * point on - */ - kbase_backend_jm_kill_running_jobs_from_kctx(kctx); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - /* AS transaction begin */ - mutex_lock(&kbdev->mmu_hw_mutex); - - /* switch to UNMAPPED mode, will abort all jobs and stop - * any hw counter dumping - */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_mmu_disable(kctx); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - mutex_unlock(&kbdev->mmu_hw_mutex); - - /* AS transaction end */ - /* Clear down the fault */ - kbase_mmu_hw_clear_fault(kbdev, as, KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); - kbase_mmu_hw_enable_fault(kbdev, as, KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); - -} - -/** - * kbase_mmu_interrupt_process() - Process a bus or page fault. - * @kbdev: The kbase_device the fault happened on - * @kctx: The kbase_context for the faulting address space if one was - * found. - * @as: The address space that has the fault - * @fault: Data relating to the fault - * - * This function will process a fault on a specific address space - */ -static void kbase_mmu_interrupt_process(struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbase_as *as, struct kbase_fault *fault) -{ - unsigned long flags; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - dev_dbg(kbdev->dev, "Entering %s kctx %pK, as %pK\n", __func__, (void *)kctx, (void *)as); - - if (!kctx) { - if (kbase_as_has_bus_fault(as, fault)) { - dev_warn( - kbdev->dev, - "Bus error in AS%u at PA 0x%pK with no context present! Spurious IRQ or SW Design Error?\n", - as->number, (void *)(uintptr_t)fault->addr); - } else { - dev_warn( - kbdev->dev, - "Page fault in AS%u at VA 0x%016llx with no context present! Spurious IRQ or SW Design Error?\n", - as->number, fault->addr); - } - /* Since no ctx was found, the MMU must be disabled. */ - WARN_ON(as->current_setup.transtab); - - if (kbase_as_has_bus_fault(as, fault)) { - kbase_mmu_hw_clear_fault(kbdev, as, KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); - kbase_mmu_hw_enable_fault(kbdev, as, KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); - } else if (kbase_as_has_page_fault(as, fault)) { - kbase_mmu_hw_clear_fault(kbdev, as, KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); - kbase_mmu_hw_enable_fault(kbdev, as, KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); - } - - return; - } - - if (kbase_as_has_bus_fault(as, fault)) { - struct kbasep_js_device_data *js_devdata = &kbdev->js_data; - - /* - * hw counters dumping in progress, signal the - * other thread that it failed - */ - spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - if ((kbdev->hwcnt.kctx == kctx) && - (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DUMPING)) - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_FAULT; - - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - - /* - * Stop the kctx from submitting more jobs and cause it - * to be scheduled out/rescheduled when all references - * to it are released - */ - kbasep_js_clear_submit_allowed(js_devdata, kctx); - - dev_warn(kbdev->dev, "Bus error in AS%u at PA=0x%pK, IPA=0x%pK\n", as->number, - (void *)(uintptr_t)fault->addr, (void *)(uintptr_t)fault->extra_addr); - - /* - * We need to switch to UNMAPPED mode - but we do this in a - * worker so that we can sleep - */ - WARN_ON(!queue_work(as->pf_wq, &as->work_busfault)); - atomic_inc(&kbdev->faults_pending); - } else { - WARN_ON(!queue_work(as->pf_wq, &as->work_pagefault)); - atomic_inc(&kbdev->faults_pending); - } - - dev_dbg(kbdev->dev, "Leaving %s kctx %pK, as %pK\n", __func__, (void *)kctx, (void *)as); -} - -static void validate_protected_page_fault(struct kbase_device *kbdev) -{ - /* GPUs which support (native) protected mode shall not report page - * fault addresses unless it has protected debug mode and protected - * debug mode is turned on - */ - u32 protected_debug_mode = 0; - - if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE)) { - protected_debug_mode = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_STATUS)) & - GPU_STATUS_GPU_DBG_ENABLED; - } - - if (!protected_debug_mode) { - /* fault_addr should never be reported in protected mode. - * However, we just continue by printing an error message - */ - dev_err(kbdev->dev, "Fault address reported in protected mode\n"); - } -} - -void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat) -{ - const int num_as = 16; - const int busfault_shift = MMU_PAGE_FAULT_FLAGS; - const int pf_shift = 0; - const unsigned long as_bit_mask = (1UL << num_as) - 1; - unsigned long flags; - u32 new_mask; - u32 tmp, bf_bits, pf_bits; - - dev_dbg(kbdev->dev, "Entering %s irq_stat %u\n", __func__, irq_stat); - /* bus faults */ - bf_bits = (irq_stat >> busfault_shift) & as_bit_mask; - /* page faults (note: Ignore ASes with both pf and bf) */ - pf_bits = ((irq_stat >> pf_shift) & as_bit_mask) & ~bf_bits; - - if (WARN_ON(kbdev == NULL)) - return; - - /* remember current mask */ - spin_lock_irqsave(&kbdev->mmu_mask_change, flags); - new_mask = kbase_reg_read32(kbdev, MMU_CONTROL_ENUM(IRQ_MASK)); - /* mask interrupts for now */ - kbase_reg_write32(kbdev, MMU_CONTROL_ENUM(IRQ_MASK), 0); - spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); - - while (bf_bits | pf_bits) { - struct kbase_as *as; - unsigned int as_no; - struct kbase_context *kctx; - struct kbase_fault *fault; - - /* - * the while logic ensures we have a bit set, no need to check - * for not-found here - */ - as_no = (unsigned int)ffs((int)(bf_bits | pf_bits)) - 1; - as = &kbdev->as[as_no]; - - /* find the fault type */ - if (bf_bits & (1UL << as_no)) - fault = &as->bf_data; - else - fault = &as->pf_data; - - /* - * Refcount the kctx ASAP - it shouldn't disappear anyway, since - * Bus/Page faults _should_ only occur whilst jobs are running, - * and a job causing the Bus/Page fault shouldn't complete until - * the MMU is updated - */ - kctx = kbase_ctx_sched_as_to_ctx_refcount(kbdev, as_no); - - /* find faulting address */ - fault->addr = kbase_reg_read64(kbdev, MMU_AS_OFFSET(as_no, FAULTADDRESS)); - /* Mark the fault protected or not */ - fault->protected_mode = kbdev->protected_mode; - - if (kbdev->protected_mode && fault->addr) { - /* check if address reporting is allowed */ - validate_protected_page_fault(kbdev); - } - - /* report the fault to debugfs */ - kbase_as_fault_debugfs_new(kbdev, as_no); - - /* record the fault status */ - fault->status = kbase_reg_read32(kbdev, MMU_AS_OFFSET(as_no, FAULTSTATUS)); - fault->extra_addr = kbase_reg_read64(kbdev, MMU_AS_OFFSET(as_no, FAULTEXTRA)); - - if (kbase_as_has_bus_fault(as, fault)) { - /* Mark bus fault as handled. - * Note that a bus fault is processed first in case - * where both a bus fault and page fault occur. - */ - bf_bits &= ~(1UL << as_no); - - /* remove the queued BF (and PF) from the mask */ - new_mask &= ~(MMU_BUS_ERROR(as_no) | MMU_PAGE_FAULT(as_no)); - } else { - /* Mark page fault as handled */ - pf_bits &= ~(1UL << as_no); - - /* remove the queued PF from the mask */ - new_mask &= ~MMU_PAGE_FAULT(as_no); - } - - /* Process the interrupt for this address space */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_mmu_interrupt_process(kbdev, kctx, as, fault); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - } - - /* reenable interrupts */ - spin_lock_irqsave(&kbdev->mmu_mask_change, flags); - tmp = kbase_reg_read32(kbdev, MMU_CONTROL_ENUM(IRQ_MASK)); - new_mask |= tmp; - kbase_reg_write32(kbdev, MMU_CONTROL_ENUM(IRQ_MASK), new_mask); - spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); - - dev_dbg(kbdev->dev, "Leaving %s irq_stat %u\n", __func__, irq_stat); -} - -int kbase_mmu_as_init(struct kbase_device *kbdev, unsigned int i) -{ - kbdev->as[i].number = i; - kbdev->as[i].bf_data.addr = 0ULL; - kbdev->as[i].pf_data.addr = 0ULL; - - kbdev->as[i].pf_wq = alloc_workqueue("mali_mmu%u", 0, 0, i); - if (!kbdev->as[i].pf_wq) - return -ENOMEM; - - INIT_WORK(&kbdev->as[i].work_pagefault, kbase_mmu_page_fault_worker); - INIT_WORK(&kbdev->as[i].work_busfault, kbase_mmu_bus_fault_worker); - - return 0; -} diff --git a/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu.c b/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu.c index 2e06ec90442e..7db1dfdc6d10 100644 --- a/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu.c +++ b/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2025 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -43,9 +43,6 @@ #include #include #include -#if !MALI_USE_CSF -#include -#endif #include #include @@ -323,12 +320,8 @@ static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mm /* Small wrapper function to factor out GPU-dependent context releasing */ static void release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) { -#if MALI_USE_CSF CSTD_UNUSED(kbdev); kbase_ctx_sched_release_ctx_lock(kctx); -#else /* MALI_USE_CSF */ - kbasep_js_runpool_release_ctx(kbdev, kctx); -#endif /* MALI_USE_CSF */ } /** @@ -357,7 +350,6 @@ static bool mmu_flush_cache_on_gpu_ctrl(struct kbase_device *kbdev) * * Issue a cache flush physical range command. */ -#if MALI_USE_CSF static void mmu_flush_pa_range(struct kbase_device *kbdev, phys_addr_t phys, size_t nr_bytes, enum kbase_mmu_op_type op) { @@ -378,7 +370,6 @@ static void mmu_flush_pa_range(struct kbase_device *kbdev, phys_addr_t phys, siz if (kbase_gpu_cache_flush_pa_range_and_busy_wait(kbdev, phys, nr_bytes, flush_op)) dev_err(kbdev->dev, "Flush for physical address range did not complete"); } -#endif /** * mmu_invalidate() - Perform an invalidate operation on MMU caches. @@ -500,13 +491,7 @@ static void mmu_flush_invalidate(struct kbase_device *kbdev, struct kbase_contex if (!kctx) { mmu_flush_invalidate_as(kbdev, &kbdev->as[as_nr], op_param); } else { -#if !MALI_USE_CSF - mutex_lock(&kbdev->js_data.queue_mutex); - ctx_is_in_runpool = kbase_ctx_sched_inc_refcount(kctx); - mutex_unlock(&kbdev->js_data.queue_mutex); -#else ctx_is_in_runpool = kbase_ctx_sched_inc_refcount_if_as_valid(kctx); -#endif /* !MALI_USE_CSF */ if (ctx_is_in_runpool) { KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); @@ -682,6 +667,12 @@ static bool kbase_mmu_handle_isolated_pgd_page(struct kbase_device *kbdev, return page_is_isolated; } +static struct kbase_mem_pool *kbase_mmu_get_pgd_pool(struct kbase_device *kbdev, + struct kbase_mmu_table *mmut) +{ + return (mmut->kctx) ? &mmut->kctx->pgd_mem_pool : &kbdev->pgd_mem_pool; +} + /** * kbase_mmu_free_pgd() - Free memory of the page directory * @@ -718,7 +709,7 @@ static void kbase_mmu_free_pgd(struct kbase_device *kbdev, struct kbase_mmu_tabl page_is_isolated = kbase_mmu_handle_isolated_pgd_page(kbdev, mmut, p); if (likely(!page_is_isolated)) { - kbase_mem_pool_free(&kbdev->mem_pools.small[mmut->group_id], p, true); + kbase_mem_pool_free(kbase_mmu_get_pgd_pool(kbdev, mmut), p, true); kbase_mmu_account_freed_pgd(kbdev, mmut); } } @@ -810,82 +801,12 @@ static size_t reg_grow_calc_extra_pages(struct kbase_device *kbdev, struct kbase */ remainder = minimum_extra % multiple; -#if !MALI_USE_CSF - if (reg->flags & KBASE_REG_TILER_ALIGN_TOP) { - /* multiple is based from the top of the initial commit, which - * has been allocated in such a way that (start_pfn + - * initial_commit) is already aligned to multiple. Hence the - * pfn for the end of committed memory will also be aligned to - * multiple - */ - size_t initial_commit = reg->initial_commit; - - if (fault_rel_pfn < initial_commit) { - /* this case is just to catch in case it's been - * recommitted by userspace to be smaller than the - * initial commit - */ - minimum_extra = initial_commit - reg_current_size; - remainder = 0; - } else { - /* same as calculating - * (fault_rel_pfn - initial_commit + 1) - */ - size_t pages_after_initial = - minimum_extra + reg_current_size - initial_commit; - - remainder = pages_after_initial % multiple; - } - } -#endif /* !MALI_USE_CSF */ - if (remainder == 0) return minimum_extra; return minimum_extra + multiple - remainder; } -/** - * estimate_pool_space_required - Determine how much a pool should be grown by to support a future - * allocation - * @pool: The memory pool to check, including its linked pools - * @pages_required: Number of small pages require for the pool to support a future allocation - * - * The value returned is accounting for the size of @pool and the size of each memory pool linked to - * @pool. Hence, the caller should use @pool and (if not already satisfied) all its linked pools to - * allocate from. - * - * Note: this is only an estimate, because even during the calculation the memory pool(s) involved - * can be updated to be larger or smaller. Hence, the result is only a guide as to whether an - * allocation could succeed, or an estimate of the correct amount to grow the pool by. The caller - * should keep attempting an allocation and then re-growing with a new value queried form this - * function until the allocation succeeds. - * - * Return: an estimate of the amount of extra small pages in @pool that are required to satisfy an - * allocation, or 0 if @pool (including its linked pools) is likely to already satisfy the - * allocation. - */ -static size_t estimate_pool_space_required(struct kbase_mem_pool *pool, const size_t pages_required) -{ - size_t pages_still_required; - - for (pages_still_required = pages_required; pool != NULL && pages_still_required; - pool = pool->next_pool) { - size_t pool_size_small; - - kbase_mem_pool_lock(pool); - - pool_size_small = kbase_mem_pool_size(pool) << pool->order; - if (pool_size_small >= pages_still_required) - pages_still_required = 0; - else - pages_still_required -= pool_size_small; - - kbase_mem_pool_unlock(pool); - } - return pages_still_required; -} - /** * page_fault_try_alloc - Try to allocate memory from a context pool * @kctx: Context pointer @@ -898,11 +819,10 @@ static size_t estimate_pool_space_required(struct kbase_mem_pool *pool, const si * @fallback_to_small: Whether fallback to small pages or not * @prealloc_sas: Pointer to kbase_sub_alloc structures * - * This function will try to allocate as many pages as possible from the context pool, then if - * required will try to allocate the remaining pages from the device pool. + * This function will try to allocate as many pages as possible from the pool * - * This function will not allocate any new memory beyond that is already present in the context or - * device pools. This is because it is intended to be called whilst the thread has acquired the + * This function will not allocate any new memory beyond that is already present in the context. + * This is because it is intended to be called whilst the thread has acquired the * region list lock with kbase_gpu_vm_lock(), and a large enough memory allocation whilst that is * held could invoke the OoM killer and cause an effective deadlock with kbase_cpu_vm_close(). * @@ -915,152 +835,91 @@ static bool page_fault_try_alloc(struct kbase_context *kctx, struct kbase_va_reg size_t new_pages, size_t *pages_to_grow, bool *grow_2mb_pool, bool fallback_to_small, struct kbase_sub_alloc **prealloc_sas) { - size_t total_gpu_pages_alloced = 0; - size_t total_cpu_pages_alloced = 0; - struct kbase_mem_pool *pool, *root_pool; + size_t gpu_pages_alloced = 0; + size_t cpu_pages_alloced = 0; + struct kbase_mem_pool *pool; bool alloc_failed = false; - size_t pages_still_required; - size_t total_mempools_free_small = 0; + size_t pool_size_small; + size_t pages_to_alloc_small_per_alloc; + struct tagged_addr *gpu_pages; lockdep_assert_held(&kctx->reg_lock); lockdep_assert_held(&kctx->mem_partials_lock); - if (WARN_ON(region->gpu_alloc->group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS)) { - /* Do not try to grow the memory pool */ - *pages_to_grow = 0; + *pages_to_grow = 0; + + if (WARN_ON(region->gpu_alloc->group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS)) return false; - } if (kbase_is_large_pages_enabled() && new_pages >= NUM_PAGES_IN_2MB_LARGE_PAGE && !fallback_to_small) { - root_pool = &kctx->mem_pools.large[region->gpu_alloc->group_id]; + pool = &kctx->mem_pools.large[region->gpu_alloc->group_id]; *grow_2mb_pool = true; } else { - root_pool = &kctx->mem_pools.small[region->gpu_alloc->group_id]; + pool = &kctx->mem_pools.small[region->gpu_alloc->group_id]; *grow_2mb_pool = false; } + pages_to_alloc_small_per_alloc = new_pages; if (region->gpu_alloc != region->cpu_alloc) - new_pages *= 2; + new_pages <<= 1; + kbase_mem_pool_lock(pool); /* Determine how many pages are in the pools before trying to allocate. * Don't attempt to allocate & free if the allocation can't succeed. */ - pages_still_required = estimate_pool_space_required(root_pool, new_pages); - - if (pages_still_required) { + pool_size_small = kbase_mem_pool_size(pool) << pool->order; + if (pool_size_small < new_pages) { /* Insufficient pages in pools. Don't try to allocate - just * request a grow. */ - *pages_to_grow = pages_still_required; - + *pages_to_grow = new_pages - pool_size_small; + kbase_mem_pool_unlock(pool); return false; } - /* Since we're not holding any of the mempool locks, the amount of memory in the pools may - * change between the above estimate and the actual allocation. - */ - pages_still_required = new_pages; - for (pool = root_pool; pool != NULL && pages_still_required; pool = pool->next_pool) { - size_t pool_size_small; - size_t pages_to_alloc_small; - size_t pages_to_alloc_small_per_alloc; + gpu_pages = kbase_alloc_phy_pages_helper_locked( + region->gpu_alloc, pool, pages_to_alloc_small_per_alloc, &prealloc_sas[0]); - kbase_mem_pool_lock(pool); + if (!gpu_pages) + alloc_failed = true; + else + gpu_pages_alloced = pages_to_alloc_small_per_alloc; - /* Allocate as much as possible from this pool*/ - pool_size_small = kbase_mem_pool_size(pool) << pool->order; - total_mempools_free_small += pool_size_small; - pages_to_alloc_small = MIN(pages_still_required, pool_size_small); - if (region->gpu_alloc == region->cpu_alloc) - pages_to_alloc_small_per_alloc = pages_to_alloc_small; + if (!alloc_failed && region->gpu_alloc != region->cpu_alloc) { + struct tagged_addr *cpu_pages = kbase_alloc_phy_pages_helper_locked( + region->cpu_alloc, pool, pages_to_alloc_small_per_alloc, &prealloc_sas[1]); + + if (!cpu_pages) + alloc_failed = true; else - pages_to_alloc_small_per_alloc = pages_to_alloc_small >> 1; - - if (pages_to_alloc_small) { - struct tagged_addr *gpu_pages = kbase_alloc_phy_pages_helper_locked( - region->gpu_alloc, pool, pages_to_alloc_small_per_alloc, - &prealloc_sas[0]); - - if (!gpu_pages) - alloc_failed = true; - else - total_gpu_pages_alloced += pages_to_alloc_small_per_alloc; - - if (!alloc_failed && region->gpu_alloc != region->cpu_alloc) { - struct tagged_addr *cpu_pages = kbase_alloc_phy_pages_helper_locked( - region->cpu_alloc, pool, pages_to_alloc_small_per_alloc, - &prealloc_sas[1]); - - if (!cpu_pages) - alloc_failed = true; - else - total_cpu_pages_alloced += pages_to_alloc_small_per_alloc; - } - } - - kbase_mem_pool_unlock(pool); - - if (alloc_failed) { - WARN_ON(!pages_still_required); - WARN_ON(pages_to_alloc_small >= pages_still_required); - WARN_ON(pages_to_alloc_small_per_alloc >= pages_still_required); - break; - } - - pages_still_required -= pages_to_alloc_small; + cpu_pages_alloced = pages_to_alloc_small_per_alloc; } - if (pages_still_required) { + kbase_mem_pool_unlock(pool); + + if (alloc_failed) { /* Allocation was unsuccessful. We have dropped the mem_pool lock after allocation, * so must in any case use kbase_free_phy_pages_helper() rather than * kbase_free_phy_pages_helper_locked() */ - if (total_gpu_pages_alloced > 0) - kbase_free_phy_pages_helper(region->gpu_alloc, total_gpu_pages_alloced); - if (region->gpu_alloc != region->cpu_alloc && total_cpu_pages_alloced > 0) - kbase_free_phy_pages_helper(region->cpu_alloc, total_cpu_pages_alloced); - - if (alloc_failed) { - /* Note that in allocating from the above memory pools, we always ensure - * never to request more than is available in each pool with the pool's - * lock held. Hence failing to allocate in such situations would be unusual - * and we should cancel the growth instead (as re-growing the memory pool - * might not fix the situation) - */ - dev_warn( - kctx->kbdev->dev, - "Page allocation failure of %zu pages: managed %zu pages, mempool (inc linked pools) had %zu pages available", - new_pages, total_gpu_pages_alloced + total_cpu_pages_alloced, - total_mempools_free_small); - *pages_to_grow = 0; - } else { - /* Tell the caller to try to grow the memory pool - * - * Freeing pages above may have spilled or returned them to the OS, so we - * have to take into account how many are still in the pool before giving a - * new estimate for growth required of the pool. We can just re-estimate a - * new value. - */ - pages_still_required = estimate_pool_space_required(root_pool, new_pages); - if (pages_still_required) { - *pages_to_grow = pages_still_required; - } else { - /* It's possible another thread could've grown the pool to be just - * big enough after we rolled back the allocation. Request at least - * one more page to ensure the caller doesn't fail the growth by - * conflating it with the alloc_failed case above - */ - *pages_to_grow = 1u; - } - } + if (gpu_pages_alloced > 0) + kbase_free_phy_pages_helper(region->gpu_alloc, gpu_pages_alloced); + if (region->gpu_alloc != region->cpu_alloc && cpu_pages_alloced > 0) + kbase_free_phy_pages_helper(region->cpu_alloc, cpu_pages_alloced); + /* Failing to allocate in such situation is unusual, + * because the pool was supposed to have enough memory pages, + * and we should cancel the growth as re-growing the memory pool + * might not fix the situation. + */ + dev_warn( + kctx->kbdev->dev, + "Page allocation failure of %zu pages: managed %zu pages, mempool had %zu pages available", + new_pages, gpu_pages_alloced + cpu_pages_alloced, pool_size_small); return false; } - /* Allocation was successful. No pages to grow, return success. */ - *pages_to_grow = 0; - return true; } @@ -1116,9 +975,6 @@ void kbase_mmu_page_fault_worker(struct work_struct *data) KBASE_DEBUG_ASSERT(kctx->kbdev == kbdev); #if MALI_JIT_PRESSURE_LIMIT_BASE -#if !MALI_USE_CSF - mutex_lock(&kctx->jctx.lock); -#endif #endif /* check if we still have GPU */ @@ -1145,10 +1001,6 @@ void kbase_mmu_page_fault_worker(struct work_struct *data) case AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSLATION_FAULT_3: fallthrough; case AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSLATION_FAULT_4: -#if !MALI_USE_CSF - fallthrough; - case AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSLATION_FAULT_IDENTITY: -#endif /* need to check against the region to handle this one */ break; @@ -1162,23 +1014,6 @@ void kbase_mmu_page_fault_worker(struct work_struct *data) kbase_mmu_report_fault_and_kill(kctx, faulting_as, "Permission failure", fault); goto fault_done; -#if !MALI_USE_CSF - case AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSTAB_BUS_FAULT_0: - fallthrough; - case AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSTAB_BUS_FAULT_1: - fallthrough; - case AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSTAB_BUS_FAULT_2: - fallthrough; - case AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSTAB_BUS_FAULT_3: - kbase_mmu_report_fault_and_kill(kctx, faulting_as, "Translation table bus fault", - fault); - goto fault_done; -#endif - -#if !MALI_USE_CSF - case AS_FAULTSTATUS_EXCEPTION_TYPE_ACCESS_FLAG_0: - fallthrough; -#endif case AS_FAULTSTATUS_EXCEPTION_TYPE_ACCESS_FLAG_1: fallthrough; case AS_FAULTSTATUS_EXCEPTION_TYPE_ACCESS_FLAG_2: @@ -1188,19 +1023,8 @@ void kbase_mmu_page_fault_worker(struct work_struct *data) dev_warn(kbdev->dev, "Access flag unexpectedly set"); goto fault_done; -#if MALI_USE_CSF case AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_IN: fallthrough; -#else - case AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_IN0: - fallthrough; - case AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_IN1: - fallthrough; - case AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_IN2: - fallthrough; - case AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_IN3: - fallthrough; -#endif case AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_OUT0: fallthrough; case AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_OUT1: @@ -1218,16 +1042,6 @@ void kbase_mmu_page_fault_worker(struct work_struct *data) case AS_FAULTSTATUS_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_FAULT_2: fallthrough; case AS_FAULTSTATUS_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_FAULT_3: -#if !MALI_USE_CSF - fallthrough; - case AS_FAULTSTATUS_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_NONCACHEABLE_0: - fallthrough; - case AS_FAULTSTATUS_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_NONCACHEABLE_1: - fallthrough; - case AS_FAULTSTATUS_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_NONCACHEABLE_2: - fallthrough; - case AS_FAULTSTATUS_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_NONCACHEABLE_3: -#endif kbase_mmu_report_fault_and_kill(kctx, faulting_as, "Memory attributes fault", fault); goto fault_done; @@ -1557,9 +1371,6 @@ fault_done: kbase_jit_done_phys_increase(kctx, pages_trimmed); kbase_gpu_vm_unlock(kctx); } -#if !MALI_USE_CSF - mutex_unlock(&kctx->jctx.lock); -#endif #endif for (i = 0; i != ARRAY_SIZE(prealloc_sas); ++i) @@ -1592,6 +1403,7 @@ static phys_addr_t kbase_mmu_alloc_pgd(struct kbase_device *kbdev, struct kbase_ u64 *page; struct page *p; phys_addr_t pgd; + struct kbase_mem_pool *pgd_mem_pool = kbase_mmu_get_pgd_pool(kbdev, mmut); lockdep_assert_held(&mmut->mmu_lock); @@ -1601,7 +1413,7 @@ static phys_addr_t kbase_mmu_alloc_pgd(struct kbase_device *kbdev, struct kbase_ return pgd; #endif - p = kbase_mem_pool_alloc(&kbdev->mem_pools.small[mmut->group_id]); + p = kbase_mem_pool_alloc(pgd_mem_pool); if (!p) return KBASE_INVALID_PHYSICAL_ADDRESS; @@ -1648,7 +1460,7 @@ static phys_addr_t kbase_mmu_alloc_pgd(struct kbase_device *kbdev, struct kbase_ return pgd; alloc_free: - kbase_mem_pool_free(&kbdev->mem_pools.small[mmut->group_id], p, false); + kbase_mem_pool_free(pgd_mem_pool, p, false); return KBASE_INVALID_PHYSICAL_ADDRESS; } @@ -1940,11 +1752,7 @@ static void mmu_flush_invalidate_insert_pages(struct kbase_device *kbdev, op_param.kctx_id = mmut->kctx ? mmut->kctx->id : 0xFFFFFFFF; op_param.flush_skip_levels = pgd_level_to_skip_flush(dirty_pgds); -#if MALI_USE_CSF as_nr = mmut->kctx ? mmut->kctx->as_nr : MCU_AS_NR; -#else - WARN_ON(!mmut->kctx); -#endif /* MMU cache flush strategy depends on whether GPU control commands for * flushing physical address ranges are supported. The new physical pages @@ -2150,6 +1958,7 @@ static int mmu_insert_alloc_pgds(struct kbase_device *kbdev, struct kbase_mmu_ta { int err = 0; int i; + struct kbase_mem_pool *pgd_mem_pool = kbase_mmu_get_pgd_pool(kbdev, mmut); lockdep_assert_held(&mmut->mmu_lock); @@ -2162,8 +1971,7 @@ static int mmu_insert_alloc_pgds(struct kbase_device *kbdev, struct kbase_mmu_ta if (new_pgds[i] != KBASE_INVALID_PHYSICAL_ADDRESS) break; mutex_unlock(&mmut->mmu_lock); - err = kbase_mem_pool_grow(&kbdev->mem_pools.small[mmut->group_id], - (size_t)level_high, NULL); + err = kbase_mem_pool_grow(pgd_mem_pool, (size_t)level_high, NULL); mutex_lock(&mmut->mmu_lock); if (err) { dev_err(kbdev->dev, "%s: kbase_mem_pool_grow() returned error %d", @@ -2410,6 +2218,10 @@ static void kbase_mmu_progress_migration_on_insert(struct tagged_addr phys, if (!kbase_is_page_migration_enabled()) return; + /* Metadata not created */ + if (!page_md) + return; + spin_lock(&page_md->migrate_lock); /* If no GPU va region is given: the metadata provided are @@ -2490,10 +2302,15 @@ u64 kbase_mmu_create_ate(struct kbase_device *const kbdev, struct tagged_addr co { u64 entry; unsigned int pte_flags = 0; + const bool cpu_access = flags & (KBASE_REG_CPU_RD | KBASE_REG_CPU_WR); kbdev->mmu_mode->entry_set_ate(&entry, phy, flags, level); - if ((flags & KBASE_REG_GPU_CACHED) && !(flags & KBASE_REG_CPU_CACHED)) + /* Address-Table Entries (ATEs) that are GPU-cached but CPU uncached are flagged as a + * Mismatched Memory Attribute (MMA) violation, as per A5.3.2 of the AMBA protocol + * specification. Depending on the system, these might need special handling: + */ + if ((flags & KBASE_REG_GPU_CACHED) && cpu_access && !(flags & KBASE_REG_CPU_CACHED)) pte_flags |= BIT(MMA_VIOLATION); return kbdev->mgm_dev->ops.mgm_update_gpu_pte(kbdev->mgm_dev, (unsigned int)group_id, @@ -2844,9 +2661,6 @@ KBASE_EXPORT_TEST_API(kbase_mmu_update); void kbase_mmu_disable_as(struct kbase_device *kbdev, int as_nr) { lockdep_assert_held(&kbdev->hwaccess_lock); -#if !MALI_USE_CSF - lockdep_assert_held(&kbdev->mmu_hw_mutex); -#endif kbdev->mmu_mode->disable_as(kbdev, as_nr); } @@ -2881,7 +2695,6 @@ void kbase_mmu_disable(struct kbase_context *kctx) op_param.kctx_id = kctx->id; op_param.mmu_sync_info = mmu_sync_info; -#if MALI_USE_CSF /* 0xF value used to prevent skipping of any levels when flushing */ if (mmu_flush_cache_on_gpu_ctrl(kbdev)) op_param.flush_skip_levels = pgd_level_to_skip_flush(0xF); @@ -2916,36 +2729,6 @@ void kbase_mmu_disable(struct kbase_context *kctx) dev_err(kbdev->dev, "Failed to unlock AS %d for ctx %d_%d", kctx->as_nr, kctx->tgid, kctx->id); } -#else - lockdep_assert_held(&kctx->kbdev->mmu_hw_mutex); - - CSTD_UNUSED(lock_err); - - /* - * The address space is being disabled, drain all knowledge of it out - * from the caches as pages and page tables might be freed after this. - * - * The job scheduler code will already be holding the locks and context - * so just do the flush. - */ - flush_err = kbase_mmu_hw_do_flush(kbdev, &kbdev->as[kctx->as_nr], &op_param); - if (flush_err) { - dev_err(kbdev->dev, - "Flush for GPU page table update did not complete to disable AS %d for ctx %d_%d", - kctx->as_nr, kctx->tgid, kctx->id); - /* GPU reset would have been triggered by the flush function */ - } - - kbdev->mmu_mode->disable_as(kbdev, kctx->as_nr); - - /* - * JM GPUs has some L1 read only caches that need to be invalidated - * with START_FLUSH configuration. Purge the MMU disabled kctx from - * the slot_rb tracking field so such invalidation is performed when - * a new katom is executed on the affected slots. - */ - kbase_backend_slot_kctx_purge_locked(kbdev, kctx); -#endif } KBASE_EXPORT_TEST_API(kbase_mmu_disable); @@ -3049,9 +2832,7 @@ static void mmu_flush_invalidate_teardown_pages(struct kbase_device *kbdev, } else if (op_param->op == KBASE_MMU_OP_FLUSH_MEM) { /* Full cache flush through the GPU_CONTROL */ mmu_flush_invalidate_on_gpu_ctrl(kbdev, kctx, as_nr, op_param); - } -#if MALI_USE_CSF - else { + } else { /* Partial GPU cache flush of the pages that were unmapped */ unsigned long irq_flags; unsigned int i; @@ -3067,10 +2848,6 @@ static void mmu_flush_invalidate_teardown_pages(struct kbase_device *kbdev, spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); } } -#else - CSTD_UNUSED(phys); - CSTD_UNUSED(phys_page_nr); -#endif } static int kbase_mmu_teardown_pgd_pages(struct kbase_device *kbdev, struct kbase_mmu_table *mmut, @@ -3506,13 +3283,6 @@ static int kbase_mmu_update_pages_common(struct kbase_device *kbdev, struct kbas const enum kbase_caller_mmu_sync_info mmu_sync_info = CALLER_MMU_ASYNC; int as_nr; -#if !MALI_USE_CSF - if (unlikely(kctx == NULL)) - return -EINVAL; - - as_nr = kctx->as_nr; - mmut = &kctx->mmu; -#else if (kctx) { mmut = &kctx->mmu; as_nr = kctx->as_nr; @@ -3520,7 +3290,6 @@ static int kbase_mmu_update_pages_common(struct kbase_device *kbdev, struct kbas mmut = &kbdev->csf.mcu_mmu; as_nr = MCU_AS_NR; } -#endif err = kbase_mmu_update_pages_no_flush(kbdev, mmut, vpfn, phys, nr, flags, group_id, &dirty_pgds); @@ -3551,13 +3320,11 @@ int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, struct tagged_a return kbase_mmu_update_pages_common(kctx->kbdev, kctx, vpfn, phys, nr, flags, group_id); } -#if MALI_USE_CSF int kbase_mmu_update_csf_mcu_pages(struct kbase_device *kbdev, u64 vpfn, struct tagged_addr *phys, size_t nr, unsigned long flags, int const group_id) { return kbase_mmu_update_pages_common(kbdev, NULL, vpfn, phys, nr, flags, group_id); } -#endif /* MALI_USE_CSF */ static void mmu_page_migration_transaction_begin(struct kbase_device *kbdev) { @@ -3753,13 +3520,11 @@ static int mmu_migrate_pgd_sub_page(phys_addr_t old_pgd_phys, phys_addr_t new_pg ret = kbase_mmu_hw_do_lock(kbdev, as, &op_param); if (!ret) { -#if MALI_USE_CSF if (mmu_flush_cache_on_gpu_ctrl(kbdev)) ret = kbase_gpu_cache_flush_pa_range_and_busy_wait( kbdev, old_pgd_phys, GPU_PAGE_SIZE, GPU_COMMAND_FLUSH_PA_RANGE_CLN_INV_L2_LSC); else -#endif ret = kbase_gpu_cache_flush_and_busy_wait( kbdev, GPU_COMMAND_CACHE_CLN_INV_L2_LSC); } @@ -4159,13 +3924,11 @@ int kbase_mmu_migrate_data_page(struct tagged_addr old_phys, struct tagged_addr ret = kbase_mmu_hw_do_lock(kbdev, as, &op_param); if (!ret) { -#if MALI_USE_CSF if (mmu_flush_cache_on_gpu_ctrl(kbdev)) ret = kbase_gpu_cache_flush_pa_range_and_busy_wait( kbdev, as_phys_addr_t(old_phys), PAGE_SIZE, GPU_COMMAND_FLUSH_PA_RANGE_CLN_INV_L2_LSC); else -#endif ret = kbase_gpu_cache_flush_and_busy_wait( kbdev, GPU_COMMAND_CACHE_CLN_INV_L2_LSC); } @@ -4411,9 +4174,10 @@ int kbase_mmu_init(struct kbase_device *const kbdev, struct kbase_mmu_table *con */ while (mmut->pgd == KBASE_INVALID_PHYSICAL_ADDRESS) { int err; + struct kbase_mem_pool *pgd_mem_pool = kbase_mmu_get_pgd_pool(kbdev, mmut); - err = kbase_mem_pool_grow(&kbdev->mem_pools.small[mmut->group_id], - MIDGARD_MMU_BOTTOMLEVEL, kctx ? kctx->task : NULL); + err = kbase_mem_pool_grow(pgd_mem_pool, MIDGARD_MMU_BOTTOMLEVEL, + kctx ? kctx->task : NULL); if (err) { kbase_mmu_term(kbdev, mmut); return -ENOMEM; @@ -4454,7 +4218,6 @@ void kbase_mmu_as_term(struct kbase_device *kbdev, unsigned int i) void kbase_mmu_flush_pa_range(struct kbase_device *kbdev, struct kbase_context *kctx, phys_addr_t phys, size_t size, enum kbase_mmu_op_type flush_op) { -#if MALI_USE_CSF unsigned long irq_flags; spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); @@ -4462,13 +4225,6 @@ void kbase_mmu_flush_pa_range(struct kbase_device *kbdev, struct kbase_context * kbdev->pm.backend.gpu_ready && (!kctx || kctx->as_nr >= 0)) mmu_flush_pa_range(kbdev, phys, size, KBASE_MMU_OP_FLUSH_PT); spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); -#else - CSTD_UNUSED(kbdev); - CSTD_UNUSED(kctx); - CSTD_UNUSED(phys); - CSTD_UNUSED(size); - CSTD_UNUSED(flush_op); -#endif } #ifdef CONFIG_MALI_VECTOR_DUMP @@ -4656,22 +4412,11 @@ void kbase_mmu_bus_fault_worker(struct work_struct *data) return; } -#if MALI_USE_CSF /* Before the GPU power off, wait is done for the completion of * in-flight MMU fault work items. So GPU is expected to remain * powered up whilst the bus fault handling is being done. */ kbase_gpu_report_bus_fault_and_kill(kctx, faulting_as, fault); -#else - /* NOTE: If GPU already powered off for suspend, - * we don't need to switch to unmapped - */ - if (!kbase_pm_context_active_handle_suspend(kbdev, - KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { - kbase_gpu_report_bus_fault_and_kill(kctx, faulting_as, fault); - kbase_pm_context_idle(kbdev); - } -#endif release_ctx(kbdev, kctx); diff --git a/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu.h b/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu.h index f0a418002247..c952346efa21 100644 --- a/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu.h +++ b/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu.h @@ -215,7 +215,6 @@ int kbase_mmu_teardown_imported_pages(struct kbase_device *kbdev, struct kbase_m int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, struct tagged_addr *phys, size_t nr, unsigned long flags, int const group_id); -#if MALI_USE_CSF /** * kbase_mmu_update_csf_mcu_pages - Update MCU mappings with changes of phys and flags * @@ -233,7 +232,6 @@ int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, struct tagged_a */ int kbase_mmu_update_csf_mcu_pages(struct kbase_device *kbdev, u64 vpfn, struct tagged_addr *phys, size_t nr, unsigned long flags, int const group_id); -#endif /** * kbase_mmu_migrate_data_page - Migrate GPU mappings and content of data pages between memory pages diff --git a/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu_faults_decoder.c b/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu_faults_decoder.c index 548d88cf216e..16057a29bb14 100644 --- a/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu_faults_decoder.c +++ b/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu_faults_decoder.c @@ -25,11 +25,7 @@ #include #include -#if MALI_USE_CSF #include -#else -#include -#endif #include #include @@ -76,7 +72,6 @@ const char *fault_source_id_internal_requester_get_str(struct kbase_device *kbde } else return "Load/store cache"; } else if (!strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "tiler")) { -#if MALI_USE_CSF if (utlb_id == 0) { if (access_type == AS_FAULTSTATUS_ACCESS_TYPE_READ) return decode_fault_source_tiler_r_t( @@ -86,25 +81,14 @@ const char *fault_source_id_internal_requester_get_str(struct kbase_device *kbde ir, kbdev->gpu_props.gpu_id.arch_id); } else return "The polygon list writer. No further details."; -#else - return (utlb_id == 0) ? "Anything other than the polygon list writer" : - "The polygon list writer"; -#endif - } -#if MALI_USE_CSF - else if (!strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "csf")) { + } else if (!strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "csf")) { if (access_type == AS_FAULTSTATUS_ACCESS_TYPE_READ) return decode_fault_source_csf_r_t(ir, kbdev->gpu_props.gpu_id.arch_id); else return decode_fault_source_csf_w_t(ir, kbdev->gpu_props.gpu_id.arch_id); - } -#else - else if (!strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "jm")) - return decode_fault_source_jm_t(ir, kbdev->gpu_props.gpu_id.arch_id); -#endif - else if (!strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "I2c") || - !strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "memsys") || - !strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "mmu")) { + } else if (!strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "I2c") || + !strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "memsys") || + !strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "mmu")) { return "Not used"; } diff --git a/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu_hw_direct.c b/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu_hw_direct.c index e531444ccb24..6051550b1ef7 100644 --- a/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu_hw_direct.c +++ b/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu_hw_direct.c @@ -29,7 +29,6 @@ #include #include -#if MALI_USE_CSF /** * mmu_has_flush_skip_pgd_levels() - Check if the GPU has the feature * AS_LOCKADDR_FLUSH_SKIP_LEVELS @@ -45,7 +44,6 @@ static bool mmu_has_flush_skip_pgd_levels(struct kbase_gpu_props const *gpu_prop { return gpu_props->gpu_id.arch_id >= GPU_ID_ARCH_MAKE(12, 0, 4); } -#endif /** * lock_region() - Generate lockaddr to lock memory region in MMU @@ -145,11 +143,9 @@ static int lock_region(struct kbase_gpu_props const *gpu_props, u64 *lockaddr, *lockaddr = lockaddr_base & ~((1ull << lockaddr_size_log2) - 1); *lockaddr |= lockaddr_size_log2 - 1; -#if MALI_USE_CSF if (mmu_has_flush_skip_pgd_levels(gpu_props)) *lockaddr = AS_LOCKADDR_FLUSH_SKIP_LEVELS_SET(*lockaddr, op_param->flush_skip_levels); -#endif return 0; } @@ -210,7 +206,6 @@ static int write_cmd(struct kbase_device *kbdev, unsigned int as_nr, u32 cmd) return status; } -#if MALI_USE_CSF static int wait_l2_power_trans_complete(struct kbase_device *kbdev) { u32 val; @@ -316,7 +311,6 @@ static int apply_hw_issue_GPU2019_3901_wa(struct kbase_device *kbdev, u32 *mmu_c return ret; } #endif /* !IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI) */ -#endif /* MALI_USE_CSF */ void kbase_mmu_hw_configure(struct kbase_device *kbdev, struct kbase_as *as) { @@ -324,9 +318,6 @@ void kbase_mmu_hw_configure(struct kbase_device *kbdev, struct kbase_as *as) u64 transcfg = 0; lockdep_assert_held(&kbdev->hwaccess_lock); -#if !MALI_USE_CSF - lockdep_assert_held(&kbdev->mmu_hw_mutex); -#endif transcfg = current_setup->transcfg; @@ -343,18 +334,19 @@ void kbase_mmu_hw_configure(struct kbase_device *kbdev, struct kbase_as *as) transcfg = AS_TRANSCFG_PTW_SH_SET(transcfg, AS_TRANSCFG_PTW_SH_OUTER_SHAREABLE); } + kbase_reg_write64(kbdev, MMU_AS_OFFSET(as->number, TRANSCFG), transcfg); kbase_reg_write64(kbdev, MMU_AS_OFFSET(as->number, TRANSTAB), current_setup->transtab); kbase_reg_write64(kbdev, MMU_AS_OFFSET(as->number, MEMATTR), current_setup->memattr); KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG(kbdev, as, current_setup->transtab, current_setup->memattr, transcfg); + KBASE_TLSTREAM_JD_AS_INFO(kbdev, as->number, current_setup->transtab, + current_setup->memattr, transcfg); write_cmd(kbdev, as->number, AS_COMMAND_COMMAND_UPDATE); -#if MALI_USE_CSF /* Wait for UPDATE command to complete */ wait_ready(kbdev, as->number); -#endif } /** @@ -465,6 +457,7 @@ int kbase_mmu_hw_do_unlock_no_addr(struct kbase_device *kbdev, struct kbase_as * if (WARN_ON(kbdev == NULL) || WARN_ON(as == NULL)) return -EINVAL; + ret = write_cmd(kbdev, as->number, AS_COMMAND_COMMAND_UNLOCK); /* Wait for UNLOCK command to complete */ @@ -491,6 +484,7 @@ int kbase_mmu_hw_do_unlock(struct kbase_device *kbdev, struct kbase_as *as, if (WARN_ON(kbdev == NULL) || WARN_ON(as == NULL)) return -EINVAL; + ret = mmu_hw_set_lock_addr(kbdev, as->number, &lock_addr, op_param); if (!ret) @@ -528,7 +522,7 @@ int kbase_mmu_hw_do_flush(struct kbase_device *kbdev, struct kbase_as *as, if (ret) return ret; -#if MALI_USE_CSF && !IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI) +#if !IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI) /* WA for the KBASE_HW_ISSUE_GPU2019_3901. */ if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_GPU2019_3901) && mmu_cmd == AS_COMMAND_COMMAND_FLUSH_MEM) { @@ -553,12 +547,10 @@ int kbase_mmu_hw_do_flush(struct kbase_device *kbdev, struct kbase_as *as, if (likely(!ret)) { mmu_command_instr(kbdev, op_param->kctx_id, mmu_cmd, lock_addr, op_param->mmu_sync_info); -#if MALI_USE_CSF if (flush_op == KBASE_MMU_OP_FLUSH_MEM && kbdev->pm.backend.apply_hw_issue_TITANHW_2938_wa && kbdev->pm.backend.l2_state == KBASE_L2_PEND_OFF) ret = wait_l2_power_trans_complete(kbdev); -#endif } return ret; @@ -587,6 +579,7 @@ int kbase_mmu_hw_do_flush_on_gpu_ctrl(struct kbase_device *kbdev, struct kbase_a if (flush_op == KBASE_MMU_OP_FLUSH_PT) gpu_cmd = GPU_COMMAND_CACHE_CLN_INV_L2; + /* 1. Issue MMU_AS_CONTROL.COMMAND.LOCK operation. */ ret = mmu_hw_do_lock(kbdev, as, op_param); if (ret) @@ -598,14 +591,12 @@ int kbase_mmu_hw_do_flush_on_gpu_ctrl(struct kbase_device *kbdev, struct kbase_a /* 3. Issue MMU_AS_CONTROL.COMMAND.UNLOCK operation. */ ret2 = kbase_mmu_hw_do_unlock_no_addr(kbdev, as, op_param); -#if MALI_USE_CSF if (!ret && !ret2) { if (flush_op == KBASE_MMU_OP_FLUSH_MEM && kbdev->pm.backend.apply_hw_issue_TITANHW_2938_wa && kbdev->pm.backend.l2_state == KBASE_L2_PEND_OFF) ret = wait_l2_power_trans_complete(kbdev); } -#endif return ret ?: ret2; } @@ -629,20 +620,14 @@ void kbase_mmu_hw_clear_fault(struct kbase_device *kbdev, struct kbase_as *as, /* Clear the page (and bus fault IRQ as well in case one occurred) */ pf_bf_mask = MMU_PAGE_FAULT(as->number); -#if !MALI_USE_CSF - if (type == KBASE_MMU_FAULT_TYPE_BUS || type == KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED) - pf_bf_mask |= MMU_BUS_ERROR(as->number); -#endif kbase_reg_write32(kbdev, MMU_CONTROL_ENUM(IRQ_CLEAR), pf_bf_mask); -#if MALI_USE_CSF /* For valid page faults, this function is called just before unblocking the MMU (which * would in turn unblock the MCU firmware) and so this is an opportune location to * update the page fault counter value in firmware visible memory. */ if (likely(type == KBASE_MMU_FAULT_TYPE_PAGE) && kbdev->csf.page_fault_cnt_ptr) *kbdev->csf.page_fault_cnt_ptr = ++kbdev->csf.page_fault_cnt; -#endif unlock: spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); @@ -670,10 +655,6 @@ void kbase_mmu_hw_enable_fault(struct kbase_device *kbdev, struct kbase_as *as, irq_mask = kbase_reg_read32(kbdev, MMU_CONTROL_ENUM(IRQ_MASK)) | MMU_PAGE_FAULT(as->number); -#if !MALI_USE_CSF - if (type == KBASE_MMU_FAULT_TYPE_BUS || type == KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED) - irq_mask |= MMU_BUS_ERROR(as->number); -#endif kbase_reg_write32(kbdev, MMU_CONTROL_ENUM(IRQ_MASK), irq_mask); unlock: diff --git a/drivers/gpu/arm/valhall/platform/devicetree/mali_kbase_clk_rate_trace.c b/drivers/gpu/arm/valhall/platform/devicetree/mali_kbase_clk_rate_trace.c index f0995a4cb22e..b3a9331b1c75 100644 --- a/drivers/gpu/arm/valhall/platform/devicetree/mali_kbase_clk_rate_trace.c +++ b/drivers/gpu/arm/valhall/platform/devicetree/mali_kbase_clk_rate_trace.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2015-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2015-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -24,19 +24,15 @@ #include #include "mali_kbase_config_platform.h" -#if MALI_USE_CSF #include -#endif static void *enumerate_gpu_clk(struct kbase_device *kbdev, unsigned int index) { if (index >= kbdev->nr_clocks) return NULL; -#if MALI_USE_CSF if (of_machine_is_compatible("arm,juno")) WARN_ON(kbdev->nr_clocks != 1); -#endif return kbdev->clocks[index]; } @@ -44,7 +40,6 @@ static void *enumerate_gpu_clk(struct kbase_device *kbdev, unsigned int index) static unsigned long get_gpu_clk_rate(struct kbase_device *kbdev, void *gpu_clk_handle) { CSTD_UNUSED(kbdev); -#if MALI_USE_CSF /* On Juno fpga platforms, the GPU clock rate is reported as 600 MHZ at * the boot time. Then after the first call to kbase_devfreq_target() * the clock rate is reported as 450 MHZ and the frequency does not @@ -60,7 +55,6 @@ static unsigned long get_gpu_clk_rate(struct kbase_device *kbdev, void *gpu_clk_ */ if (of_machine_is_compatible("arm,juno")) return arch_timer_get_cntfrq(); -#endif return clk_get_rate((struct clk *)gpu_clk_handle); } @@ -79,11 +73,9 @@ static int gpu_clk_notifier_register(struct kbase_device *kbdev, void *gpu_clk_h sizeof(((struct kbase_gpu_clk_notifier_data *)0)->gpu_clk_handle), "mismatch in the size of clk member"); -#if MALI_USE_CSF /* Frequency is fixed on Juno platforms */ if (of_machine_is_compatible("arm,juno")) return 0; -#endif return clk_notifier_register((struct clk *)gpu_clk_handle, nb); } @@ -93,10 +85,8 @@ static void gpu_clk_notifier_unregister(struct kbase_device *kbdev, void *gpu_cl { CSTD_UNUSED(kbdev); -#if MALI_USE_CSF if (of_machine_is_compatible("arm,juno")) return; -#endif clk_notifier_unregister((struct clk *)gpu_clk_handle, nb); } diff --git a/drivers/gpu/arm/valhall/platform/devicetree/mali_kbase_config_devicetree.c b/drivers/gpu/arm/valhall/platform/devicetree/mali_kbase_config_devicetree.c index 7c07f4f96f56..02bcc94d9d61 100644 --- a/drivers/gpu/arm/valhall/platform/devicetree/mali_kbase_config_devicetree.c +++ b/drivers/gpu/arm/valhall/platform/devicetree/mali_kbase_config_devicetree.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2015-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2015-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -42,12 +42,7 @@ void kbase_platform_unregister(void) #endif #ifdef CONFIG_MALI_VALHALL_DVFS -#if MALI_USE_CSF int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation) -#else -int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation, u32 util_gl_share, - u32 util_cl_share[2]) -#endif { return 1; } diff --git a/drivers/gpu/arm/valhall/platform/devicetree/mali_kbase_runtime_pm.c b/drivers/gpu/arm/valhall/platform/devicetree/mali_kbase_runtime_pm.c index b580bfee0078..ef060fabdbaf 100644 --- a/drivers/gpu/arm/valhall/platform/devicetree/mali_kbase_runtime_pm.c +++ b/drivers/gpu/arm/valhall/platform/devicetree/mali_kbase_runtime_pm.c @@ -60,7 +60,15 @@ static void disable_gpu_power_control(struct kbase_device *kbdev) ; else if (__clk_is_enabled(kbdev->clocks[i])) { clk_disable_unprepare(kbdev->clocks[i]); - WARN_ON(__clk_is_enabled(kbdev->clocks[i])); + if (__clk_is_enabled(kbdev->clocks[i])) + /* + * Clock won't be disabled when: + * 1. the clock_count for clock[i] is not reaching 0. + * 2. the clocks[i] is fixed-clock. + * Printing logs just for debugging purpose. + */ + dev_dbg(kbdev->dev, "%s Clock %d (%s) is not disabled.\n", __func__, + i, __clk_get_name(kbdev->clocks[i])); } } @@ -77,15 +85,12 @@ static void disable_gpu_power_control(struct kbase_device *kbdev) static int pm_callback_power_on(struct kbase_device *kbdev) { - int ret = 1; /* Assume GPU has been powered off */ - int error; unsigned long flags; dev_dbg(kbdev->dev, "%s %pK\n", __func__, (void *)kbdev->dev->pm_domain); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); WARN_ON(kbase_io_is_gpu_powered(kbdev)); -#if MALI_USE_CSF if (likely(kbdev->csf.firmware_inited)) { WARN_ON(!kbdev->pm.active_count); WARN_ON(kbdev->pm.runtime_active); @@ -93,27 +98,8 @@ static int pm_callback_power_on(struct kbase_device *kbdev) spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); enable_gpu_power_control(kbdev); - CSTD_UNUSED(error); -#else - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#ifdef KBASE_PM_RUNTIME - error = pm_runtime_get_sync(kbdev->dev); - if (error == 1) { - /* - * Let core know that the chip has not been - * powered off, so we can save on re-initialization. - */ - ret = 0; - } - dev_dbg(kbdev->dev, "pm_runtime_get_sync returned %d\n", error); -#else - enable_gpu_power_control(kbdev); -#endif /* KBASE_PM_RUNTIME */ - -#endif /* MALI_USE_CSF */ - - return ret; + return 1; } static void pm_callback_power_off(struct kbase_device *kbdev) @@ -124,37 +110,26 @@ static void pm_callback_power_off(struct kbase_device *kbdev) spin_lock_irqsave(&kbdev->hwaccess_lock, flags); WARN_ON(kbase_io_is_gpu_powered(kbdev)); -#if MALI_USE_CSF if (likely(kbdev->csf.firmware_inited)) { -#ifdef CONFIG_MALI_VALHALL_DEBUG - WARN_ON(kbase_csf_scheduler_get_nr_active_csgs(kbdev)); -#endif + if (IS_ENABLED(CONFIG_MALI_VALHALL_DEBUG)) + WARN_ON(kbase_csf_scheduler_get_nr_active_csgs(kbdev) && + !kbase_io_is_aw_removed(kbdev)); WARN_ON(kbdev->pm.backend.mcu_state != KBASE_MCU_OFF); } spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); /* Power down the GPU immediately */ disable_gpu_power_control(kbdev); -#else /* MALI_USE_CSF */ - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - -#ifdef KBASE_PM_RUNTIME - pm_runtime_mark_last_busy(kbdev->dev); - pm_runtime_put_autosuspend(kbdev->dev); -#else - /* Power down the GPU immediately as runtime PM is disabled */ - disable_gpu_power_control(kbdev); -#endif -#endif /* MALI_USE_CSF */ } -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) static void pm_callback_runtime_gpu_active(struct kbase_device *kbdev) { unsigned long flags; int error; lockdep_assert_held(&kbdev->pm.lock); + if (!IS_ENABLED(CONFIG_PM)) + return; spin_lock_irqsave(&kbdev->hwaccess_lock, flags); WARN_ON(!kbase_io_is_gpu_powered(kbdev)); @@ -179,6 +154,8 @@ static void pm_callback_runtime_gpu_idle(struct kbase_device *kbdev) unsigned long flags; lockdep_assert_held(&kbdev->pm.lock); + if (!IS_ENABLED(CONFIG_PM)) + return; dev_dbg(kbdev->dev, "%s", __func__); @@ -193,14 +170,14 @@ static void pm_callback_runtime_gpu_idle(struct kbase_device *kbdev) pm_runtime_put_autosuspend(kbdev->dev); kbdev->pm.runtime_active = false; } -#endif -#ifdef KBASE_PM_RUNTIME static int kbase_device_runtime_init(struct kbase_device *kbdev) { int ret = 0; dev_dbg(kbdev->dev, "%s\n", __func__); + if (!IS_ENABLED(CONFIG_PM)) + return 0; pm_runtime_set_autosuspend_delay(kbdev->dev, AUTO_SUSPEND_DELAY); pm_runtime_use_autosuspend(kbdev->dev); @@ -211,11 +188,14 @@ static int kbase_device_runtime_init(struct kbase_device *kbdev) if (!pm_runtime_enabled(kbdev->dev)) { dev_warn(kbdev->dev, "pm_runtime not enabled"); ret = -EINVAL; - } else if (atomic_read(&kbdev->dev->power.usage_count)) { + } +#if IS_ENABLED(CONFIG_PM) + else if (atomic_read(&kbdev->dev->power.usage_count)) { dev_warn(kbdev->dev, "%s: Device runtime usage count unexpectedly non zero %d", __func__, atomic_read(&kbdev->dev->power.usage_count)); ret = -EINVAL; } +#endif return ret; } @@ -223,32 +203,26 @@ static int kbase_device_runtime_init(struct kbase_device *kbdev) static void kbase_device_runtime_disable(struct kbase_device *kbdev) { dev_dbg(kbdev->dev, "%s\n", __func__); - + if (!IS_ENABLED(CONFIG_PM)) + return; +#if IS_ENABLED(CONFIG_PM) if (atomic_read(&kbdev->dev->power.usage_count)) dev_warn(kbdev->dev, "%s: Device runtime usage count unexpectedly non zero %d", __func__, atomic_read(&kbdev->dev->power.usage_count)); - +#endif pm_runtime_disable(kbdev->dev); } -#endif /* KBASE_PM_RUNTIME */ static int pm_callback_runtime_on(struct kbase_device *kbdev) { dev_dbg(kbdev->dev, "%s\n", __func__); -#if !MALI_USE_CSF - enable_gpu_power_control(kbdev); -#endif return 0; } static void pm_callback_runtime_off(struct kbase_device *kbdev) { dev_dbg(kbdev->dev, "%s\n", __func__); - -#if !MALI_USE_CSF - disable_gpu_power_control(kbdev); -#endif } static void pm_callback_resume(struct kbase_device *kbdev) @@ -269,23 +243,10 @@ struct kbase_pm_callback_conf pm_callbacks = { .power_off_callback = pm_callback_power_off, .power_suspend_callback = pm_callback_suspend, .power_resume_callback = pm_callback_resume, -#ifdef KBASE_PM_RUNTIME .power_runtime_init_callback = kbase_device_runtime_init, .power_runtime_term_callback = kbase_device_runtime_disable, .power_runtime_on_callback = pm_callback_runtime_on, .power_runtime_off_callback = pm_callback_runtime_off, -#else /* KBASE_PM_RUNTIME */ - .power_runtime_init_callback = NULL, - .power_runtime_term_callback = NULL, - .power_runtime_on_callback = NULL, - .power_runtime_off_callback = NULL, -#endif /* KBASE_PM_RUNTIME */ - -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) .power_runtime_gpu_idle_callback = pm_callback_runtime_gpu_idle, .power_runtime_gpu_active_callback = pm_callback_runtime_gpu_active, -#else - .power_runtime_gpu_idle_callback = NULL, - .power_runtime_gpu_active_callback = NULL, -#endif }; diff --git a/drivers/gpu/arm/valhall/platform/meson/mali_kbase_config_meson.c b/drivers/gpu/arm/valhall/platform/meson/mali_kbase_config_meson.c index 7c07f4f96f56..02bcc94d9d61 100644 --- a/drivers/gpu/arm/valhall/platform/meson/mali_kbase_config_meson.c +++ b/drivers/gpu/arm/valhall/platform/meson/mali_kbase_config_meson.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2015-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2015-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -42,12 +42,7 @@ void kbase_platform_unregister(void) #endif #ifdef CONFIG_MALI_VALHALL_DVFS -#if MALI_USE_CSF int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation) -#else -int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation, u32 util_gl_share, - u32 util_cl_share[2]) -#endif { return 1; } diff --git a/drivers/gpu/arm/valhall/platform/meson/mali_kbase_runtime_pm.c b/drivers/gpu/arm/valhall/platform/meson/mali_kbase_runtime_pm.c index bd3b4b5e2aa3..e51e251c8380 100644 --- a/drivers/gpu/arm/valhall/platform/meson/mali_kbase_runtime_pm.c +++ b/drivers/gpu/arm/valhall/platform/meson/mali_kbase_runtime_pm.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2015-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2015-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -126,7 +126,15 @@ static void disable_gpu_power_control(struct kbase_device *kbdev) ; else if (__clk_is_enabled(kbdev->clocks[i])) { clk_disable_unprepare(kbdev->clocks[i]); - WARN_ON(__clk_is_enabled(kbdev->clocks[i])); + if (__clk_is_enabled(kbdev->clocks[i])) + /* + * Clock won't be disabled when: + * 1. the clock_count for clock[i] is not reaching 0. + * 2. the clocks[i] is fixed-clock. + * Printing logs just for debugging purpose. + */ + dev_dbg(kbdev->dev, "%s Clock %d (%s) is not disabled.\n", __func__, + i, __clk_get_name(kbdev->clocks[i])); } } @@ -147,19 +155,18 @@ static int pm_callback_power_on(struct kbase_device *kbdev) dev_dbg(kbdev->dev, "%s %pK\n", __func__, (void *)kbdev->dev->pm_domain); -#ifdef KBASE_PM_RUNTIME - error = pm_runtime_get_sync(kbdev->dev); - if (error == 1) { - /* - * Let core know that the chip has not been - * powered off, so we can save on re-initialization. - */ - ret = 0; - } - dev_dbg(kbdev->dev, "pm_runtime_get_sync returned %d\n", error); -#else - enable_gpu_power_control(kbdev); -#endif + if (IS_ENABLED(CONFIG_PM)) { + error = pm_runtime_get_sync(kbdev->dev); + if (error == 1) { + /* + * Let core know that the chip has not been + * powered off, so we can save on re-initialization. + */ + ret = 0; + } + dev_dbg(kbdev->dev, "pm_runtime_get_sync returned %d\n", error); + } else + enable_gpu_power_control(kbdev); return ret; } @@ -168,21 +175,22 @@ static void pm_callback_power_off(struct kbase_device *kbdev) { dev_dbg(kbdev->dev, "%s\n", __func__); -#ifdef KBASE_PM_RUNTIME - pm_runtime_mark_last_busy(kbdev->dev); - pm_runtime_put_autosuspend(kbdev->dev); -#else - /* Power down the GPU immediately as runtime PM is disabled */ - disable_gpu_power_control(kbdev); -#endif + if (IS_ENABLED(CONFIG_PM)) { + pm_runtime_mark_last_busy(kbdev->dev); + pm_runtime_put_autosuspend(kbdev->dev); + } else { + /* Power down the GPU immediately as runtime PM is disabled */ + disable_gpu_power_control(kbdev); + } } -#ifdef KBASE_PM_RUNTIME static int kbase_device_runtime_init(struct kbase_device *kbdev) { int ret = 0; dev_dbg(kbdev->dev, "%s\n", __func__); + if (!IS_ENABLED(CONFIG_PM)) + return 0; pm_runtime_set_autosuspend_delay(kbdev->dev, AUTO_SUSPEND_DELAY); pm_runtime_use_autosuspend(kbdev->dev); @@ -193,11 +201,14 @@ static int kbase_device_runtime_init(struct kbase_device *kbdev) if (!pm_runtime_enabled(kbdev->dev)) { dev_warn(kbdev->dev, "pm_runtime not enabled"); ret = -EINVAL; - } else if (atomic_read(&kbdev->dev->power.usage_count)) { + } +#if IS_ENABLED(CONFIG_PM) + else if (atomic_read(&kbdev->dev->power.usage_count)) { dev_warn(kbdev->dev, "%s: Device runtime usage count unexpectedly non zero %d", __func__, atomic_read(&kbdev->dev->power.usage_count)); ret = -EINVAL; } +#endif /* allocate resources for reset */ if (!ret) @@ -209,28 +220,32 @@ static int kbase_device_runtime_init(struct kbase_device *kbdev) static void kbase_device_runtime_disable(struct kbase_device *kbdev) { dev_dbg(kbdev->dev, "%s\n", __func__); + if (!IS_ENABLED(CONFIG_PM)) + return; +#if IS_ENABLED(CONFIG_PM) if (atomic_read(&kbdev->dev->power.usage_count)) dev_warn(kbdev->dev, "%s: Device runtime usage count unexpectedly non zero %d", __func__, atomic_read(&kbdev->dev->power.usage_count)); +#endif pm_runtime_disable(kbdev->dev); } -#endif /* KBASE_PM_RUNTIME */ static int pm_callback_runtime_on(struct kbase_device *kbdev) { dev_dbg(kbdev->dev, "%s\n", __func__); + if (IS_ENABLED(CONFIG_PM)) + enable_gpu_power_control(kbdev); - enable_gpu_power_control(kbdev); return 0; } static void pm_callback_runtime_off(struct kbase_device *kbdev) { dev_dbg(kbdev->dev, "%s\n", __func__); - - disable_gpu_power_control(kbdev); + if (IS_ENABLED(CONFIG_PM)) + disable_gpu_power_control(kbdev); } static void pm_callback_resume(struct kbase_device *kbdev) @@ -251,15 +266,8 @@ struct kbase_pm_callback_conf pm_callbacks = { .power_suspend_callback = pm_callback_suspend, .power_resume_callback = pm_callback_resume, .soft_reset_callback = pm_callback_soft_reset, -#ifdef KBASE_PM_RUNTIME .power_runtime_init_callback = kbase_device_runtime_init, .power_runtime_term_callback = kbase_device_runtime_disable, .power_runtime_on_callback = pm_callback_runtime_on, .power_runtime_off_callback = pm_callback_runtime_off, -#else /* KBASE_PM_RUNTIME */ - .power_runtime_init_callback = NULL, - .power_runtime_term_callback = NULL, - .power_runtime_on_callback = NULL, - .power_runtime_off_callback = NULL, -#endif /* KBASE_PM_RUNTIME */ }; diff --git a/drivers/gpu/arm/valhall/platform/rk/mali_kbase_config_rk.c b/drivers/gpu/arm/valhall/platform/rk/mali_kbase_config_rk.c index f87a6f50503a..628be3a40c86 100755 --- a/drivers/gpu/arm/valhall/platform/rk/mali_kbase_config_rk.c +++ b/drivers/gpu/arm/valhall/platform/rk/mali_kbase_config_rk.c @@ -18,9 +18,7 @@ #include #include -#if MALI_USE_CSF #include -#endif #include #include diff --git a/drivers/gpu/arm/valhall/platform/vexpress/mali_kbase_config_vexpress.c b/drivers/gpu/arm/valhall/platform/vexpress/mali_kbase_config_vexpress.c index c655aa62b526..e1bfe2f26f77 100644 --- a/drivers/gpu/arm/valhall/platform/vexpress/mali_kbase_config_vexpress.c +++ b/drivers/gpu/arm/valhall/platform/vexpress/mali_kbase_config_vexpress.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2011-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -63,12 +63,7 @@ struct kbase_platform_config *kbase_get_platform_config(void) } #ifdef CONFIG_MALI_VALHALL_DVFS -#if MALI_USE_CSF int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation) -#else -int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation, u32 util_gl_share, - u32 util_cl_share[2]) -#endif { return 1; } diff --git a/drivers/gpu/arm/valhall/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c b/drivers/gpu/arm/valhall/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c index d8a566d9d3c0..45ddda005d11 100644 --- a/drivers/gpu/arm/valhall/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c +++ b/drivers/gpu/arm/valhall/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2011-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -62,12 +62,7 @@ struct kbase_platform_config *kbase_get_platform_config(void) } #ifdef CONFIG_MALI_VALHALL_DVFS -#if MALI_USE_CSF int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation) -#else -int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation, u32 util_gl_share, - u32 util_cl_share[2]) -#endif { return 1; } diff --git a/drivers/gpu/arm/valhall/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c b/drivers/gpu/arm/valhall/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c index 63e990cdf6cf..3285277ccfb0 100644 --- a/drivers/gpu/arm/valhall/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c +++ b/drivers/gpu/arm/valhall/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2011-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -62,12 +62,7 @@ struct kbase_platform_config *kbase_get_platform_config(void) } #ifdef CONFIG_MALI_VALHALL_DVFS -#if MALI_USE_CSF int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation) -#else -int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation, u32 util_gl_share, - u32 util_cl_share[2]) -#endif { return 1; } diff --git a/drivers/gpu/arm/valhall/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c b/drivers/gpu/arm/valhall/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c index 0598d4397e2a..1939e3f3de48 100644 --- a/drivers/gpu/arm/valhall/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c +++ b/drivers/gpu/arm/valhall/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c @@ -176,9 +176,6 @@ static void kutf_set_pm_ctx_active(struct kutf_context *context) kbase_pm_context_active(data->kbdev); kbase_pm_wait_for_desired_state(data->kbdev); -#if !MALI_USE_CSF - kbase_pm_request_gpu_cycle_counter(data->kbdev); -#endif } static void kutf_set_pm_ctx_idle(struct kutf_context *context) @@ -187,9 +184,6 @@ static void kutf_set_pm_ctx_idle(struct kutf_context *context) if (WARN_ON(data->pm_ctx_cnt > 0)) return; -#if !MALI_USE_CSF - kbase_pm_release_gpu_cycle_counter(data->kbdev); -#endif kbase_pm_context_idle(data->kbdev); } diff --git a/drivers/gpu/arm/valhall/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c b/drivers/gpu/arm/valhall/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c index 6a143bb01616..4f7d781c0589 100644 --- a/drivers/gpu/arm/valhall/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c +++ b/drivers/gpu/arm/valhall/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c @@ -85,7 +85,6 @@ static irqreturn_t kbase_gpu_irq_custom_handler(int irq, void *data) u64 tval; bool has_test_irq = val & test_irq; -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface) { status_reg_enum = HOST_POWER_ENUM(PWR_IRQ_STATUS); clear_reg_enum = HOST_POWER_ENUM(PWR_IRQ_CLEAR); @@ -93,7 +92,6 @@ static irqreturn_t kbase_gpu_irq_custom_handler(int irq, void *data) val = kbase_reg_read32(kbdev, status_reg_enum); has_test_irq = val & test_irq; } -#endif if (has_test_irq) { tval = ktime_get_real_ns(); @@ -201,12 +199,10 @@ static void mali_kutf_irq_latency(struct kutf_context *context) triggered = false; /* Trigger fake IRQ */ -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface) { reg_enum = HOST_POWER_ENUM(PWR_IRQ_RAWSTAT); test_irq = PWR_IRQ_POWER_CHANGED_SINGLE; } -#endif kbase_reg_write32(kbdev, reg_enum, test_irq); if (wait_event_timeout(wait, triggered, IRQ_TIMEOUT) == 0) { diff --git a/drivers/gpu/arm/valhall/thirdparty/mali_kbase_mmap.c b/drivers/gpu/arm/valhall/thirdparty/mali_kbase_mmap.c index 162f82e5493a..8a71ed685de4 100644 --- a/drivers/gpu/arm/valhall/thirdparty/mali_kbase_mmap.c +++ b/drivers/gpu/arm/valhall/thirdparty/mali_kbase_mmap.c @@ -487,21 +487,6 @@ unsigned long kbase_context_get_unmapped_area(struct kbase_context *const kctx, align_mask = align_offset - 1; is_shader_code = true; } -#if !MALI_USE_CSF - } else if (reg->flags & KBASE_REG_TILER_ALIGN_TOP) { - unsigned long extension_bytes = - (unsigned long)(reg->extension << PAGE_SHIFT); - /* kbase_check_alloc_sizes() already satisfies - * these checks, but they're here to avoid - * maintenance hazards due to the assumptions - * involved - */ - WARN_ON(reg->extension > (ULONG_MAX >> PAGE_SHIFT)); - WARN_ON(reg->initial_commit > (ULONG_MAX >> PAGE_SHIFT)); - WARN_ON(!is_power_of_2(extension_bytes)); - align_mask = extension_bytes - 1; - align_offset = extension_bytes - (reg->initial_commit << PAGE_SHIFT); -#endif /* !MALI_USE_CSF */ } else if (reg->flags & KBASE_REG_GPU_VA_SAME_4GB_PAGE) { is_same_4gb_page = true; } diff --git a/drivers/gpu/arm/valhall/tl/Kbuild b/drivers/gpu/arm/valhall/tl/Kbuild index 35c3fabfc609..3dce47d3c0de 100644 --- a/drivers/gpu/arm/valhall/tl/Kbuild +++ b/drivers/gpu/arm/valhall/tl/Kbuild @@ -22,11 +22,6 @@ valhall_kbase-y += \ tl/mali_kbase_timeline.o \ tl/mali_kbase_timeline_io.o \ tl/mali_kbase_tlstream.o \ - tl/mali_kbase_tracepoints.o + tl/mali_kbase_tracepoints.o \ + tl/backend/mali_kbase_timeline_csf.o - -ifeq ($(CONFIG_MALI_VALHALL_CSF_SUPPORT),y) - valhall_kbase-y += tl/backend/mali_kbase_timeline_csf.o -else - valhall_kbase-y += tl/backend/mali_kbase_timeline_jm.o -endif diff --git a/drivers/gpu/arm/valhall/tl/backend/mali_kbase_timeline_csf.c b/drivers/gpu/arm/valhall/tl/backend/mali_kbase_timeline_csf.c index 8c65b62ae46d..60880d1cff13 100644 --- a/drivers/gpu/arm/valhall/tl/backend/mali_kbase_timeline_csf.c +++ b/drivers/gpu/arm/valhall/tl/backend/mali_kbase_timeline_csf.c @@ -33,12 +33,10 @@ void kbase_create_timeline_objects(struct kbase_device *kbdev) struct kbase_timeline *timeline = kbdev->timeline; struct kbase_tlstream *summary = &kbdev->timeline->streams[TL_STREAM_TYPE_OBJ_SUMMARY]; u32 const num_sb_entries = kbdev->gpu_props.gpu_id.arch_major >= 11 ? 16 : 8; - u32 const supports_gpu_sleep = -#ifdef KBASE_PM_RUNTIME - test_bit(KBASE_GPU_SUPPORTS_GPU_SLEEP, &kbdev->pm.backend.gpu_sleep_allowed); -#else - false; -#endif /* KBASE_PM_RUNTIME */ + u32 const supports_gpu_sleep = IS_ENABLED(CONFIG_PM) ? + test_bit(KBASE_GPU_SUPPORTS_GPU_SLEEP, + &kbdev->pm.backend.gpu_sleep_allowed) : + false; /* Summarize the Address Space objects. */ for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) diff --git a/drivers/gpu/arm/valhall/tl/backend/mali_kbase_timeline_jm.c b/drivers/gpu/arm/valhall/tl/backend/mali_kbase_timeline_jm.c deleted file mode 100644 index 3e9e6e864125..000000000000 --- a/drivers/gpu/arm/valhall/tl/backend/mali_kbase_timeline_jm.c +++ /dev/null @@ -1,86 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include -#include -#include - -#include - -void kbase_create_timeline_objects(struct kbase_device *kbdev) -{ - unsigned int lpu_id; - int as_nr; - struct kbase_context *kctx; - struct kbase_timeline *timeline = kbdev->timeline; - struct kbase_tlstream *summary = &timeline->streams[TL_STREAM_TYPE_OBJ_SUMMARY]; - - /* Summarize the LPU objects. */ - for (lpu_id = 0; lpu_id < kbdev->gpu_props.num_job_slots; lpu_id++) { - void *lpu = &kbdev->gpu_props.js_features[lpu_id]; - - __kbase_tlstream_tl_new_lpu(summary, lpu, lpu_id, 0); - } - - /* Summarize the Address Space objects. */ - for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) - __kbase_tlstream_tl_new_as(summary, &kbdev->as[as_nr], (u32)as_nr); - - /* Create GPU object and make it retain all LPUs and address spaces. */ - __kbase_tlstream_tl_new_gpu(summary, kbdev, kbdev->id, kbdev->gpu_props.num_cores); - - for (lpu_id = 0; lpu_id < kbdev->gpu_props.num_job_slots; lpu_id++) { - void *lpu = &kbdev->gpu_props.js_features[lpu_id]; - __kbase_tlstream_tl_lifelink_lpu_gpu(summary, lpu, kbdev); - } - - for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) - __kbase_tlstream_tl_lifelink_as_gpu(summary, &kbdev->as[as_nr], kbdev); - - /* Lock the context list, to ensure no changes to the list are made - * while we're summarizing the contexts and their contents. - */ - mutex_lock(&timeline->tl_kctx_list_lock); - - /* For each context in the device... */ - list_for_each_entry(kctx, &timeline->tl_kctx_list, tl_kctx_list_node) { - /* Summarize the context itself */ - __kbase_tlstream_tl_new_ctx(summary, kctx, kctx->id, (u32)(kctx->tgid)); - } - - /* Reset body stream buffers while holding the kctx lock. - * This ensures we can't fire both summary and normal tracepoints for - * the same objects. - * If we weren't holding the lock, it's possible that the summarized - * objects could have been created, destroyed, or used after we - * constructed the summary stream tracepoints, but before we reset - * the body stream, resulting in losing those object event tracepoints. - */ - kbase_timeline_streams_body_reset(timeline); - - mutex_unlock(&timeline->tl_kctx_list_lock); - - /* Static object are placed into summary packet that needs to be - * transmitted first. Flush all streams to make it available to - * user space. - */ - kbase_timeline_streams_flush(timeline); -} diff --git a/drivers/gpu/arm/valhall/tl/mali_kbase_timeline.c b/drivers/gpu/arm/valhall/tl/mali_kbase_timeline.c index 96f780945a4b..22de4c8a7e9a 100644 --- a/drivers/gpu/arm/valhall/tl/mali_kbase_timeline.c +++ b/drivers/gpu/arm/valhall/tl/mali_kbase_timeline.c @@ -94,9 +94,8 @@ static void kbasep_timeline_autoflush_timer_callback(struct timer_list *timer) int kbase_timeline_init(struct kbase_timeline **timeline, atomic_t *timeline_flags) { struct kbase_timeline *result; -#if MALI_USE_CSF struct kbase_tlstream *csffw_stream; -#endif + int i; if (!timeline || !timeline_flags) return -EINVAL; @@ -109,6 +108,9 @@ int kbase_timeline_init(struct kbase_timeline **timeline, atomic_t *timeline_fla mutex_init(&result->streams_buf_lock); init_waitqueue_head(&result->event_queue); + for (i = 0; i < TL_STREAM_TYPE_COUNT; i++) + spin_lock_init(&result->streams[i].lock); + /* Initialize the kctx list */ mutex_init(&result->tl_kctx_list_lock); INIT_LIST_HEAD(&result->tl_kctx_list); @@ -118,10 +120,8 @@ int kbase_timeline_init(struct kbase_timeline **timeline, atomic_t *timeline_fla kbase_timer_setup(&result->autoflush_timer, kbasep_timeline_autoflush_timer_callback); result->timeline_flags = timeline_flags; -#if MALI_USE_CSF csffw_stream = &result->streams[TL_STREAM_TYPE_CSFFW]; kbase_csf_tl_reader_init(&result->csf_tl_reader, csffw_stream); -#endif *timeline = result; return 0; @@ -134,9 +134,7 @@ void kbase_timeline_term(struct kbase_timeline *timeline) if (!timeline) return; -#if MALI_USE_CSF kbase_csf_tl_reader_term(&timeline->csf_tl_reader); -#endif WARN_ON(!list_empty(&timeline->tl_kctx_list)); @@ -208,7 +206,6 @@ int kbase_timeline_acquire(struct kbase_device *kbdev, u32 flags) if (atomic_cmpxchg(timeline->timeline_flags, 0, (int)timeline_flags)) return -EBUSY; -#if MALI_USE_CSF if (flags & BASE_TLSTREAM_ENABLE_CSFFW_TRACEPOINTS) { err = kbase_csf_tl_reader_start(&timeline->csf_tl_reader, kbdev); if (err) { @@ -216,7 +213,6 @@ int kbase_timeline_acquire(struct kbase_device *kbdev, u32 flags) return err; } } -#endif /* Reset and initialize header streams. */ kbase_tlstream_reset(&timeline->streams[TL_STREAM_TYPE_OBJ_SUMMARY]); @@ -224,18 +220,6 @@ int kbase_timeline_acquire(struct kbase_device *kbdev, u32 flags) timeline->obj_header_btc = obj_desc_header_size; timeline->aux_header_btc = aux_desc_header_size; -#if !MALI_USE_CSF - /* If job dumping is enabled, readjust the software event's - * timeout as the default value of 3 seconds is often - * insufficient. - */ - if (flags & BASE_TLSTREAM_JOB_DUMPING_ENABLED) { - dev_info(kbdev->dev, - "Job dumping is enabled, readjusting the software event's timeout\n"); - atomic_set(&kbdev->js_data.soft_job_timeout_ms, 1800000); - } -#endif /* !MALI_USE_CSF */ - /* Summary stream was cleared during acquire. * Create static timeline objects that will be * read by client. @@ -285,9 +269,7 @@ void kbase_timeline_release(struct kbase_timeline *timeline) if (time_to_sleep > 0) msleep_interruptible(time_to_sleep); -#if MALI_USE_CSF kbase_csf_tl_reader_stop(&timeline->csf_tl_reader); -#endif /* Stop autoflush timer before releasing access to streams. */ atomic_set(&timeline->autoflush_timer_active, 0); @@ -305,14 +287,12 @@ int kbase_timeline_streams_flush(struct kbase_timeline *timeline) if (WARN_ON(!timeline)) return -EINVAL; -#if MALI_USE_CSF { int ret = kbase_csf_tl_reader_flush_buffer(&timeline->csf_tl_reader); if (ret > 0) has_bytes = true; } -#endif for (stype = 0; stype < TL_STREAM_TYPE_COUNT; stype++) { nbytes = kbase_tlstream_flush_stream(&timeline->streams[stype]); @@ -326,9 +306,7 @@ void kbase_timeline_streams_body_reset(struct kbase_timeline *timeline) { kbase_tlstream_reset(&timeline->streams[TL_STREAM_TYPE_OBJ]); kbase_tlstream_reset(&timeline->streams[TL_STREAM_TYPE_AUX]); -#if MALI_USE_CSF kbase_tlstream_reset(&timeline->streams[TL_STREAM_TYPE_CSFFW]); -#endif } void kbase_timeline_pre_kbase_context_destroy(struct kbase_context *kctx) @@ -370,9 +348,7 @@ void kbase_timeline_post_kbase_context_create(struct kbase_context *kctx) * never in parallel with it. If fired in parallel, we could get * duplicate creation tracepoints. */ -#if MALI_USE_CSF KBASE_TLSTREAM_TL_KBASE_NEW_CTX(kbdev, kctx->id, kbdev->id); -#endif /* Trace with the AOM tracepoint even in CSF for dumping */ KBASE_TLSTREAM_TL_NEW_CTX(kbdev, kctx, kctx->id, 0); @@ -385,9 +361,7 @@ void kbase_timeline_post_kbase_context_destroy(struct kbase_context *kctx) /* Trace with the AOM tracepoint even in CSF for dumping */ KBASE_TLSTREAM_TL_DEL_CTX(kbdev, kctx); -#if MALI_USE_CSF KBASE_TLSTREAM_TL_KBASE_DEL_CTX(kbdev, kctx->id); -#endif /* Flush the timeline stream, so the user can see the termination * tracepoints being fired. diff --git a/drivers/gpu/arm/valhall/tl/mali_kbase_timeline_io.c b/drivers/gpu/arm/valhall/tl/mali_kbase_timeline_io.c index 522fd9e0964a..e3b4b0da17ce 100644 --- a/drivers/gpu/arm/valhall/tl/mali_kbase_timeline_io.c +++ b/drivers/gpu/arm/valhall/tl/mali_kbase_timeline_io.c @@ -146,11 +146,8 @@ static int kbasep_timeline_io_packet_pending(struct kbase_timeline *timeline, */ static int kbasep_timeline_has_header_data(struct kbase_timeline *timeline) { - return timeline->obj_header_btc || timeline->aux_header_btc -#if MALI_USE_CSF - || timeline->csf_tl_reader.tl_header.btc -#endif - ; + return timeline->obj_header_btc || timeline->aux_header_btc || + timeline->csf_tl_reader.tl_header.btc; } /** @@ -217,12 +214,10 @@ static inline int kbasep_timeline_copy_headers(struct kbase_timeline *timeline, if (copy_stream_header(buffer, size, copy_len, aux_desc_header, aux_desc_header_size, &timeline->aux_header_btc)) return -1; -#if MALI_USE_CSF if (copy_stream_header(buffer, size, copy_len, timeline->csf_tl_reader.tl_header.data, timeline->csf_tl_reader.tl_header.size, &timeline->csf_tl_reader.tl_header.btc)) return -1; -#endif return 0; } diff --git a/drivers/gpu/arm/valhall/tl/mali_kbase_timeline_priv.h b/drivers/gpu/arm/valhall/tl/mali_kbase_timeline_priv.h index 59848ba7bf0f..ddcdc5816ca2 100644 --- a/drivers/gpu/arm/valhall/tl/mali_kbase_timeline_priv.h +++ b/drivers/gpu/arm/valhall/tl/mali_kbase_timeline_priv.h @@ -25,10 +25,8 @@ #include #include "mali_kbase_tlstream.h" -#if MALI_USE_CSF #include "csf/mali_kbase_csf_tl_reader.h" #include "csf/mali_kbase_csf_trace_buffer.h" -#endif #include #include @@ -74,9 +72,7 @@ struct kbase_timeline { size_t obj_header_btc; size_t aux_header_btc; ktime_t last_acquire_time; -#if MALI_USE_CSF struct kbase_csf_tl_reader csf_tl_reader; -#endif }; void kbase_create_timeline_objects(struct kbase_device *kbdev); diff --git a/drivers/gpu/arm/valhall/tl/mali_kbase_tlstream.c b/drivers/gpu/arm/valhall/tl/mali_kbase_tlstream.c index 117417c30183..1f504e253140 100644 --- a/drivers/gpu/arm/valhall/tl/mali_kbase_tlstream.c +++ b/drivers/gpu/arm/valhall/tl/mali_kbase_tlstream.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2015-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2015-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -120,14 +120,12 @@ static const struct { TL_PACKET_TYPE_BODY, TL_STREAM_ID_KERNEL, }, -#if MALI_USE_CSF { TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_OBJ, TL_PACKET_TYPE_BODY, TL_STREAM_ID_CSFFW, }, -#endif }; void kbase_tlstream_init(struct kbase_tlstream *stream, enum tl_stream_type stream_type, @@ -138,8 +136,6 @@ void kbase_tlstream_init(struct kbase_tlstream *stream, enum tl_stream_type stre KBASE_DEBUG_ASSERT(stream); KBASE_DEBUG_ASSERT(stream_type < TL_STREAM_TYPE_COUNT); - spin_lock_init(&stream->lock); - /* All packets carrying tracepoints shall be numbered. */ if (tl_stream_cfg[stream_type].pkt_type == TL_PACKET_TYPE_BODY) stream->numbered = 1; @@ -219,6 +215,7 @@ char *kbase_tlstream_msgbuf_acquire(struct kbase_tlstream *stream, size_t msg_si size_t wb_size; KBASE_DEBUG_ASSERT(PACKET_SIZE - PACKET_HEADER_SIZE - PACKET_NUMBER_SIZE >= msg_size); + WARN_ON(!stream->buffer); spin_lock_irqsave(&stream->lock, *flags); @@ -259,6 +256,9 @@ size_t kbase_tlstream_flush_stream(struct kbase_tlstream *stream) size_t wb_size; size_t min_size = PACKET_HEADER_SIZE; + if (!stream->buffer) + return 0; + if (stream->numbered) min_size += PACKET_NUMBER_SIZE; diff --git a/drivers/gpu/arm/valhall/tl/mali_kbase_tlstream.h b/drivers/gpu/arm/valhall/tl/mali_kbase_tlstream.h index d5745796e8e2..a74f5c044db4 100644 --- a/drivers/gpu/arm/valhall/tl/mali_kbase_tlstream.h +++ b/drivers/gpu/arm/valhall/tl/mali_kbase_tlstream.h @@ -99,9 +99,7 @@ enum tl_stream_type { TL_STREAM_TYPE_OBJ_SUMMARY = TL_STREAM_TYPE_FIRST, TL_STREAM_TYPE_OBJ, TL_STREAM_TYPE_AUX, -#if MALI_USE_CSF TL_STREAM_TYPE_CSFFW, -#endif TL_STREAM_TYPE_COUNT }; diff --git a/drivers/gpu/arm/valhall/tl/mali_kbase_tracepoints.c b/drivers/gpu/arm/valhall/tl/mali_kbase_tracepoints.c index 2cc556583d8f..a8052716dee8 100644 --- a/drivers/gpu/arm/valhall/tl/mali_kbase_tracepoints.c +++ b/drivers/gpu/arm/valhall/tl/mali_kbase_tracepoints.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2025 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -83,7 +83,7 @@ enum tl_msg_id_obj { KBASE_TL_ATTRIB_ATOM_PRIORITY, KBASE_TL_ATTRIB_ATOM_STATE, KBASE_TL_ATTRIB_ATOM_PRIORITIZED, - KBASE_TL_ATTRIB_ATOM_JIT, + KBASE_JD_AS_INFO, KBASE_TL_KBASE_NEW_DEVICE, KBASE_TL_KBASE_GPUCMDQUEUE_KICK, KBASE_TL_KBASE_DEVICE_PROGRAM_CSG, @@ -354,10 +354,10 @@ enum tl_msg_id_obj { "atom caused priority change", \ "@p", \ "atom") \ - TRACEPOINT_DESC(KBASE_TL_ATTRIB_ATOM_JIT, \ - "jit done for atom", \ - "@pLLILILLL", \ - "atom,edit_addr,new_addr,jit_flags,mem_flags,j_id,com_pgs,extent,va_pgs") \ + TRACEPOINT_DESC(KBASE_JD_AS_INFO, \ + "address space attributes", \ + "@ILLL", \ + "as_nr,transtab,memattr,transcfg") \ TRACEPOINT_DESC(KBASE_TL_KBASE_NEW_DEVICE, \ "New KBase Device", \ "@IIIIIIII", \ @@ -2043,30 +2043,20 @@ void __kbase_tlstream_tl_attrib_atom_prioritized( kbase_tlstream_msgbuf_release(stream, acq_flags); } -void __kbase_tlstream_tl_attrib_atom_jit( +void __kbase_tlstream_jd_as_info( struct kbase_tlstream *stream, - const void *atom, - u64 edit_addr, - u64 new_addr, - u32 jit_flags, - u64 mem_flags, - u32 j_id, - u64 com_pgs, - u64 extent, - u64 va_pgs + u32 as_nr, + u64 transtab, + u64 memattr, + u64 transcfg ) { - const u32 msg_id = KBASE_TL_ATTRIB_ATOM_JIT; + const u32 msg_id = KBASE_JD_AS_INFO; const size_t msg_size = sizeof(msg_id) + sizeof(u64) - + sizeof(atom) - + sizeof(edit_addr) - + sizeof(new_addr) - + sizeof(jit_flags) - + sizeof(mem_flags) - + sizeof(j_id) - + sizeof(com_pgs) - + sizeof(extent) - + sizeof(va_pgs) + + sizeof(as_nr) + + sizeof(transtab) + + sizeof(memattr) + + sizeof(transcfg) ; char *buffer; unsigned long acq_flags; @@ -2077,23 +2067,13 @@ void __kbase_tlstream_tl_attrib_atom_jit( pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); pos = kbasep_serialize_timestamp(buffer, pos); pos = kbasep_serialize_bytes(buffer, - pos, &atom, sizeof(atom)); + pos, &as_nr, sizeof(as_nr)); pos = kbasep_serialize_bytes(buffer, - pos, &edit_addr, sizeof(edit_addr)); + pos, &transtab, sizeof(transtab)); pos = kbasep_serialize_bytes(buffer, - pos, &new_addr, sizeof(new_addr)); + pos, &memattr, sizeof(memattr)); pos = kbasep_serialize_bytes(buffer, - pos, &jit_flags, sizeof(jit_flags)); - pos = kbasep_serialize_bytes(buffer, - pos, &mem_flags, sizeof(mem_flags)); - pos = kbasep_serialize_bytes(buffer, - pos, &j_id, sizeof(j_id)); - pos = kbasep_serialize_bytes(buffer, - pos, &com_pgs, sizeof(com_pgs)); - pos = kbasep_serialize_bytes(buffer, - pos, &extent, sizeof(extent)); - pos = kbasep_serialize_bytes(buffer, - pos, &va_pgs, sizeof(va_pgs)); + pos, &transcfg, sizeof(transcfg)); kbase_tlstream_msgbuf_release(stream, acq_flags); } diff --git a/drivers/gpu/arm/valhall/tl/mali_kbase_tracepoints.h b/drivers/gpu/arm/valhall/tl/mali_kbase_tracepoints.h index 2763866758c0..09678688805f 100644 --- a/drivers/gpu/arm/valhall/tl/mali_kbase_tracepoints.h +++ b/drivers/gpu/arm/valhall/tl/mali_kbase_tracepoints.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2025 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -382,17 +382,12 @@ void __kbase_tlstream_tl_attrib_atom_prioritized( const void *atom ); -void __kbase_tlstream_tl_attrib_atom_jit( +void __kbase_tlstream_jd_as_info( struct kbase_tlstream *stream, - const void *atom, - u64 edit_addr, - u64 new_addr, - u32 jit_flags, - u64 mem_flags, - u32 j_id, - u64 com_pgs, - u64 extent, - u64 va_pgs + u32 as_nr, + u64 transtab, + u64 memattr, + u64 transcfg ); void __kbase_tlstream_tl_kbase_new_device( @@ -1961,45 +1956,30 @@ struct kbase_tlstream; } while (0) /** - * KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT - jit done for atom + * KBASE_TLSTREAM_JD_AS_INFO - address space attributes * * @kbdev: Kbase device - * @atom: Atom identifier - * @edit_addr: Address edited by jit - * @new_addr: Address placed into the edited location - * @jit_flags: Flags specifying the special requirements for the JIT allocation. - * @mem_flags: Flags defining the properties of a memory region - * @j_id: Unique ID provided by the caller, this is used to pair allocation and free requests. - * @com_pgs: The minimum number of physical pages which should back the allocation. - * @extent: Granularity of physical pages to grow the allocation by during a fault. - * @va_pgs: The minimum number of virtual pages required + * @as_nr: Address space number + * @transtab: Configuration of the TRANSTAB register + * @memattr: Configuration of the MEMATTR register + * @transcfg: Configuration of the TRANSCFG register (or zero if not present) */ -#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT( \ +#define KBASE_TLSTREAM_JD_AS_INFO( \ kbdev, \ - atom, \ - edit_addr, \ - new_addr, \ - jit_flags, \ - mem_flags, \ - j_id, \ - com_pgs, \ - extent, \ - va_pgs \ + as_nr, \ + transtab, \ + memattr, \ + transcfg \ ) \ do { \ u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_JOB_DUMPING_ENABLED) \ - __kbase_tlstream_tl_attrib_atom_jit( \ + __kbase_tlstream_jd_as_info( \ __TL_DISPATCH_STREAM(kbdev, obj), \ - atom, \ - edit_addr, \ - new_addr, \ - jit_flags, \ - mem_flags, \ - j_id, \ - com_pgs, \ - extent, \ - va_pgs \ + as_nr, \ + transtab, \ + memattr, \ + transcfg \ ); \ } while (0) @@ -2016,7 +1996,6 @@ struct kbase_tlstream; * @kbase_device_supports_gpu_sleep: Whether GPU sleep is supported * @kbase_device_has_neural_engine: Whether neural engine is supported */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_NEW_DEVICE( \ kbdev, \ kbase_device_id, \ @@ -2043,20 +2022,6 @@ struct kbase_tlstream; kbase_device_has_neural_engine \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_NEW_DEVICE( \ - kbdev, \ - kbase_device_id, \ - kbase_device_gpu_core_count, \ - kbase_device_max_num_csgs, \ - kbase_device_as_count, \ - kbase_device_sb_entry_count, \ - kbase_device_has_cross_stream_sync, \ - kbase_device_supports_gpu_sleep, \ - kbase_device_has_neural_engine \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_GPUCMDQUEUE_KICK - Kernel receives a request to process new GPU queue instructions @@ -2065,7 +2030,6 @@ struct kbase_tlstream; * @kernel_ctx_id: Unique ID for the KBase Context * @buffer_gpu_addr: Address of the GPU queue's command buffer */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_GPUCMDQUEUE_KICK( \ kbdev, \ kernel_ctx_id, \ @@ -2080,14 +2044,6 @@ struct kbase_tlstream; buffer_gpu_addr \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_GPUCMDQUEUE_KICK( \ - kbdev, \ - kernel_ctx_id, \ - buffer_gpu_addr \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_DEVICE_PROGRAM_CSG - CSG is programmed to a slot @@ -2099,7 +2055,6 @@ struct kbase_tlstream; * @kbase_device_csg_slot_index: The index of the slot in the scheduler being programmed * @kbase_device_csg_slot_resuming: Whether the csg is being resumed */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_DEVICE_PROGRAM_CSG( \ kbdev, \ kbase_device_id, \ @@ -2120,17 +2075,6 @@ struct kbase_tlstream; kbase_device_csg_slot_resuming \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_DEVICE_PROGRAM_CSG( \ - kbdev, \ - kbase_device_id, \ - kernel_ctx_id, \ - gpu_cmdq_grp_handle, \ - kbase_device_csg_slot_index, \ - kbase_device_csg_slot_resuming \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_DEVICE_DEPROGRAM_CSG - CSG is deprogrammed from a slot @@ -2139,7 +2083,6 @@ struct kbase_tlstream; * @kbase_device_id: The ID of the physical hardware * @kbase_device_csg_slot_index: The index of the slot in the scheduler whose CSG is being deprogrammed */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_DEVICE_DEPROGRAM_CSG( \ kbdev, \ kbase_device_id, \ @@ -2154,14 +2097,6 @@ struct kbase_tlstream; kbase_device_csg_slot_index \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_DEVICE_DEPROGRAM_CSG( \ - kbdev, \ - kbase_device_id, \ - kbase_device_csg_slot_index \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_DEVICE_PROTM_ENTER_CSG - CSG slot is entering protected mode @@ -2170,7 +2105,6 @@ struct kbase_tlstream; * @kbase_device_id: The ID of the physical hardware * @kbase_device_csg_slot_index: The index of the slot in the scheduler whose CSG has entered PMODE */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_DEVICE_PROTM_ENTER_CSG( \ kbdev, \ kbase_device_id, \ @@ -2185,14 +2119,6 @@ struct kbase_tlstream; kbase_device_csg_slot_index \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_DEVICE_PROTM_ENTER_CSG( \ - kbdev, \ - kbase_device_id, \ - kbase_device_csg_slot_index \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_DEVICE_HALTING_CSG - CSG is halting @@ -2202,7 +2128,6 @@ struct kbase_tlstream; * @kbase_device_csg_slot_index: The index of the slot in the scheduler whose CSG is being halted * @kbase_device_csg_slot_suspending: Whether the csg is being suspended */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_DEVICE_HALTING_CSG( \ kbdev, \ kbase_device_id, \ @@ -2219,15 +2144,6 @@ struct kbase_tlstream; kbase_device_csg_slot_suspending \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_DEVICE_HALTING_CSG( \ - kbdev, \ - kbase_device_id, \ - kbase_device_csg_slot_index, \ - kbase_device_csg_slot_suspending \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_DEVICE_SUSPEND_CSG - CSG is suspended @@ -2236,7 +2152,6 @@ struct kbase_tlstream; * @kbase_device_id: The ID of the physical hardware * @kbase_device_csg_slot_index: The index of the slot in the scheduler whose CSG is being suspended */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_DEVICE_SUSPEND_CSG( \ kbdev, \ kbase_device_id, \ @@ -2251,14 +2166,6 @@ struct kbase_tlstream; kbase_device_csg_slot_index \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_DEVICE_SUSPEND_CSG( \ - kbdev, \ - kbase_device_id, \ - kbase_device_csg_slot_index \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_DEVICE_CSG_IDLE - KBase device is notified that CSG is idle. @@ -2267,7 +2174,6 @@ struct kbase_tlstream; * @kbase_device_id: The ID of the physical hardware * @kbase_device_csg_slot_index: The index of the slot in the scheduler whose CSG for which we are receiving an idle notification */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_DEVICE_CSG_IDLE( \ kbdev, \ kbase_device_id, \ @@ -2282,14 +2188,6 @@ struct kbase_tlstream; kbase_device_csg_slot_index \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_DEVICE_CSG_IDLE( \ - kbdev, \ - kbase_device_id, \ - kbase_device_csg_slot_index \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_NEW_CTX - New KBase Context @@ -2298,7 +2196,6 @@ struct kbase_tlstream; * @kernel_ctx_id: Unique ID for the KBase Context * @kbase_device_id: The ID of the physical hardware */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_NEW_CTX( \ kbdev, \ kernel_ctx_id, \ @@ -2313,14 +2210,6 @@ struct kbase_tlstream; kbase_device_id \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_NEW_CTX( \ - kbdev, \ - kernel_ctx_id, \ - kbase_device_id \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_DEL_CTX - Delete KBase Context @@ -2328,7 +2217,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kernel_ctx_id: Unique ID for the KBase Context */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_DEL_CTX( \ kbdev, \ kernel_ctx_id \ @@ -2341,13 +2229,6 @@ struct kbase_tlstream; kernel_ctx_id \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_DEL_CTX( \ - kbdev, \ - kernel_ctx_id \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_NEW_KCPUQUEUE - New KCPU Queue @@ -2358,7 +2239,6 @@ struct kbase_tlstream; * @kernel_ctx_id: Unique ID for the KBase Context * @kcpuq_num_pending_cmds: Number of commands already enqueued in the KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_NEW_KCPUQUEUE( \ kbdev, \ kcpu_queue, \ @@ -2377,16 +2257,6 @@ struct kbase_tlstream; kcpuq_num_pending_cmds \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_NEW_KCPUQUEUE( \ - kbdev, \ - kcpu_queue, \ - kcpu_queue_id, \ - kernel_ctx_id, \ - kcpuq_num_pending_cmds \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_DEL_KCPUQUEUE - Delete KCPU Queue @@ -2394,7 +2264,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_DEL_KCPUQUEUE( \ kbdev, \ kcpu_queue \ @@ -2407,13 +2276,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_DEL_KCPUQUEUE( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL - KCPU Queue enqueues Signal on Fence @@ -2422,7 +2284,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @fence: Fence object handle */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL( \ kbdev, \ kcpu_queue, \ @@ -2437,14 +2298,6 @@ struct kbase_tlstream; fence \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL( \ - kbdev, \ - kcpu_queue, \ - fence \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT - KCPU Queue enqueues Wait on Fence @@ -2453,7 +2306,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @fence: Fence object handle */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT( \ kbdev, \ kcpu_queue, \ @@ -2468,14 +2320,6 @@ struct kbase_tlstream; fence \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT( \ - kbdev, \ - kcpu_queue, \ - fence \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT - KCPU Queue enqueues Wait on Cross Queue Sync Object @@ -2486,7 +2330,6 @@ struct kbase_tlstream; * @compare_value: Semaphore value that should be exceeded for the WAIT to pass * @inherit_error: Flag which indicates if the CQS object error state should be inherited by the queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT( \ kbdev, \ kcpu_queue, \ @@ -2505,16 +2348,6 @@ struct kbase_tlstream; inherit_error \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT( \ - kbdev, \ - kcpu_queue, \ - cqs_obj_gpu_addr, \ - compare_value, \ - inherit_error \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET - KCPU Queue enqueues Set on Cross Queue Sync Object @@ -2523,7 +2356,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @cqs_obj_gpu_addr: CQS Object GPU pointer */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET( \ kbdev, \ kcpu_queue, \ @@ -2538,14 +2370,6 @@ struct kbase_tlstream; cqs_obj_gpu_addr \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET( \ - kbdev, \ - kcpu_queue, \ - cqs_obj_gpu_addr \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT_OPERATION - KCPU Queue enqueues Wait Operation on Cross Queue Sync Object @@ -2558,7 +2382,6 @@ struct kbase_tlstream; * @data_type: Data type of a CQS Object's value * @inherit_error: Flag which indicates if the CQS object error state should be inherited by the queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT_OPERATION( \ kbdev, \ kcpu_queue, \ @@ -2581,18 +2404,6 @@ struct kbase_tlstream; inherit_error \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT_OPERATION( \ - kbdev, \ - kcpu_queue, \ - cqs_obj_gpu_addr, \ - compare_value, \ - condition, \ - data_type, \ - inherit_error \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET_OPERATION - KCPU Queue enqueues Set Operation on Cross Queue Sync Object @@ -2604,7 +2415,6 @@ struct kbase_tlstream; * @operation: Operation type performed on semaphore value (SET or ADD) * @data_type: Data type of a CQS Object's value */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET_OPERATION( \ kbdev, \ kcpu_queue, \ @@ -2625,17 +2435,6 @@ struct kbase_tlstream; data_type \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET_OPERATION( \ - kbdev, \ - kcpu_queue, \ - cqs_obj_gpu_addr, \ - value, \ - operation, \ - data_type \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT - KCPU Queue enqueues Map Import @@ -2644,7 +2443,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @map_import_buf_gpu_addr: Map import buffer GPU pointer */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT( \ kbdev, \ kcpu_queue, \ @@ -2659,14 +2457,6 @@ struct kbase_tlstream; map_import_buf_gpu_addr \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT( \ - kbdev, \ - kcpu_queue, \ - map_import_buf_gpu_addr \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT - KCPU Queue enqueues Unmap Import @@ -2675,7 +2465,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @map_import_buf_gpu_addr: Map import buffer GPU pointer */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT( \ kbdev, \ kcpu_queue, \ @@ -2690,14 +2479,6 @@ struct kbase_tlstream; map_import_buf_gpu_addr \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT( \ - kbdev, \ - kcpu_queue, \ - map_import_buf_gpu_addr \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE - KCPU Queue enqueues Unmap Import ignoring reference count @@ -2706,7 +2487,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @map_import_buf_gpu_addr: Map import buffer GPU pointer */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE( \ kbdev, \ kcpu_queue, \ @@ -2721,14 +2501,6 @@ struct kbase_tlstream; map_import_buf_gpu_addr \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE( \ - kbdev, \ - kcpu_queue, \ - map_import_buf_gpu_addr \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC - Begin array of KCPU Queue enqueues JIT Alloc @@ -2736,7 +2508,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ kbdev, \ kcpu_queue \ @@ -2749,13 +2520,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_ALLOC - Array item of KCPU Queue enqueues JIT Alloc @@ -2772,7 +2536,6 @@ struct kbase_tlstream; * @jit_alloc_flags: Flags specifying the special requirements for the JIT allocation * @jit_alloc_usage_id: A hint about which allocation should be reused. The kernel should attempt to use a previous allocation with the same usage_id */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ kbdev, \ kcpu_queue, \ @@ -2803,22 +2566,6 @@ struct kbase_tlstream; jit_alloc_usage_id \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ - kbdev, \ - kcpu_queue, \ - jit_alloc_gpu_alloc_addr_dest, \ - jit_alloc_va_pages, \ - jit_alloc_commit_pages, \ - jit_alloc_extent, \ - jit_alloc_jit_id, \ - jit_alloc_bin_id, \ - jit_alloc_max_allocations, \ - jit_alloc_flags, \ - jit_alloc_usage_id \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC - End array of KCPU Queue enqueues JIT Alloc @@ -2826,7 +2573,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ kbdev, \ kcpu_queue \ @@ -2839,13 +2585,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE - Begin array of KCPU Queue enqueues JIT Free @@ -2853,7 +2592,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE( \ kbdev, \ kcpu_queue \ @@ -2866,13 +2604,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_FREE - Array item of KCPU Queue enqueues JIT Free @@ -2881,7 +2612,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @jit_alloc_jit_id: Unique ID provided by the caller, this is used to pair allocation and free requests. Zero is not a valid value */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_FREE( \ kbdev, \ kcpu_queue, \ @@ -2896,14 +2626,6 @@ struct kbase_tlstream; jit_alloc_jit_id \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_FREE( \ - kbdev, \ - kcpu_queue, \ - jit_alloc_jit_id \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE - End array of KCPU Queue enqueues JIT Free @@ -2911,7 +2633,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE( \ kbdev, \ kcpu_queue \ @@ -2924,13 +2645,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_ERROR_BARRIER - KCPU Queue enqueues Error Barrier @@ -2938,7 +2652,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_ERROR_BARRIER( \ kbdev, \ kcpu_queue \ @@ -2951,13 +2664,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_ERROR_BARRIER( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_GROUP_SUSPEND - KCPU Queue enqueues Group Suspend @@ -2967,7 +2673,6 @@ struct kbase_tlstream; * @group_suspend_buf: Pointer to the suspend buffer structure * @gpu_cmdq_grp_handle: GPU Command Queue Group handle which will match userspace */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_GROUP_SUSPEND( \ kbdev, \ kcpu_queue, \ @@ -2984,15 +2689,6 @@ struct kbase_tlstream; gpu_cmdq_grp_handle \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_GROUP_SUSPEND( \ - kbdev, \ - kcpu_queue, \ - group_suspend_buf, \ - gpu_cmdq_grp_handle \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START - KCPU Queue starts a Signal on Fence @@ -3000,7 +2696,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START( \ kbdev, \ kcpu_queue \ @@ -3013,13 +2708,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END - KCPU Queue ends a Signal on Fence @@ -3028,7 +2716,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END( \ kbdev, \ kcpu_queue, \ @@ -3043,14 +2730,6 @@ struct kbase_tlstream; execute_error \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END( \ - kbdev, \ - kcpu_queue, \ - execute_error \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START - KCPU Queue starts a Wait on Fence @@ -3058,7 +2737,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START( \ kbdev, \ kcpu_queue \ @@ -3071,13 +2749,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END - KCPU Queue ends a Wait on Fence @@ -3086,7 +2757,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END( \ kbdev, \ kcpu_queue, \ @@ -3101,14 +2771,6 @@ struct kbase_tlstream; execute_error \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END( \ - kbdev, \ - kcpu_queue, \ - execute_error \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START - KCPU Queue starts a Wait on Cross Queue Sync Object @@ -3116,7 +2778,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START( \ kbdev, \ kcpu_queue \ @@ -3129,13 +2790,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END - KCPU Queue ends a Wait on Cross Queue Sync Object @@ -3144,7 +2798,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END( \ kbdev, \ kcpu_queue, \ @@ -3159,14 +2812,6 @@ struct kbase_tlstream; execute_error \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END( \ - kbdev, \ - kcpu_queue, \ - execute_error \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET - KCPU Queue executes a Set on Cross Queue Sync Object @@ -3175,7 +2820,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET( \ kbdev, \ kcpu_queue, \ @@ -3190,14 +2834,6 @@ struct kbase_tlstream; execute_error \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET( \ - kbdev, \ - kcpu_queue, \ - execute_error \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_OPERATION_START - KCPU Queue starts a Wait Operation on Cross Queue Sync Object @@ -3205,7 +2841,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_OPERATION_START( \ kbdev, \ kcpu_queue \ @@ -3218,13 +2853,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_OPERATION_START( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_OPERATION_END - KCPU Queue ends a Wait Operation on Cross Queue Sync Object @@ -3233,7 +2861,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_OPERATION_END( \ kbdev, \ kcpu_queue, \ @@ -3248,14 +2875,6 @@ struct kbase_tlstream; execute_error \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_OPERATION_END( \ - kbdev, \ - kcpu_queue, \ - execute_error \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET_OPERATION - KCPU Queue executes a Set Operation on Cross Queue Sync Object @@ -3264,7 +2883,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET_OPERATION( \ kbdev, \ kcpu_queue, \ @@ -3279,14 +2897,6 @@ struct kbase_tlstream; execute_error \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET_OPERATION( \ - kbdev, \ - kcpu_queue, \ - execute_error \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START - KCPU Queue starts a Map Import @@ -3294,7 +2904,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START( \ kbdev, \ kcpu_queue \ @@ -3307,13 +2916,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END - KCPU Queue ends a Map Import @@ -3322,7 +2924,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END( \ kbdev, \ kcpu_queue, \ @@ -3337,14 +2938,6 @@ struct kbase_tlstream; execute_error \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END( \ - kbdev, \ - kcpu_queue, \ - execute_error \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START - KCPU Queue starts an Unmap Import @@ -3352,7 +2945,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START( \ kbdev, \ kcpu_queue \ @@ -3365,13 +2957,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END - KCPU Queue ends an Unmap Import @@ -3380,7 +2965,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END( \ kbdev, \ kcpu_queue, \ @@ -3395,14 +2979,6 @@ struct kbase_tlstream; execute_error \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END( \ - kbdev, \ - kcpu_queue, \ - execute_error \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START - KCPU Queue starts an Unmap Import ignoring reference count @@ -3410,7 +2986,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START( \ kbdev, \ kcpu_queue \ @@ -3423,13 +2998,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END - KCPU Queue ends an Unmap Import ignoring reference count @@ -3438,7 +3006,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END( \ kbdev, \ kcpu_queue, \ @@ -3453,14 +3020,6 @@ struct kbase_tlstream; execute_error \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END( \ - kbdev, \ - kcpu_queue, \ - execute_error \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START - KCPU Queue starts an array of JIT Allocs @@ -3468,7 +3027,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START( \ kbdev, \ kcpu_queue \ @@ -3481,13 +3039,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END - Begin array of KCPU Queue ends an array of JIT Allocs @@ -3495,7 +3046,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ kbdev, \ kcpu_queue \ @@ -3508,13 +3058,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END - Array item of KCPU Queue ends an array of JIT Allocs @@ -3525,7 +3068,6 @@ struct kbase_tlstream; * @jit_alloc_gpu_alloc_addr: The JIT allocated GPU virtual address * @jit_alloc_mmu_flags: The MMU flags for the JIT allocation */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ kbdev, \ kcpu_queue, \ @@ -3544,16 +3086,6 @@ struct kbase_tlstream; jit_alloc_mmu_flags \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ - kbdev, \ - kcpu_queue, \ - execute_error, \ - jit_alloc_gpu_alloc_addr, \ - jit_alloc_mmu_flags \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END - End array of KCPU Queue ends an array of JIT Allocs @@ -3561,7 +3093,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ kbdev, \ kcpu_queue \ @@ -3574,13 +3105,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START - KCPU Queue starts an array of JIT Frees @@ -3588,7 +3112,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START( \ kbdev, \ kcpu_queue \ @@ -3601,13 +3124,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END - Begin array of KCPU Queue ends an array of JIT Frees @@ -3615,7 +3131,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ kbdev, \ kcpu_queue \ @@ -3628,13 +3143,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END - Array item of KCPU Queue ends an array of JIT Frees @@ -3644,7 +3152,6 @@ struct kbase_tlstream; * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero * @jit_free_pages_used: The actual number of pages used by the JIT allocation */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ kbdev, \ kcpu_queue, \ @@ -3661,15 +3168,6 @@ struct kbase_tlstream; jit_free_pages_used \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ - kbdev, \ - kcpu_queue, \ - execute_error, \ - jit_free_pages_used \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END - End array of KCPU Queue ends an array of JIT Frees @@ -3677,7 +3175,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ kbdev, \ kcpu_queue \ @@ -3690,13 +3187,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_ERROR_BARRIER - KCPU Queue executes an Error Barrier @@ -3704,7 +3194,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_ERROR_BARRIER( \ kbdev, \ kcpu_queue \ @@ -3717,13 +3206,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_ERROR_BARRIER( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_START - KCPU Queue starts a group suspend @@ -3731,7 +3213,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_START( \ kbdev, \ kcpu_queue \ @@ -3744,13 +3225,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_START( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_END - KCPU Queue ends a group suspend @@ -3759,7 +3233,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_END( \ kbdev, \ kcpu_queue, \ @@ -3774,14 +3247,6 @@ struct kbase_tlstream; execute_error \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_END( \ - kbdev, \ - kcpu_queue, \ - execute_error \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_DEVICE_L2_CORE_STATE - KBase device updates L2 Core state @@ -3790,7 +3255,6 @@ struct kbase_tlstream; * @kbase_device_id: The ID of the physical hardware * @new_state: New state */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_DEVICE_L2_CORE_STATE( \ kbdev, \ kbase_device_id, \ @@ -3805,14 +3269,6 @@ struct kbase_tlstream; new_state \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_DEVICE_L2_CORE_STATE( \ - kbdev, \ - kbase_device_id, \ - new_state \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_DEVICE_MCU_STATE - KBase device updates MCU state @@ -3821,7 +3277,6 @@ struct kbase_tlstream; * @kbase_device_id: The ID of the physical hardware * @new_state: New state */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_DEVICE_MCU_STATE( \ kbdev, \ kbase_device_id, \ @@ -3836,14 +3291,6 @@ struct kbase_tlstream; new_state \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_DEVICE_MCU_STATE( \ - kbdev, \ - kbase_device_id, \ - new_state \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_DEVICE_SHADER_CORE_STATE - KBase device updates Shader Core state @@ -3852,7 +3299,6 @@ struct kbase_tlstream; * @kbase_device_id: The ID of the physical hardware * @new_state: New state */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_DEVICE_SHADER_CORE_STATE( \ kbdev, \ kbase_device_id, \ @@ -3867,14 +3313,6 @@ struct kbase_tlstream; new_state \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_DEVICE_SHADER_CORE_STATE( \ - kbdev, \ - kbase_device_id, \ - new_state \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_DISABLING - CSF FW is being disabled @@ -3882,7 +3320,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @csffw_cycle: Cycle number of a CSFFW event */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_DISABLING( \ kbdev, \ csffw_cycle \ @@ -3895,13 +3332,6 @@ struct kbase_tlstream; csffw_cycle \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_DISABLING( \ - kbdev, \ - csffw_cycle \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_OFF - CSF FW is off @@ -3909,7 +3339,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @csffw_cycle: Cycle number of a CSFFW event */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_OFF( \ kbdev, \ csffw_cycle \ @@ -3922,13 +3351,6 @@ struct kbase_tlstream; csffw_cycle \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_OFF( \ - kbdev, \ - csffw_cycle \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW - An overflow has happened with the CSFFW Timeline stream @@ -3937,7 +3359,6 @@ struct kbase_tlstream; * @csffw_timestamp: Timestamp of a CSFFW event * @csffw_cycle: Cycle number of a CSFFW event */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW( \ kbdev, \ csffw_timestamp, \ @@ -3952,14 +3373,6 @@ struct kbase_tlstream; csffw_cycle \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW( \ - kbdev, \ - csffw_timestamp, \ - csffw_cycle \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_AUX_PM_STATE - PM state diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_drv.h b/drivers/gpu/drm/rockchip/rockchip_drm_drv.h index 6d0d223ee17c..665f65181963 100644 --- a/drivers/gpu/drm/rockchip/rockchip_drm_drv.h +++ b/drivers/gpu/drm/rockchip/rockchip_drm_drv.h @@ -274,6 +274,7 @@ struct rockchip_crtc_state { int output_mode; int output_bpc; int output_flags; + int data_map_mode; bool enable_afbc; /** * @splice_mode: enabled when display a hdisplay > 4096 on rk3588 diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c index 5c3f82df2bac..9eac6c767165 100644 --- a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c +++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c @@ -4176,6 +4176,15 @@ static void vop_crtc_atomic_enable(struct drm_crtc *crtc, if (vop->lut_active) vop_crtc_load_lut(crtc); + if (s->data_map_mode != -1) { + if (s->output_if & VOP_OUTPUT_IF_BT1120) + VOP_CTRL_SET(vop, bt1120_data_map_mode, s->data_map_mode); + else if (s->output_if & VOP_OUTPUT_IF_BT656) + VOP_CTRL_SET(vop, bt656_data_map_mode, s->data_map_mode); + else if (vop->mcu_timing.mcu_pix_total) + VOP_CTRL_SET(vop, mcu_data_map_mode, s->data_map_mode); + } + if (vop->mcu_timing.mcu_pix_total) { /* * For RK3576_LITE/RK3506/RV1126B diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop.h b/drivers/gpu/drm/rockchip/rockchip_drm_vop.h index 5f5ea7ccfcac..e2b26d7c7c1b 100644 --- a/drivers/gpu/drm/rockchip/rockchip_drm_vop.h +++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop.h @@ -458,14 +458,17 @@ struct vop_ctrl { struct vop_reg mcu_type; struct vop_reg mcu_rw_bypass_port; struct vop_reg mcu_force_rdn; + struct vop_reg mcu_data_map_mode; /* bt1120 */ struct vop_reg bt1120_uv_swap; struct vop_reg bt1120_yc_swap; struct vop_reg bt1120_en; + struct vop_reg bt1120_data_map_mode; /* bt656 */ struct vop_reg bt656_en; + struct vop_reg bt656_data_map_mode; struct vop_reg reg_done_frm; struct vop_reg cfg_done; diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c index 96fccea8e2ca..35451b2fbf9a 100644 --- a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c +++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c @@ -12104,9 +12104,9 @@ static void vop2_setup_dly_for_vp(struct vop2_video_port *vp) struct drm_crtc *crtc = &vp->rockchip_crtc.crtc; struct rockchip_crtc_state *vcstate = to_rockchip_crtc_state(crtc->state); struct drm_display_mode *adjusted_mode = &crtc->state->adjusted_mode; - u16 hsync_len = adjusted_mode->crtc_hsync_end - adjusted_mode->crtc_hsync_start; - u16 hdisplay = adjusted_mode->crtc_hdisplay; u32 bg_dly = vp_data->pre_scan_max_dly[0]; + u16 hsync_len; + u16 hdisplay; u32 pre_scan_dly; if (vp_data->hdr_table) { @@ -12131,9 +12131,9 @@ static void vop2_setup_dly_for_vp(struct vop2_video_port *vp) if (vp->splice_mode_right) { vcstate = to_rockchip_crtc_state(left_vp->rockchip_crtc.crtc.state); adjusted_mode = &left_vp->rockchip_crtc.crtc.state->adjusted_mode; - hsync_len = adjusted_mode->crtc_hsync_end - adjusted_mode->crtc_hsync_start; - hdisplay = adjusted_mode->crtc_hdisplay; } + hsync_len = adjusted_mode->crtc_hsync_end - adjusted_mode->crtc_hsync_start; + hdisplay = adjusted_mode->crtc_hdisplay; /* * splice mode: hdisplay must roundup as 4 pixel, diff --git a/drivers/gpu/drm/rockchip/rockchip_rgb.c b/drivers/gpu/drm/rockchip/rockchip_rgb.c index 86a53924a7c9..89e48f4f2a4d 100644 --- a/drivers/gpu/drm/rockchip/rockchip_rgb.c +++ b/drivers/gpu/drm/rockchip/rockchip_rgb.c @@ -144,6 +144,7 @@ struct rockchip_rgb { u8 id; u32 max_dclk_rate; u32 mcu_pix_total; + int data_map_mode; struct device *dev; struct device_node *np_mcu_panel; struct drm_panel *panel; @@ -391,6 +392,7 @@ rockchip_rgb_encoder_atomic_check(struct drm_encoder *encoder, break; } + s->data_map_mode = rgb->data_map_mode; s->output_type = DRM_MODE_CONNECTOR_DPI; s->bus_flags = info->bus_flags; s->tv_state = &conn_state->tv; @@ -1060,6 +1062,10 @@ static int rockchip_rgb_probe(struct platform_device *pdev) id = 0; rgb->data_sync_bypass = of_property_read_bool(dev->of_node, "rockchip,data-sync-bypass"); + if (of_property_read_u32(dev->of_node, "rockchip,data-map-mode", &rgb->data_map_mode)) + rgb->data_map_mode = -1; + if (rgb->data_map_mode < 0 || rgb->data_map_mode > 3) + rgb->data_map_mode = -1; fwnode_mcu_panel = device_get_named_child_node(dev, "mcu-panel"); if (fwnode_mcu_panel) { diff --git a/drivers/gpu/drm/rockchip/rockchip_vop_reg.c b/drivers/gpu/drm/rockchip/rockchip_vop_reg.c index b882d8f020ab..cbd3431d2586 100644 --- a/drivers/gpu/drm/rockchip/rockchip_vop_reg.c +++ b/drivers/gpu/drm/rockchip/rockchip_vop_reg.c @@ -1946,6 +1946,9 @@ static const struct vop_ctrl rv1126b_ctrl_data = { .bt656_en = VOP_REG(RK3366_LIT_DSP_CTRL0, 0x1, 6), .mcu_force_rdn = VOP_REG(RV1126B_DSP_CTRL1, 0x1, 21), + .mcu_data_map_mode = VOP_REG(RV1126B_DSP_CTRL1, 0x1, 20), + .bt1120_data_map_mode = VOP_REG(RV1126B_DSP_CTRL1, 0x3, 18), + .bt656_data_map_mode = VOP_REG(RV1126B_DSP_CTRL1, 0x3, 16), .dsp_interlace = VOP_REG(RK3366_LIT_DSP_CTRL2, 0x1, 0), .dither_up_en = VOP_REG(RK3366_LIT_DSP_CTRL2, 0x1, 2), diff --git a/drivers/input/touchscreen/gt1x/gt1x_generic.c b/drivers/input/touchscreen/gt1x/gt1x_generic.c index 3840444c0131..abbf764b3fd1 100644 --- a/drivers/input/touchscreen/gt1x/gt1x_generic.c +++ b/drivers/input/touchscreen/gt1x/gt1x_generic.c @@ -2354,18 +2354,21 @@ s32 gt1x_init(void) ret = gt1x_get_chip_type(); if (ret != 0) { GTP_ERROR("Get chip type failed!"); + goto init_err; } /* read version information */ ret = gt1x_read_version(>1x_version); if (ret != 0) { GTP_ERROR("Get verision failed!"); + goto init_err; } /* init and send configs */ ret = gt1x_init_panel(); if (ret != 0) { GTP_ERROR("Init panel failed."); + goto init_err; } gt1x_workqueue = create_singlethread_workqueue("gt1x_workthread"); @@ -2403,6 +2406,8 @@ s32 gt1x_init(void) #if GTP_WITH_STYLUS gt1x_pen_init(); #endif + +init_err: if (ret != 0) gt1x_power_switch(SWITCH_OFF); diff --git a/drivers/media/i2c/sc450ai.c b/drivers/media/i2c/sc450ai.c index 17794f7ed050..443ebbbbeb14 100644 --- a/drivers/media/i2c/sc450ai.c +++ b/drivers/media/i2c/sc450ai.c @@ -1217,6 +1217,7 @@ static const struct sc450ai_mode supported_modes_2lane[] = { .hts_def = 0x2ee * 4, .vts_def = 0x0638, .bus_fmt = MEDIA_BUS_FMT_SBGGR10_1X10, + .global_reg_list = sc450ai_global_regs_2lane, .reg_list = sc450ai_linear_10_2688x1520_30fps_2lane_regs, .hdr_mode = NO_HDR, .mclk = 27000000, diff --git a/drivers/media/platform/rockchip/cif/capture.c b/drivers/media/platform/rockchip/cif/capture.c index dd4cf893139e..ca3db74adb8e 100644 --- a/drivers/media/platform/rockchip/cif/capture.c +++ b/drivers/media/platform/rockchip/cif/capture.c @@ -6171,6 +6171,7 @@ void rkcif_free_rx_buf(struct rkcif_stream *stream, int buf_num) if (buf->dbufs.is_init) v4l2_subdev_call(sd, core, ioctl, RKISP_VICAP_CMD_RX_BUFFER_FREE, &buf->dbufs); + rkcif_free_reserved_mem_buf(dev, buf); memset(buf, 0, sizeof(*buf)); buf->dummy.is_free = true; } diff --git a/drivers/media/platform/rockchip/cif/common.c b/drivers/media/platform/rockchip/cif/common.c index 9f5cdb3e037d..61ec92440b8b 100644 --- a/drivers/media/platform/rockchip/cif/common.c +++ b/drivers/media/platform/rockchip/cif/common.c @@ -343,17 +343,42 @@ int rkcif_alloc_reserved_mem_buf(struct rkcif_device *dev, struct rkcif_rx_buffe { struct rkcif_dummy_buffer *dummy = &buf->dummy; u32 reserved_mem = 0; + struct dma_buf_attachment *dba; + struct sg_table *sgt; + dma_addr_t dma; + int ret = 0; + u32 dma_addr = 0; if (dev->pre_buf_num) reserved_mem = SHARED_MEM_RESERVED_HEAD_SIZE; - dummy->dma_addr = reserved_mem + dev->resmem_pa + dummy->size * buf->buf_idx; - if (dummy->dma_addr + dummy->size > dev->resmem_pa + dev->resmem_size) + dma_addr = reserved_mem + dev->resmem_pa + dummy->size * buf->buf_idx; + if (dma_addr + dummy->size > dev->resmem_pa + dev->resmem_size) return -EINVAL; - buf->dbufs.dma = dummy->dma_addr; + buf->dbufs.dma = dma_addr; buf->dbufs.is_resmem = true; - buf->shmem.shm_start = dummy->dma_addr; + buf->shmem.shm_start = dma_addr; buf->shmem.shm_size = dummy->size; dummy->dbuf = rkcif_shm_alloc(&buf->shmem); + buf->dbufs.dbuf = dummy->dbuf; + if (dev->hw_dev->iommu_en) { + dba = dma_buf_attach(dummy->dbuf, dev->hw_dev->dev); + if (IS_ERR(dba)) { + ret = PTR_ERR(dba); + goto err_alloc; + } + dummy->dba = dba; + sgt = dma_buf_map_attachment(dba, DMA_BIDIRECTIONAL); + if (IS_ERR(sgt)) { + ret = PTR_ERR(sgt); + goto err_alloc; + } + dummy->sgt = sgt; + dma = sg_dma_address(sgt->sgl); + get_dma_buf(dummy->dbuf); + dummy->dma_addr = dma; + } else { + dummy->dma_addr = dma_addr; + } if (dummy->is_need_vaddr) { struct iosys_map map; @@ -361,6 +386,11 @@ int rkcif_alloc_reserved_mem_buf(struct rkcif_device *dev, struct rkcif_rx_buffe dummy->vaddr = map.vaddr; } return 0; +err_alloc: + v4l2_info(&dev->v4l2_dev, + "can't match dma_buf 0x%x with iommu\n", + (u32)dummy->dma_addr); + return ret; } void rkcif_free_reserved_mem_buf(struct rkcif_device *dev, struct rkcif_rx_buffer *buf) @@ -396,13 +426,30 @@ void rkcif_free_reserved_mem_buf(struct rkcif_device *dev, struct rkcif_rx_buffe if (buf->dbufs.is_init) v4l2_subdev_call(sd, core, ioctl, RKISP_VICAP_CMD_RX_BUFFER_FREE, &buf->dbufs); + if (dev->hw_dev->iommu_en) { + if (dummy->dba) { + if (dummy->sgt) { + dma_buf_unmap_attachment(dummy->dba, dummy->sgt, + DMA_BIDIRECTIONAL); + dummy->sgt = NULL; + } + dma_buf_detach(dummy->dbuf, dummy->dba); + dma_buf_put(dummy->dbuf); + dummy->dba = NULL; + } + } if (dummy->is_need_vaddr) dummy->dbuf->ops->vunmap(dummy->dbuf, NULL); -#ifdef CONFIG_VIDEO_ROCKCHIP_THUNDER_BOOT_ISP - free_reserved_area(phys_to_virt(buf->shmem.shm_start), - phys_to_virt(buf->shmem.shm_start + buf->shmem.shm_size), - -1, "rkisp_thunderboot"); -#endif + dma_buf_put(dummy->dbuf); buf->dummy.is_free = true; } +void rkcif_free_reserved_mem(u32 start, u32 size) +{ +#ifdef CONFIG_VIDEO_ROCKCHIP_THUNDER_BOOT_ISP + free_reserved_area(phys_to_virt(start), + phys_to_virt(start + size), + -1, "rkisp_thunderboot"); +#endif +} + diff --git a/drivers/media/platform/rockchip/cif/common.h b/drivers/media/platform/rockchip/cif/common.h index 7bdfe4f40223..8745df4508f8 100644 --- a/drivers/media/platform/rockchip/cif/common.h +++ b/drivers/media/platform/rockchip/cif/common.h @@ -25,6 +25,7 @@ void rkcif_free_common_dummy_buf(struct rkcif_device *dev, struct rkcif_dummy_bu int rkcif_alloc_reserved_mem_buf(struct rkcif_device *dev, struct rkcif_rx_buffer *buf); void rkcif_free_reserved_mem_buf(struct rkcif_device *dev, struct rkcif_rx_buffer *buf); +void rkcif_free_reserved_mem(u32 start, u32 size); #endif /* _RKCIF_COMMON_H */ diff --git a/drivers/media/platform/rockchip/cif/hw.h b/drivers/media/platform/rockchip/cif/hw.h index 537dc154b329..c0ecbf2ab78c 100644 --- a/drivers/media/platform/rockchip/cif/hw.h +++ b/drivers/media/platform/rockchip/cif/hw.h @@ -81,6 +81,8 @@ struct rkcif_dummy_buffer { struct vb2_queue vb2_queue; struct list_head list; struct dma_buf *dbuf; + struct dma_buf_attachment *dba; + struct sg_table *sgt; dma_addr_t dma_addr; struct page **pages; void *mem_priv; diff --git a/drivers/media/platform/rockchip/cif/subdev-itf.c b/drivers/media/platform/rockchip/cif/subdev-itf.c index 26a6423d88f2..d47d38be3b5f 100644 --- a/drivers/media/platform/rockchip/cif/subdev-itf.c +++ b/drivers/media/platform/rockchip/cif/subdev-itf.c @@ -98,6 +98,7 @@ static void sditf_buffree_work(struct work_struct *work) if (rx_buf) { list_del(&rx_buf->list_free); rkcif_free_reserved_mem_buf(priv->cif_dev, rx_buf); + rkcif_free_reserved_mem(rx_buf->shmem.shm_start, rx_buf->shmem.shm_size); memset(rx_buf, 0, sizeof(*rx_buf)); rx_buf->dummy.is_free = true; } @@ -1444,7 +1445,7 @@ static int sditf_s_rx_buffer(struct v4l2_subdev *sd, if (cif_dev->is_thunderboot || cif_dev->is_rtt_suspend || cif_dev->is_aov_reserved) - dma_sync_single_for_device(cif_dev->dev, + dma_sync_single_for_device(cif_dev->hw_dev->dev, rx_buf->dummy.dma_addr + rx_buf->dummy.size - stream->pixm.plane_fmt[0].bytesperline * 3, stream->pixm.plane_fmt[0].bytesperline * 3, @@ -1453,6 +1454,7 @@ static int sditf_s_rx_buffer(struct v4l2_subdev *sd, cif_dev->hw_dev->mem_ops->prepare(rx_buf->dummy.mem_priv); } } + spin_unlock_irqrestore(&stream->vbq_lock, flags); if (dbufs->is_switch && dbufs->type == BUF_SHORT) { if (stream->is_in_vblank || !stream->dma_en) { @@ -1465,7 +1467,6 @@ static int sditf_s_rx_buffer(struct v4l2_subdev *sd, v4l2_dbg(3, rkcif_debug, &cif_dev->v4l2_dev, "switch to online mode\n"); } - spin_unlock_irqrestore(&stream->vbq_lock, flags); spin_lock_irqsave(&stream->cifdev->stream_spinlock, flags); stream->is_finish_single_cap = true; diff --git a/drivers/media/platform/rockchip/isp/rkisp.c b/drivers/media/platform/rockchip/isp/rkisp.c index 05cce6ab4776..b51d6fc3ca95 100644 --- a/drivers/media/platform/rockchip/isp/rkisp.c +++ b/drivers/media/platform/rockchip/isp/rkisp.c @@ -3477,7 +3477,7 @@ static int rkisp_rx_buf_pool_init(struct rkisp_device *dev, "%s type:0x%x first:%d dbufs[%d]:%p", __func__, dbufs->type, dbufs->is_first, i, dbufs); - if (dbufs->is_resmem) { + if (dbufs->is_resmem && !dev->hw_dev->is_mmu) { dma = dbufs->dma; goto end; } diff --git a/drivers/media/platform/rockchip/vpss/stream_v20.c b/drivers/media/platform/rockchip/vpss/stream_v20.c index aa01566bcc93..fb2defee33fc 100644 --- a/drivers/media/platform/rockchip/vpss/stream_v20.c +++ b/drivers/media/platform/rockchip/vpss/stream_v20.c @@ -1947,12 +1947,10 @@ static void rkvpss_stream_stop(struct rkvpss_stream *stream) int ret; stream->stopping = true; - if (atomic_read(&dev->pipe_stream_cnt) > 0) { - ret = wait_event_timeout(stream->done, !stream->streaming, - msecs_to_jiffies(300)); - if (!ret) - v4l2_warn(&dev->v4l2_dev, "%s id:%d timeout\n", __func__, stream->id); - } + ret = wait_event_timeout(stream->done, !stream->streaming, + msecs_to_jiffies(300)); + if (!ret) + v4l2_warn(&dev->v4l2_dev, "%s id:%d timeout\n", __func__, stream->id); stream->stopping = false; stream->streaming = false; if (stream->ops->disable_mi) diff --git a/drivers/mfd/rk808.c b/drivers/mfd/rk808.c index 3e9094c8cc0e..d89f979133ad 100644 --- a/drivers/mfd/rk808.c +++ b/drivers/mfd/rk808.c @@ -1400,8 +1400,10 @@ static int rk808_probe(struct i2c_client *client, const struct mfd_cell *cells; u8 on_source = 0, off_source = 0; unsigned int on, off; + u32 pmic_id_mask = RK8XX_ID_MSK; int nr_pre_init_regs; int nr_cells; + int pmic_id; int msb, lsb; unsigned char pmic_id_msb, pmic_id_lsb; int ret; @@ -1423,6 +1425,7 @@ static int rk808_probe(struct i2c_client *client, } else if (of_device_is_compatible(np, "rockchip,rk801")) { pmic_id_msb = RK801_ID_MSB; pmic_id_lsb = RK801_ID_LSB; + pmic_id_mask = RK801_ID_MSK; } else { pmic_id_msb = RK808_ID_MSB; pmic_id_lsb = RK808_ID_LSB; @@ -1443,8 +1446,9 @@ static int rk808_probe(struct i2c_client *client, return lsb; } - rk808->variant = ((msb << 8) | lsb) & RK8XX_ID_MSK; - dev_info(&client->dev, "chip id: 0x%x\n", (unsigned int)rk808->variant); + pmic_id = (msb << 8) | lsb; + rk808->variant = pmic_id & RK8XX_ID_MSK; + dev_info(&client->dev, "chip id: 0x%x\n", pmic_id & pmic_id_mask); switch (rk808->variant) { case RK801_ID: diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig index 34cbfb83a3af..39c3e6ed001e 100644 --- a/drivers/misc/Kconfig +++ b/drivers/misc/Kconfig @@ -14,6 +14,7 @@ config RK803 Driver for RK803 which is used for driving porjector and IR flood LED. source "drivers/misc/rockchip/Kconfig" +source "drivers/misc/vehicle/Kconfig" config LT7911D_FB_NOTIFIER tristate "Lontium LT7911D FB Notifier" diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile index 1306058114ab..2faa854af422 100644 --- a/drivers/misc/Makefile +++ b/drivers/misc/Makefile @@ -6,6 +6,7 @@ obj-y += rk628/ obj-$(CONFIG_RK803) += rk803.o obj-y += rockchip/ +obj-y += vehicle/ obj-$(CONFIG_LT7911D_FB_NOTIFIER) += lt7911d-fb-notifier.o obj-$(CONFIG_IBM_ASM) += ibmasm/ obj-$(CONFIG_IBMVMC) += ibmvmc.o diff --git a/drivers/misc/vehicle/Kconfig b/drivers/misc/vehicle/Kconfig new file mode 100644 index 000000000000..64b7eba956e2 --- /dev/null +++ b/drivers/misc/vehicle/Kconfig @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# vehicle configuration +# + +menu "misc vehicle setting" + +config VEHICLE_CORE + tristate "ROCKCHIP VEHICLE core support" + help + If you say yes here you get support for the RK + vehicle core module. + +config VEHICLE_DUMMY + tristate "RK VEHICLE DUMMY support" + select EXTCON + depends on VEHICLE_CORE + help + If you say yes here you get support for the RK dummy MCU + which simulate the vehicle signal. + +config VEHICLE_ADC + tristate "VEHICLE ADC support" + depends on VEHICLE_CORE + help + If you say yes here you get support for the vehicle with adc + which simulate the vehicle signal. + +config VEHICLE_GPIO + tristate "VEHICLE GPIO support" + depends on VEHICLE_CORE + help + If you say yes here you get support for the vehicle with gpio + which simulate the vehicle signal. + +config VEHICLE_SPI + tristate "VEHICLE SPI support" + depends on VEHICLE_CORE + help + If you say yes here you get support for the vehicle with spi bus + which simulate the vehicle signal. + +config VEHICLE_SPI_PROTOCOL + tristate "VEHICLE SPI_PROTOCOL support" + depends on VEHICLE_SPI + help + If you say yes here, you can get support for vehicle spi protocol. + +config VEHICLE_GPIO_MCU_EXPANDER + tristate "VEHICLE MCU GPIO EXPANDER support" + help + If you say yes here you get support for the mcu gpio expander. + +config VEHICLE_DRIVER_OREO + bool "RK vehicle driver for android oreo" + help + If you say yes here, you can get support for vehicle protocol + in android o, otherwise it support android p. + +endmenu diff --git a/drivers/misc/vehicle/Makefile b/drivers/misc/vehicle/Makefile new file mode 100644 index 000000000000..5d245fc6964a --- /dev/null +++ b/drivers/misc/vehicle/Makefile @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for vehicle driver +# + +ccflags-y := -std=gnu99 -Wno-declaration-after-statement -DPB_SYSTEM_HEADER=\"pb_system.h\" -I$(srctree)/drivers/misc/vehicle/nanopb-c/ + +obj-$(CONFIG_VEHICLE_CORE) += vehicle-core.o +vehicle-core-objs = \ + nanopb-c/pb_common.o \ + nanopb-c/pb_decode.o \ + nanopb-c/pb_encode.o \ + vehicle_core.o \ + vehicle_protocol_callback.o \ + vehiclehalproto.pb.o + +obj-$(CONFIG_VEHICLE_DUMMY) += \ + vehicle_dummy_hw.o + +obj-$(CONFIG_VEHICLE_ADC) += \ + vehicle-adc.o + +obj-$(CONFIG_VEHICLE_GPIO) += \ + vehicle-gpio.o + +obj-$(CONFIG_VEHICLE_SPI) += \ + vehicle-spi.o diff --git a/drivers/misc/vehicle/core.h b/drivers/misc/vehicle/core.h new file mode 100644 index 000000000000..4dbac12a73e5 --- /dev/null +++ b/drivers/misc/vehicle/core.h @@ -0,0 +1,374 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * core.h -- core define for mfd display arch + * + * Copyright (c) 2023-2028 Rockchip Electronics Co., Ltd. + * + * Author: luowei + * + */ + +#ifndef __MFD_VEHICLE_CORE_H__ +#define __MFD_VEHICLE_CORE_H__ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + + +#include "vehicle_core.h" + +/* + * if enable all the debug information, + * there will be much log. + * + * so suggest set CONFIG_LOG_BUF_SHIFT to 18 + */ +#define VEHICLE_DEBUG + + +#ifdef VEHICLE_DEBUG +#define VEHICLE_DBG(x...) pr_info(x) +#else +#define VEHICLE_DBG(x...) no_printk(x) +#endif + +#define MCU_MAX_REGS 8 +struct mcu_gpio_chip { + const char *name; + struct platform_device *pdev; + struct gpio_chip gpio_chip; + struct regmap *regmap; + struct regulator *regulator; + unsigned int ngpio; + u8 backup_regs[MCU_MAX_REGS]; + + struct gpio_desc *reset_gpio_desc; + int reset_gpio_irq; +}; + +struct vehicle; +enum vehicle_hw_type { + VEHICLE_HW_TYPE_INVALID = 0, + + VEHICLE_HW_TYPE_ADC, + VEHICLE_HW_TYPE_GPIO, + VEHICLE_HW_TYPE_I2C, + VEHICLE_HW_TYPE_SPI, + VEHICLE_HW_TYPE_UART, + VEHICLE_HW_TYPE_CHIP_MCU, + + VEHICLE_HW_TYPE_END, +}; + +/* VEHICLE is in state parking */ +#define GEAR_0 1 +/* VEHICLE is in state reverse */ +#define GEAR_1 2 +/* VEHICLE is in state neutral */ +#define GEAR_3 3 +/* VEHICLE is in state driver */ +#define GEAR_2 4 +/* no turn signal */ +#define TURN_0 0 +/* left turn signal */ +#define TURN_1 1 +/* right turn signal */ +#define TURN_2 2 + +#define POWER_REQ_STATE_ON 0 +#define POWER_REQ_STATE_SHUTDOWN_PREPARE 1 +#define POWER_REQ_STATE_CANCEL_SHUTDOWN 2 +#define POWER_REQ_STATE_FINISHED 3 + +#define POWER_REQ_PARAM_SHUTDOWN_IMMEDIATELY 1 +#define POWER_REQ_PARAM_CAN_SLEEP 2 +#define POWER_REQ_PARAM_SHUTDOWN_ONLY 3 +#define POWER_REQ_PARAM_SLEEP_IMMEDIATELY 4 +#define POWER_REQ_PARAM_HIBERNATE_IMMEDIATELY 5 +#define POWER_REQ_PARAM_CAN_HIBERNATE 6 + +/* temperature set from hardware on Android OREO and PIE uses below indexes */ +#define AC_TEMP_LEFT_INDEX 1 +#define AC_TEMP_RIGHT_INDEX 4 + +/* temperature set from APP on Android PIE uses below indexes */ +#define PIE_AC_TEMP_LEFT_INDEX 49 +#define PIE_AC_TEMP_RIGHT_INDEX 68 + + +struct vehicle_hw_data { + const char *name; + enum vehicle_hw_type vehicle_hw_type; + + int (*hw_init)(struct vehicle *vehicle); + int (*data_update)(struct vehicle *vehicle); + int (*suspend)(struct vehicle *vehicle); + int (*resume)(struct vehicle *vehicle); +}; + +struct vehicle_event_data { + u32 gear; + u32 turn; + u32 temp_right; + u32 temp_left; + u32 fan_direction; + u32 fan_speed; + u32 defrost_left; + u32 defrost_right; + u32 ac_on; + u32 auto_on; + u32 hvac_on; + u32 recirc_on; + u32 power_req_state; + u32 power_req_param; + u32 seat_temp_left; + u32 seat_temp_right; +}; + +struct vehicle_gpio { + struct mutex wq_lock; + int use_delay_work; + struct delayed_work irq_work; + struct workqueue_struct *vehicle_wq; + struct delayed_work vehicle_delay_work; + + enum vehicle_hw_type type; + struct device *dev; + struct platform_device *pdev; + struct vehicle *parent; + struct regmap *regmap; + int irq; + + struct gpio_desc *gear_gpio_park; + struct gpio_desc *gear_gpio_reverse; + struct gpio_desc *gear_gpio_neutral; + struct gpio_desc *gear_gpio_drive; + struct gpio_desc *gear_gpio_manual; + struct gpio_desc *turn_gpio; + struct gpio_desc *temp_right_gpio; + struct gpio_desc *temp_left_gpio; + struct gpio_desc *fan_direction_gpio; + struct gpio_desc *fan_speed_gpio; + struct gpio_desc *defrost_left_gpio; + struct gpio_desc *defrost_right_gpio; + struct gpio_desc *ac_on_gpio; + struct gpio_desc *auto_on_gpio; + struct gpio_desc *hvac_on_gpio; + struct gpio_desc *recirc_on_gpio; + struct gpio_desc *power_req_state_gpio; + struct gpio_desc *power_req_param_gpio; + struct gpio_desc *seat_temp_left_gpio; + struct gpio_desc *seat_temp_right_gpio; + + struct vehicle_hw_data *hw_data; +}; + +struct vehicle_chip_mcu { + enum vehicle_hw_type type; + struct device *dev; + struct vehicle *parent; + struct regmap *regmap; + +}; + +struct vehicle_spi { + struct mutex wq_lock; + int use_delay_work; + struct delayed_work irq_work; + struct workqueue_struct *vehicle_wq; + struct delayed_work vehicle_delay_work; + + enum vehicle_hw_type hw_type; + struct device *dev; + struct platform_device *pdev; + struct spi_device *spi; + int irq; + struct vehicle *parent; + struct regmap *regmap; + + struct mcu_gpio_chip *gpio_mcu; + + char *rx_buf; + int rx_len; + char *tx_buf; + int tx_len; + + struct vehicle_event_data spi_data; + struct vehicle_hw_data *hw_data; +}; + +struct vehicle_uart { + struct mutex wq_lock; + int use_delay_work; + struct delayed_work irq_work; + struct workqueue_struct *vehicle_wq; + struct delayed_work vehicle_delay_work; + + enum vehicle_hw_type hw_type; + struct device *dev; + struct platform_device *pdev; + struct vehicle *parent; + struct regmap *regmap; + + struct vehicle_event_data uart_data; + struct vehicle_hw_data *hw_data; +}; + +struct vehicle_dummy { + struct mutex wq_lock; + int use_delay_work; + struct delayed_work irq_work; + struct workqueue_struct *vehicle_wq; + struct delayed_work vehicle_delay_work; + + enum vehicle_hw_type hw_type; + struct device *dev; + struct platform_device *pdev; + struct vehicle *parent; + struct regmap *regmap; + + struct vehicle_event_data dummy_hw_data; +}; + +struct vehicle_adc { + struct mutex wq_lock; + int use_delay_work; + struct delayed_work irq_work; + struct workqueue_struct *vehicle_wq; + struct delayed_work vehicle_delay_work; + + enum vehicle_hw_type type; + struct device *dev; + struct platform_device *pdev; + struct vehicle *parent; + struct regmap *regmap; + int irq; + + struct iio_channel *gear_adc_chn; + struct iio_channel *turn_left_adc_chn; + struct iio_channel *turn_right_adc_chn; + struct iio_channel *temp_right_adc_chn; + struct iio_channel *temp_left_adc_chn; + struct iio_channel *fan_direction_adc_chn; + struct iio_channel *fan_speed_adc_chn; + struct iio_channel *defrost_left_adc_chn; + struct iio_channel *defrost_right_adc_chn; + struct iio_channel *ac_on_adc_chn; + struct iio_channel *auto_on_adc_chn; + struct iio_channel *hvac_on_adc_chn; + struct iio_channel *recirc_on_adc_chn; + struct iio_channel *power_req_state_adc_chn; + struct iio_channel *power_req_param_adc_chn; + struct iio_channel *seat_temp_left_adc_chn; + struct iio_channel *seat_temp_right_adc_chn; + + struct vehicle_hw_data *hw_data; +}; + +struct vehicle_i2c { + struct mutex wq_lock; + int use_delay_work; + struct delayed_work irq_work; + struct workqueue_struct *vehicle_wq; + struct delayed_work vehicle_delay_work; + + enum vehicle_hw_type hw_type; + struct device *dev; + struct platform_device *pdev; + struct vehicle *parent; + struct regmap *regmap; + + struct i2c_client *client; + struct vehicle_event_data i2c_data; + struct vehicle_hw_data *hw_data; +}; + +struct vehicle { + enum vehicle_hw_type hw_type; + struct device *dev; + struct platform_device *pdev; + + struct vehicle_event_data vehicle_data; + + struct vehicle_adc *vehicle_adc; + struct vehicle_gpio *vehicle_gpio; + struct vehicle_i2c *vehicle_i2c; + struct vehicle_spi *vehicle_spi; + struct vehicle_uart *vehicle_uart; + struct vehicle_dummy *vehicle_dummy; + struct vehicle_chip_mcu *vehicle_chip_mcu; +}; + +extern struct vehicle_hw_data vehicle_adc_data; +extern struct vehicle_hw_data vehicle_gpio_data; +extern struct vehicle_hw_data vehicle_i2c_data; +extern struct vehicle_hw_data vehicle_spi_data; +extern struct vehicle_hw_data vehicle_uart_data; +extern struct vehicle_hw_data vehicle_chip_mcu_data; + +extern struct vehicle *g_vehicle_hw; +extern void vehicle_set_property(u16 prop, u8 index, u32 value, u32 param); +extern int gpio_mcu_register(struct spi_device *spi); +extern int vehicle_spi_write_slt(struct vehicle *vehicle, const void *txbuf, size_t n); +extern int vehicle_spi_read_slt(struct vehicle *vehicle, void *rxbuf, size_t n); + +#endif diff --git a/drivers/misc/vehicle/nanopb-c/Makefile b/drivers/misc/vehicle/nanopb-c/Makefile new file mode 100644 index 000000000000..3e2725cd98e5 --- /dev/null +++ b/drivers/misc/vehicle/nanopb-c/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile file pb_encode pb_decode pb_common +# + +ccflags-y := -Wall -W -Wno-unused-parameter -DPB_SYSTEM_HEADER=\"pb_system.h\" diff --git a/drivers/misc/vehicle/nanopb-c/pb.h b/drivers/misc/vehicle/nanopb-c/pb.h new file mode 100644 index 000000000000..d74a6a60acd0 --- /dev/null +++ b/drivers/misc/vehicle/nanopb-c/pb.h @@ -0,0 +1,643 @@ +/* + * Copyright (c) 2011 Petteri Aimonen + * + * This software is provided 'as-is', without any express or + * implied warranty. In no event will the authors be held liable + * for any damages arising from the use of this software. + * + * Permission is granted to anyone to use this software for any + * purpose, including commercial applications, and to alter it and + * redistribute it freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you + * must not claim that you wrote the original software. If you use + * this software in a product, an acknowledgment in the product + * documentation would be appreciated but is not required. + * + * 2. Altered source versions must be plainly marked as such, and + * must not be misrepresented as being the original software. + * + * 3. This notice may not be removed or altered from any source + * distribution. + */ +/* Common parts of the nanopb library. Most of these are quite low-level + * stuff. For the high-level interface, see pb_encode.h and pb_decode.h. + */ + +#ifndef PB_H_INCLUDED +#define PB_H_INCLUDED + +/***************************************************************** + * Nanopb compilation time options. You can change these here by * + * uncommenting the lines, or on the compiler command line. * + *****************************************************************/ + +/* Enable support for dynamically allocated fields */ +/* #define PB_ENABLE_MALLOC 1 */ + +/* Define this if your CPU / compiler combination does not support + * unaligned memory access to packed structures. + */ +/* #define PB_NO_PACKED_STRUCTS 1 */ + +/* Increase the number of required fields that are tracked. + * A compiler warning will tell if you need this. + */ +/* #define PB_MAX_REQUIRED_FIELDS 256 */ + +/* Add support for tag numbers > 255 and fields larger than 255 bytes. */ +/* #define PB_FIELD_16BIT 1 */ + +/* Add support for tag numbers > 65536 and fields larger than 65536 bytes. */ +/* #define PB_FIELD_32BIT 1 */ + +/* Disable support for error messages in order to save some code space. */ +/* #define PB_NO_ERRMSG 1 */ + +/* Disable support for custom streams (support only memory buffers). */ +/* #define PB_BUFFER_ONLY 1 */ + +/* Switch back to the old-style callback function signature. + * This was the default until nanopb-0.2.1. + */ +/* #define PB_OLD_CALLBACK_STYLE */ + + +/****************************************************************** + * You usually don't need to change anything below this line. * + * Feel free to look around and use the defined macros, though. * + ****************************************************************** + */ + + +/* Version of the nanopb library. Just in case you want to check it in + * your own program. + */ +#define NANOPB_VERSION (nanopb-0.3.9.1) + +/* Include all the system headers needed by nanopb. You will need the + * definitions of the following: + * - strlen, memcpy, memset functions + * - [u]int_least8_t, uint_fast8_t, [u]int_least16_t, [u]int32_t, [u]int64_t + * - size_t + * - bool + * + * If you don't have the standard header files, you can instead provide + * a custom header that defines or includes all this. In that case, + * define PB_SYSTEM_HEADER to the path of this file. + */ +#ifdef PB_SYSTEM_HEADER +#include PB_SYSTEM_HEADER +#else +#include +#include +#include +#include + +#ifdef PB_ENABLE_MALLOC +#include +#endif +#endif + +/* Macro for defining packed structures (compiler dependent). + * This just reduces memory requirements, but is not required. + */ +#if defined(PB_NO_PACKED_STRUCTS) + /* Disable struct packing */ +#define PB_PACKED_STRUCT_START +#define PB_PACKED_STRUCT_END +#define pb_packed +#elif defined(__GNUC__) || defined(__clang__) + /* For GCC and clang */ +#define PB_PACKED_STRUCT_START +#define PB_PACKED_STRUCT_END +#define pb_packed __attribute__((packed)) +#elif defined(__ICCARM__) || defined(__CC_ARM) + /* For IAR ARM and Keil MDK-ARM compilers */ +#define PB_PACKED_STRUCT_START _Pragma("pack(push, 1)") +#define PB_PACKED_STRUCT_END _Pragma("pack(pop)") +#define pb_packed +#elif defined(_MSC_VER) && (_MSC_VER >= 1500) + /* For Microsoft Visual C++ */ +#define PB_PACKED_STRUCT_START __pragma(pack(push, 1)) +#define PB_PACKED_STRUCT_END __pragma(pack(pop)) +#define pb_packed +#else + /* Unknown compiler */ +#define PB_PACKED_STRUCT_START +#define PB_PACKED_STRUCT_END +#define pb_packed +#endif + +/* Handly macro for suppressing unreferenced-parameter compiler warnings. */ +#ifndef PB_UNUSED +#define PB_UNUSED(x) ((void)(x)) +#endif + +/* Compile-time assertion, used for checking compatible compilation options. + * If this does not work properly on your compiler, use + * #define PB_NO_STATIC_ASSERT to disable it. + * + * But before doing that, check carefully the error message / place where it + * comes from to see if the error has a real cause. Unfortunately the error + * message is not always very clear to read, but you can see the reason better + * in the place where the PB_STATIC_ASSERT macro was called. + */ +#ifndef PB_NO_STATIC_ASSERT +#ifndef PB_STATIC_ASSERT +#define PB_STATIC_ASSERT(COND, MSG) \ + typedef char PB_STATIC_ASSERT_MSG(MSG, __LINE__, __COUNTER__)[(COND) ? 1 : -1]; +#define PB_STATIC_ASSERT_MSG(MSG, LINE, COUNTER) PB_STATIC_ASSERT_MSG_(MSG, LINE, COUNTER) +#define PB_STATIC_ASSERT_MSG_(MSG, LINE, COUNTER) pb_static_assertion_##MSG##LINE##COUNTER +#endif +#ifndef STATIC_ASSERT +#define STATIC_ASSERT PB_STATIC_ASSERT +#define STATIC_ASSERT_MSG PB_STATIC_ASSERT_MSG +#define STATIC_ASSERT_MSG_ PB_STATIC_ASSERT_MSG_ +#endif +#else +#define PB_STATIC_ASSERT(COND, MSG) +#endif + +/* Number of required fields to keep track of. */ +#ifndef PB_MAX_REQUIRED_FIELDS +#define PB_MAX_REQUIRED_FIELDS 64 +#endif + +#if PB_MAX_REQUIRED_FIELDS < 64 +#error You should not lower PB_MAX_REQUIRED_FIELDS from the default value (64). +#endif + +/* List of possible field types. These are used in the autogenerated code. + * Least-significant 4 bits tell the scalar type + * Most-significant 4 bits specify repeated/required/packed etc. + */ + +typedef uint_least8_t pb_type_t; + +/**** Field data types ****/ + +/* Numeric types */ +#define PB_LTYPE_VARINT 0x00 /* int32, int64, enum, bool */ +#define PB_LTYPE_UVARINT 0x01 /* uint32, uint64 */ +#define PB_LTYPE_SVARINT 0x02 /* sint32, sint64 */ +#define PB_LTYPE_FIXED32 0x03 /* fixed32, sfixed32, float */ +#define PB_LTYPE_FIXED64 0x04 /* fixed64, sfixed64, double */ + +/* Marker for last packable field type. */ +#define PB_LTYPE_LAST_PACKABLE 0x04 + +/* Byte array with pre-allocated buffer. + * data_size is the length of the allocated PB_BYTES_ARRAY structure. + */ +#define PB_LTYPE_BYTES 0x05 + +/* String with pre-allocated buffer. + * data_size is the maximum length. + */ +#define PB_LTYPE_STRING 0x06 + +/* Submessage + * submsg_fields is pointer to field descriptions + */ +#define PB_LTYPE_SUBMESSAGE 0x07 + +/* Extension pseudo-field + * The field contains a pointer to pb_extension_t + */ +#define PB_LTYPE_EXTENSION 0x08 + +/* Byte array with inline, pre-allocated byffer. + * data_size is the length of the inline, allocated buffer. + * This differs from PB_LTYPE_BYTES by defining the element as + * pb_byte_t[data_size] rather than pb_bytes_array_t. + */ +#define PB_LTYPE_FIXED_LENGTH_BYTES 0x09 + +/* Number of declared LTYPES */ +#define PB_LTYPES_COUNT 0x0A +#define PB_LTYPE_MASK 0x0F + +/**** Field repetition rules ****/ + +#define PB_HTYPE_REQUIRED 0x00 +#define PB_HTYPE_OPTIONAL 0x10 +#define PB_HTYPE_REPEATED 0x20 +#define PB_HTYPE_ONEOF 0x30 +#define PB_HTYPE_MASK 0x30 + +/**** Field allocation types ****/ + +#define PB_ATYPE_STATIC 0x00 +#define PB_ATYPE_POINTER 0x80 +#define PB_ATYPE_CALLBACK 0x40 +#define PB_ATYPE_MASK 0xC0 + +#define PB_ATYPE(x) ((x) & PB_ATYPE_MASK) +#define PB_HTYPE(x) ((x) & PB_HTYPE_MASK) +#define PB_LTYPE(x) ((x) & PB_LTYPE_MASK) + +/* Data type used for storing sizes of struct fields + * and array counts. + */ +#if defined(PB_FIELD_32BIT) +typedef uint32_t pb_size_t; +typedef int32_t pb_ssize_t; +#elif defined(PB_FIELD_16BIT) +typedef uint_least16_t pb_size_t; +typedef int_least16_t pb_ssize_t; +#else +typedef uint_least8_t pb_size_t; +typedef int_least8_t pb_ssize_t; +#endif +#define PB_SIZE_MAX ((pb_size_t)-1) + +/* Data type for storing encoded data and other byte streams. + * This typedef exists to support platforms where uint8_t does not exist. + * You can regard it as equivalent on uint8_t on other platforms. + */ +typedef uint_least8_t pb_byte_t; + +/* This structure is used in auto-generated constants + * to specify struct fields. + * You can change field sizes if you need structures + * larger than 256 bytes or field tags larger than 256. + * The compiler should complain if your .proto has such + * structures. Fix that by defining PB_FIELD_16BIT or + * PB_FIELD_32BIT. + */ +PB_PACKED_STRUCT_START +typedef struct pb_field_s pb_field_t; +struct pb_field_s { + pb_size_t tag; + pb_type_t type; + pb_size_t data_offset; /* Offset of field data, relative to previous field. */ + pb_ssize_t size_offset; /* Offset of array size or has-boolean, relative to data */ + pb_size_t data_size; /* Data size in bytes for a single item */ + pb_size_t array_size; /* Maximum number of entries in array */ + + /* Field definitions for submessage + * OR default value for all other non-array, non-callback types + * If null, then field will zeroed. + */ + const void *ptr; +} pb_packed; +PB_PACKED_STRUCT_END + +/* Make sure that the standard integer types are of the expected sizes. + * Otherwise fixed32/fixed64 fields can break. + * + * If you get errors here, it probably means that your stdint.h is not + * correct for your platform. + */ +#ifndef PB_WITHOUT_64BIT +PB_STATIC_ASSERT(sizeof(int64_t) == 2 * sizeof(int32_t), INT64_T_WRONG_SIZE) +PB_STATIC_ASSERT(sizeof(uint64_t) == 2 * sizeof(uint32_t), UINT64_T_WRONG_SIZE) +#endif + +/* This structure is used for 'bytes' arrays. + * It has the number of bytes in the beginning, and after that an array. + * Note that actual structs used will have a different length of bytes array. + */ +#define PB_BYTES_ARRAY_T(n) struct { pb_size_t size; pb_byte_t bytes[n]; } +#define PB_BYTES_ARRAY_T_ALLOCSIZE(n) ((size_t)n + offsetof(pb_bytes_array_t, bytes)) + +struct pb_bytes_array_s { + pb_size_t size; + pb_byte_t bytes[1]; +}; +typedef struct pb_bytes_array_s pb_bytes_array_t; + +/* This structure is used for giving the callback function. + * It is stored in the message structure and filled in by the method that + * calls pb_decode. + * + * The decoding callback will be given a limited-length stream + * If the wire type was string, the length is the length of the string. + * If the wire type was a varint/fixed32/fixed64, the length is the length + * of the actual value. + * The function may be called multiple times (especially for repeated types, + * but also otherwise if the message happens to contain the field multiple + * times.) + * + * The encoding callback will receive the actual output stream. + * It should write all the data in one call, including the field tag and + * wire type. It can write multiple fields. + * + * The callback can be null if you want to skip a field. + */ +typedef struct pb_istream_s pb_istream_t; +typedef struct pb_ostream_s pb_ostream_t; +typedef struct pb_callback_s pb_callback_t; +struct pb_callback_s { +#ifdef PB_OLD_CALLBACK_STYLE + /* Deprecated since nanopb-0.2.1 */ +union { + bool (*decode)(pb_istream_t *stream, const pb_field_t *field, void *arg); + bool (*encode)(pb_ostream_t *stream, const pb_field_t *field, const void *arg); +} funcs; +#else + /* New function signature, which allows modifying arg contents in callback. */ +union { + bool (*decode)(pb_istream_t *stream, const pb_field_t *field, void **arg); + bool (*encode)(pb_ostream_t *stream, const pb_field_t *field, void * const *arg); +} funcs; +#endif + + /* Free arg for use by callback */ + void *arg; +}; + +/* Wire types. Library user needs these only in encoder callbacks. */ +typedef enum { + PB_WT_VARINT = 0, + PB_WT_64BIT = 1, + PB_WT_STRING = 2, + PB_WT_32BIT = 5 +} pb_wire_type_t; + +/* Structure for defining the handling of unknown/extension fields. + * Usually the pb_extension_type_t structure is automatically generated, + * while the pb_extension_t structure is created by the user. However, + * if you want to catch all unknown fields, you can also create a custom + * pb_extension_type_t with your own callback. + */ +typedef struct pb_extension_type_s pb_extension_type_t; +typedef struct pb_extension_s pb_extension_t; +struct pb_extension_type_s { + /* Called for each unknown field in the message. + * If you handle the field, read off all of its data and return true. + * If you do not handle the field, do not read anything and return true. + * If you run into an error, return false. + * Set to NULL for default handler. + */ + bool (*decode)(pb_istream_t *stream, pb_extension_t *extension, + uint32_t tag, pb_wire_type_t wire_type); + + /* Called once after all regular fields have been encoded. + * If you have something to write, do so and return true. + * If you do not have anything to write, just return true. + * If you run into an error, return false. + * Set to NULL for default handler. + */ + bool (*encode)(pb_ostream_t *stream, const pb_extension_t *extension); + + /* Free field for use by the callback. */ + const void *arg; +}; + +struct pb_extension_s { + /* Type describing the extension field. Usually you'll initialize + * this to a pointer to the automatically generated structure. + */ + const pb_extension_type_t *type; + + /* Destination for the decoded data. This must match the datatype + * of the extension field. + */ + void *dest; + + /* Pointer to the next extension handler, or NULL. + * If this extension does not match a field, the next handler is + * automatically called. + */ + pb_extension_t *next; + + /* The decoder sets this to true if the extension was found. + * Ignored for encoding. + */ + bool found; +}; + +/* Memory allocation functions to use. You can define pb_realloc and + * pb_free to custom functions if you want. + */ +#ifdef PB_ENABLE_MALLOC +#ifndef pb_realloc +#define pb_realloc(ptr, size) realloc(ptr, size) +#endif +#ifndef pb_free +#define pb_free(ptr) free(ptr) +#endif +#endif + +/* This is used to inform about need to regenerate .pb.h/.pb.c files. */ +#define PB_PROTO_HEADER_VERSION 30 + +/* These macros are used to declare pb_field_t's in the constant array. */ +/* Size of a structure member, in bytes. */ +#define pb_membersize(st, m) ((sizeof((st *)0)->m)) +/* Number of entries in an array. */ +#define pb_arraysize(st, m) (pb_membersize(st, m) / pb_membersize(st, m[0])) +/* Delta from start of one member to the start of another member. */ +#define pb_delta(st, m1, m2) ((int)offsetof(st, m1) - (int)offsetof(st, m2)) +/* Marks the end of the field list */ +#define PB_LAST_FIELD {0, (pb_type_t) 0, 0, 0, 0, 0, 0} + +/* Macros for filling in the data_offset field */ +/* data_offset for first field in a message */ +#define PB_DATAOFFSET_FIRST(st, m1, m2) (offsetof(st, m1)) +/* data_offset for subsequent fields */ +#define PB_DATAOFFSET_OTHER(st, m1, m2) \ + (offsetof(st, m1) - offsetof(st, m2) - pb_membersize(st, m2)) +/* data offset for subsequent fields inside a union (oneof) */ +#define PB_DATAOFFSET_UNION(st, m1, m2) (PB_SIZE_MAX) +/* Choose first/other based on m1 == m2 (deprecated, remains for backwards compatibility) */ +#define PB_DATAOFFSET_CHOOSE(st, m1, m2) \ + ((int)(offsetof(st, m1) == offsetof(st, m2)) \ + ? (PB_DATAOFFSET_FIRST(st, m1, m2)) \ + : (PB_DATAOFFSET_OTHER(st, m1, m2))) + +/* Required fields are the simplest. They just have delta (padding) from + * previous field end, and the size of the field. Pointer is used for + * submessages and default values. + */ +#define PB_REQUIRED_STATIC(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_REQUIRED | ltype, \ + fd, 0, pb_membersize(st, m), 0, ptr} + +/* Optional fields add the delta to the has_ variable. */ +#define PB_OPTIONAL_STATIC(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_OPTIONAL | ltype, \ + fd, \ + pb_delta(st, has_ ## m, m), \ + pb_membersize(st, m), 0, ptr} + +#define PB_SINGULAR_STATIC(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_OPTIONAL | ltype, \ + fd, 0, pb_membersize(st, m), 0, ptr} + +/* Repeated fields have a _count field and also the maximum number of entries. */ +#define PB_REPEATED_STATIC(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_REPEATED | ltype, \ + fd, \ + pb_delta(st, m ## _count, m), \ + pb_membersize(st, m[0]), \ + pb_arraysize(st, m), ptr} + +/* Allocated fields carry the size of the actual data, not the pointer */ +#define PB_REQUIRED_POINTER(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_POINTER | PB_HTYPE_REQUIRED | ltype, \ + fd, 0, pb_membersize(st, m[0]), 0, ptr} + +/* Optional fields don't need a has_ variable, as information would be redundant */ +#define PB_OPTIONAL_POINTER(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_POINTER | PB_HTYPE_OPTIONAL | ltype, \ + fd, 0, pb_membersize(st, m[0]), 0, ptr} + +/* Same as optional fields*/ +#define PB_SINGULAR_POINTER(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_POINTER | PB_HTYPE_OPTIONAL | ltype, \ + fd, 0, pb_membersize(st, m[0]), 0, ptr} + +/* Repeated fields have a _count field and a pointer to array of pointers */ +#define PB_REPEATED_POINTER(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_POINTER | PB_HTYPE_REPEATED | ltype, \ + fd, pb_delta(st, m ## _count, m), \ + pb_membersize(st, m[0]), 0, ptr} + +/* Callbacks are much like required fields except with special datatype. */ +#define PB_REQUIRED_CALLBACK(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_CALLBACK | PB_HTYPE_REQUIRED | ltype, \ + fd, 0, pb_membersize(st, m), 0, ptr} + +#define PB_OPTIONAL_CALLBACK(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_CALLBACK | PB_HTYPE_OPTIONAL | ltype, \ + fd, 0, pb_membersize(st, m), 0, ptr} + +#define PB_SINGULAR_CALLBACK(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_CALLBACK | PB_HTYPE_OPTIONAL | ltype, \ + fd, 0, pb_membersize(st, m), 0, ptr} + +#define PB_REPEATED_CALLBACK(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_CALLBACK | PB_HTYPE_REPEATED | ltype, \ + fd, 0, pb_membersize(st, m), 0, ptr} + +/* Optional extensions don't have the has_ field, as that would be redundant. + * Furthermore, the combination of OPTIONAL without has_ field is used + * for indicating proto3 style fields. Extensions exist in proto2 mode only, + * so they should be encoded according to proto2 rules. To avoid the conflict, + * extensions are marked as REQUIRED instead. + */ +#define PB_OPTEXT_STATIC(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_REQUIRED | ltype, \ + 0, \ + 0, \ + pb_membersize(st, m), 0, ptr} + +#define PB_OPTEXT_POINTER(tag, st, m, fd, ltype, ptr) \ + PB_OPTIONAL_POINTER(tag, st, m, fd, ltype, ptr) + +#define PB_OPTEXT_CALLBACK(tag, st, m, fd, ltype, ptr) \ + PB_OPTIONAL_CALLBACK(tag, st, m, fd, ltype, ptr) + +/* The mapping from protobuf types to LTYPEs is done using these macros. */ +#define PB_LTYPE_MAP_BOOL PB_LTYPE_VARINT +#define PB_LTYPE_MAP_BYTES PB_LTYPE_BYTES +#define PB_LTYPE_MAP_DOUBLE PB_LTYPE_FIXED64 +#define PB_LTYPE_MAP_ENUM PB_LTYPE_VARINT +#define PB_LTYPE_MAP_UENUM PB_LTYPE_UVARINT +#define PB_LTYPE_MAP_FIXED32 PB_LTYPE_FIXED32 +#define PB_LTYPE_MAP_FIXED64 PB_LTYPE_FIXED64 +#define PB_LTYPE_MAP_FLOAT PB_LTYPE_FIXED32 +#define PB_LTYPE_MAP_INT32 PB_LTYPE_VARINT +#define PB_LTYPE_MAP_INT64 PB_LTYPE_VARINT +#define PB_LTYPE_MAP_MESSAGE PB_LTYPE_SUBMESSAGE +#define PB_LTYPE_MAP_SFIXED32 PB_LTYPE_FIXED32 +#define PB_LTYPE_MAP_SFIXED64 PB_LTYPE_FIXED64 +#define PB_LTYPE_MAP_SINT32 PB_LTYPE_SVARINT +#define PB_LTYPE_MAP_SINT64 PB_LTYPE_SVARINT +#define PB_LTYPE_MAP_STRING PB_LTYPE_STRING +#define PB_LTYPE_MAP_UINT32 PB_LTYPE_UVARINT +#define PB_LTYPE_MAP_UINT64 PB_LTYPE_UVARINT +#define PB_LTYPE_MAP_EXTENSION PB_LTYPE_EXTENSION +#define PB_LTYPE_MAP_FIXED_LENGTH_BYTES PB_LTYPE_FIXED_LENGTH_BYTES + +/* This is the actual macro used in field descriptions. + * It takes these arguments: + * - Field tag number + * - Field type: BOOL, BYTES, DOUBLE, ENUM, UENUM, FIXED32, FIXED64, + * FLOAT, INT32, INT64, MESSAGE, SFIXED32, SFIXED64 + * SINT32, SINT64, STRING, UINT32, UINT64 or EXTENSION + * - Field rules: REQUIRED, OPTIONAL or REPEATED + * - Allocation: STATIC, CALLBACK or POINTER + * - Placement: FIRST or OTHER, depending on if this is the first field in structure. + * - Message name + * - Field name + * - Previous field name (or field name again for first field) + * - Pointer to default value or submsg fields. + */ + +#define PB_FIELD(tag, type, rules, allocation, placement, message, field, prevfield, ptr) \ + PB_ ## rules ## _ ## allocation(tag, message, field, \ + PB_DATAOFFSET_ ## placement(message, field, prevfield), \ + PB_LTYPE_MAP_ ## type, ptr) +#define PB_FIELD2 PB_FIELD + +/* Field description for repeated static fixed count fields.*/ +#define PB_REPEATED_FIXED_COUNT(tag, type, placement, message, field, prevfield, ptr) \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_REPEATED | PB_LTYPE_MAP_ ## type, \ + PB_DATAOFFSET_ ## placement(message, field, prevfield), \ + 0, \ + pb_membersize(message, field[0]), \ + pb_arraysize(message, field), ptr} + +/* Field description for oneof fields. This requires taking into account the + * union name also, that's why a separate set of macros is needed. + */ +#define PB_ONEOF_STATIC(u, tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_ONEOF | ltype, \ + fd, pb_delta(st, which_ ## u, u.m), \ + pb_membersize(st, u.m), 0, ptr} + +#define PB_ONEOF_POINTER(u, tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_POINTER | PB_HTYPE_ONEOF | ltype, \ + fd, pb_delta(st, which_ ## u, u.m), \ + pb_membersize(st, u.m[0]), 0, ptr} + +#define PB_ONEOF_FIELD(union_name, tag, type, rules, \ + allocation, placement, message, field, prevfield, ptr) \ + PB_ONEOF_ ## allocation(union_name, tag, message, field, \ + PB_DATAOFFSET_ ## placement(message, union_name.field, prevfield), \ + PB_LTYPE_MAP_ ## type, ptr) + +#define PB_ANONYMOUS_ONEOF_STATIC(u, tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_ONEOF | ltype, \ + fd, pb_delta(st, which_ ## u, m), \ + pb_membersize(st, m), 0, ptr} + +#define PB_ANONYMOUS_ONEOF_POINTER(u, tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_POINTER | PB_HTYPE_ONEOF | ltype, \ + fd, pb_delta(st, which_ ## u, m), \ + pb_membersize(st, m[0]), 0, ptr} + +#define PB_ANONYMOUS_ONEOF_FIELD(union_name, tag, type, rules, \ + allocation, placement, message, field, prevfield, ptr) \ + PB_ANONYMOUS_ONEOF_ ## allocation(union_name, tag, message, field, \ + PB_DATAOFFSET_ ## placement(message, field, prevfield), \ + PB_LTYPE_MAP_ ## type, ptr) + +/* These macros are used for giving out error messages. + * They are mostly a debugging aid; the main error information + * is the true/false return value from functions. + * Some code space can be saved by disabling the error + * messages if not used. + * + * PB_SET_ERROR() sets the error message if none has been set yet. + * msg must be a constant string literal. + * PB_GET_ERROR() always returns a pointer to a string. + * PB_RETURN_ERROR() sets the error and returns false from current + * function. + */ +#ifdef PB_NO_ERRMSG +#define PB_SET_ERROR(stream, msg) PB_UNUSED(stream) +#define PB_GET_ERROR(stream) "(errmsg disabled)" +#else +#define PB_SET_ERROR(stream, msg) \ + (stream->errmsg = (stream)->errmsg ? (stream)->errmsg : (msg)) +#define PB_GET_ERROR(stream) ((stream)->errmsg ? (stream)->errmsg : "(none)") +#endif + +#define PB_RETURN_ERROR(stream, msg) return (PB_SET_ERROR(stream, msg), false) + +#endif diff --git a/drivers/misc/vehicle/nanopb-c/pb_common.c b/drivers/misc/vehicle/nanopb-c/pb_common.c new file mode 100644 index 000000000000..51555d4f8f90 --- /dev/null +++ b/drivers/misc/vehicle/nanopb-c/pb_common.c @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2011 Petteri Aimonen + * + * This software is provided 'as-is', without any express or + * implied warranty. In no event will the authors be held liable + * for any damages arising from the use of this software. + * + * Permission is granted to anyone to use this software for any + * purpose, including commercial applications, and to alter it and + * redistribute it freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you + * must not claim that you wrote the original software. If you use + * this software in a product, an acknowledgment in the product + * documentation would be appreciated but is not required. + * + * 2. Altered source versions must be plainly marked as such, and + * must not be misrepresented as being the original software. + * + * 3. This notice may not be removed or altered from any source + * distribution. + */ +/* pb_common.c: Common support functions for pb_encode.c and pb_decode.c. + * + * 2014 Petteri Aimonen + */ + +#include "pb_common.h" + +bool pb_field_iter_begin(pb_field_iter_t *iter, const pb_field_t *fields, void *dest_struct) +{ + iter->start = fields; + iter->pos = fields; + iter->required_field_index = 0; + iter->dest_struct = dest_struct; + iter->pData = (char *)dest_struct + iter->pos->data_offset; + iter->pSize = (char *)iter->pData + iter->pos->size_offset; + + return (iter->pos->tag != 0); +} + +bool pb_field_iter_next(pb_field_iter_t *iter) +{ + const pb_field_t *prev_field = iter->pos; + + if (prev_field->tag == 0) + /* Handle empty message types, where the first field is already the terminator. + * In other cases, the iter->pos never points to the terminator. + */ + return false; + + iter->pos++; + + if (iter->pos->tag == 0) { + /* Wrapped back to beginning, reinitialize */ + (void)pb_field_iter_begin(iter, iter->start, iter->dest_struct); + return false; + } else { + /* Increment the pointers based on previous field size */ + size_t prev_size = prev_field->data_size; + + if (PB_HTYPE(prev_field->type) == PB_HTYPE_ONEOF && + PB_HTYPE(iter->pos->type) == PB_HTYPE_ONEOF && + iter->pos->data_offset == PB_SIZE_MAX) + /* Don't advance pointers inside unions */ + return true; + else if (PB_ATYPE(prev_field->type) == PB_ATYPE_STATIC && + PB_HTYPE(prev_field->type) == PB_HTYPE_REPEATED) + /* In static arrays, the data_size tells the size of a single entry and + * array_size is the number of entries + */ + prev_size *= prev_field->array_size; + else if (PB_ATYPE(prev_field->type) == PB_ATYPE_POINTER) + /* Pointer fields always have a constant size in the main structure. + * The data_size only applies to the dynamically allocated area. + */ + prev_size = sizeof(void *); + + if (PB_HTYPE(prev_field->type) == PB_HTYPE_REQUIRED) + /* Count the required fields, in order to check their presence in the + * decoder. + */ + iter->required_field_index++; + + iter->pData = (char *)iter->pData + prev_size + iter->pos->data_offset; + iter->pSize = (char *)iter->pData + iter->pos->size_offset; + + return true; + } +} + +bool pb_field_iter_find(pb_field_iter_t *iter, uint32_t tag) +{ + const pb_field_t *start = iter->pos; + + do { + if (iter->pos->tag == tag && + PB_LTYPE(iter->pos->type) != PB_LTYPE_EXTENSION) + /* Found the wanted field */ + return true; + + (void)pb_field_iter_next(iter); + } while (iter->pos != start); + + /* Searched all the way back to start, and found nothing. */ + return false; +} + diff --git a/drivers/misc/vehicle/nanopb-c/pb_common.h b/drivers/misc/vehicle/nanopb-c/pb_common.h new file mode 100644 index 000000000000..9f0edf002f92 --- /dev/null +++ b/drivers/misc/vehicle/nanopb-c/pb_common.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2011 Petteri Aimonen + * + * This software is provided 'as-is', without any express or + * implied warranty. In no event will the authors be held liable + * for any damages arising from the use of this software. + * + * Permission is granted to anyone to use this software for any + * purpose, including commercial applications, and to alter it and + * redistribute it freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you + * must not claim that you wrote the original software. If you use + * this software in a product, an acknowledgment in the product + * documentation would be appreciated but is not required. + * + * 2. Altered source versions must be plainly marked as such, and + * must not be misrepresented as being the original software. + * + * 3. This notice may not be removed or altered from any source + * distribution. + */ +/* pb_common.h: Common support functions for pb_encode.c and pb_decode.c. + * These functions are rarely needed by applications directly. + */ + +#ifndef PB_COMMON_H_INCLUDED +#define PB_COMMON_H_INCLUDED + +#include "pb.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* Iterator for pb_field_t list */ +struct pb_field_iter_s { + const pb_field_t *start; /* Start of the pb_field_t array */ + const pb_field_t *pos; /* Current position of the iterator */ + /* Zero-based index that counts only the required fields */ + unsigned int required_field_index; + void *dest_struct; /* Pointer to start of the structure */ + void *pData; /* Pointer to current field value */ + void *pSize; /* Pointer to count/has field */ +}; +typedef struct pb_field_iter_s pb_field_iter_t; + +/* Initialize the field iterator structure to beginning. + * Returns false if the message type is empty. + */ +bool pb_field_iter_begin(pb_field_iter_t *iter, const pb_field_t *fields, void *dest_struct); + +/* Advance the iterator to the next field. + * Returns false when the iterator wraps back to the first field. + */ +bool pb_field_iter_next(pb_field_iter_t *iter); + +/* Advance the iterator until it points at a field with the given tag. + * Returns false if no such field exists. + */ +bool pb_field_iter_find(pb_field_iter_t *iter, uint32_t tag); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif + diff --git a/drivers/misc/vehicle/nanopb-c/pb_decode.c b/drivers/misc/vehicle/nanopb-c/pb_decode.c new file mode 100644 index 000000000000..f2b6334b1f14 --- /dev/null +++ b/drivers/misc/vehicle/nanopb-c/pb_decode.c @@ -0,0 +1,1503 @@ +/* + * Copyright (c) 2011 Petteri Aimonen + * + * This software is provided 'as-is', without any express or + * implied warranty. In no event will the authors be held liable + * for any damages arising from the use of this software. + * + * Permission is granted to anyone to use this software for any + * purpose, including commercial applications, and to alter it and + * redistribute it freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you + * must not claim that you wrote the original software. If you use + * this software in a product, an acknowledgment in the product + * documentation would be appreciated but is not required. + * + * 2. Altered source versions must be plainly marked as such, and + * must not be misrepresented as being the original software. + * + * 3. This notice may not be removed or altered from any source + * distribution. + */ +/* pb_decode.c -- decode a protobuf using minimal resources + * + * 2011 Petteri Aimonen + */ + +/* Use the GCC warn_unused_result attribute to check that all return values + * are propagated correctly. On other compilers and gcc before 3.4.0 just + * ignore the annotation. + */ +#if (!defined(__GNUC__)) || (__GNUC__ < 3) || ((__GNUC__ == 3) && (__GNUC_MINOR__ < 4)) + #define checkreturn +#else + #define checkreturn __attribute__((warn_unused_result)) +#endif + +#include "pb.h" +#include "pb_decode.h" +#include "pb_common.h" + +/************************************** + * Declarations internal to this file * + **************************************/ + +typedef bool (*pb_decoder_t)(pb_istream_t *stream, const pb_field_t *field, void *dest) checkreturn; + +static bool checkreturn buf_read(pb_istream_t *stream, pb_byte_t *buf, size_t count); +static bool checkreturn read_raw_value(pb_istream_t *stream, + pb_wire_type_t wire_type, + pb_byte_t *buf, + size_t *size); +static bool checkreturn decode_static_field(pb_istream_t *stream, + pb_wire_type_t wire_type, + pb_field_iter_t *iter); +static bool checkreturn decode_callback_field(pb_istream_t *stream, + pb_wire_type_t wire_type, + pb_field_iter_t *iter); +static bool checkreturn decode_field(pb_istream_t *stream, + pb_wire_type_t wire_type, + pb_field_iter_t *iter); +static void iter_from_extension(pb_field_iter_t *iter, + pb_extension_t *extension); +static bool checkreturn default_extension_decoder(pb_istream_t *stream, + pb_extension_t *extension, + uint32_t tag, + pb_wire_type_t wire_type); +static bool checkreturn decode_extension(pb_istream_t *stream, + uint32_t tag, + pb_wire_type_t wire_type, + pb_field_iter_t *iter); +static bool checkreturn find_extension_field(pb_field_iter_t *iter); +static void pb_field_set_to_default(pb_field_iter_t *iter); +static void pb_message_set_to_defaults(const pb_field_t fields[], + void *dest_struct); +static bool checkreturn pb_dec_varint(pb_istream_t *stream, + const pb_field_t *field, + void *dest); +static bool checkreturn pb_decode_varint32_eof(pb_istream_t *stream, + uint32_t *dest, + bool *eof); +static bool checkreturn pb_dec_uvarint(pb_istream_t *stream, + const pb_field_t *field, + void *dest); +static bool checkreturn pb_dec_svarint(pb_istream_t *stream, + const pb_field_t *field, void *dest); +static bool checkreturn pb_dec_fixed32(pb_istream_t *stream, + const pb_field_t *field, + void *dest); +static bool checkreturn pb_dec_fixed64(pb_istream_t *stream, + const pb_field_t *field, + void *dest); +static bool checkreturn pb_dec_bytes(pb_istream_t *stream, + const pb_field_t *field, + void *dest); +static bool checkreturn pb_dec_string(pb_istream_t *stream, + const pb_field_t *field, + void *dest); +static bool checkreturn pb_dec_submessage(pb_istream_t *stream, + const pb_field_t *field, + void *dest); +static bool checkreturn pb_dec_fixed_length_bytes(pb_istream_t *stream, + const pb_field_t *field, + void *dest); +static bool checkreturn pb_skip_varint(pb_istream_t *stream); +static bool checkreturn pb_skip_string(pb_istream_t *stream); + +#ifdef PB_ENABLE_MALLOC +static bool checkreturn allocate_field(pb_istream_t *stream, void *pData, + size_t data_size, size_t array_size); +static bool checkreturn pb_release_union_field(pb_istream_t *stream, + pb_field_iter_t *iter); +static void pb_release_single_field(const pb_field_iter_t *iter); +#endif + +#ifdef PB_WITHOUT_64BIT +#define pb_int64_t int32_t +#define pb_uint64_t uint32_t +#else +#define pb_int64_t int64_t +#define pb_uint64_t uint64_t +#endif + +/* --- Function pointers to field decoders --- + * Order in the array must match pb_action_t LTYPE numbering. + */ +static const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT] = { + &pb_dec_varint, + &pb_dec_uvarint, + &pb_dec_svarint, + &pb_dec_fixed32, + &pb_dec_fixed64, + + &pb_dec_bytes, + &pb_dec_string, + &pb_dec_submessage, + NULL, /* extensions */ + &pb_dec_fixed_length_bytes +}; + +/******************************* + * pb_istream_t implementation * + *******************************/ + +static bool checkreturn buf_read(pb_istream_t *stream, pb_byte_t *buf, size_t count) +{ + size_t i; + + const pb_byte_t *source = (const pb_byte_t *)stream->state; + + stream->state = (pb_byte_t *)stream->state + count; + if (buf != NULL) { + for (i = 0; i < count; i++) + buf[i] = source[i]; + } + + return true; +} + +bool checkreturn pb_read(pb_istream_t *stream, pb_byte_t *buf, size_t count) +{ +#ifndef PB_BUFFER_ONLY + if (buf == NULL && stream->callback != buf_read) { + /* Skip input bytes */ + pb_byte_t tmp[16]; + + while (count > 16) { + if (!pb_read(stream, tmp, 16)) + return false; + + count -= 16; + } + + return pb_read(stream, tmp, count); + } +#endif + + if (stream->bytes_left < count) + PB_RETURN_ERROR(stream, "end-of-stream"); + +#ifndef PB_BUFFER_ONLY + if (!stream->callback(stream, buf, count)) + PB_RETURN_ERROR(stream, "io error"); +#else + if (!buf_read(stream, buf, count)) + return false; +#endif + + stream->bytes_left -= count; + return true; +} + +/* Read a single byte from input stream. buf may not be NULL. + * This is an optimization for the varint decoding. + */ +static bool checkreturn pb_readbyte(pb_istream_t *stream, pb_byte_t *buf) +{ + if (stream->bytes_left == 0) + PB_RETURN_ERROR(stream, "end-of-stream"); + +#ifndef PB_BUFFER_ONLY + if (!stream->callback(stream, buf, 1)) + PB_RETURN_ERROR(stream, "io error"); +#else + *buf = *(const pb_byte_t *)stream->state; + stream->state = (pb_byte_t *)stream->state + 1; +#endif + + stream->bytes_left--; + + return true; +} + +pb_istream_t pb_istream_from_buffer(const pb_byte_t *buf, size_t bufsize) +{ + pb_istream_t stream; + /* Cast away the const from buf without a compiler error. We are + * careful to use it only in a const manner in the callbacks. + */ + union { + void *state; + const void *c_state; + } state; + +#ifdef PB_BUFFER_ONLY + stream.callback = NULL; +#else + stream.callback = &buf_read; +#endif + state.c_state = buf; + stream.state = state.state; + stream.bytes_left = bufsize; +#ifndef PB_NO_ERRMSG + stream.errmsg = NULL; +#endif + return stream; +} + +/******************** + * Helper functions * + ********************/ + +static bool checkreturn pb_decode_varint32_eof(pb_istream_t *stream, uint32_t *dest, bool *eof) +{ + pb_byte_t byte; + uint32_t result; + + if (!pb_readbyte(stream, &byte)) { + if (stream->bytes_left == 0) { + if (eof) + *eof = true; + } + + return false; + } + + if ((byte & 0x80) == 0) + /* Quick case, 1 byte value */ + result = byte; + else { + /* Multibyte case */ + uint_fast8_t bitpos = 7; + + result = byte & 0x7F; + + do { + if (!pb_readbyte(stream, &byte)) + return false; + + if (bitpos >= 32) { + /* The varint could have trailing 0x80 bytes, + * or 0xFF for negative. + */ + uint8_t sign_extension = (bitpos < 63) ? 0xFF : 0x01; + + if ((byte & 0x7F) != 0x00 && + ((result >> 31) == 0 || byte != sign_extension)) + PB_RETURN_ERROR(stream, "varint overflow"); + } else + result |= (uint32_t)(byte & 0x7F) << bitpos; + + bitpos = (uint_fast8_t)(bitpos + 7); + } while (byte & 0x80); + + if (bitpos == 35 && (byte & 0x70) != 0) + /* The last byte was at bitpos=28, so only bottom 4 bits fit. */ + PB_RETURN_ERROR(stream, "varint overflow"); + } + + *dest = result; + return true; +} + +bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) +{ + return pb_decode_varint32_eof(stream, dest, NULL); +} + +#ifndef PB_WITHOUT_64BIT +bool checkreturn pb_decode_varint(pb_istream_t *stream, uint64_t *dest) +{ + pb_byte_t byte; + uint_fast8_t bitpos = 0; + uint64_t result = 0; + + do { + if (bitpos >= 64) + PB_RETURN_ERROR(stream, "varint overflow"); + + if (!pb_readbyte(stream, &byte)) + return false; + + result |= (uint64_t)(byte & 0x7F) << bitpos; + bitpos = (uint_fast8_t)(bitpos + 7); + } while (byte & 0x80); + + *dest = result; + return true; +} +#endif + +bool checkreturn pb_skip_varint(pb_istream_t *stream) +{ + pb_byte_t byte; + + do { + if (!pb_read(stream, &byte, 1)) + return false; + } while (byte & 0x80); + return true; +} + +bool checkreturn pb_skip_string(pb_istream_t *stream) +{ + uint32_t length; + + if (!pb_decode_varint32(stream, &length)) + return false; + + return pb_read(stream, NULL, length); +} + +bool checkreturn pb_decode_tag(pb_istream_t *stream, + pb_wire_type_t *wire_type, uint32_t *tag, bool *eof) +{ + uint32_t temp; + *eof = false; + *wire_type = (pb_wire_type_t) 0; + *tag = 0; + + if (!pb_decode_varint32_eof(stream, &temp, eof)) + return false; + + if (temp == 0) { + *eof = true; /* Special feature: allow 0-terminated messages. */ + return false; + } + + *tag = temp >> 3; + *wire_type = (pb_wire_type_t)(temp & 7); + + return true; +} + +bool checkreturn pb_skip_field(pb_istream_t *stream, pb_wire_type_t wire_type) +{ + switch (wire_type) { + case PB_WT_VARINT: return pb_skip_varint(stream); + case PB_WT_64BIT: return pb_read(stream, NULL, 8); + case PB_WT_STRING: return pb_skip_string(stream); + case PB_WT_32BIT: return pb_read(stream, NULL, 4); + default: + PB_RETURN_ERROR(stream, "invalid wire_type"); + } +} + +/* Read a raw value to buffer, for the purpose of passing it to callback as + * a substream. Size is maximum size on call, and actual size on return. + */ +static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire_type, + pb_byte_t *buf, size_t *size) +{ + size_t max_size = *size; + + switch (wire_type) { + case PB_WT_VARINT: + *size = 0; + do { + (*size)++; + if (*size > max_size) + return false; + if (!pb_read(stream, buf, 1)) + return false; + } while (*buf++ & 0x80); + return true; + + case PB_WT_64BIT: + *size = 8; + return pb_read(stream, buf, 8); + + case PB_WT_32BIT: + *size = 4; + return pb_read(stream, buf, 4); + + case PB_WT_STRING: + /* Calling read_raw_value with a PB_WT_STRING is an error. + * Explicitly handle this case and fallthrough to default to avoid + * compiler warnings. + */ + + default: + PB_RETURN_ERROR(stream, "invalid wire_type"); + } +} + +/* Decode string length from stream and return a substream with limited length. + * Remember to close the substream using pb_close_string_substream(). + */ +bool checkreturn pb_make_string_substream(pb_istream_t *stream, pb_istream_t *substream) +{ + uint32_t size; + + if (!pb_decode_varint32(stream, &size)) + return false; + + *substream = *stream; + if (substream->bytes_left < size) + PB_RETURN_ERROR(stream, "parent stream too short"); + + substream->bytes_left = size; + stream->bytes_left -= size; + return true; +} + +bool checkreturn pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream) +{ + if (substream->bytes_left) { + if (!pb_read(substream, NULL, substream->bytes_left)) + return false; + } + + stream->state = substream->state; + +#ifndef PB_NO_ERRMSG + stream->errmsg = substream->errmsg; +#endif + return true; +} + +/************************* + * Decode a single field * + *************************/ + +static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, + pb_field_iter_t *iter) +{ + pb_type_t type; + pb_decoder_t func; + + type = iter->pos->type; + func = PB_DECODERS[PB_LTYPE(type)]; + + switch (PB_HTYPE(type)) { + case PB_HTYPE_REQUIRED: + return func(stream, iter->pos, iter->pData); + + case PB_HTYPE_OPTIONAL: + if (iter->pSize != iter->pData) + *(bool *)iter->pSize = true; + return func(stream, iter->pos, iter->pData); + + case PB_HTYPE_REPEATED: + if (wire_type == PB_WT_STRING + && PB_LTYPE(type) <= PB_LTYPE_LAST_PACKABLE) { + /* Packed array */ + bool status = true; + pb_size_t *size = (pb_size_t *)iter->pSize; + pb_istream_t substream; + + if (!pb_make_string_substream(stream, &substream)) + return false; + + while (substream.bytes_left > 0 && *size < iter->pos->array_size) { + void *pItem = (char *)iter->pData + iter->pos->data_size * (*size); + + if (!func(&substream, iter->pos, pItem)) { + status = false; + break; + } + (*size)++; + } + + if (substream.bytes_left != 0) + PB_RETURN_ERROR(stream, "array overflow"); + if (!pb_close_string_substream(stream, &substream)) + return false; + + return status; + } else { + /* Repeated field */ + pb_size_t *size = (pb_size_t *)iter->pSize; + char *pItem = (char *)iter->pData + iter->pos->data_size * (*size); + + if ((*size)++ >= iter->pos->array_size) + PB_RETURN_ERROR(stream, "array overflow"); + + return func(stream, iter->pos, pItem); + } + + case PB_HTYPE_ONEOF: + *(pb_size_t *)iter->pSize = iter->pos->tag; + if (PB_LTYPE(type) == PB_LTYPE_SUBMESSAGE) { + /* We memset to zero so that any callbacks are set to NULL. + * Then set any default values. + */ + memset(iter->pData, 0, iter->pos->data_size); + pb_message_set_to_defaults((const pb_field_t *)iter->pos->ptr, iter->pData); + } + return func(stream, iter->pos, iter->pData); + + default: + PB_RETURN_ERROR(stream, "invalid field type"); + } +} + +#ifdef PB_ENABLE_MALLOC +/* Allocate storage for the field and store the pointer at iter->pData. + * array_size is the number of entries to reserve in an array. + * Zero size is not allowed, use pb_free() for releasing. + */ +static bool checkreturn allocate_field(pb_istream_t *stream, void *pData, size_t data_size, + size_t array_size) +{ + void *ptr = *(void **)pData; + + if (data_size == 0 || array_size == 0) + PB_RETURN_ERROR(stream, "invalid size"); + + /* Check for multiplication overflows. + * This code avoids the costly division if the sizes are small enough. + * Multiplication is safe as long as only half of bits are set + * in either multiplicand. + */ + const size_t check_limit = (size_t)1 << (sizeof(size_t) * 4); + + if (data_size >= check_limit || array_size >= check_limit) { + const size_t size_max = (size_t)-1; + + if (size_max / array_size < data_size) + PB_RETURN_ERROR(stream, "size too large"); + } + + /* Allocate new or expand previous allocation */ + /* Note: on failure the old pointer will remain in the structure, + * the message must be freed by caller also on error return. + */ + ptr = pb_realloc(ptr, array_size * data_size); + if (ptr == NULL) + PB_RETURN_ERROR(stream, "realloc failed"); + + *(void **)pData = ptr; + return true; +} + +/* Clear a newly allocated item in case it contains a pointer, or is a submessage. */ +static void initialize_pointer_field(void *pItem, pb_field_iter_t *iter) +{ + if (PB_LTYPE(iter->pos->type) == PB_LTYPE_STRING || + PB_LTYPE(iter->pos->type) == PB_LTYPE_BYTES) + *(void **)pItem = NULL; + else if (PB_LTYPE(iter->pos->type) == PB_LTYPE_SUBMESSAGE) { + /* We memset to zero so that any callbacks are set to NULL. + * Then set any default values. + */ + memset(pItem, 0, iter->pos->data_size); + pb_message_set_to_defaults((const pb_field_t *) iter->pos->ptr, pItem); + } +} +#endif + +static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_t wire_type, + pb_field_iter_t *iter) +{ +#ifndef PB_ENABLE_MALLOC + PB_UNUSED(wire_type); + PB_UNUSED(iter); + PB_RETURN_ERROR(stream, "no malloc support"); +#else + pb_type_t type; + pb_decoder_t func; + + type = iter->pos->type; + func = PB_DECODERS[PB_LTYPE(type)]; + + switch (PB_HTYPE(type)) { + case PB_HTYPE_REQUIRED: + case PB_HTYPE_OPTIONAL: + case PB_HTYPE_ONEOF: + if (PB_LTYPE(type) == PB_LTYPE_SUBMESSAGE && + *(void **)iter->pData != NULL) + /* Duplicate field, have to release the old allocation first. */ + pb_release_single_field(iter); + + if (PB_HTYPE(type) == PB_HTYPE_ONEOF) + *(pb_size_t *)iter->pSize = iter->pos->tag; + + if (PB_LTYPE(type) == PB_LTYPE_STRING || + PB_LTYPE(type) == PB_LTYPE_BYTES) + return func(stream, iter->pos, iter->pData); + else { + if (!allocate_field(stream, iter->pData, iter->pos->data_size, 1)) + return false; + + initialize_pointer_field(*(void **)iter->pData, iter); + return func(stream, iter->pos, *(void **)iter->pData); + } + + case PB_HTYPE_REPEATED: + if (wire_type == PB_WT_STRING + && PB_LTYPE(type) <= PB_LTYPE_LAST_PACKABLE) { + /* Packed array, multiple items come in at once. */ + bool status = true; + pb_size_t *size = (pb_size_t *)iter->pSize; + size_t allocated_size = *size; + void *pItem; + pb_istream_t substream; + + if (!pb_make_string_substream(stream, &substream)) + return false; + + while (substream.bytes_left) { + if ((size_t)*size + 1 > allocated_size) { + /* Allocate more storage. This tries to guess the + * number of remaining entries. Round the division + * upwards. + */ + allocated_size += (substream.bytes_left - 1) / + iter->pos->data_size + 1; + + if (!allocate_field(&substream, iter->pData, + iter->pos->data_size, allocated_size)) { + status = false; + break; + } + } + + /* Decode the array entry */ + pItem = *(char **)iter->pData + iter->pos->data_size * (*size); + initialize_pointer_field(pItem, iter); + if (!func(&substream, iter->pos, pItem)) { + status = false; + break; + } + + if (*size == PB_SIZE_MAX) { +#ifndef PB_NO_ERRMSG + stream->errmsg = "too many array entries"; +#endif + status = false; + break; + } + + (*size)++; + } + if (!pb_close_string_substream(stream, &substream)) + return false; + + return status; + } else { + /* Normal repeated field, i.e. only one item at a time. */ + pb_size_t *size = (pb_size_t *)iter->pSize; + void *pItem; + + if (*size == PB_SIZE_MAX) + PB_RETURN_ERROR(stream, "too many array entries"); + + (*size)++; + if (!allocate_field(stream, iter->pData, iter->pos->data_size, *size)) + return false; + + pItem = *(char **)iter->pData + iter->pos->data_size * (*size - 1); + initialize_pointer_field(pItem, iter); + return func(stream, iter->pos, pItem); + } + + default: + PB_RETURN_ERROR(stream, "invalid field type"); + } +#endif +} + +static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, + pb_field_iter_t *iter) +{ + pb_callback_t *pCallback = (pb_callback_t *)iter->pData; + +#ifdef PB_OLD_CALLBACK_STYLE + void *arg = pCallback->arg; +#else + void **arg = &(pCallback->arg); +#endif + + if (pCallback == NULL || pCallback->funcs.decode == NULL) + return pb_skip_field(stream, wire_type); + + if (wire_type == PB_WT_STRING) { + pb_istream_t substream; + + if (!pb_make_string_substream(stream, &substream)) + return false; + + do { + if (!pCallback->funcs.decode(&substream, iter->pos, arg)) + PB_RETURN_ERROR(stream, "callback failed"); + } while (substream.bytes_left); + + if (!pb_close_string_substream(stream, &substream)) + return false; + + return true; + } else { + /* Copy the single scalar value to stack. + * This is required so that we can limit the stream length, + * which in turn allows to use same callback for packed and + * not-packed fields. + */ + pb_istream_t substream; + pb_byte_t buffer[10]; + size_t size = sizeof(buffer); + + if (!read_raw_value(stream, wire_type, buffer, &size)) + return false; + substream = pb_istream_from_buffer(buffer, size); + + return pCallback->funcs.decode(&substream, iter->pos, arg); + } +} + +static bool checkreturn decode_field(pb_istream_t *stream, + pb_wire_type_t wire_type, pb_field_iter_t *iter) +{ +#ifdef PB_ENABLE_MALLOC + /* When decoding an oneof field, check if there is old data that must be + * released first. + */ + if (PB_HTYPE(iter->pos->type) == PB_HTYPE_ONEOF) { + if (!pb_release_union_field(stream, iter)) + return false; + } +#endif + + switch (PB_ATYPE(iter->pos->type)) { + case PB_ATYPE_STATIC: + return decode_static_field(stream, wire_type, iter); + + case PB_ATYPE_POINTER: + return decode_pointer_field(stream, wire_type, iter); + + case PB_ATYPE_CALLBACK: + return decode_callback_field(stream, wire_type, iter); + + default: + PB_RETURN_ERROR(stream, "invalid field type"); + } +} + +static void iter_from_extension(pb_field_iter_t *iter, pb_extension_t *extension) +{ + /* Fake a field iterator for the extension field. + * It is not actually safe to advance this iterator, but decode_field + * will not even try to. + */ + const pb_field_t *field = (const pb_field_t *)extension->type->arg; + (void)pb_field_iter_begin(iter, field, extension->dest); + iter->pData = extension->dest; + iter->pSize = &extension->found; + + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) + /* For pointer extensions, the pointer is stored directly + * in the extension structure. This avoids having an extra + * indirection. + */ + iter->pData = &extension->dest; +} + +/* Default handler for extension fields. Expects a pb_field_t structure + * in extension->type->arg. + */ +static bool checkreturn default_extension_decoder(pb_istream_t *stream, + pb_extension_t *extension, + uint32_t tag, pb_wire_type_t wire_type) +{ + const pb_field_t *field = (const pb_field_t *)extension->type->arg; + pb_field_iter_t iter; + + if (field->tag != tag) + return true; + + iter_from_extension(&iter, extension); + extension->found = true; + return decode_field(stream, wire_type, &iter); +} + +/* Try to decode an unknown field as an extension field. Tries each extension + * decoder in turn, until one of them handles the field or loop ends. + */ +static bool checkreturn decode_extension(pb_istream_t *stream, + uint32_t tag, pb_wire_type_t wire_type, + pb_field_iter_t *iter) +{ + pb_extension_t *extension = *(pb_extension_t * const *)iter->pData; + size_t pos = stream->bytes_left; + + while (extension != NULL && pos == stream->bytes_left) { + bool status; + + if (extension->type->decode) + status = extension->type->decode(stream, extension, tag, wire_type); + else + status = default_extension_decoder(stream, extension, tag, wire_type); + + if (!status) + return false; + + extension = extension->next; + } + + return true; +} + +/* Step through the iterator until an extension field is found or until all + * entries have been checked. There can be only one extension field per + * message. Returns false if no extension field is found. + */ +static bool checkreturn find_extension_field(pb_field_iter_t *iter) +{ + const pb_field_t *start = iter->pos; + + do { + if (PB_LTYPE(iter->pos->type) == PB_LTYPE_EXTENSION) + return true; + (void)pb_field_iter_next(iter); + } while (iter->pos != start); + + return false; +} + +/* Initialize message fields to default values, recursively */ +static void pb_field_set_to_default(pb_field_iter_t *iter) +{ + pb_type_t type; + + type = iter->pos->type; + if (PB_LTYPE(type) == PB_LTYPE_EXTENSION) { + pb_extension_t *ext = *(pb_extension_t * const *)iter->pData; + + while (ext != NULL) { + pb_field_iter_t ext_iter; + + ext->found = false; + iter_from_extension(&ext_iter, ext); + pb_field_set_to_default(&ext_iter); + ext = ext->next; + } + } else if (PB_ATYPE(type) == PB_ATYPE_STATIC) { + bool init_data = true; + + if (PB_HTYPE(type) == PB_HTYPE_OPTIONAL && iter->pSize != iter->pData) + /* Set has_field to false. Still initialize the optional field + * itself also. + */ + *(bool *)iter->pSize = false; + else if (PB_HTYPE(type) == PB_HTYPE_REPEATED || + PB_HTYPE(type) == PB_HTYPE_ONEOF) { + /* REPEATED: Set array count to 0, no need to initialize contents. + *ONEOF: Set which_field to 0. + */ + *(pb_size_t *)iter->pSize = 0; + init_data = false; + } + + if (init_data) { + if (PB_LTYPE(iter->pos->type) == PB_LTYPE_SUBMESSAGE) + /* Initialize submessage to defaults */ + pb_message_set_to_defaults((const pb_field_t *) iter->pos->ptr, + iter->pData); + else if (iter->pos->ptr != NULL) + /* Initialize to default value */ + memcpy(iter->pData, iter->pos->ptr, iter->pos->data_size); + else + /* Initialize to zeros */ + memset(iter->pData, 0, iter->pos->data_size); + } + } else if (PB_ATYPE(type) == PB_ATYPE_POINTER) { + /* Initialize the pointer to NULL. */ + *(void **)iter->pData = NULL; + + /* Initialize array count to 0. */ + if (PB_HTYPE(type) == PB_HTYPE_REPEATED || + PB_HTYPE(type) == PB_HTYPE_ONEOF) + *(pb_size_t *)iter->pSize = 0; + } +} + +static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_struct) +{ + pb_field_iter_t iter; + + if (!pb_field_iter_begin(&iter, fields, dest_struct)) + return; /* Empty message type */ + + do { + pb_field_set_to_default(&iter); + } while (pb_field_iter_next(&iter)); +} + +/********************* + * Decode all fields * + *********************/ + +bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[], + void *dest_struct) +{ + uint32_t fields_seen[(PB_MAX_REQUIRED_FIELDS + 31) / 32] = {0, 0}; + const uint32_t allbits = ~(uint32_t)0; + uint32_t extension_range_start = 0; + pb_field_iter_t iter; + + /* 'fixed_count_field' and 'fixed_count_size' track position of a repeated fixed + * count field. This can only handle _one_ repeated fixed count field that + * is unpacked and unordered among other (non repeated fixed count) fields. + */ + const pb_field_t *fixed_count_field = NULL; + pb_size_t fixed_count_size = 0; + + /* Return value ignored, as empty message types will be correctly handled by + * pb_field_iter_find() anyway. + */ + (void)pb_field_iter_begin(&iter, fields, dest_struct); + + while (stream->bytes_left) { + uint32_t tag; + pb_wire_type_t wire_type; + bool eof; + + if (!pb_decode_tag(stream, &wire_type, &tag, &eof)) { + if (eof) + break; + else + return false; + } + + if (!pb_field_iter_find(&iter, tag)) { + /* No match found, check if it matches an extension. */ + if (tag >= extension_range_start) { + if (!find_extension_field(&iter)) + extension_range_start = (uint32_t)-1; + else + extension_range_start = iter.pos->tag; + + if (tag >= extension_range_start) { + size_t pos = stream->bytes_left; + + if (!decode_extension(stream, tag, wire_type, &iter)) + return false; + + if (pos != stream->bytes_left) + /* The field was handled */ + continue; + } + } + + /* No match found, skip data */ + if (!pb_skip_field(stream, wire_type)) + return false; + continue; + } + + /* If a repeated fixed count field was found, get size from + * 'fixed_count_field' as there is no counter contained in the struct. + */ + if (PB_HTYPE(iter.pos->type) == PB_HTYPE_REPEATED + && iter.pSize == iter.pData) { + if (fixed_count_field != iter.pos) { + /* If the new fixed count field does not match the previous one, + * check that the previous one is NULL or that it finished + * receiving all the expected data. + */ + if (fixed_count_field != NULL && + fixed_count_size != fixed_count_field->array_size) + PB_RETURN_ERROR(stream, "wrong size for fixed count field"); + + fixed_count_field = iter.pos; + fixed_count_size = 0; + } + + iter.pSize = &fixed_count_size; + } + + if (PB_HTYPE(iter.pos->type) == PB_HTYPE_REQUIRED + && iter.required_field_index < PB_MAX_REQUIRED_FIELDS) { + uint32_t tmp = ((uint32_t)1 << (iter.required_field_index & 31)); + + fields_seen[iter.required_field_index >> 5] |= tmp; + } + + if (!decode_field(stream, wire_type, &iter)) + return false; + } + + /* Check that all elements of the last decoded fixed count field were present. */ + if (fixed_count_field != NULL && + fixed_count_size != fixed_count_field->array_size) + PB_RETURN_ERROR(stream, "wrong size for fixed count field"); + + /* Check that all required fields were present. */ + /* First figure out the number of required fields by + * seeking to the end of the field array. Usually we + * are already close to end after decoding. + */ + unsigned int req_field_count; + pb_type_t last_type; + unsigned int i; + + do { + req_field_count = iter.required_field_index; + last_type = iter.pos->type; + } while (pb_field_iter_next(&iter)); + + /* Fixup if last field was also required. */ + if (PB_HTYPE(last_type) == PB_HTYPE_REQUIRED && iter.pos->tag != 0) + req_field_count++; + + if (req_field_count > PB_MAX_REQUIRED_FIELDS) + req_field_count = PB_MAX_REQUIRED_FIELDS; + + if (req_field_count > 0) { + /* Check the whole words */ + for (i = 0; i < (req_field_count >> 5); i++) { + if (fields_seen[i] != allbits) + PB_RETURN_ERROR(stream, "missing required field"); + } + + /* Check the remaining bits (if any) */ + if ((req_field_count & 31) != 0) { + if (fields_seen[req_field_count >> 5] != + (allbits >> (32 - (req_field_count & 31)))) + PB_RETURN_ERROR(stream, "missing required field"); + } + } + + return true; +} + +bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) +{ + bool status; + + pb_message_set_to_defaults(fields, dest_struct); + status = pb_decode_noinit(stream, fields, dest_struct); + +#ifdef PB_ENABLE_MALLOC + if (!status) + pb_release(fields, dest_struct); +#endif + + return status; +} + +bool pb_decode_delimited_noinit(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) +{ + pb_istream_t substream; + bool status; + + if (!pb_make_string_substream(stream, &substream)) + return false; + + status = pb_decode_noinit(&substream, fields, dest_struct); + + if (!pb_close_string_substream(stream, &substream)) + return false; + return status; +} + +bool pb_decode_delimited(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) +{ + pb_istream_t substream; + bool status; + + if (!pb_make_string_substream(stream, &substream)) + return false; + + status = pb_decode(&substream, fields, dest_struct); + + if (!pb_close_string_substream(stream, &substream)) + return false; + return status; +} + +bool pb_decode_nullterminated(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) +{ + /* This behaviour will be separated in nanopb-0.4.0, see issue #278. */ + return pb_decode(stream, fields, dest_struct); +} + +#ifdef PB_ENABLE_MALLOC +/* Given an oneof field, if there has already been a field inside this oneof, + * release it before overwriting with a different one. + */ +static bool pb_release_union_field(pb_istream_t *stream, pb_field_iter_t *iter) +{ + pb_size_t old_tag = *(pb_size_t *)iter->pSize; /* Previous which_ value */ + pb_size_t new_tag = iter->pos->tag; /* New which_ value */ + + if (old_tag == 0) + return true; /* Ok, no old data in union */ + + if (old_tag == new_tag) + return true; /* Ok, old data is of same type => merge */ + + /* Release old data. The find can fail if the message struct contains + * invalid data. + */ + if (!pb_field_iter_find(iter, old_tag)) + PB_RETURN_ERROR(stream, "invalid union tag"); + + pb_release_single_field(iter); + + /* Restore iterator to where it should be. + * This shouldn't fail unless the pb_field_t structure is corrupted. + */ + if (!pb_field_iter_find(iter, new_tag)) + PB_RETURN_ERROR(stream, "iterator error"); + + return true; +} + +static void pb_release_single_field(const pb_field_iter_t *iter) +{ + pb_type_t type; + + type = iter->pos->type; + if (PB_HTYPE(type) == PB_HTYPE_ONEOF) { + if (*(pb_size_t *)iter->pSize != iter->pos->tag) + return; /* This is not the current field in the union */ + } + + /* Release anything contained inside an extension or submsg. + * This has to be done even if the submsg itself is statically + * allocated. + */ + if (PB_LTYPE(type) == PB_LTYPE_EXTENSION) { + /* Release fields from all extensions in the linked list */ + pb_extension_t *ext = *(pb_extension_t **)iter->pData; + + while (ext != NULL) { + pb_field_iter_t ext_iter; + + iter_from_extension(&ext_iter, ext); + pb_release_single_field(&ext_iter); + ext = ext->next; + } + } else if (PB_LTYPE(type) == PB_LTYPE_SUBMESSAGE) { + /* Release fields in submessage or submsg array */ + void *pItem = iter->pData; + pb_size_t count = 1; + + if (PB_ATYPE(type) == PB_ATYPE_POINTER) + pItem = *(void **)iter->pData; + + if (PB_HTYPE(type) == PB_HTYPE_REPEATED) { + if (PB_ATYPE(type) == PB_ATYPE_STATIC && iter->pSize == iter->pData) + /* No _count field so use size of the array */ + count = iter->pos->array_size; + else + count = *(pb_size_t *)iter->pSize; + + if (PB_ATYPE(type) == PB_ATYPE_STATIC && count > iter->pos->array_size) + /* Protect against corrupted _count fields */ + count = iter->pos->array_size; + } + + if (pItem) { + while (count--) { + pb_release((const pb_field_t *)iter->pos->ptr, pItem); + pItem = (char *)pItem + iter->pos->data_size; + } + } + } + + if (PB_ATYPE(type) == PB_ATYPE_POINTER) { + if (PB_HTYPE(type) == PB_HTYPE_REPEATED && + (PB_LTYPE(type) == PB_LTYPE_STRING || + PB_LTYPE(type) == PB_LTYPE_BYTES)) { + /* Release entries in repeated string or bytes array */ + void **pItem = *(void ***)iter->pData; + pb_size_t count = *(pb_size_t *)iter->pSize; + + while (count--) { + pb_free(*pItem); + *pItem++ = NULL; + } + } + + if (PB_HTYPE(type) == PB_HTYPE_REPEATED) + /* We are going to release the array, so set the size to 0 */ + *(pb_size_t *)iter->pSize = 0; + + /* Release main item */ + pb_free(*(void **)iter->pData); + *(void **)iter->pData = NULL; + } +} + +void pb_release(const pb_field_t fields[], void *dest_struct) +{ + pb_field_iter_t iter; + + if (!dest_struct) + return; /* Ignore NULL pointers, similar to free() */ + + if (!pb_field_iter_begin(&iter, fields, dest_struct)) + return; /* Empty message type */ + + do { + pb_release_single_field(&iter); + } while (pb_field_iter_next(&iter)); +} +#endif + +/* Field decoders */ + +bool pb_decode_svarint(pb_istream_t *stream, pb_int64_t *dest) +{ + pb_uint64_t value; + + if (!pb_decode_varint(stream, &value)) + return false; + + if (value & 1) + *dest = (pb_int64_t)(~(value >> 1)); + else + *dest = (pb_int64_t)(value >> 1); + + return true; +} + +bool pb_decode_fixed32(pb_istream_t *stream, void *dest) +{ + pb_byte_t bytes[4]; + + if (!pb_read(stream, bytes, 4)) + return false; + + *(uint32_t *)dest = ((uint32_t)bytes[0] << 0) | + ((uint32_t)bytes[1] << 8) | + ((uint32_t)bytes[2] << 16) | + ((uint32_t)bytes[3] << 24); + return true; +} + +#ifndef PB_WITHOUT_64BIT +bool pb_decode_fixed64(pb_istream_t *stream, void *dest) +{ + pb_byte_t bytes[8]; + + if (!pb_read(stream, bytes, 8)) + return false; + + *(uint64_t *)dest = ((uint64_t)bytes[0] << 0) | + ((uint64_t)bytes[1] << 8) | + ((uint64_t)bytes[2] << 16) | + ((uint64_t)bytes[3] << 24) | + ((uint64_t)bytes[4] << 32) | + ((uint64_t)bytes[5] << 40) | + ((uint64_t)bytes[6] << 48) | + ((uint64_t)bytes[7] << 56); + + return true; +} +#endif + +static bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + pb_uint64_t value; + pb_int64_t svalue; + pb_int64_t clamped; + + if (!pb_decode_varint(stream, &value)) + return false; + + /* See issue 97: Google's C++ protobuf allows negative varint values to + * be cast as int32_t, instead of the int64_t that should be used when + * encoding. Previous nanopb versions had a bug in encoding. In order to + * not break decoding of such messages, we cast <=32 bit fields to + * int32_t first to get the sign correct. + */ + if (field->data_size == sizeof(pb_int64_t)) + svalue = (pb_int64_t)value; + else + svalue = (int32_t)value; + + /* Cast to the proper field size, while checking for overflows */ + if (field->data_size == sizeof(pb_int64_t)) + clamped = *(pb_int64_t *)dest = svalue; + else if (field->data_size == sizeof(int32_t)) + clamped = *(int32_t *)dest = (int32_t)svalue; + else if (field->data_size == sizeof(int_least16_t)) + clamped = *(int_least16_t *)dest = (int_least16_t)svalue; + else if (field->data_size == sizeof(int_least8_t)) + clamped = *(int_least8_t *)dest = (int_least8_t)svalue; + else + PB_RETURN_ERROR(stream, "invalid data_size"); + + if (clamped != svalue) + PB_RETURN_ERROR(stream, "integer too large"); + + return true; +} + +static bool checkreturn pb_dec_uvarint(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + pb_uint64_t value, clamped; + + if (!pb_decode_varint(stream, &value)) + return false; + + /* Cast to the proper field size, while checking for overflows */ + if (field->data_size == sizeof(pb_uint64_t)) + clamped = *(pb_uint64_t *)dest = value; + else if (field->data_size == sizeof(uint32_t)) + clamped = *(uint32_t *)dest = (uint32_t)value; + else if (field->data_size == sizeof(uint_least16_t)) + clamped = *(uint_least16_t *)dest = (uint_least16_t)value; + else if (field->data_size == sizeof(uint_least8_t)) + clamped = *(uint_least8_t *)dest = (uint_least8_t)value; + else + PB_RETURN_ERROR(stream, "invalid data_size"); + + if (clamped != value) + PB_RETURN_ERROR(stream, "integer too large"); + + return true; +} + +static bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + pb_int64_t value, clamped; + + if (!pb_decode_svarint(stream, &value)) + return false; + + /* Cast to the proper field size, while checking for overflows */ + if (field->data_size == sizeof(pb_int64_t)) + clamped = *(pb_int64_t *)dest = value; + else if (field->data_size == sizeof(int32_t)) + clamped = *(int32_t *)dest = (int32_t)value; + else if (field->data_size == sizeof(int_least16_t)) + clamped = *(int_least16_t *)dest = (int_least16_t)value; + else if (field->data_size == sizeof(int_least8_t)) + clamped = *(int_least8_t *)dest = (int_least8_t)value; + else + PB_RETURN_ERROR(stream, "invalid data_size"); + + if (clamped != value) + PB_RETURN_ERROR(stream, "integer too large"); + + return true; +} + +static bool checkreturn pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + PB_UNUSED(field); + return pb_decode_fixed32(stream, dest); +} + +static bool checkreturn pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + PB_UNUSED(field); +#ifndef PB_WITHOUT_64BIT + return pb_decode_fixed64(stream, dest); +#else + PB_UNUSED(dest); + PB_RETURN_ERROR(stream, "no 64bit support"); +#endif +} + +static bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + uint32_t size; + size_t alloc_size; + pb_bytes_array_t *bdest; + + if (!pb_decode_varint32(stream, &size)) + return false; + + if (size > PB_SIZE_MAX) + PB_RETURN_ERROR(stream, "bytes overflow"); + + alloc_size = PB_BYTES_ARRAY_T_ALLOCSIZE(size); + if (size > alloc_size) + PB_RETURN_ERROR(stream, "size too large"); + + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) { +#ifndef PB_ENABLE_MALLOC + PB_RETURN_ERROR(stream, "no malloc support"); +#else + if (!allocate_field(stream, dest, alloc_size, 1)) + return false; + bdest = *(pb_bytes_array_t **)dest; +#endif + } else { + if (alloc_size > field->data_size) + PB_RETURN_ERROR(stream, "bytes overflow"); + bdest = (pb_bytes_array_t *)dest; + } + + bdest->size = (pb_size_t)size; + return pb_read(stream, bdest->bytes, size); +} + +static bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + uint32_t size; + size_t alloc_size; + bool status; + + if (!pb_decode_varint32(stream, &size)) + return false; + + /* Space for null terminator */ + alloc_size = size + 1; + + if (alloc_size < size) + PB_RETURN_ERROR(stream, "size too large"); + + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) { +#ifndef PB_ENABLE_MALLOC + PB_RETURN_ERROR(stream, "no malloc support"); +#else + if (!allocate_field(stream, dest, alloc_size, 1)) + return false; + dest = *(void **)dest; +#endif + } else { + if (alloc_size > field->data_size) + PB_RETURN_ERROR(stream, "string overflow"); + } + + status = pb_read(stream, (pb_byte_t *)dest, size); + *((pb_byte_t *)dest + size) = 0; + return status; +} + +static bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + bool status; + pb_istream_t substream; + const pb_field_t *submsg_fields = (const pb_field_t *)field->ptr; + + if (!pb_make_string_substream(stream, &substream)) + return false; + + if (field->ptr == NULL) + PB_RETURN_ERROR(stream, "invalid field descriptor"); + + /* New array entries need to be initialized, while required and optional + * submessages have already been initialized in the top-level pb_decode. + */ + if (PB_HTYPE(field->type) == PB_HTYPE_REPEATED) + status = pb_decode(&substream, submsg_fields, dest); + else + status = pb_decode_noinit(&substream, submsg_fields, dest); + + if (!pb_close_string_substream(stream, &substream)) + return false; + return status; +} + +static bool checkreturn pb_dec_fixed_length_bytes(pb_istream_t *stream, const pb_field_t *field, + void *dest) +{ + uint32_t size; + + if (!pb_decode_varint32(stream, &size)) + return false; + + if (size > PB_SIZE_MAX) + PB_RETURN_ERROR(stream, "bytes overflow"); + + if (size == 0) { + /* As a special case, treat empty bytes string + * as all zeros for fixed_length_bytes. + */ + memset(dest, 0, field->data_size); + return true; + } + + if (size != field->data_size) + PB_RETURN_ERROR(stream, "incorrect fixed length bytes size"); + + return pb_read(stream, (pb_byte_t *)dest, field->data_size); +} diff --git a/drivers/misc/vehicle/nanopb-c/pb_decode.h b/drivers/misc/vehicle/nanopb-c/pb_decode.h new file mode 100644 index 000000000000..55dee5750e98 --- /dev/null +++ b/drivers/misc/vehicle/nanopb-c/pb_decode.h @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2011 Petteri Aimonen + * + * This software is provided 'as-is', without any express or + * implied warranty. In no event will the authors be held liable + * for any damages arising from the use of this software. + * + * Permission is granted to anyone to use this software for any + * purpose, including commercial applications, and to alter it and + * redistribute it freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you + * must not claim that you wrote the original software. If you use + * this software in a product, an acknowledgment in the product + * documentation would be appreciated but is not required. + * + * 2. Altered source versions must be plainly marked as such, and + * must not be misrepresented as being the original software. + * + * 3. This notice may not be removed or altered from any source + * distribution. + */ +/* pb_decode.h: Functions to decode protocol buffers. Depends on pb_decode.c. + * The main function is pb_decode. You also need an input stream, and the + * field descriptions created by nanopb_generator.py. + */ + +#ifndef PB_DECODE_H_INCLUDED +#define PB_DECODE_H_INCLUDED + +#include "pb.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* Structure for defining custom input streams. You will need to provide + * a callback function to read the bytes from your storage, which can be + * for example a file or a network socket. + * + * The callback must conform to these rules: + * + * 1) Return false on IO errors. This will cause decoding to abort. + * 2) You can use state to store your own data (e.g. buffer pointer), + * and rely on pb_read to verify that no-body reads past bytes_left. + * 3) Your callback may be used with substreams, in which case bytes_left + * is different than from the main stream. Don't use bytes_left to compute + * any pointers. + */ +struct pb_istream_s { +#ifdef PB_BUFFER_ONLY + /* Callback pointer is not used in buffer-only configuration. + * Having an int pointer here allows binary compatibility but + * gives an error if someone tries to assign callback function. + */ + int *callback; +#else + bool (*callback)(pb_istream_t *stream, pb_byte_t *buf, size_t count); +#endif + + void *state; /* Free field for use by callback implementation */ + size_t bytes_left; + +#ifndef PB_NO_ERRMSG + const char *errmsg; +#endif +}; + +/*************************** + * Main decoding functions * + ***************************/ +/* Decode a single protocol buffers message from input stream into a C structure. + * Returns true on success, false on any failure. + * The actual struct pointed to by dest must match the description in fields. + * Callback fields of the destination structure must be initialized by caller. + * All other fields will be initialized by this function. + * + * Example usage: + * MyMessage msg = {}; + * uint8_t buffer[64]; + * pb_istream_t stream; + * + * // ... read some data into buffer ... + * + * stream = pb_istream_from_buffer(buffer, count); + * pb_decode(&stream, MyMessage_fields, &msg); + */ +bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); + +/* Same as pb_decode, except does not initialize the destination structure + * to default values. This is slightly faster if you need no default values + * and just do memset(struct, 0, sizeof(struct)) yourself. + * + * This can also be used for 'merging' two messages, i.e. update only the + * fields that exist in the new message. + * + * Note: If this function returns with an error, it will not release any + * dynamically allocated fields. You will need to call pb_release() yourself. + */ +bool pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); + +/* Same as pb_decode, except expects the stream to start with the message size + * encoded as varint. Corresponds to parseDelimitedFrom() in Google's + * protobuf API. + */ +bool pb_decode_delimited(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); + +/* Same as pb_decode_delimited, except that it does not initialize the destination structure. + * See pb_decode_noinit + */ +bool pb_decode_delimited_noinit(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); + +/* Same as pb_decode, except allows the message to be terminated with a null byte. + * NOTE: Until nanopb-0.4.0, pb_decode() also allows null-termination. This behaviour + * is not supported in most other protobuf implementations, so pb_decode_delimited() + * is a better option for compatibility. + */ +bool pb_decode_nullterminated(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); + +#ifdef PB_ENABLE_MALLOC +/* Release any allocated pointer fields. If you use dynamic allocation, you should + * call this for any successfully decoded message when you are done with it. If + * pb_decode() returns with an error, the message is already released. + */ +void pb_release(const pb_field_t fields[], void *dest_struct); +#endif + + +/************************************** + * Functions for manipulating streams * + **************************************/ + +/* Create an input stream for reading from a memory buffer. + * + * Alternatively, you can use a custom stream that reads directly from e.g. + * a file or a network socket. + */ +pb_istream_t pb_istream_from_buffer(const pb_byte_t *buf, size_t bufsize); + +/* Function to read from a pb_istream_t. You can use this if you need to + * read some custom header data, or to read data in field callbacks. + */ +bool pb_read(pb_istream_t *stream, pb_byte_t *buf, size_t count); + + +/************************************************ + * Helper functions for writing field callbacks * + ************************************************/ + +/* Decode the tag for the next field in the stream. Gives the wire type and + * field tag. At end of the message, returns false and sets eof to true. + */ +bool pb_decode_tag(pb_istream_t *stream, pb_wire_type_t *wire_type, uint32_t *tag, bool *eof); + +/* Skip the field payload data, given the wire type. */ +bool pb_skip_field(pb_istream_t *stream, pb_wire_type_t wire_type); + +/* Decode an integer in the varint format. This works for bool, enum, int32, + * int64, uint32 and uint64 field types. + */ +#ifndef PB_WITHOUT_64BIT +bool pb_decode_varint(pb_istream_t *stream, uint64_t *dest); +#else +#define pb_decode_varint pb_decode_varint32 +#endif + +/* Decode an integer in the varint format. This works for bool, enum, int32, + * and uint32 field types. + */ +bool pb_decode_varint32(pb_istream_t *stream, uint32_t *dest); + +/* Decode an integer in the zig-zagged svarint format. This works for sint32 + * and sint64. + */ +#ifndef PB_WITHOUT_64BIT +bool pb_decode_svarint(pb_istream_t *stream, int64_t *dest); +#else +bool pb_decode_svarint(pb_istream_t *stream, int32_t *dest); +#endif + +/* Decode a fixed32, sfixed32 or float value. You need to pass a pointer to + * a 4-byte wide C variable. + */ +bool pb_decode_fixed32(pb_istream_t *stream, void *dest); + +#ifndef PB_WITHOUT_64BIT +/* Decode a fixed64, sfixed64 or double value. You need to pass a pointer to + * a 8-byte wide C variable. + */ +bool pb_decode_fixed64(pb_istream_t *stream, void *dest); +#endif + +/* Make a limited-length substream for reading a PB_WT_STRING field. */ +bool pb_make_string_substream(pb_istream_t *stream, pb_istream_t *substream); +bool pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif diff --git a/drivers/misc/vehicle/nanopb-c/pb_encode.c b/drivers/misc/vehicle/nanopb-c/pb_encode.c new file mode 100644 index 000000000000..35bae5f2e08d --- /dev/null +++ b/drivers/misc/vehicle/nanopb-c/pb_encode.c @@ -0,0 +1,889 @@ +/* + * Copyright (c) 2011 Petteri Aimonen + * + * This software is provided 'as-is', without any express or + * implied warranty. In no event will the authors be held liable + * for any damages arising from the use of this software. + * + * Permission is granted to anyone to use this software for any + * purpose, including commercial applications, and to alter it and + * redistribute it freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you + * must not claim that you wrote the original software. If you use + * this software in a product, an acknowledgment in the product + * documentation would be appreciated but is not required. + * + * 2. Altered source versions must be plainly marked as such, and + * must not be misrepresented as being the original software. + * + * 3. This notice may not be removed or altered from any source + * distribution. + */ +/* pb_encode.c -- encode a protobuf using minimal resources + * + * 2011 Petteri Aimonen + */ + +#include "pb.h" +#include "pb_encode.h" +#include "pb_common.h" + +/* Use the GCC warn_unused_result attribute to check that all return values + * are propagated correctly. On other compilers and gcc before 3.4.0 just + * ignore the annotation. + */ +#if !defined(__GNUC__) || (__GNUC__ < 3) || (__GNUC__ == 3 && __GNUC_MINOR__ < 4) + #define checkreturn +#else + #define checkreturn __attribute__((warn_unused_result)) +#endif + +/************************************** + * Declarations internal to this file * + **************************************/ +typedef bool (*pb_encoder_t)(pb_ostream_t *stream, const pb_field_t *field, + const void *src) checkreturn; + +static bool checkreturn buf_write(pb_ostream_t *stream, const pb_byte_t *buf, size_t count); +static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *field, + const void *pData, size_t count, pb_encoder_t func); +static bool checkreturn encode_field(pb_ostream_t *stream, + const pb_field_t *field, const void *pData); +static bool checkreturn default_extension_encoder(pb_ostream_t *stream, + const pb_extension_t *extension); +static bool checkreturn encode_extension_field(pb_ostream_t *stream, const pb_field_t *field, + const void *pData); +static void *pb_const_cast(const void *p); +static bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, + const void *src); +static bool checkreturn pb_enc_uvarint(pb_ostream_t *stream, const pb_field_t *field, + const void *src); +static bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, + const void *src); +static bool checkreturn pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, + const void *src); +static bool checkreturn pb_enc_fixed64(pb_ostream_t *stream, const pb_field_t *field, + const void *src); +static bool checkreturn pb_enc_bytes(pb_ostream_t *stream, + const pb_field_t *field, const void *src); +static bool checkreturn pb_enc_string(pb_ostream_t *stream, + const pb_field_t *field, const void *src); +static bool checkreturn pb_enc_submessage(pb_ostream_t *stream, + const pb_field_t *field, const void *src); +static bool checkreturn pb_enc_fixed_length_bytes(pb_ostream_t *stream, + const pb_field_t *field, const void *src); + +#ifdef PB_WITHOUT_64BIT +#define pb_int64_t int32_t +#define pb_uint64_t uint32_t + +static bool checkreturn pb_encode_negative_varint(pb_ostream_t *stream, pb_uint64_t value); +#else +#define pb_int64_t int64_t +#define pb_uint64_t uint64_t +#endif + +/* --- Function pointers to field encoders --- + * Order in the array must match pb_action_t LTYPE numbering. + */ +static const pb_encoder_t PB_ENCODERS[PB_LTYPES_COUNT] = { + &pb_enc_varint, + &pb_enc_uvarint, + &pb_enc_svarint, + &pb_enc_fixed32, + &pb_enc_fixed64, + + &pb_enc_bytes, + &pb_enc_string, + &pb_enc_submessage, + NULL, /* extensions */ + &pb_enc_fixed_length_bytes +}; + +/******************************* + * pb_ostream_t implementation * + *******************************/ + +static bool checkreturn buf_write(pb_ostream_t *stream, const pb_byte_t *buf, + size_t count) +{ + size_t i; + pb_byte_t *dest = (pb_byte_t *)stream->state; + + stream->state = dest + count; + + for (i = 0; i < count; i++) + dest[i] = buf[i]; + + return true; +} + +pb_ostream_t pb_ostream_from_buffer(pb_byte_t *buf, size_t bufsize) +{ + pb_ostream_t stream; + +#ifdef PB_BUFFER_ONLY + stream.callback = (void *)1; /* Just a marker value */ +#else + stream.callback = &buf_write; +#endif + stream.state = buf; + stream.max_size = bufsize; + stream.bytes_written = 0; +#ifndef PB_NO_ERRMSG + stream.errmsg = NULL; +#endif + return stream; +} + +bool checkreturn pb_write(pb_ostream_t *stream, const pb_byte_t *buf, size_t count) +{ + if (stream->callback != NULL) { + if (stream->bytes_written + count > stream->max_size) + PB_RETURN_ERROR(stream, "stream full"); + +#ifdef PB_BUFFER_ONLY + if (!buf_write(stream, buf, count)) + PB_RETURN_ERROR(stream, "io error"); +#else + if (!stream->callback(stream, buf, count)) + PB_RETURN_ERROR(stream, "io error"); +#endif + } + + stream->bytes_written += count; + return true; +} + +/************************* + * Encode a single field * + *************************/ + +/* Encode a static array. Handles the size calculations and possible packing. */ +static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *field, + const void *pData, size_t count, pb_encoder_t func) +{ + size_t i; + const void *p; + size_t size; + + if (count == 0) + return true; + + if (PB_ATYPE(field->type) != PB_ATYPE_POINTER && count > field->array_size) + PB_RETURN_ERROR(stream, "array max size exceeded"); + + /* We always pack arrays if the datatype allows it. */ + if (PB_LTYPE(field->type) <= PB_LTYPE_LAST_PACKABLE) { + if (!pb_encode_tag(stream, PB_WT_STRING, field->tag)) + return false; + + /* Determine the total size of packed array. */ + if (PB_LTYPE(field->type) == PB_LTYPE_FIXED32) + size = 4 * count; + else if (PB_LTYPE(field->type) == PB_LTYPE_FIXED64) + size = 8 * count; + else { + pb_ostream_t sizestream = PB_OSTREAM_SIZING; + + p = pData; + for (i = 0; i < count; i++) { + if (!func(&sizestream, field, p)) + return false; + p = (const char *)p + field->data_size; + } + size = sizestream.bytes_written; + } + + if (!pb_encode_varint(stream, (pb_uint64_t)size)) + return false; + + if (stream->callback == NULL) + return pb_write(stream, NULL, size); /* Just sizing.. */ + + /* Write the data */ + p = pData; + for (i = 0; i < count; i++) { + if (!func(stream, field, p)) + return false; + p = (const char *)p + field->data_size; + } + } else { + p = pData; + for (i = 0; i < count; i++) { + if (!pb_encode_tag_for_field(stream, field)) + return false; + + /* Normally the data is stored directly in the array entries, but + * for pointer-type string and bytes fields, the array entries are + * actually pointers themselves also. So we have to dereference once + * more to get to the actual data. + */ + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER && + (PB_LTYPE(field->type) == PB_LTYPE_STRING || + PB_LTYPE(field->type) == PB_LTYPE_BYTES)) { + if (!func(stream, field, *(const void * const *)p)) + return false; + } else { + if (!func(stream, field, p)) + return false; + } + p = (const char *)p + field->data_size; + } + } + + return true; +} + +/* In proto3, all fields are optional and are only encoded + * if their value is "non-zero". + * This function implements the check for the zero value. + */ +static bool pb_check_proto3_default_value(const pb_field_t *field, const void *pData) +{ + pb_type_t type = field->type; + const void *pSize = (const char *)pData + field->size_offset; + + if (PB_HTYPE(type) == PB_HTYPE_REQUIRED) + /* Required proto2 fields inside proto3 submessage, pretty rare case */ + return false; + else if (PB_HTYPE(type) == PB_HTYPE_REPEATED) + /* Repeated fields inside proto3 submessage: present if count != 0 */ + return *(const pb_size_t *)pSize == 0; + else if (PB_HTYPE(type) == PB_HTYPE_ONEOF) + /* Oneof fields */ + return *(const pb_size_t *)pSize == 0; + else if (PB_HTYPE(type) == PB_HTYPE_OPTIONAL && field->size_offset) + /* Proto2 optional fields inside proto3 submessage */ + return *(const bool *)pSize == false; + + /* Rest is proto3 singular fields */ + + if (PB_ATYPE(type) == PB_ATYPE_STATIC) { + if (PB_LTYPE(type) == PB_LTYPE_BYTES) { + const pb_bytes_array_t *bytes = (const pb_bytes_array_t *)pData; + + return bytes->size == 0; + } else if (PB_LTYPE(type) == PB_LTYPE_STRING) + return *(const char *)pData == '\0'; + else if (PB_LTYPE(type) == PB_LTYPE_FIXED_LENGTH_BYTES) + /* Fixed length bytes is only empty if its length is fixed + * as 0. Which would be pretty strange, but we can check + * it anyway. + */ + return field->data_size == 0; + else if (PB_LTYPE(type) == PB_LTYPE_SUBMESSAGE) { + /* Check all fields in the submessage to find if any of them + * are non-zero. The comparison cannot be done byte-per-byte + * because the C struct may contain padding bytes that must + * be skipped. + */ + pb_field_iter_t iter; + + if (pb_field_iter_begin(&iter, (const pb_field_t *)field->ptr, + pb_const_cast(pData))) { + do { + if (!pb_check_proto3_default_value(iter.pos, iter.pData)) + return false; + } while (pb_field_iter_next(&iter)); + } + return true; + } + } + + /* Catch-all branch that does byte-per-byte comparison for zero value. + * + * This is for all pointer fields, and for static PB_LTYPE_VARINT, + * UVARINT, SVARINT, FIXED32, FIXED64, EXTENSION fields, and also + * callback fields. These all have integer or pointer value which + * can be compared with 0. + */ + pb_size_t i; + const char *p = (const char *)pData; + + for (i = 0; i < field->data_size; i++) { + if (p[i] != 0) + return false; + } + + return true; + +} + +/* Encode a field with static or pointer allocation, i.e. one whose data + * is available to the encoder directly. + */ +static bool checkreturn encode_basic_field(pb_ostream_t *stream, + const pb_field_t *field, const void *pData) +{ + pb_encoder_t func; + bool implicit_has; + const void *pSize = &implicit_has; + + func = PB_ENCODERS[PB_LTYPE(field->type)]; + + if (field->size_offset) + /* Static optional, repeated or oneof field */ + pSize = (const char *)pData + field->size_offset; + else if (PB_HTYPE(field->type) == PB_HTYPE_OPTIONAL) + /* Proto3 style field, optional but without explicit has_ field. */ + implicit_has = !pb_check_proto3_default_value(field, pData); + else + /* Required field, always present */ + implicit_has = true; + + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) { + /* pData is a pointer to the field, which contains pointer to + * the data. If the 2nd pointer is NULL, it is interpreted as if + * the has_field was false. + */ + pData = *(const void * const *)pData; + implicit_has = (pData != NULL); + } + + switch (PB_HTYPE(field->type)) { + case PB_HTYPE_REQUIRED: + if (!pData) + PB_RETURN_ERROR(stream, "missing required field"); + if (!pb_encode_tag_for_field(stream, field)) + return false; + if (!func(stream, field, pData)) + return false; + break; + + case PB_HTYPE_OPTIONAL: + if (*(const bool *)pSize) { + if (!pb_encode_tag_for_field(stream, field)) + return false; + + if (!func(stream, field, pData)) + return false; + } + break; + + case PB_HTYPE_REPEATED: { + pb_size_t count; + + if (field->size_offset != 0) + count = *(const pb_size_t *)pSize; + else + count = field->array_size; + + if (!encode_array(stream, field, pData, count, func)) + return false; + break; + } + + case PB_HTYPE_ONEOF: + if (*(const pb_size_t *)pSize == field->tag) { + if (!pb_encode_tag_for_field(stream, field)) + return false; + + if (!func(stream, field, pData)) + return false; + } + break; + + default: + PB_RETURN_ERROR(stream, "invalid field type"); + } + + return true; +} + +/* Encode a field with callback semantics. + * This means that a user function is + * called to provide and encode the actual data. + */ +static bool checkreturn encode_callback_field(pb_ostream_t *stream, + const pb_field_t *field, + const void *pData) +{ + const pb_callback_t *callback = (const pb_callback_t *)pData; + +#ifdef PB_OLD_CALLBACK_STYLE + const void *arg = callback->arg; +#else + void * const *arg = &(callback->arg); +#endif + + if (callback->funcs.encode != NULL) { + if (!callback->funcs.encode(stream, field, arg)) + PB_RETURN_ERROR(stream, "callback error"); + } + + return true; +} + +/* Encode a single field of any callback or static type. */ +static bool checkreturn encode_field(pb_ostream_t *stream, + const pb_field_t *field, const void *pData) +{ + switch (PB_ATYPE(field->type)) { + case PB_ATYPE_STATIC: + case PB_ATYPE_POINTER: + return encode_basic_field(stream, field, pData); + + case PB_ATYPE_CALLBACK: + return encode_callback_field(stream, field, pData); + + default: + PB_RETURN_ERROR(stream, "invalid field type"); + } +} + +/* Default handler for extension fields. Expects to have a pb_field_t + * pointer in the extension->type->arg field. + */ +static bool checkreturn default_extension_encoder(pb_ostream_t *stream, + const pb_extension_t *extension) +{ + const pb_field_t *field = (const pb_field_t *)extension->type->arg; + + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) { + /* For pointer extensions, the pointer is stored directly + * in the extension structure. This avoids having an extra + * indirection. + */ + return encode_field(stream, field, &extension->dest); + } else + return encode_field(stream, field, extension->dest); +} + +/* Walk through all the registered extensions and give them a chance + * to encode themselves. + */ +static bool checkreturn encode_extension_field(pb_ostream_t *stream, const pb_field_t *field, + const void *pData) +{ + const pb_extension_t *extension = *(const pb_extension_t * const *)pData; + + PB_UNUSED(field); + while (extension) { + bool status; + + if (extension->type->encode) + status = extension->type->encode(stream, extension); + else + status = default_extension_encoder(stream, extension); + + if (!status) + return false; + + extension = extension->next; + } + + return true; +} + +/********************* + * Encode all fields * + *********************/ + +static void *pb_const_cast(const void *p) +{ + /* Note: this casts away const, + * in order to use the common field iterator + * logic for both encoding and decoding. + */ + union { + void *p1; + const void *p2; + } t; + + t.p2 = p; + return t.p1; +} + +bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], + const void *src_struct) +{ + pb_field_iter_t iter; + + if (!pb_field_iter_begin(&iter, fields, pb_const_cast(src_struct))) + return true; /* Empty message type */ + + do { + if (PB_LTYPE(iter.pos->type) == PB_LTYPE_EXTENSION) { + /* Special case for the extension field placeholder */ + if (!encode_extension_field(stream, iter.pos, iter.pData)) + return false; + } else { + /* Regular field */ + if (!encode_field(stream, iter.pos, iter.pData)) + return false; + } + } while (pb_field_iter_next(&iter)); + + return true; +} + +bool pb_encode_delimited(pb_ostream_t *stream, const pb_field_t fields[], + const void *src_struct) +{ + return pb_encode_submessage(stream, fields, src_struct); +} + +bool pb_encode_nullterminated(pb_ostream_t *stream, const pb_field_t fields[], + const void *src_struct) +{ + const pb_byte_t zero = 0; + + if (!pb_encode(stream, fields, src_struct)) + return false; + + return pb_write(stream, &zero, 1); +} + +bool pb_get_encoded_size(size_t *size, const pb_field_t fields[], + const void *src_struct) +{ + pb_ostream_t stream = PB_OSTREAM_SIZING; + + if (!pb_encode(&stream, fields, src_struct)) + return false; + + *size = stream.bytes_written; + return true; +} + +/******************** + * Helper functions * + ********************/ + +#ifdef PB_WITHOUT_64BIT +bool checkreturn pb_encode_negative_varint(pb_ostream_t *stream, pb_uint64_t value) +{ + pb_byte_t buffer[10]; + size_t i = 0; + size_t compensation = 32;/* we need to compensate 32 bits all set to 1 */ + + while (value) { + buffer[i] = (pb_byte_t)((value & 0x7F) | 0x80); + value >>= 7; + if (compensation) { + /* re-set all the compensation bits we can or need */ + size_t bits = compensation > 7 ? 7 : compensation; + + value ^= (pb_uint64_t)((0xFFu >> (8 - bits)) << 25); + /* set the number of bits needed on the lowest + * of the most significant 7 bits + */ + compensation -= bits; + } + i++; + } + buffer[i - 1] &= 0x7F; /* Unset top bit on last byte */ + + return pb_write(stream, buffer, i); +} +#endif + +bool checkreturn pb_encode_varint(pb_ostream_t *stream, pb_uint64_t value) +{ + pb_byte_t buffer[10]; + size_t i = 0; + + if (value <= 0x7F) { + pb_byte_t v = (pb_byte_t)value; + + return pb_write(stream, &v, 1); + } + + while (value) { + buffer[i] = (pb_byte_t)((value & 0x7F) | 0x80); + value >>= 7; + i++; + } + buffer[i-1] &= 0x7F; /* Unset top bit on last byte */ + + return pb_write(stream, buffer, i); +} + +bool checkreturn pb_encode_svarint(pb_ostream_t *stream, pb_int64_t value) +{ + pb_uint64_t zigzagged; + + if (value < 0) + zigzagged = ~((pb_uint64_t)value << 1); + else + zigzagged = (pb_uint64_t)value << 1; + + return pb_encode_varint(stream, zigzagged); +} + +bool checkreturn pb_encode_fixed32(pb_ostream_t *stream, const void *value) +{ + uint32_t val = *(const uint32_t *)value; + pb_byte_t bytes[4]; + + bytes[0] = (pb_byte_t)(val & 0xFF); + bytes[1] = (pb_byte_t)((val >> 8) & 0xFF); + bytes[2] = (pb_byte_t)((val >> 16) & 0xFF); + bytes[3] = (pb_byte_t)((val >> 24) & 0xFF); + + return pb_write(stream, bytes, 4); +} + +#ifndef PB_WITHOUT_64BIT +bool checkreturn pb_encode_fixed64(pb_ostream_t *stream, const void *value) +{ + uint64_t val = *(const uint64_t *)value; + pb_byte_t bytes[8]; + + bytes[0] = (pb_byte_t)(val & 0xFF); + bytes[1] = (pb_byte_t)((val >> 8) & 0xFF); + bytes[2] = (pb_byte_t)((val >> 16) & 0xFF); + bytes[3] = (pb_byte_t)((val >> 24) & 0xFF); + bytes[4] = (pb_byte_t)((val >> 32) & 0xFF); + bytes[5] = (pb_byte_t)((val >> 40) & 0xFF); + bytes[6] = (pb_byte_t)((val >> 48) & 0xFF); + bytes[7] = (pb_byte_t)((val >> 56) & 0xFF); + + return pb_write(stream, bytes, 8); +} +#endif + +bool checkreturn pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, + uint32_t field_number) +{ + pb_uint64_t tag = ((pb_uint64_t)field_number << 3) | wiretype; + + return pb_encode_varint(stream, tag); +} + +bool checkreturn pb_encode_tag_for_field(pb_ostream_t *stream, const pb_field_t *field) +{ + pb_wire_type_t wiretype; + + switch (PB_LTYPE(field->type)) { + case PB_LTYPE_VARINT: + case PB_LTYPE_UVARINT: + case PB_LTYPE_SVARINT: + wiretype = PB_WT_VARINT; + break; + + case PB_LTYPE_FIXED32: + wiretype = PB_WT_32BIT; + break; + + case PB_LTYPE_FIXED64: + wiretype = PB_WT_64BIT; + break; + + case PB_LTYPE_BYTES: + case PB_LTYPE_STRING: + case PB_LTYPE_SUBMESSAGE: + case PB_LTYPE_FIXED_LENGTH_BYTES: + wiretype = PB_WT_STRING; + break; + + default: + PB_RETURN_ERROR(stream, "invalid field type"); + } + + return pb_encode_tag(stream, wiretype, field->tag); +} + +bool checkreturn pb_encode_string(pb_ostream_t *stream, const pb_byte_t *buffer, size_t size) +{ + if (!pb_encode_varint(stream, (pb_uint64_t)size)) + return false; + + return pb_write(stream, buffer, size); +} + +bool checkreturn pb_encode_submessage(pb_ostream_t *stream, const pb_field_t fields[], + const void *src_struct) +{ + /* First calculate the message size using a non-writing substream. */ + pb_ostream_t substream = PB_OSTREAM_SIZING; + size_t size; + bool status; + + if (!pb_encode(&substream, fields, src_struct)) { +#ifndef PB_NO_ERRMSG + stream->errmsg = substream.errmsg; +#endif + return false; + } + + size = substream.bytes_written; + + if (!pb_encode_varint(stream, (pb_uint64_t)size)) + return false; + + if (stream->callback == NULL) + return pb_write(stream, NULL, size); /* Just sizing */ + + if (stream->bytes_written + size > stream->max_size) + PB_RETURN_ERROR(stream, "stream full"); + + /* Use a substream to verify that a callback doesn't write more than + * what it did the first time. + */ + substream.callback = stream->callback; + substream.state = stream->state; + substream.max_size = size; + substream.bytes_written = 0; +#ifndef PB_NO_ERRMSG + substream.errmsg = NULL; +#endif + + status = pb_encode(&substream, fields, src_struct); + + stream->bytes_written += substream.bytes_written; + stream->state = substream.state; +#ifndef PB_NO_ERRMSG + stream->errmsg = substream.errmsg; +#endif + + if (substream.bytes_written != size) + PB_RETURN_ERROR(stream, "submsg size changed"); + + return status; +} + +/* Field encoders */ + +static bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, + const void *src) +{ + pb_int64_t value = 0; + + if (field->data_size == sizeof(int_least8_t)) + value = *(const int_least8_t *)src; + else if (field->data_size == sizeof(int_least16_t)) + value = *(const int_least16_t *)src; + else if (field->data_size == sizeof(int32_t)) + value = *(const int32_t *)src; + else if (field->data_size == sizeof(pb_int64_t)) + value = *(const pb_int64_t *)src; + else + PB_RETURN_ERROR(stream, "invalid data_size"); + +#ifdef PB_WITHOUT_64BIT + if (value < 0) + return pb_encode_negative_varint(stream, (pb_uint64_t)value); +#endif + return pb_encode_varint(stream, (pb_uint64_t)value); +} + +static bool checkreturn pb_enc_uvarint(pb_ostream_t *stream, const pb_field_t *field, + const void *src) +{ + pb_uint64_t value = 0; + + if (field->data_size == sizeof(uint_least8_t)) + value = *(const uint_least8_t *)src; + else if (field->data_size == sizeof(uint_least16_t)) + value = *(const uint_least16_t *)src; + else if (field->data_size == sizeof(uint32_t)) + value = *(const uint32_t *)src; + else if (field->data_size == sizeof(pb_uint64_t)) + value = *(const pb_uint64_t *)src; + else + PB_RETURN_ERROR(stream, "invalid data_size"); + + return pb_encode_varint(stream, value); +} + +static bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, + const void *src) +{ + pb_int64_t value = 0; + + if (field->data_size == sizeof(int_least8_t)) + value = *(const int_least8_t *)src; + else if (field->data_size == sizeof(int_least16_t)) + value = *(const int_least16_t *)src; + else if (field->data_size == sizeof(int32_t)) + value = *(const int32_t *)src; + else if (field->data_size == sizeof(pb_int64_t)) + value = *(const pb_int64_t *)src; + else + PB_RETURN_ERROR(stream, "invalid data_size"); + + return pb_encode_svarint(stream, value); +} + +static bool checkreturn pb_enc_fixed64(pb_ostream_t *stream, const pb_field_t *field, + const void *src) +{ + PB_UNUSED(field); +#ifndef PB_WITHOUT_64BIT + return pb_encode_fixed64(stream, src); +#else + PB_UNUSED(src); + PB_RETURN_ERROR(stream, "no 64bit support"); +#endif +} + +static bool checkreturn pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, + const void *src) +{ + PB_UNUSED(field); + return pb_encode_fixed32(stream, src); +} + +static bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, + const void *src) +{ + const pb_bytes_array_t *bytes = NULL; + + bytes = (const pb_bytes_array_t *)src; + + if (src == NULL) { + /* Treat null pointer as an empty bytes field */ + return pb_encode_string(stream, NULL, 0); + } + + if (PB_ATYPE(field->type) == PB_ATYPE_STATIC && + PB_BYTES_ARRAY_T_ALLOCSIZE(bytes->size) > field->data_size) + PB_RETURN_ERROR(stream, "bytes size exceeded"); + + return pb_encode_string(stream, bytes->bytes, bytes->size); +} + +static bool checkreturn pb_enc_string(pb_ostream_t *stream, + const pb_field_t *field, + const void *src) +{ + size_t size = 0; + size_t max_size = field->data_size; + const char *p = (const char *)src; + + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) + max_size = (size_t)-1; + + if (src == NULL) + size = 0; /* Treat null pointer as an empty string */ + else { + /* strnlen() is not always available, so just use a loop */ + while (size < max_size && *p != '\0') { + size++; + p++; + } + } + + return pb_encode_string(stream, (const pb_byte_t *)src, size); +} + +static bool checkreturn pb_enc_submessage(pb_ostream_t *stream, + const pb_field_t *field, + const void *src) +{ + if (field->ptr == NULL) + PB_RETURN_ERROR(stream, "invalid field descriptor"); + + return pb_encode_submessage(stream, (const pb_field_t *)field->ptr, src); +} + +static bool checkreturn pb_enc_fixed_length_bytes(pb_ostream_t *stream, + const pb_field_t *field, + const void *src) +{ + return pb_encode_string(stream, (const pb_byte_t *)src, field->data_size); +} + diff --git a/drivers/misc/vehicle/nanopb-c/pb_encode.h b/drivers/misc/vehicle/nanopb-c/pb_encode.h new file mode 100644 index 000000000000..0ad04e494304 --- /dev/null +++ b/drivers/misc/vehicle/nanopb-c/pb_encode.h @@ -0,0 +1,204 @@ +/* + * Copyright (c) 2011 Petteri Aimonen + * + * This software is provided 'as-is', without any express or + * implied warranty. In no event will the authors be held liable + * for any damages arising from the use of this software. + * + * Permission is granted to anyone to use this software for any + * purpose, including commercial applications, and to alter it and + * redistribute it freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you + * must not claim that you wrote the original software. If you use + * this software in a product, an acknowledgment in the product + * documentation would be appreciated but is not required. + * + * 2. Altered source versions must be plainly marked as such, and + * must not be misrepresented as being the original software. + * + * 3. This notice may not be removed or altered from any source + * distribution. + */ +/* pb_encode.h: Functions to encode protocol buffers. Depends on pb_encode.c. + * The main function is pb_encode. You also need an output stream, and the + * field descriptions created by nanopb_generator.py. + */ + +#ifndef PB_ENCODE_H_INCLUDED +#define PB_ENCODE_H_INCLUDED + +#include "pb.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* Structure for defining custom output streams. You will need to provide + * a callback function to write the bytes to your storage, which can be + * for example a file or a network socket. + * + * The callback must conform to these rules: + * + * 1) Return false on IO errors. This will cause encoding to abort. + * 2) You can use state to store your own data (e.g. buffer pointer). + * 3) pb_write will update bytes_written after your callback runs. + * 4) Substreams will modify max_size and bytes_written. Don't use them + * to calculate any pointers. + */ +struct pb_ostream_s { +#ifdef PB_BUFFER_ONLY + /* Callback pointer is not used in buffer-only configuration. + * Having an int pointer here allows binary compatibility but + * gives an error if someone tries to assign callback function. + * Also, NULL pointer marks a 'sizing stream' that does not + * write anything. + */ + int *callback; +#else + bool (*callback)(pb_ostream_t *stream, const pb_byte_t *buf, size_t count); +#endif + void *state; /* Free field for use by callback implementation. */ + size_t max_size; /* Limit number of output bytes written (or use SIZE_MAX). */ + size_t bytes_written; /* Number of bytes written so far. */ + +#ifndef PB_NO_ERRMSG + const char *errmsg; +#endif +}; + +/*************************** + * Main encoding functions * + ***************************/ + +/* Encode a single protocol buffers message from C structure into a stream. + * Returns true on success, false on any failure. + * The actual struct pointed to by src_struct must match the description in fields. + * All required fields in the struct are assumed to have been filled in. + * + * Example usage: + * MyMessage msg = {}; + * uint8_t buffer[64]; + * pb_ostream_t stream; + * + * msg.field1 = 42; + * stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + * pb_encode(&stream, MyMessage_fields, &msg); + */ +bool pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct); + +/* Same as pb_encode, but prepends the length of the message as a varint. + * Corresponds to writeDelimitedTo() in Google's protobuf API. + */ +bool pb_encode_delimited(pb_ostream_t *stream, const pb_field_t fields[], + const void *src_struct); + +/* Same as pb_encode, but appends a null byte to the message for termination. + * NOTE: This behaviour is not supported in most other protobuf implementations, + * so pb_encode_delimited() + * is a better option for compatibility. + */ +bool pb_encode_nullterminated(pb_ostream_t *stream, const pb_field_t fields[], + const void *src_struct); + +/* Encode the message to get the size of the encoded data, but do not store + * the data. + */ +bool pb_get_encoded_size(size_t *size, const pb_field_t fields[], + const void *src_struct); + +/************************************** + * Functions for manipulating streams * + **************************************/ + +/* Create an output stream for writing into a memory buffer. + * The number of bytes written can be found in stream.bytes_written after + * encoding the message. + * + * Alternatively, you can use a custom stream that writes directly to e.g. + * a file or a network socket. + */ +pb_ostream_t pb_ostream_from_buffer(pb_byte_t *buf, size_t bufsize); + +/* Pseudo-stream for measuring the size of a message without actually storing + * the encoded data. + * + * Example usage: + * MyMessage msg = {}; + * pb_ostream_t stream = PB_OSTREAM_SIZING; + * pb_encode(&stream, MyMessage_fields, &msg); + * printf("Message size is %d\n", stream.bytes_written); + */ +#ifndef PB_NO_ERRMSG +#define PB_OSTREAM_SIZING {0, 0, 0, 0, 0} +#else +#define PB_OSTREAM_SIZING {0, 0, 0, 0} +#endif + +/* Function to write into a pb_ostream_t stream. You can use this if you need + * to append or prepend some custom headers to the message. + */ +bool pb_write(pb_ostream_t *stream, const pb_byte_t *buf, size_t count); + + +/************************************************ + * Helper functions for writing field callbacks * + ************************************************/ + +/* Encode field header based on type and field number defined in the field + * structure. Call this from the callback before writing out field contents. + */ +bool pb_encode_tag_for_field(pb_ostream_t *stream, const pb_field_t *field); + +/* Encode field header by manually specifying wire type. You need to use this + * if you want to write out packed arrays from a callback field. + */ +bool pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, + uint32_t field_number); + +/* Encode an integer in the varint format. + * This works for bool, enum, int32, int64, uint32 and uint64 field types. + */ +#ifndef PB_WITHOUT_64BIT +bool pb_encode_varint(pb_ostream_t *stream, uint64_t value); +#else +bool pb_encode_varint(pb_ostream_t *stream, uint32_t value); +#endif + +/* Encode an integer in the zig-zagged svarint format. + * This works for sint32 and sint64. + */ +#ifndef PB_WITHOUT_64BIT +bool pb_encode_svarint(pb_ostream_t *stream, int64_t value); +#else +bool pb_encode_svarint(pb_ostream_t *stream, int32_t value); +#endif + +/* Encode a string or bytes type field. For strings, pass strlen(s) as size. */ +bool pb_encode_string(pb_ostream_t *stream, const pb_byte_t *buffer, size_t size); + +/* Encode a fixed32, sfixed32 or float value. + * You need to pass a pointer to a 4-byte wide C variable. + */ +bool pb_encode_fixed32(pb_ostream_t *stream, const void *value); + +#ifndef PB_WITHOUT_64BIT +/* Encode a fixed64, sfixed64 or double value. + * You need to pass a pointer to a 8-byte wide C variable. + */ +bool pb_encode_fixed64(pb_ostream_t *stream, const void *value); +#endif + +/* Encode a submessage field. + * You need to pass the pb_field_t array and pointer to struct, just like + * with pb_encode(). This internally encodes the submessage twice, first to + * calculate message size and then to actually write it out. + */ +bool pb_encode_submessage(pb_ostream_t *stream, const pb_field_t fields[], + const void *src_struct); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif diff --git a/drivers/misc/vehicle/nanopb-c/pb_system.h b/drivers/misc/vehicle/nanopb-c/pb_system.h new file mode 100644 index 000000000000..2166e4b7256b --- /dev/null +++ b/drivers/misc/vehicle/nanopb-c/pb_system.h @@ -0,0 +1,27 @@ +/* + * Copyright 2018 NXP + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +#ifndef PB_SYSTEM_H +#define PB_SYSTEM_H + +#include + +typedef u8 uint_least8_t; +typedef s8 int_least8_t; +typedef u8 uint_fast8_t; + +typedef u16 uint_least16_t; +typedef s16 int_least16_t; + +#endif diff --git a/drivers/misc/vehicle/vehicle-adc.c b/drivers/misc/vehicle/vehicle-adc.c new file mode 100644 index 000000000000..0477f52ac061 --- /dev/null +++ b/drivers/misc/vehicle/vehicle-adc.c @@ -0,0 +1,274 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * vehicle-adc.c -- I2C register interface access for adc + * + * Copyright (c) 2025-2030 Rockchip Electronics Co., Ltd. + * + * Author: LuoWei + */ + +#include "core.h" + +static int vehicle_adc_init_channel(struct vehicle *vehicle, struct iio_channel **channel, + const char *name) +{ + int err = 0; + enum iio_chan_type type; + struct device *dev = vehicle->vehicle_adc->dev; + + struct iio_channel *tmp_channel = devm_iio_channel_get(dev, name); + + if (IS_ERR(tmp_channel)) + return PTR_ERR(tmp_channel); + + if (!tmp_channel->indio_dev) + return -ENXIO; + + err = iio_get_channel_type(tmp_channel, &type); + if (err < 0) + return err; + + if (type != IIO_VOLTAGE) { + dev_err(dev, "Incompatible channel type %d\n", type); + return -EINVAL; + } + + *channel = tmp_channel; + + return err; +} + +static int vehicle_adc_update_data(struct vehicle *vehicle) +{ + int val; + struct device *dev = vehicle->vehicle_adc->dev; + + vehicle->vehicle_data.turn = TURN_0; + if (vehicle->vehicle_adc->gear_adc_chn) { + if (iio_read_channel_raw(vehicle->vehicle_adc->gear_adc_chn, &val) < 0) + dev_err(dev, "Failed to read gear adc value\n"); + + if (val < 200) + vehicle->vehicle_data.gear = GEAR_0; + else if (val > 200 && val < 1200) + vehicle->vehicle_data.gear = GEAR_1; + else if (val > 1200 && val < 2200) + vehicle->vehicle_data.gear = GEAR_3; + else if (val > 2200 && val < 3200) + vehicle->vehicle_data.gear = GEAR_2; + } + + if (vehicle->vehicle_adc->turn_left_adc_chn) { + if (iio_read_channel_raw(vehicle->vehicle_adc->turn_left_adc_chn, &val) < 0) + dev_err(dev, "Failed to read turn adc value\n"); + + if (val < 100) + vehicle->vehicle_data.turn = TURN_1; + } + + if (vehicle->vehicle_adc->turn_right_adc_chn) { + if (iio_read_channel_raw(vehicle->vehicle_adc->turn_right_adc_chn, &val) < 0) + dev_err(dev, "Failed to read turn adc value\n"); + + if (val < 100) + vehicle->vehicle_data.turn = TURN_2; + } + + vehicle_set_property(VEHICLE_GEAR, 0, vehicle->vehicle_data.gear, 0); + vehicle_set_property(VEHICLE_TURN_SIGNAL, 0, vehicle->vehicle_data.turn, 0); + + dev_info(dev, "gear %u turn %u\n", vehicle->vehicle_data.gear, + vehicle->vehicle_data.turn); + + /* to do others adc */ + return 0; +} + +static void vehicle_adc_delay_work_func(struct work_struct *work) +{ + struct vehicle_adc *vehicle_adc = container_of(work, struct vehicle_adc, + vehicle_delay_work.work); + struct device *dev = vehicle_adc->dev; + + vehicle_adc_update_data(g_vehicle_hw); + + if (vehicle_adc->use_delay_work) + queue_delayed_work(vehicle_adc->vehicle_wq, &vehicle_adc->vehicle_delay_work, + msecs_to_jiffies(1000)); + + dev_info(dev, "%s\n", __func__); +} + +static irqreturn_t vehicle_adc_irq_handle(int irq, void *_data) +{ + struct vehicle_adc *vehicle_adc = _data; + + queue_delayed_work(vehicle_adc->vehicle_wq, &vehicle_adc->vehicle_delay_work, + msecs_to_jiffies(10)); + + return IRQ_HANDLED; +} + +static int vehicle_adc_irq_init(struct vehicle_adc *vehicle_adc) +{ + struct device *dev = vehicle_adc->dev; + int ret; + + vehicle_adc->irq = platform_get_irq(vehicle_adc->pdev, 0); + if (vehicle_adc->irq < 0) { + dev_err(dev, "failed to get irq\n"); + return -ENODEV; + } + ret = devm_request_threaded_irq(dev, vehicle_adc->irq, + vehicle_adc_irq_handle, NULL, + IRQF_TRIGGER_HIGH, + dev_name(dev), vehicle_adc); + + if (ret < 0) { + dev_err(dev, "error: irq %d\n", vehicle_adc->irq); + return ret; + } + + return 0; +} + +static int adc_hw_init(struct vehicle *vehicle) +{ + int err = 0; + struct device *dev = vehicle->dev; + + err = vehicle_adc_init_channel(vehicle, + &vehicle->vehicle_adc->gear_adc_chn, + "gear"); + if (err) + dev_err(dev, "failed to get turn adc channel.\n"); + + err = vehicle_adc_init_channel(vehicle, + &vehicle->vehicle_adc->turn_left_adc_chn, + "turn_left"); + if (err) + dev_err(dev, "failed to get turn_left adc channel.\n"); + + err = vehicle_adc_init_channel(vehicle, + &vehicle->vehicle_adc->turn_right_adc_chn, + "turn_right"); + if (err) + dev_err(dev, "failed to get turn_right adc channel.\n"); + + vehicle->vehicle_adc->vehicle_wq = alloc_ordered_workqueue("%s", + WQ_MEM_RECLAIM | WQ_FREEZABLE, + "vehicle-adc-wq"); + mutex_init(&vehicle->vehicle_adc->wq_lock); + INIT_DELAYED_WORK(&vehicle->vehicle_adc->vehicle_delay_work, vehicle_adc_delay_work_func); + + //vehicle->vehicle_adc->use_delay_work = + // of_property_read_bool(vehicle->dev->of_node, "use-delay-work"); + vehicle->vehicle_adc->use_delay_work = 1; + if (vehicle->vehicle_adc->use_delay_work) { + queue_delayed_work(vehicle->vehicle_adc->vehicle_wq, + &vehicle->vehicle_adc->vehicle_delay_work, + msecs_to_jiffies(100)); + VEHICLE_DBG("%s: vehicle_adc->use_delay_work=%d\n", __func__, + vehicle->vehicle_adc->use_delay_work); + } else { + vehicle_adc_irq_init(vehicle->vehicle_adc); + VEHICLE_DBG("%s: vehicle_adc->use_delay_work=%d\n", __func__, + vehicle->vehicle_adc->use_delay_work); + } + + return 0; +} + +static int adc_pm_suspend(struct vehicle *vehicle) +{ + return 0; +} + +static int adc_pm_resume(struct vehicle *vehicle) +{ + return 0; +} + +struct vehicle_hw_data vehicle_adc_data = { + .name = "vehicle-adc", + .vehicle_hw_type = VEHICLE_HW_TYPE_ADC, + .data_update = vehicle_adc_update_data, + .hw_init = adc_hw_init, + .suspend = adc_pm_suspend, + .resume = adc_pm_resume, +}; +EXPORT_SYMBOL_GPL(vehicle_adc_data); + +static int vehicle_adc_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct vehicle_adc *vehicle_adc; + + if (!pdev) + return -ENOMEM; + + if (g_vehicle_hw == NULL) + return -ENOMEM; + + vehicle_adc = devm_kzalloc(dev, sizeof(*vehicle_adc), GFP_KERNEL); + if (vehicle_adc == NULL) + return -ENOMEM; + + platform_set_drvdata(pdev, vehicle_adc); + vehicle_adc->hw_data = &vehicle_adc_data; + vehicle_adc->dev = dev; + vehicle_adc->pdev = pdev; + + g_vehicle_hw->vehicle_adc = vehicle_adc; + + adc_hw_init(g_vehicle_hw); + + pr_info("%s successfully\n", __func__); + + return 0; +} + +static int vehicle_adc_remove(struct platform_device *pdev) +{ + struct vehicle_adc *vehicle_adc = platform_get_drvdata(pdev); + + destroy_workqueue(vehicle_adc->vehicle_wq); + return 0; +} + +static const struct of_device_id vehicle_adc_id[] = { + { .compatible = "rockchip,vehicle-adc", }, + { .compatible = "rockchip,vehicle-dummy-adc", }, + {}, +}; + +static struct platform_driver vehicle_adc_device_driver = { + .probe = vehicle_adc_probe, + .remove = vehicle_adc_remove, + .driver = { + .name = "vehicle-adc", + .of_match_table = vehicle_adc_id, + } +}; + +static int vehicle_adc_init(void) +{ + int err; + + err = platform_driver_register(&vehicle_adc_device_driver); + if (err) + pr_err("Failed to register vehicle driver\n"); + + pr_info("%s successfully\n", __func__); + return err; +} + +static void __exit vehicle_adc_exit(void) +{ + platform_driver_unregister(&vehicle_adc_device_driver); +} + +postcore_initcall(vehicle_adc_init); +module_exit(vehicle_adc_exit); + +MODULE_LICENSE("GPL"); diff --git a/drivers/misc/vehicle/vehicle-gpio.c b/drivers/misc/vehicle/vehicle-gpio.c new file mode 100644 index 000000000000..f10ed247e400 --- /dev/null +++ b/drivers/misc/vehicle/vehicle-gpio.c @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * vehicle-gpio.c -- I2C register interface access for gpio + * + * Copyright (c) 2025-2030 Rockchip Electronics Co., Ltd. + * + * Author: LuoWei + */ + +#include "core.h" + +static int vehicle_gpio_init_io(struct vehicle_gpio *vehicle_gpio) +{ + int err = 0; + struct device *dev = vehicle_gpio->dev; + + vehicle_gpio->gear_gpio_reverse = devm_gpiod_get_optional(dev, "reverse", GPIOD_IN); + if (IS_ERR(vehicle_gpio->gear_gpio_reverse)) { + dev_err(dev, "failed to get gpio reverse\n"); + err = PTR_ERR(vehicle_gpio->gear_gpio_reverse); + } + + vehicle_gpio->gear_gpio_park = devm_gpiod_get_optional(dev, "park", GPIOD_IN); + if (IS_ERR(vehicle_gpio->gear_gpio_park)) { + dev_err(dev, "failed to get gpio park\n"); + err = PTR_ERR(vehicle_gpio->gear_gpio_park); + } + + return err; +} + +static int vehicle_gpio_update_data(struct vehicle *vehicle) +{ + int gear = 0; + int park_value = 0; + int reverse_value = 0; + struct device *dev = vehicle->vehicle_gpio->dev; + + if (vehicle->vehicle_gpio->gear_gpio_park) + park_value = !!gpiod_get_value(vehicle->vehicle_gpio->gear_gpio_park); + + if (vehicle->vehicle_gpio->gear_gpio_reverse) + reverse_value = !!gpiod_get_value(vehicle->vehicle_gpio->gear_gpio_reverse); + + dev_info(dev, "vehicle gpio %d %d\n", park_value, reverse_value); + + if (park_value && reverse_value) + vehicle->vehicle_data.gear = GEAR_2; + else if (!park_value && reverse_value) + vehicle->vehicle_data.gear = GEAR_1; + else if (park_value && !reverse_value) + vehicle->vehicle_data.gear = GEAR_0; + else + vehicle->vehicle_data.gear = GEAR_3; + + vehicle_set_property(VEHICLE_GEAR, 0, vehicle->vehicle_data.gear, 0); + vehicle_set_property(VEHICLE_TURN_SIGNAL, 0, vehicle->vehicle_data.turn, 0); + + dev_info(dev, "gear %u turn %u\n", vehicle->vehicle_data.gear, + vehicle->vehicle_data.turn); + + /* to do others gpio*/ + + return gear; +} + +static void vehicle_gpio_delay_work_func(struct work_struct *work) +{ + struct vehicle_gpio *vehicle_gpio = container_of(work, struct vehicle_gpio, + vehicle_delay_work.work); + struct device *dev = vehicle_gpio->dev; + + vehicle_gpio_update_data(g_vehicle_hw); + + if (vehicle_gpio->use_delay_work) + queue_delayed_work(vehicle_gpio->vehicle_wq, &vehicle_gpio->vehicle_delay_work, + msecs_to_jiffies(1000)); + + dev_info(dev, "%s\n", __func__); +} + +static irqreturn_t vehicle_gpio_irq_handle(int irq, void *_data) +{ + struct vehicle_gpio *vehicle_gpio = _data; + + queue_delayed_work(vehicle_gpio->vehicle_wq, &vehicle_gpio->vehicle_delay_work, + msecs_to_jiffies(10)); + + return IRQ_HANDLED; +} + +static int vehicle_gpio_irq_init(struct vehicle_gpio *vehicle_gpio) +{ + struct device *dev = vehicle_gpio->dev; + int ret; + + vehicle_gpio->irq = platform_get_irq(vehicle_gpio->pdev, 0); + if (vehicle_gpio->irq < 0) { + dev_err(dev, "failed to get irq\n"); + return -ENODEV; + } + ret = devm_request_threaded_irq(dev, vehicle_gpio->irq, + vehicle_gpio_irq_handle, NULL, + IRQF_TRIGGER_HIGH, + dev_name(dev), vehicle_gpio); + + if (ret < 0) { + dev_err(dev, "error: irq %d\n", vehicle_gpio->irq); + return ret; + } + + return 0; +} + +static int gpio_hw_init(struct vehicle *vehicle) +{ + vehicle_gpio_init_io(vehicle->vehicle_gpio); + vehicle->vehicle_gpio->vehicle_wq = + alloc_ordered_workqueue("%s", WQ_MEM_RECLAIM | WQ_FREEZABLE, + "vehicle-gpio-wq"); + mutex_init(&vehicle->vehicle_gpio->wq_lock); + INIT_DELAYED_WORK(&vehicle->vehicle_gpio->vehicle_delay_work, vehicle_gpio_delay_work_func); + + vehicle->vehicle_gpio->use_delay_work = + of_property_read_bool(vehicle->vehicle_gpio->dev->of_node, "use-delay-work"); + if (vehicle->vehicle_gpio->use_delay_work) { + queue_delayed_work(vehicle->vehicle_gpio->vehicle_wq, + &vehicle->vehicle_gpio->vehicle_delay_work, + msecs_to_jiffies(100)); + VEHICLE_DBG("%s: vehicle_gpio->use_delay_work=%d\n", __func__, + vehicle->vehicle_gpio->use_delay_work); + } else { + vehicle_gpio_irq_init(vehicle->vehicle_gpio); + VEHICLE_DBG("%s: vehicle_gpio->use_delay_work=%d\n", __func__, + vehicle->vehicle_gpio->use_delay_work); + } + + return 0; +} + +static int gpio_pm_suspend(struct vehicle *vehicle) +{ + return 0; +} + +static int gpio_pm_resume(struct vehicle *vehicle) +{ + return 0; +} + +struct vehicle_hw_data vehicle_gpio_data = { + .name = "vehicle-gpio", + .vehicle_hw_type = VEHICLE_HW_TYPE_GPIO, + .data_update = vehicle_gpio_update_data, + .hw_init = gpio_hw_init, + .suspend = gpio_pm_suspend, + .resume = gpio_pm_resume, +}; +EXPORT_SYMBOL_GPL(vehicle_gpio_data); + +static int vehicle_gpio_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct vehicle_gpio *vehicle_gpio; + + if (!pdev) + return -ENOMEM; + + if (g_vehicle_hw == NULL) + return -ENOMEM; + + vehicle_gpio = devm_kzalloc(dev, sizeof(*vehicle_gpio), GFP_KERNEL); + if (vehicle_gpio == NULL) + return -ENOMEM; + + platform_set_drvdata(pdev, vehicle_gpio); + vehicle_gpio->hw_data = &vehicle_gpio_data; + vehicle_gpio->dev = dev; + vehicle_gpio->pdev = pdev; + + g_vehicle_hw->vehicle_gpio = vehicle_gpio; + + gpio_hw_init(g_vehicle_hw); + + return 0; +} + +static int vehicle_gpio_remove(struct platform_device *pdev) +{ + struct vehicle_gpio *vehicle_gpio = platform_get_drvdata(pdev); + + destroy_workqueue(vehicle_gpio->vehicle_wq); + return 0; +} + +static const struct of_device_id vehicle_gpio_id[] = { + { .compatible = "rockchip,vehicle-gpio", }, + { .compatible = "rockchip,vehicle-dummy-gpio", }, + {}, +}; + +static struct platform_driver vehicle_gpio_device_driver = { + .probe = vehicle_gpio_probe, + .remove = vehicle_gpio_remove, + .driver = { + .name = "vehicle-gpio", + .of_match_table = vehicle_gpio_id, + } +}; + +static int vehicle_gpio_init(void) +{ + int err; + + err = platform_driver_register(&vehicle_gpio_device_driver); + if (err) + pr_err("Failed to register vehicle driver\n"); + + return err; +} + +static void __exit vehicle_gpio_exit(void) +{ + platform_driver_unregister(&vehicle_gpio_device_driver); +} + +postcore_initcall(vehicle_gpio_init); +module_exit(vehicle_gpio_exit); + +MODULE_LICENSE("GPL"); diff --git a/drivers/misc/vehicle/vehicle-spi.c b/drivers/misc/vehicle/vehicle-spi.c new file mode 100644 index 000000000000..1aca58977e64 --- /dev/null +++ b/drivers/misc/vehicle/vehicle-spi.c @@ -0,0 +1,413 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * vehicle-spi.c -- I2C register interface access for spi + * + * Copyright (c) 2025-2030 Rockchip Electronics Co., Ltd. + * + * Author: LuoWei + */ + +#include "core.h" +#ifdef CONFIG_VEHICLE_SPI_PROTOCOL +#include "vehicle_spi_protocol.h" +#endif + +static struct completion spi_complete; +#define SPI_TIMEOUT_MS 20 + +static void spi_complete_callback(void *arg) +{ + complete(&spi_complete); +} + +int vehicle_spi_write_slt(struct vehicle *vehicle, const void *txbuf, size_t n) +{ + int ret = -1; + struct spi_device *spi = NULL; + struct spi_transfer t = { + .tx_buf = txbuf, + .len = n, + .bits_per_word = 8, + }; + struct spi_message m; + + spi = vehicle->vehicle_spi->spi; + reinit_completion(&spi_complete); + spi_message_init(&m); + spi_message_add_tail(&t, &m); + m.complete = spi_complete_callback; + ret = spi_async(spi, &m); + + if (ret) { + dev_err(&spi->dev, "SPI write async error: %d\n", ret); + return ret; + } + + if (!wait_for_completion_timeout(&spi_complete, msecs_to_jiffies(SPI_TIMEOUT_MS))) { + dev_err(&spi->dev, "SPI write operation timed out\n"); + return -ETIMEDOUT; + } + + return ret; +} + +int vehicle_spi_read_slt(struct vehicle *vehicle, void *rxbuf, size_t n) +{ + int ret = -1; + struct spi_device *spi = NULL; + struct spi_transfer t = { + .rx_buf = rxbuf, + .len = n, + .bits_per_word = 8, + }; + struct spi_message m; + + spi = vehicle->vehicle_spi->spi; + reinit_completion(&spi_complete); + spi_message_init(&m); + spi_message_add_tail(&t, &m); + m.complete = spi_complete_callback; + ret = spi_async(spi, &m); + if (ret) { + dev_err(&spi->dev, "SPI read async error: %d\n", ret); + return ret; + } + + if (!wait_for_completion_timeout(&spi_complete, msecs_to_jiffies(SPI_TIMEOUT_MS))) { + dev_err(&spi->dev, "SPI read operation timed out\n"); + return -ETIMEDOUT; + } + + return ret; +} + +static int vehicle_spi_update_data(struct vehicle *vehicle) +{ + int i = 0; + unsigned int times = 1, size = 12; + unsigned long us = 0, bytes = 0; + unsigned char *rxbuf = NULL; + ktime_t start_time; + ktime_t end_time; + ktime_t cost_time; + struct device *dev = vehicle->vehicle_spi->dev; + + rxbuf = kzalloc(size, GFP_KERNEL); + + if (!rxbuf) + return -ENOMEM; + + start_time = ktime_get(); + for (i = 0; i < times; i++) +#ifndef CONFIG_VEHICLE_SPI_PROTOCOL + vehicle_spi_read_slt(vehicle, rxbuf, size); +#else + Analyze_read_data(vehicle, rxbuf, size); +#endif + end_time = ktime_get(); + cost_time = ktime_sub(end_time, start_time); + us = ktime_to_us(cost_time); + + bytes = size * times * 1; + bytes = bytes * 1000 / us; + pr_info("spi read %d*%d cost %ldus speed:%ldKB/S\n", size, times, us, bytes); + print_hex_dump(KERN_ERR, "SPI RX: ", + DUMP_PREFIX_OFFSET, + 16, + 1, + rxbuf, + size, + 1); + + kfree(rxbuf); + vehicle_set_property(VEHICLE_GEAR, 0, vehicle->vehicle_data.gear, 0); + vehicle_set_property(VEHICLE_TURN_SIGNAL, 0, vehicle->vehicle_data.turn, 0); + + dev_info(dev, "gear %u turn %u\n", vehicle->vehicle_data.gear, vehicle->vehicle_data.turn); + + /* to do others spi */ + return 0; +} + + +static void vehicle_spi_delay_work_func(struct work_struct *work) +{ + struct vehicle_spi *vehicle_spi = container_of(work, struct vehicle_spi, + vehicle_delay_work.work); + struct device *dev = vehicle_spi->dev; + + vehicle_spi_update_data(g_vehicle_hw); + + if (vehicle_spi->use_delay_work) + queue_delayed_work(vehicle_spi->vehicle_wq, &vehicle_spi->vehicle_delay_work, + msecs_to_jiffies(1000)); + + dev_info(dev, "%s end\n", __func__); +} + +static irqreturn_t vehicle_spi_irq_handle(int irq, void *_data) +{ + struct vehicle_spi *vehicle_spi = _data; + + queue_delayed_work(vehicle_spi->vehicle_wq, &vehicle_spi->vehicle_delay_work, + msecs_to_jiffies(0)); + + return IRQ_HANDLED; +} + +static int vehicle_spi_irq_init(struct vehicle_spi *vehicle_spi) +{ + struct device *dev = vehicle_spi->dev; + struct gpio_desc *irq_gpio_desc; + int ret; + + irq_gpio_desc = devm_gpiod_get_optional(dev, "irq", GPIOD_IN); + if (IS_ERR_OR_NULL(irq_gpio_desc)) + dev_warn(dev, "Failed to request irq-gpio\n"); + else { + vehicle_spi->irq = gpiod_to_irq(irq_gpio_desc); + ret = devm_request_threaded_irq(dev, vehicle_spi->irq, + vehicle_spi_irq_handle, NULL, + IRQF_TRIGGER_FALLING, + dev_name(dev), vehicle_spi); + + if (ret < 0) { + dev_err(dev, "error: irq %d\n", vehicle_spi->irq); + gpio_free(vehicle_spi->irq); + return ret; + } + } + + return 0; +} + +static int spi_hw_init(struct vehicle *vehicle) +{ + vehicle->vehicle_spi->vehicle_wq = alloc_ordered_workqueue("%s", + WQ_MEM_RECLAIM | WQ_FREEZABLE, + "vehicle-spi-wq"); + mutex_init(&vehicle->vehicle_spi->wq_lock); + INIT_DELAYED_WORK(&vehicle->vehicle_spi->vehicle_delay_work, + vehicle_spi_delay_work_func); + + vehicle->vehicle_spi->use_delay_work = + of_property_read_bool(vehicle->vehicle_spi->dev->of_node, "use-delay-work"); + + if (vehicle->vehicle_spi->use_delay_work) { + queue_delayed_work(vehicle->vehicle_spi->vehicle_wq, + &vehicle->vehicle_spi->vehicle_delay_work, + msecs_to_jiffies(100)); + VEHICLE_DBG("%s: vehicle_spi->use_delay_work=%d\n", __func__, + vehicle->vehicle_spi->use_delay_work); + } else { + vehicle_spi_irq_init(vehicle->vehicle_spi); + VEHICLE_DBG("%s: vehicle_spi->use_delay_work=%d\n", __func__, + vehicle->vehicle_spi->use_delay_work); + } + + return 0; +} + +static int spi_pm_suspend(struct vehicle *vehicle) +{ + return 0; +} + +static int spi_pm_resume(struct vehicle *vehicle) +{ + return 0; +} + +struct vehicle_hw_data vehicle_spi_data = { + .name = "vehicle-spi", + .vehicle_hw_type = VEHICLE_HW_TYPE_SPI, + .data_update = vehicle_spi_update_data, + .hw_init = spi_hw_init, + .suspend = spi_pm_suspend, + .resume = spi_pm_resume, +}; +EXPORT_SYMBOL_GPL(vehicle_spi_data); + +static ssize_t spi_test_write(struct file *file, + const char __user *buf, size_t n, loff_t *offset) +{ + int argc = 0, i; + char tmp[64]; + char *argv[16]; + char *cmd, *data; + unsigned int id = 0, times = 0, size = 0, cmd_spi = 0; + unsigned long us = 0, bytes = 0; + char *txbuf = NULL; + ktime_t start_time; + ktime_t end_time; + ktime_t cost_time; + + if (n >= sizeof(tmp)) { + pr_info("%s error size > 64\n", __func__); + return -EINVAL; + } + + memset(tmp, 0, sizeof(tmp)); + if (copy_from_user(tmp, buf, n)) + return -EFAULT; + cmd = tmp; + data = tmp; + + memset(argv, 0, sizeof(argv)); + + while (data < (tmp + n)) { + data = strstr(data, " "); + if (!data) + break; + *data = 0; + argv[argc] = ++data; + argc++; + if (argc >= 16) + break; + } + + tmp[n - 1] = 0; + + if (!strcmp(cmd, "write")) { + if (kstrtoint(argv[0], 10, &id) < 0) + return -EFAULT; + + if (kstrtoint(argv[1], 10, ×) < 0) + return -EFAULT; + + if (kstrtoint(argv[2], 10, &size) < 0) + return -EFAULT; + + if (kstrtoint(argv[3], 16, &cmd_spi) < 0) + return -EFAULT; + + txbuf = kzalloc(size, GFP_KERNEL); + if (!txbuf) + return n; + + for (i = 0; i < size; i++) { + if (kstrtoint(argv[4+i], 16, (int *)(txbuf+i)) < 0) + return -EFAULT; + } + + start_time = ktime_get(); + for (i = 0; i < times; i++) +#ifndef CONFIG_VEHICLE_SPI_PROTOCOL + vehicle_spi_write_slt(g_vehicle_hw, txbuf, size); +#else + Analyze_write_data(g_vehicle_hw, (unsigned char)cmd_spi, txbuf, size); +#endif + end_time = ktime_get(); + cost_time = ktime_sub(end_time, start_time); + us = ktime_to_us(cost_time); + + bytes = size * times * 1; + bytes = bytes * 1000 / us; + pr_info("spi write %d*%d cost %ldus speed:%ldKB/S\n", size, times, us, bytes); + kfree(txbuf); + } + return n; +} +static const struct file_operations spi_test_fops = { + .write = spi_test_write, +}; + +static struct miscdevice spi_test_misc = { + .minor = MISC_DYNAMIC_MINOR, + .name = "spi_misc_test", + .fops = &spi_test_fops, +}; + +static int vehicle_spi_probe(struct spi_device *spi) +{ + struct device *dev = &spi->dev; + struct vehicle_spi *vehicle_spi; + int ret = 0; + int id = 0; + + if (!spi) + return -ENOMEM; + + if (g_vehicle_hw == NULL) + return -ENOMEM; + + vehicle_spi = devm_kzalloc(dev, sizeof(*vehicle_spi), GFP_KERNEL); + if (vehicle_spi == NULL) + return -ENOMEM; + + spi_set_drvdata(spi, vehicle_spi); + + vehicle_spi->hw_data = &vehicle_spi_data; + spi->bits_per_word = 8; + vehicle_spi->dev = dev; + vehicle_spi->spi = spi; + g_vehicle_hw->vehicle_spi = vehicle_spi; + + ret = spi_setup(spi); + if (ret < 0) { + dev_err(dev, "ERR: fail to setup spi\n"); + return -1; + } + + if (device_property_read_u32(&spi->dev, "id", &id)) { + dev_warn(&spi->dev, "fail to get id, default set 0\n"); + id = 0; + } + + init_completion(&spi_complete); + spi_hw_init(g_vehicle_hw); + +#ifdef CONFIG_VEHICLE_GPIO_MCU_EXPANDER + gpio_mcu_register(spi); +#endif + + return 0; +} + +static void vehicle_spi_remove(struct spi_device *spi) +{ + struct vehicle_spi *vehicle_spi = spi_get_drvdata(spi); + + destroy_workqueue(vehicle_spi->vehicle_wq); +} + +#ifdef CONFIG_OF +static const struct of_device_id vehicle_spi_id[] = { + { .compatible = "rockchip,vehicle-spi", }, + {}, +}; +#endif + +static struct spi_driver vehicle_spi_device_driver = { + .probe = vehicle_spi_probe, + .remove = vehicle_spi_remove, + .driver = { + .name = "vehicle-spi", + .of_match_table = vehicle_spi_id, + } +}; + +static int vehicle_spi_init(void) +{ + int err; + + misc_register(&spi_test_misc); + err = spi_register_driver(&vehicle_spi_device_driver); + if (err) { + pr_err("Failed to register vehicle spi driver\n"); + misc_deregister(&spi_test_misc); + } + + return err; +} +module_init(vehicle_spi_init); + +static void __exit vehicle_spi_exit(void) +{ + misc_deregister(&spi_test_misc); + spi_unregister_driver(&vehicle_spi_device_driver); +} + +module_exit(vehicle_spi_exit); + +MODULE_LICENSE("GPL"); diff --git a/drivers/misc/vehicle/vehicle_core.c b/drivers/misc/vehicle/vehicle_core.c new file mode 100644 index 000000000000..5a800e0f88c4 --- /dev/null +++ b/drivers/misc/vehicle/vehicle_core.c @@ -0,0 +1,418 @@ +/* + * Copyright 2018 NXP + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "core.h" + +#include "pb_encode.h" +#include "pb_decode.h" +#include "pb.h" +#include "vehiclehalproto.pb.h" +#include "vehicle_protocol_callback.h" +#include "vehicle_core.h" + +#define PROTOCOL_ID 30 + +struct vehicle *g_vehicle_hw; +EXPORT_SYMBOL(g_vehicle_hw); +static struct sock *nlsk; +static int g_user_pid; + +struct vehicle_core_drvdata { + const struct hw_prop_ops *prop_ops; +}; + +/* param in command PWR_REQ */ +enum vehicle_power_request_param { + AP_POWER_REQUEST_PARAM_SHUTDOWN_IMMEDIATELY = 1, + AP_POWER_REQUEST_PARAM_CAN_SLEEP, + AP_POWER_REQUEST_PARAM_SHUTDOWN_ONLY, + AP_POWER_REQUEST_PARAM_SLEEP_IMMEDIATELY, + AP_POWER_REQUEST_PARAM_HIBERNATE_IMMEDIATELY, + AP_POWER_REQUEST_PARAM_CAN_HIBERNATE, +}; + +/* state in command PWR_REQ */ +enum vehicle_power_request_state { + AP_POWER_REQUEST_STATE_ON = 0, + AP_POWER_REQUEST_STATE_SHUTDOWN_PREPARE, + AP_POWER_REQUEST_STATE_CANCEL_SHUTDOWN, + AP_POWER_REQUEST_STATE_FINISHED, +}; + +static struct vehicle_core_drvdata *vehicle_core; +static struct vehicle_property_set property_encode; +static struct vehicle_property_set property_decode; +static struct vehicle_power_req power_req_encode; + +void vehicle_hw_prop_ops_register(const struct hw_prop_ops *prop_ops) +{ + if (!prop_ops) + return; + + if (vehicle_core) + vehicle_core->prop_ops = prop_ops; +} +EXPORT_SYMBOL_GPL(vehicle_hw_prop_ops_register); + +static int vehicle_send_message_core(u32 prop, u32 area, u32 value) +{ + if (vehicle_core && vehicle_core->prop_ops && + vehicle_core->prop_ops->set_control_commands) + vehicle_core->prop_ops->set_control_commands(prop, area, value); + return 0; +} + +static int send_usrmsg(char *pbuf, uint16_t len) +{ + struct sk_buff *nl_skb; + struct nlmsghdr *nlh; + + int ret; + + if (nlsk == NULL || pbuf == NULL) { + pr_err("Invalid parameters nlsk %p pbuf %p\n", nlsk, pbuf); + return -1; + } + + nl_skb = nlmsg_new(len, GFP_ATOMIC); + if (!nl_skb) { + pr_err("netlink alloc failure\n"); + return -1; + } + + nlh = nlmsg_put(nl_skb, 0, 0, 0, len, 0); + if (nlh == NULL) { + pr_err("nlmsg_put failaure\n"); + nlmsg_free(nl_skb); + return -1; + } + + memcpy(nlmsg_data(nlh), pbuf, len); + ret = netlink_unicast(nlsk, nl_skb, g_user_pid, MSG_DONTWAIT); + + return ret; +} + +void vehicle_set_property(u16 prop, u8 index, u32 value, u32 param) +{ + char *buffer; + + buffer = kmalloc(128, GFP_KERNEL); + pr_debug("%s: prop %d, index %d, value %d\n", __func__, prop, index, value); + property_encode.value = value; + switch (prop) { + case VEHICLE_FAN_SPEED: + property_encode.prop = HVAC_FAN_SPEED; + property_encode.area_id = HVAC_ALL; + if (property_encode.value != FAN_SPEED_0 && + property_encode.value != FAN_SPEED_1 && + property_encode.value != FAN_SPEED_2 && + property_encode.value != FAN_SPEED_3 && + property_encode.value != FAN_SPEED_4 && + property_encode.value != FAN_SPEED_5) { + pr_err("input value is not correct, please type correct one\n"); + goto exit; + } + break; + case VEHICLE_FAN_DIRECTION: + property_encode.prop = HVAC_FAN_DIRECTION; + property_encode.area_id = HVAC_ALL; + if (property_encode.value != FAN_DIRECTION_0 && + property_encode.value != FAN_DIRECTION_1 && + property_encode.value != FAN_DIRECTION_2 && + property_encode.value != FAN_DIRECTION_3) { + pr_err("input value is not correct, please type correct one\n"); + goto exit; + } + break; + case VEHICLE_AUTO_ON: + property_encode.prop = HVAC_AUTO_ON; + property_encode.area_id = (u32)index; + if (property_encode.value != AUTO_ON && property_encode.value != AUTO_OFF) { + pr_err("input value is not correct, please type correct one\n"); + goto exit; + } + break; + case VEHICLE_AC: + property_encode.prop = HVAC_AC_ON; + property_encode.area_id = HVAC_ALL; + if (property_encode.value != AC_ON && property_encode.value != AC_OFF) { + pr_err("input value is not correct, please type correct one\n"); + goto exit; + } + break; + case VEHICLE_RECIRC_ON: + property_encode.prop = HVAC_RECIRC_ON; + property_encode.area_id = HVAC_ALL; + if (property_encode.value != RECIRC_ON && property_encode.value != RECIRC_OFF) { + pr_err("input value is not correct, please type correct one\n"); + goto exit; + } + break; + case VEHICLE_DEFROST: + property_encode.prop = HVAC_DEFROSTER; + property_encode.area_id = (u32)index; + if (property_encode.value != DEFROST_ON && property_encode.value != DEFROST_OFF) { + pr_err("input value is not correct, please type correct one\n"); + goto exit; + } + break; + case VEHICLE_AC_TEMP: + property_encode.prop = HVAC_TEMPERATURE_SET; + property_encode.area_id = (u32)index; + break; + case VEHICLE_HVAC_POWER_ON: + property_encode.prop = HVAC_POWER_ON; + property_encode.area_id = (u32)index; + if (property_encode.value != HVAC_ON && property_encode.value != HVAC_OFF) { + pr_err("input value is not correct, please type correct one\n"); + goto exit; + } + break; + case VEHICLE_SEAT_TEMPERATURE: + property_encode.prop = HVAC_SEAT_TEMPERATURE; + property_encode.area_id = (u32)index; + if (property_encode.value != SEAT_TEMP_0 && + property_encode.value != SEAT_TEMP_1 && + property_encode.value != SEAT_TEMP_2 && + property_encode.value != SEAT_TEMP_3) { + pr_err("input value is not correct, please type correct one\n"); + goto exit; + } + break; + case VEHICLE_GEAR: + property_encode.prop = GEAR_SELECTION; + if (value == VEHICLE_GEAR_DRIVE) + property_encode.value = VEHICLE_GEAR_DRIVE_CLIENT; + else if (value == VEHICLE_GEAR_REVERSE) + property_encode.value = VEHICLE_GEAR_REVERSE_CLIENT; + else if (value == VEHICLE_GEAR_PARKING) + property_encode.value = VEHICLE_GEAR_PARK_CLIENT; + break; + case VEHICLE_TURN_SIGNAL: + property_encode.prop = TURN_SIGNAL_STATE; + break; + case VEHICLE_POWER_STATE_REQ: + if (value != AP_POWER_REQUEST_STATE_ON && + value != AP_POWER_REQUEST_STATE_SHUTDOWN_PREPARE && + value != AP_POWER_REQUEST_STATE_CANCEL_SHUTDOWN && + value != AP_POWER_REQUEST_STATE_FINISHED) { + pr_err("AP_POWER_STATE_REQ: invalid state\n"); + goto exit; + } + if (param != AP_POWER_REQUEST_PARAM_SHUTDOWN_IMMEDIATELY && + param != AP_POWER_REQUEST_PARAM_CAN_SLEEP && + param != AP_POWER_REQUEST_PARAM_SHUTDOWN_ONLY && + param != AP_POWER_REQUEST_PARAM_SLEEP_IMMEDIATELY && + param != AP_POWER_REQUEST_PARAM_HIBERNATE_IMMEDIATELY && + param != AP_POWER_REQUEST_PARAM_CAN_HIBERNATE) { + pr_err("AP_POWER_STATE_REQ: invalid param\n"); + goto exit; + } + + power_req_encode.prop = AP_POWER_STATE_REQ; + power_req_encode.state = value; + power_req_encode.param = param; + break; + default: + pr_err("property %d is not supported\n", prop); + } + + pb_ostream_t stream; + + stream = pb_ostream_from_buffer(buffer, 128); + + emulator_EmulatorMessage send_message = {}; + + send_message.msg_type = emulator_MsgType_SET_PROPERTY_CMD; + send_message.has_status = true; + send_message.status = emulator_Status_RESULT_OK; + if (prop == VEHICLE_POWER_STATE_REQ) { + send_message.value.funcs.encode = &encode_power_state_callback; + send_message.value.arg = &power_req_encode; + } else { + send_message.value.funcs.encode = &encode_value_callback; + send_message.value.arg = &property_encode; + } + if (!pb_encode(&stream, emulator_EmulatorMessage_fields, &send_message)) + pr_err("vehicle protocol encode fail\n"); + send_usrmsg(buffer, stream.bytes_written); +exit: + kfree(buffer); +} +EXPORT_SYMBOL_GPL(vehicle_set_property); + +static void netlink_rcv_msg(struct sk_buff *skb) +{ + struct nlmsghdr *nlh = NULL; + char *umsg = NULL; + char *buffer; + bool status; + size_t len; + int i; + emulator_EmulatorMessage emulator_message; + + buffer = kmalloc(128, GFP_KERNEL); + if (skb->len >= nlmsg_total_size(0)) { + nlh = nlmsg_hdr(skb); + g_user_pid = nlh->nlmsg_pid; + umsg = NLMSG_DATA(nlh); + len = nlh->nlmsg_len - NLMSG_LENGTH(0); + if (umsg) { + for (i = 0; i < len; i++) + pr_debug("%s raw byte %d %d\n", __func__, i, umsg[i]); + memcpy(buffer, umsg, len); + pb_istream_t stream = pb_istream_from_buffer(buffer, len); + + emulator_message.prop.funcs.decode = &decode_prop_callback; + emulator_message.config.funcs.decode = &decode_config_callback; + emulator_message.value.funcs.decode = &decode_value_callback; + emulator_message.value.arg = &property_decode; + + status = pb_decode(&stream, emulator_EmulatorMessage_fields, + &emulator_message); + if (!status) + pr_err("pb_decode failed\n"); + + vehicle_send_message_core(property_decode.prop, property_decode.area_id, + property_decode.value); + } + } + kfree(buffer); +} + +static void create_netlink_vehicle(void) +{ + struct netlink_kernel_cfg cfg = { + .input = netlink_rcv_msg, + }; + + nlsk = netlink_kernel_create(&init_net, PROTOCOL_ID, &cfg); + if (nlsk == NULL) { + pr_err("netlink_kernel_create error !\n"); + return; + } +} + +static struct vehicle_core_drvdata * +vehicle_get_devtree_pdata(struct device *dev) +{ + struct vehicle_core_drvdata *ddata; + + ddata = devm_kzalloc(dev, + sizeof(*ddata), + GFP_KERNEL); + + if (!ddata) + return ERR_PTR(-ENOMEM); + + return ddata; +} + +static struct vehicle * +vehicle_get_dev_pdata(struct device *dev) +{ + struct vehicle *ddata; + + ddata = devm_kzalloc(dev, + sizeof(*ddata), + GFP_KERNEL); + if (!ddata) + return ERR_PTR(-ENOMEM); + + return ddata; +} + +static int vehicle_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct vehicle_core_drvdata *ddata; + + ddata = vehicle_get_devtree_pdata(dev); + if (IS_ERR(ddata)) + return PTR_ERR(ddata); + + g_vehicle_hw = vehicle_get_dev_pdata(dev); + if (IS_ERR(g_vehicle_hw)) + return PTR_ERR(g_vehicle_hw); + + vehicle_core = ddata; + platform_set_drvdata(pdev, ddata); + platform_set_drvdata(pdev, g_vehicle_hw); + + create_netlink_vehicle(); + return 0; +} + +static int vehicle_remove(struct platform_device *pdev) +{ + if (nlsk) + netlink_kernel_release(nlsk); + return 0; +} + +static struct platform_driver vehicle_device_driver = { + .probe = vehicle_probe, + .remove = vehicle_remove, + .driver = { + .name = "vehicle-core", + } +}; +static struct platform_device *core_pdev; + +static int vehicle_init(void) +{ + int err; + + core_pdev = platform_device_alloc("vehicle-core", -1); + if (!core_pdev) { + pr_err("Failed to allocate core vehicle device\n"); + return -ENODEV; + } + + err = platform_device_add(core_pdev); + if (err != 0) { + pr_err("Failed to register core device: %d\n", err); + platform_device_put(core_pdev); + return err; + } + + err = platform_driver_register(&vehicle_device_driver); + if (err) + pr_err("Failed to register vehicle driver\n"); + + return err; +} + +static void __exit vehicle_exit(void) +{ + platform_driver_unregister(&vehicle_device_driver); + platform_device_unregister(core_pdev); +} + +postcore_initcall(vehicle_init); +module_exit(vehicle_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("NXP Semiconductor"); +MODULE_DESCRIPTION("VEHICLE core driver"); diff --git a/drivers/misc/vehicle/vehicle_core.h b/drivers/misc/vehicle/vehicle_core.h new file mode 100644 index 000000000000..3a586c0ce75a --- /dev/null +++ b/drivers/misc/vehicle/vehicle_core.h @@ -0,0 +1,388 @@ +/* + * Copyright 2018 NXP + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +#ifndef VEHICLE_CORE_H +#define VEHICLE_CORE_H + +#include + +// seat temperature indexes +#define SEAT_TEMP_LEFT_INDEX 1 +#define SEAT_TEMP_RIGHT_INDEX 4 +// seat temperature values +#define SEAT_TEMP_0 0 // off +#define SEAT_TEMP_1 1 +#define SEAT_TEMP_2 2 +#define SEAT_TEMP_3 3 + +#define DEFROST_ON 1 +#define DEFROST_OFF 0 + +#define AC_ON 1 +#define AC_OFF 0 + +#define AUTO_ON 1 +#define AUTO_OFF 0 + +#define HVAC_ON 1 +#define HVAC_OFF 0 + +#define RECIRC_ON 1 +#define RECIRC_OFF 0 + +#define FAN_SPEED_0 1 +#define FAN_SPEED_1 2 +#define FAN_SPEED_2 3 +#define FAN_SPEED_3 4 +#define FAN_SPEED_4 5 +#define FAN_SPEED_5 6 + +#define FAN_DIRECTION_0 1 +#define FAN_DIRECTION_1 2 +#define FAN_DIRECTION_2 3 +#define FAN_DIRECTION_3 6 + +struct vehicle_property_set { + u32 prop; + u32 area_id; + u64 value; +}; + +struct vehicle_power_req { + u32 prop; + u32 state; + u32 param; +}; + +struct hw_prop_ops { + /* send the control commands to hw */ + void (*set_control_commands)(u32 prop, u32 area, u32 value); +}; + +/* areaId defines - begin */ +#define VEHICLE_AREA_SEAT_ROW_1_LEFT 1 /* 0x0001 */ +#define VEHICLE_AREA_SEAT_ROW_1_CENTER 2 /* 0x0002 */ +#define VEHICLE_AREA_SEAT_ROW_1_RIGHT 4 /* 0x0004 */ +#define VEHICLE_AREA_SEAT_ROW_2_LEFT 16 /* 0x0010 */ +#define VEHICLE_AREA_SEAT_ROW_2_CENTER 32 /* 0x0020 */ +#define VEHICLE_AREA_SEAT_ROW_2_RIGHT 64 /* 0x0040 */ +#define VEHICLE_AREA_SEAT_ROW_3_LEFT 256 /* 0x0100 */ +#define VEHICLE_AREA_SEAT_ROW_3_CENTER 512 /* 0x0200 */ +#define VEHICLE_AREA_SEAT_ROW_3_RIGHT 1024 /* 0x0400 */ + +#define HVAC_LEFT VEHICLE_AREA_SEAT_ROW_1_LEFT +#define HVAC_RIGHT VEHICLE_AREA_SEAT_ROW_1_RIGHT +//Due to VHAL migration from hidl to aidl, details in commit message +#define HVAC_ALL VEHICLE_AREA_SEAT_ROW_1_LEFT +/* areaId defines - end */ + +#define HVAC_FAN_SPEED 356517120 +#define HVAC_FAN_DIRECTION 356517121 +#define HVAC_AUTO_ON 354419978 +#define HVAC_AC_ON 354419973 +#define HVAC_RECIRC_ON 354419976 +#define HVAC_DEFROSTER 320865540 +#define HVAC_TEMPERATURE_SET 358614275 +#define HVAC_POWER_ON 354419984 + +#define TURN_SIGNAL_STATE 289408008 +#define GEAR_SELECTION 289408000 +#define CURRENT_GEAR 289408001 + +#define AP_POWER_STATE_REQ 289475072 +#define AP_POWER_STATE_REPORT 289475073 + +#define VEHICLE_GEAR_DRIVE_CLIENT 8 +#define VEHICLE_GEAR_PARK_CLIENT 4 +#define VEHICLE_GEAR_REVERSE_CLIENT 2 + +#define WATCHDOG_ALIVE 290459441 +#define HVAC_SEAT_TEMPERATURE 356517131 +#define DISPLAY_BRIGHTNESS 289409539 +#define INFO_MAKE 286261505 +#define INFO_MODEL 286261506 +#define POWER_POLICY_REQ 286265121 +#define POWER_POLICY_GROUP_REQ 286265122 +#define CURRENT_POWER_POLICY 286265123 +#define FUEL_DOOR_OPEN 287310600 +#define EV_CHARGE_PORT_OPEN 287310602 +#define EV_CHARGE_PORT_CONNECTED 287310603 +#define PARKING_BRAKE_ON 287310850 +#define PARKING_BRAKE_AUTO_APPLY 287310851 +#define FUEL_LEVEL_LOW 287310853 +#define NIGHT_MODE 287310855 +#define ABS_ACTIVE 287310858 +#define TRACTION_CONTROL_ACTIVE 287310859 +#define SUPPORT_CUSTOMIZE_VENDOR_PERMISSION 287313669 +#define INFO_MODEL_YEAR 289407235 +#define INFO_FUEL_DOOR_LOCATION 289407240 +#define INFO_EV_PORT_LOCATION 289407241 +#define ENGINE_OIL_LEVEL 289407747 +#define IGNITION_STATE 289408009 +#define HVAC_STEERING_WHEEL_HEAT 289408269 +#define HVAC_TEMPERATURE_DISPLAY_UNITS 289408270 +#define DISTANCE_DISPLAY_UNITS 289408512 +#define TIRE_PRESSURE_DISPLAY_UNITS 289408514 +#define VEHICLE_SPEED_DISPLAY_UNITS 289408517 +#define HEADLIGHTS_STATE 289410560 +#define HIGH_BEAM_LIGHTS_STATE 289410561 +#define FOG_LIGHTS_STATE 289410562 +#define PARKING_BRAKE_AUTO_APPLY 287310851 +#define HAZARD_LIGHTS_STATE 289410563 +#define HEADLIGHTS_SWITCH 289410576 +#define HIGH_BEAM_LIGHTS_SWITCH 289410577 +#define FOG_LIGHTS_SWITCH 289410578 +#define HAZARD_LIGHTS_SWITCH 289410579 +#define CLUSTER_SWITCH_UI 289410868 +#define CLUSTER_REQUEST_DISPLAY 289410871 +#define POWER_POLICY_GROUP_REQ 286265122 +#define CURRENT_POWER_POLICY 286265123 +#define ELECTRONIC_TOLL_COLLECTION_CARD_TYPE 289410873 +#define ELECTRONIC_TOLL_COLLECTION_CARD_STATUS 289410874 +#define INFO_FUEL_TYPE 289472773 +#define INFO_EV_CONNECTOR_TYPE 289472775 +#define INFO_DRIVER_SEAT 356516106 +#define INFO_EXTERIOR_DIMENSIONS 289472779 +#define INFO_MULTI_EV_PORT_LOCATIONS 289472780 +#define HW_KEY_INPUT 289475088 +#define HW_ROTARY_INPUT 289475104 +#define HW_CUSTOM_INPUT 289475120 +#define EVS_SERVICE_REQUEST 289476368 +#define CLUSTER_DISPLAY_STATE 289476405 +#define EPOCH_TIME 290457094 +#define VHAL_HEARTBEAT 290459443 +#define WHEEL_TICK 290521862 +#define INFO_FUEL_CAPACITY 291504388 +#define INFO_EV_BATTERY_CAPACITY 291504390 +#define PERF_ODOMETER 291504644 +#define PERF_VEHICLE_SPEED 291504647 +#define PERF_STEERING_ANGLE 291504649 +#define PERF_REAR_STEERING_ANGLE 291504656 +#define ENGINE_OIL_TEMP 291504900 +#define ENGINE_RPM 291504901 +#define FUEL_LEVEL 291504903 +#define RANGE_REMAINING 291504904 +#define EV_BATTERY_LEVEL 291504905 +#define EV_BATTERY_INSTANTANEOUS_CHARGE_RATE 291504908 +#define ENV_OUTSIDE_TEMPERATURE 291505923 +#define HVAC_TEMPERATURE_VALUE_SUGGESTION 291570965 +#define STORAGE_ENCRYPTION_BINDING_SEED 292554247 +#define CLUSTER_NAVIGATION_STATE 292556600 +#define INITIAL_USER_INFO 299896583 +#define SWITCH_USER 299896584 +#define CREATE_USER 299896585 +#define REMOVE_USER 299896586 +#define USER_IDENTIFICATION_ASSOCIATION 299896587 +#define WATCHDOG_TERMINATED_PROCESS 299896626 +#define CLUSTER_REPORT_STATE 299896630 +#define HVAC_ELECTRIC_DEFROSTER_ON 320865556 +#define WINDOW_LOCK 320867268 +#define VEHICLE_MAP_SERVICE 299895808 +#define WINDOW_POS 322964416 +#define HVAC_MAX_AC_ON 354419974 +#define HVAC_MAX_DEFROST_ON 354419975 +#define HVAC_DUAL_ON 354419977 +#define HVAC_AUTO_RECIRC_ON 354419986 +#define HVAC_SEAT_VENTILATION 356517139 +#define HVAC_ELECTRIC_DEFROSTER_ON 320865556 +#define SEAT_OCCUPANCY 356518832 +#define HVAC_FAN_DIRECTION_AVAILABLE 356582673 +#define DOOR_LOCK 371198722 +#define DOOR_POS 373295872 +#define TIRE_PRESSURE 392168201 +#define CRITICALLY_LOW_TIRE_PRESSURE 392168202 +#define FRONT_FOG_LIGHTS_STATE 289410875 +#define FRONT_FOG_LIGHTS_SWITCH 289410876 +#define REAR_FOG_LIGHTS_STATE 289410877 +#define REAR_FOG_LIGHTS_SWITCH 289410878 +#define EV_CHARGE_CURRENT_DRAW_LIMIT 291508031 +#define EV_CHARGE_PERCENT_LIMIT 291508032 +#define EV_CHARGE_STATE 289410881 +#define EV_CHARGE_SWITCH 287313730 +#define EV_CHARGE_TIME_REMAINING 289410883 +#define EV_REGENERATIVE_BRAKING_STATE 289410884 +#define TRAILER_PRESENT 289410885 +#define VEHICLE_CURB_WEIGHT 289410886 +#define READING_LIGHTS_SWITCH 356519684 +#define SEAT_BELT_HEIGHT_POS 356518787 +#define WINDOW_MOVE 322964417 +#define MIRROR_AUTO_TILT_ENABLED 337644359 +#define SEAT_FORE_AFT_POS 356518789 +#define EV_BATTERY_DISPLAY_UNITS 289408515 +#define SEAT_HEIGHT_MOVE 356518796 +#define INFO_VIN 286261504 +#define SEAT_HEADREST_ANGLE_POS 356518807 +#define SEAT_HEADREST_ANGLE_MOVE 356518808 +#define ENGINE_COOLANT_TEMP 291504897 +#define MIRROR_LOCK 287312708 +#define HVAC_ACTUAL_FAN_SPEED_RPM 356517135 +#define SEAT_BELT_HEIGHT_MOVE 356518788 +#define SEAT_FORE_AFT_MOVE 356518790 +#define SEAT_BACKREST_ANGLE_2_MOVE 356518794 +#define SEAT_HEIGHT_POS 356518795 +#define SEAT_TILT_POS 356518799 +#define SEAT_DEPTH_MOVE 356518798 +#define SEAT_LUMBAR_FORE_AFT_POS 356518801 +#define SEAT_EASY_ACCESS_ENABLED 354421661 +#define SEAT_LUMBAR_FORE_AFT_MOVE 356518802 +#define SEAT_AIRBAG_ENABLED 354421662 +#define AUTOMATIC_EMERGENCY_BRAKING_ENABLED 287313920 +#define SEAT_LUMBAR_SIDE_SUPPORT_POS 356518803 +#define SEAT_LUMBAR_SIDE_SUPPORT_MOVE 356518804 +#define SEAT_MEMORY_SELECT 356518784 +#define HANDS_ON_DETECTION_DRIVER_STATE 289411095 +#define SEAT_HEADREST_HEIGHT_MOVE 356518806 +#define SEAT_HEADREST_FORE_AFT_POS 356518809 +#define MIRROR_FOLD 287312709 +#define SEAT_HEADREST_FORE_AFT_MOVE 356518810 +#define GLOVE_BOX_LOCKED 354421745 +#define SEAT_TILT_MOVE 356518800 +#define MIRROR_Z_POS 339741504 +#define HVAC_SIDE_MIRROR_HEAT 339739916 +#define SEAT_LUMBAR_VERTICAL_POS 356518817 +#define SEAT_FOOTWELL_LIGHTS_SWITCH 356518812 +#define WINDSHIELD_WIPERS_STATE 322964422 +#define SEAT_DEPTH_POS 356518797 +#define SEAT_HEADREST_HEIGHT_POS_V2 356518820 +#define EV_CURRENT_BATTERY_CAPACITY 291504909 +#define EV_BRAKE_REGENERATION_LEVEL 289408012 +#define SEAT_LUMBAR_VERTICAL_MOVE 356518818 +#define SEAT_WALK_IN_POS 356518819 +#define SEAT_BELT_BUCKLED 354421634 +#define SEAT_BACKREST_ANGLE_1_POS 356518791 +#define LANE_KEEP_ASSIST_STATE 289411081 +#define LANE_CENTERING_ASSIST_ENABLED 287313930 +#define SEAT_BACKREST_ANGLE_1_MOVE 356518792 +#define SEAT_BACKREST_ANGLE_2_POS 356518793 +#define MIRROR_AUTO_FOLD_ENABLED 337644358 +#define BLIND_SPOT_WARNING_ENABLED 287313924 +#define EV_STOPPING_MODE 289408013 +#define STEERING_WHEEL_LIGHTS_STATE 289410828 +#define MIRROR_Z_MOVE 339741505 +#define SEAT_CUSHION_SIDE_SUPPORT_POS 356518815 +#define MIRROR_Y_MOVE 339741507 +#define AUTOMATIC_EMERGENCY_BRAKING_STATE 289411073 +#define FORWARD_COLLISION_WARNING_ENABLED 287313922 +#define SEAT_MEMORY_SET 356518785 +#define HANDS_ON_DETECTION_WARNING 289411096 +#define WINDSHIELD_WIPERS_PERIOD 322964421 +#define SEAT_FOOTWELL_LIGHTS_STATE 356518811 +#define DOOR_CHILD_LOCK_ENABLED 371198723 +#define DOOR_MOVE 373295873 +#define WINDSHIELD_WIPERS_SWITCH 322964423 +#define STEERING_WHEEL_DEPTH_POS 289410016 +#define GLOVE_BOX_DOOR_POS 356518896 +#define STEERING_WHEEL_HEIGHT_POS 289410018 +#define STEERING_WHEEL_THEFT_LOCK_ENABLED 287312868 +#define SEAT_CUSHION_SIDE_SUPPORT_MOVE 356518816 +#define CABIN_LIGHTS_STATE 289410817 +#define STEERING_WHEEL_EASY_ACCESS_ENABLED 287312870 +#define CABIN_LIGHTS_SWITCH 289410818 +#define STEERING_WHEEL_HEIGHT_MOVE 289410019 +#define STEERING_WHEEL_LIGHTS_SWITCH 289410829 +#define FUEL_CONSUMPTION_UNITS_DISTANCE_OVER_VOLUME 287311364 +#define MIRROR_Y_POS 339741506 +#define LOCATION_CHARACTERIZATION 289410064 +#define EMERGENCY_LANE_KEEP_ASSIST_STATE 289411086 +#define CRUISE_CONTROL_TYPE 289411088 +#define CRUISE_CONTROL_STATE 289411089 +#define STEERING_WHEEL_LOCKED 287312869 +#define ADAPTIVE_CRUISE_CONTROL_TARGET_TIME_GAP 289411092 +#define ADAPTIVE_CRUISE_CONTROL_LEAD_VEHICLE_MEASURED_DISTANCE 289411093 +#define HANDS_ON_DETECTION_ENABLED 287313942 +#define FORWARD_COLLISION_WARNING_STATE 289411075 +#define STEERING_WHEEL_DEPTH_MOVE 289410017 +#define EMERGENCY_LANE_KEEP_ASSIST_ENABLED 287313933 +#define GENERAL_SAFETY_REGULATION_COMPLIANCE 289410887 +#define LANE_CENTERING_ASSIST_STATE 289411084 +#define CRUISE_CONTROL_ENABLED 287313935 +#define PERF_VEHICLE_SPEED_DISPLAY 291504648 +#define HVAC_TEMPERATURE_CURRENT 358614274 +#define LANE_KEEP_ASSIST_ENABLED 287313928 +#define READING_LIGHTS_STATE 356519683 +#define CRUISE_CONTROL_TARGET_SPEED 291508243 +#define ENGINE_IDLE_AUTO_STOP_ENABLED 287310624 +#define BLIND_SPOT_WARNING_STATE 339742725 +#define LANE_DEPARTURE_WARNING_ENABLED 287313926 +#define FUEL_VOLUME_DISPLAY_UNITS 289408513 +#define LANE_DEPARTURE_WARNING_STATE 289411079 + +/* (int)(0x104 | VehiclePropertyGroup:: + * VENDOR | VehiclePropertyType::STRING | VehicleArea::GLOBAL); + */ +#define VENDOR_EXTENSION_STRING_PROPERTY 554696964 +/* (int)(0x101 | VehiclePropertyGroup:: + * VENDOR | VehiclePropertyType::BOOLEAN | VehicleArea::DOOR); + */ +#define VENDOR_EXTENSION_BOOLEAN_PROPERTY 639631617 +/* (int)(0x102 | VehiclePropertyGroup:: + * VENDOR | VehiclePropertyType::FLOAT | VehicleArea::SEAT); + */ +#define VENDOR_EXTENSION_FLOAT_PROPERTY 627048706 +/* (int)(0x103 | VehiclePropertyGroup:: + * VENDOR | VehiclePropertyType::INT32 | VehicleArea::WINDOW); + */ +#define VENDOR_EXTENSION_INT_PROPERTY 591397123 +/* 0x1111 | VehiclePropertyGroup:: + * VENDOR | VehicleArea::GLOBAL | VehiclePropertyType::MIXED; + */ +#define kMixedTypePropertyForTest 568332561 + +/*vehicle_event_type: stateType in command VSTATE*/ +enum vehicle_event_type { + VEHICLE_AC = 0, + VEHICLE_AUTO_ON, + VEHICLE_AC_TEMP, + VEHICLE_FAN_SPEED, + VEHICLE_FAN_DIRECTION, + VEHICLE_RECIRC_ON, + VEHICLE_HEATER, + VEHICLE_DEFROST, + VEHICLE_HVAC_POWER_ON, + VEHICLE_MUTE, + VEHICLE_VOLUME, + VEHICLE_DOOR, + VEHICLE_RVC, + VEHICLE_LIGHT, + VEHICLE_GEAR, + VEHICLE_TURN_SIGNAL, + VEHICLE_POWER_STATE_REQ, + VEHICLE_SEAT_TEMPERATURE, + VEHICLE_UNSUPPORTED = -1, +}; + +/*vehicle_event_gear: stateValue of type VEHICLE_GEAR*/ +enum vehicle_event_gear { + VEHICLE_GEAR_NONE = 0, + VEHICLE_GEAR_PARKING, + VEHICLE_GEAR_REVERSE, + VEHICLE_GEAR_NEUTRAL, + VEHICLE_GEAR_DRIVE, + VEHICLE_GEAR_FIRST, + VEHICLE_GEAR_SECOND, + VEHICLE_GEAR_SPORT, + VEHICLE_GEAR_MANUAL_1, + VEHICLE_GEAR_MANUAL_2, + VEHICLE_GEAR_MANUAL_3, + VEHICLE_GEAR_MANUAL_4, + VEHICLE_GEAR_MANUAL_5, + VEHICLE_GEAR_MANUAL_6, +}; + +extern struct net init_net; +extern void vehicle_hw_prop_ops_register(const struct hw_prop_ops *prop_ops); +extern void vehicle_set_property(u16 prop, u8 index, u32 value, u32 param); + + +#endif diff --git a/drivers/misc/vehicle/vehicle_dummy_hw.c b/drivers/misc/vehicle/vehicle_dummy_hw.c new file mode 100644 index 000000000000..90d43bb30766 --- /dev/null +++ b/drivers/misc/vehicle/vehicle_dummy_hw.c @@ -0,0 +1,916 @@ +/* + * Copyright 2018 NXP + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include "vehicle_core.h" +#include "core.h" + +/* Miscellaneous external connector */ +#define EXTCON_VEHICLE_RPMSG_REGISTER 58 +#define EXTCON_VEHICLE_RPMSG_EVENT 59 +//VEHICLE is in state parking +#define GEAR_0 1 +//VEHICLE is in state reverse +#define GEAR_1 2 +//VEHICLE is in state neutral +#define GEAR_3 3 +//VEHICLE is in state driver +#define GEAR_2 4 +// no turn signal +#define TURN_0 0 +// left turn signal +#define TURN_1 1 +// right turn signal +#define TURN_2 2 + +#define POWER_REQ_STATE_ON 0 +#define POWER_REQ_STATE_SHUTDOWN_PREPARE 1 +#define POWER_REQ_STATE_CANCEL_SHUTDOWN 2 +#define POWER_REQ_STATE_FINISHED 3 + +#define POWER_REQ_PARAM_SHUTDOWN_IMMEDIATELY 1 +#define POWER_REQ_PARAM_CAN_SLEEP 2 +#define POWER_REQ_PARAM_SHUTDOWN_ONLY 3 +#define POWER_REQ_PARAM_SLEEP_IMMEDIATELY 4 +#define POWER_REQ_PARAM_HIBERNATE_IMMEDIATELY 5 +#define POWER_REQ_PARAM_CAN_HIBERNATE 6 + +// temperature set from hardware on Android OREO and PIE uses below indexes +#define AC_TEMP_LEFT_INDEX 1 +#define AC_TEMP_RIGHT_INDEX 4 + +// temperature set from APP on Android PIE uses below indexes +#define PIE_AC_TEMP_LEFT_INDEX 49 +#define PIE_AC_TEMP_RIGHT_INDEX 68 + +#ifdef CONFIG_EXTCON +static const unsigned int rk_vehicle_dummy_extcon_register_cables[] = { + EXTCON_VEHICLE_RPMSG_REGISTER, + EXTCON_NONE, +}; +static const unsigned int rk_vehicle_dummy_extcon_event_cables[] = { + EXTCON_VEHICLE_RPMSG_EVENT, + EXTCON_NONE, +}; +static struct extcon_dev *rg_edev; +static struct extcon_dev *ev_edev; +#endif + +static struct class *vehicle_dummy_class; + +static void mcu_set_control_commands(u32 prop, u32 area, u32 value) +{ + pr_debug("%s: prop %d, area %d, value %d\n", __func__, prop, area, value); + switch (prop) { + case HVAC_FAN_SPEED: + pr_info("set fan speed with value %d\n", value); + g_vehicle_hw->vehicle_data.fan_speed = value; + break; + case HVAC_FAN_DIRECTION: + pr_info("set fan direction with value %d\n", value); + g_vehicle_hw->vehicle_data.fan_direction = value; + break; + case HVAC_AUTO_ON: + pr_info("set fan auto on with value %d\n", value); + g_vehicle_hw->vehicle_data.auto_on = value; + break; + case HVAC_AC_ON: + pr_info("set fan ac on with value %d\n", value); + g_vehicle_hw->vehicle_data.ac_on = value; + break; + case HVAC_RECIRC_ON: + pr_info("set fan recirc on with value %d\n", value); + g_vehicle_hw->vehicle_data.recirc_on = value; + break; + case HVAC_DEFROSTER: + pr_info("set defroster index %d with value %d\n", area, value); + if (area == 1) + g_vehicle_hw->vehicle_data.defrost_left = value; + else + g_vehicle_hw->vehicle_data.defrost_right = value; + break; + case HVAC_TEMPERATURE_SET: + pr_info("set temp index %d with value %d\n", area, value); + if (area == AC_TEMP_LEFT_INDEX) + g_vehicle_hw->vehicle_data.temp_left = value; + else if (area == AC_TEMP_RIGHT_INDEX) + g_vehicle_hw->vehicle_data.temp_right = value; + break; + case HVAC_POWER_ON: + pr_info("set hvac power on with value %d\n", value); + g_vehicle_hw->vehicle_data.hvac_on = value; + break; + case AP_POWER_STATE_REPORT: + pr_info("receive power state report with value %d\n", value); + break; + case AP_POWER_STATE_REQ: + // Proper action is TBD + break; + case WATCHDOG_ALIVE: + case DISPLAY_BRIGHTNESS: + // Proper action is TBD + break; + case HVAC_SEAT_TEMPERATURE: + pr_info("set seat temperature index %d with value %d\n", area, value); + if (area == SEAT_TEMP_LEFT_INDEX) + g_vehicle_hw->vehicle_data.seat_temp_left = value; + else if (area == SEAT_TEMP_RIGHT_INDEX) + g_vehicle_hw->vehicle_data.seat_temp_right = value; + else + pr_err("unknown index: %d:%d:%d!\n", prop, area, value); + break; + case GEAR_SELECTION: + case TURN_SIGNAL_STATE: + case CURRENT_GEAR: + // GEAR is handled in mxc vehicle driver + break; + case INFO_MAKE: + case INFO_MODEL: + case POWER_POLICY_REQ: + case FUEL_DOOR_OPEN: + case EV_CHARGE_PORT_OPEN: + case EV_CHARGE_PORT_CONNECTED: + case PARKING_BRAKE_ON: + case FUEL_LEVEL_LOW: + case NIGHT_MODE: + case ABS_ACTIVE: + case TRACTION_CONTROL_ACTIVE: + case SUPPORT_CUSTOMIZE_VENDOR_PERMISSION: + case INFO_MODEL_YEAR: + case INFO_FUEL_DOOR_LOCATION: + case INFO_EV_PORT_LOCATION: + case ENGINE_OIL_LEVEL: + case IGNITION_STATE: + case HVAC_STEERING_WHEEL_HEAT: + case HVAC_TEMPERATURE_DISPLAY_UNITS: + case DISTANCE_DISPLAY_UNITS: + case TIRE_PRESSURE_DISPLAY_UNITS: + case VEHICLE_SPEED_DISPLAY_UNITS: + case HEADLIGHTS_STATE: + case HIGH_BEAM_LIGHTS_STATE: + case FOG_LIGHTS_STATE: + case PARKING_BRAKE_AUTO_APPLY: + case HAZARD_LIGHTS_STATE: + case HEADLIGHTS_SWITCH: + case HIGH_BEAM_LIGHTS_SWITCH: + case FOG_LIGHTS_SWITCH: + case HAZARD_LIGHTS_SWITCH: + case CLUSTER_SWITCH_UI: + case CLUSTER_REQUEST_DISPLAY: + case POWER_POLICY_GROUP_REQ: + case CURRENT_POWER_POLICY: + case ELECTRONIC_TOLL_COLLECTION_CARD_TYPE: + case ELECTRONIC_TOLL_COLLECTION_CARD_STATUS: + case INFO_FUEL_TYPE: + case INFO_EV_CONNECTOR_TYPE: + case INFO_DRIVER_SEAT: + case INFO_EXTERIOR_DIMENSIONS: + case INFO_MULTI_EV_PORT_LOCATIONS: + case HW_KEY_INPUT: + case HW_ROTARY_INPUT: + case HW_CUSTOM_INPUT: + case EVS_SERVICE_REQUEST: + case CLUSTER_DISPLAY_STATE: + case EPOCH_TIME: + case VHAL_HEARTBEAT: + case WHEEL_TICK: + case INFO_FUEL_CAPACITY: + case INFO_EV_BATTERY_CAPACITY: + case PERF_ODOMETER: + case PERF_VEHICLE_SPEED: + case PERF_STEERING_ANGLE: + case PERF_REAR_STEERING_ANGLE: + case ENGINE_OIL_TEMP: + case ENGINE_RPM: + case FUEL_LEVEL: + case RANGE_REMAINING: + case EV_BATTERY_LEVEL: + case EV_BATTERY_INSTANTANEOUS_CHARGE_RATE: + case ENV_OUTSIDE_TEMPERATURE: + case HVAC_TEMPERATURE_VALUE_SUGGESTION: + case STORAGE_ENCRYPTION_BINDING_SEED: + case CLUSTER_NAVIGATION_STATE: + case INITIAL_USER_INFO: + case SWITCH_USER: + case CREATE_USER: + case REMOVE_USER: + case USER_IDENTIFICATION_ASSOCIATION: + case WATCHDOG_TERMINATED_PROCESS: + case CLUSTER_REPORT_STATE: + case WINDOW_LOCK: + case VEHICLE_MAP_SERVICE: + case WINDOW_POS: + case HVAC_MAX_AC_ON: + case HVAC_MAX_DEFROST_ON: + case HVAC_DUAL_ON: + case HVAC_AUTO_RECIRC_ON: + case HVAC_SEAT_VENTILATION: + case HVAC_ELECTRIC_DEFROSTER_ON: + case SEAT_OCCUPANCY: + case HVAC_FAN_DIRECTION_AVAILABLE: + case DOOR_LOCK: + case DOOR_POS: + case TIRE_PRESSURE: + case CRITICALLY_LOW_TIRE_PRESSURE: + case FRONT_FOG_LIGHTS_STATE: + case FRONT_FOG_LIGHTS_SWITCH: + case REAR_FOG_LIGHTS_STATE: + case REAR_FOG_LIGHTS_SWITCH: + case EV_CHARGE_CURRENT_DRAW_LIMIT: + case EV_CHARGE_PERCENT_LIMIT: + case EV_CHARGE_STATE: + case EV_CHARGE_SWITCH: + case EV_CHARGE_TIME_REMAINING: + case EV_REGENERATIVE_BRAKING_STATE: + case TRAILER_PRESENT: + case VEHICLE_CURB_WEIGHT: + // Proper action is TBD + break; + case VENDOR_EXTENSION_STRING_PROPERTY: + case VENDOR_EXTENSION_BOOLEAN_PROPERTY: + case VENDOR_EXTENSION_FLOAT_PROPERTY: + case VENDOR_EXTENSION_INT_PROPERTY: + case READING_LIGHTS_SWITCH: + case SEAT_BELT_HEIGHT_POS: + case WINDOW_MOVE: + case MIRROR_AUTO_TILT_ENABLED: + case SEAT_FORE_AFT_POS: + case EV_BATTERY_DISPLAY_UNITS: + case SEAT_HEIGHT_MOVE: + case INFO_VIN: + case SEAT_HEADREST_ANGLE_POS: + case SEAT_HEADREST_ANGLE_MOVE: + case ENGINE_COOLANT_TEMP: + case MIRROR_LOCK: + case HVAC_ACTUAL_FAN_SPEED_RPM: + case SEAT_BELT_HEIGHT_MOVE: + case SEAT_FORE_AFT_MOVE: + case SEAT_BACKREST_ANGLE_2_MOVE: + case SEAT_HEIGHT_POS: + case SEAT_TILT_POS: + case SEAT_DEPTH_MOVE: + case SEAT_LUMBAR_FORE_AFT_POS: + case SEAT_EASY_ACCESS_ENABLED: + case SEAT_LUMBAR_FORE_AFT_MOVE: + case SEAT_AIRBAG_ENABLED: + case AUTOMATIC_EMERGENCY_BRAKING_ENABLED: + case SEAT_LUMBAR_SIDE_SUPPORT_POS: + case SEAT_LUMBAR_SIDE_SUPPORT_MOVE: + case SEAT_MEMORY_SELECT: + case HANDS_ON_DETECTION_DRIVER_STATE: + case SEAT_HEADREST_HEIGHT_MOVE: + case SEAT_HEADREST_FORE_AFT_POS: + case MIRROR_FOLD: + case SEAT_HEADREST_FORE_AFT_MOVE: + case GLOVE_BOX_LOCKED: + case SEAT_TILT_MOVE: + case MIRROR_Z_POS: + case HVAC_SIDE_MIRROR_HEAT: + case SEAT_LUMBAR_VERTICAL_POS: + case SEAT_FOOTWELL_LIGHTS_SWITCH: + case WINDSHIELD_WIPERS_STATE: + case SEAT_DEPTH_POS: + case SEAT_HEADREST_HEIGHT_POS_V2: + case EV_CURRENT_BATTERY_CAPACITY: + case EV_BRAKE_REGENERATION_LEVEL: + case SEAT_LUMBAR_VERTICAL_MOVE: + case SEAT_WALK_IN_POS: + case SEAT_BELT_BUCKLED: + case SEAT_BACKREST_ANGLE_1_POS: + case LANE_KEEP_ASSIST_STATE: + case LANE_CENTERING_ASSIST_ENABLED: + case SEAT_BACKREST_ANGLE_1_MOVE: + case SEAT_BACKREST_ANGLE_2_POS: + case MIRROR_AUTO_FOLD_ENABLED: + case BLIND_SPOT_WARNING_ENABLED: + case EV_STOPPING_MODE: + case STEERING_WHEEL_LIGHTS_STATE: + case MIRROR_Z_MOVE: + case SEAT_CUSHION_SIDE_SUPPORT_POS: + case MIRROR_Y_MOVE: + case AUTOMATIC_EMERGENCY_BRAKING_STATE: + case FORWARD_COLLISION_WARNING_ENABLED: + case SEAT_MEMORY_SET: + case HANDS_ON_DETECTION_WARNING: + case WINDSHIELD_WIPERS_PERIOD: + case SEAT_FOOTWELL_LIGHTS_STATE: + case DOOR_CHILD_LOCK_ENABLED: + case DOOR_MOVE: + case WINDSHIELD_WIPERS_SWITCH: + case STEERING_WHEEL_DEPTH_POS: + case GLOVE_BOX_DOOR_POS: + case STEERING_WHEEL_HEIGHT_POS: + case STEERING_WHEEL_THEFT_LOCK_ENABLED: + case SEAT_CUSHION_SIDE_SUPPORT_MOVE: + case CABIN_LIGHTS_STATE: + case STEERING_WHEEL_EASY_ACCESS_ENABLED: + case CABIN_LIGHTS_SWITCH: + case STEERING_WHEEL_HEIGHT_MOVE: + case STEERING_WHEEL_LIGHTS_SWITCH: + case FUEL_CONSUMPTION_UNITS_DISTANCE_OVER_VOLUME: + case MIRROR_Y_POS: + case LOCATION_CHARACTERIZATION: + case EMERGENCY_LANE_KEEP_ASSIST_STATE: + case CRUISE_CONTROL_TYPE: + case CRUISE_CONTROL_STATE: + case STEERING_WHEEL_LOCKED: + case ADAPTIVE_CRUISE_CONTROL_TARGET_TIME_GAP: + case ADAPTIVE_CRUISE_CONTROL_LEAD_VEHICLE_MEASURED_DISTANCE: + case HANDS_ON_DETECTION_ENABLED: + case FORWARD_COLLISION_WARNING_STATE: + case STEERING_WHEEL_DEPTH_MOVE: + case EMERGENCY_LANE_KEEP_ASSIST_ENABLED: + case GENERAL_SAFETY_REGULATION_COMPLIANCE: + case LANE_CENTERING_ASSIST_STATE: + case CRUISE_CONTROL_ENABLED: + case PERF_VEHICLE_SPEED_DISPLAY: + case HVAC_TEMPERATURE_CURRENT: + case LANE_KEEP_ASSIST_ENABLED: + case READING_LIGHTS_STATE: + case CRUISE_CONTROL_TARGET_SPEED: + case ENGINE_IDLE_AUTO_STOP_ENABLED: + case BLIND_SPOT_WARNING_STATE: + case LANE_DEPARTURE_WARNING_ENABLED: + case FUEL_VOLUME_DISPLAY_UNITS: + case LANE_DEPARTURE_WARNING_STATE: + case kMixedTypePropertyForTest: + // Proper action is TBD + break; + default: + pr_err("this type is not correct: %d:%d:%d!\n", prop, area, value); + } +} + +static ssize_t turn_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%u\n", g_vehicle_hw->vehicle_data.turn); +} + +/* echo 0/1/2(none/left/right) > /sys/devices/platform/vehicle-dummy/turn*/ +static ssize_t turn_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + u32 turn; + + if (!size) + return -EINVAL; + turn = strtoul(buf, NULL, 10); + if (turn != TURN_0 && turn != TURN_1 && turn != TURN_2) { + pr_err("input value is not correct, please type correct one\n"); + return -EINVAL; + } + if (turn != g_vehicle_hw->vehicle_data.turn) { + g_vehicle_hw->vehicle_data.turn = turn; + vehicle_set_property(VEHICLE_TURN_SIGNAL, 0, turn, 0); + } + return size; +} +static DEVICE_ATTR_RW(turn); + +static ssize_t gear_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%u\n", g_vehicle_hw->vehicle_data.gear); +} + +/*echo 1/2/4(parking/reverse/drive) > /sys/devices/platform/vehicle-dummy/gear*/ +static ssize_t gear_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + u32 gear; + + if (!size) + return -EINVAL; + gear = strtoul(buf, NULL, 10); + if (gear != g_vehicle_hw->vehicle_data.gear) { + g_vehicle_hw->vehicle_data.gear = gear; + vehicle_set_property(VEHICLE_GEAR, 0, gear, 0); + if (gear != GEAR_0 && gear != GEAR_1 && gear != GEAR_2) { + pr_err("input value is not correct, please type correct one\n"); + return -EINVAL; + } +#ifdef CONFIG_EXTCON + if (gear == VEHICLE_GEAR_DRIVE) + extcon_set_state_sync(ev_edev, EXTCON_VEHICLE_RPMSG_EVENT, 0); + else if (gear == VEHICLE_GEAR_REVERSE) + extcon_set_state_sync(ev_edev, EXTCON_VEHICLE_RPMSG_EVENT, 1); +#endif + } + return size; +} + +static DEVICE_ATTR_RW(gear); + +static ssize_t temp_left_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%d\n", g_vehicle_hw->vehicle_data.temp_left); +} + +/*echo 1100713529 > /sys/devices/platform/vehicle-dummy/temp_left*/ +static ssize_t temp_left_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + u32 temp; + + if (!size) + return -EINVAL; + temp = strtoul(buf, NULL, 10); + if (temp != g_vehicle_hw->vehicle_data.temp_left) { + g_vehicle_hw->vehicle_data.temp_left = temp; + vehicle_set_property(VEHICLE_AC_TEMP, AC_TEMP_LEFT_INDEX, temp, 0); + } + return size; +} + +static DEVICE_ATTR_RW(temp_left); + +static ssize_t temp_right_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%d\n", g_vehicle_hw->vehicle_data.temp_right); +} + +/*echo 1100713529 > /sys/devices/platform/vehicle-dummy/temp_right*/ +static ssize_t temp_right_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + u32 temp; + + if (!size) + return -EINVAL; + temp = strtoul(buf, NULL, 10); + if (temp != g_vehicle_hw->vehicle_data.temp_right) { + g_vehicle_hw->vehicle_data.temp_right = temp; + vehicle_set_property(VEHICLE_AC_TEMP, AC_TEMP_RIGHT_INDEX, temp, 0); + } + return size; +} + +static DEVICE_ATTR_RW(temp_right); + +static ssize_t seat_temp_left_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%d\n", g_vehicle_hw->vehicle_data.seat_temp_left); +} + +/*echo 0/1/2/3 > /sys/devices/platform/vehicle-dummy/seat_temp_left*/ +static ssize_t seat_temp_left_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + u32 seat_temp; + + if (!size) + return -EINVAL; + seat_temp = strtoul(buf, NULL, 10); + + if (seat_temp != g_vehicle_hw->vehicle_data.seat_temp_left) { + g_vehicle_hw->vehicle_data.seat_temp_left = seat_temp; + vehicle_set_property(VEHICLE_SEAT_TEMPERATURE, SEAT_TEMP_LEFT_INDEX, seat_temp, 0); + } + return size; +} + +static DEVICE_ATTR_RW(seat_temp_left); + +static ssize_t seat_temp_right_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%d\n", g_vehicle_hw->vehicle_data.seat_temp_right); +} + +/*echo 0/1/2/3 > /sys/devices/platform/vehicle-dummy/seat_temp_right*/ +static ssize_t seat_temp_right_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + u32 seat_temp; + + if (!size) + return -EINVAL; + seat_temp = strtoul(buf, NULL, 10); + if (seat_temp != SEAT_TEMP_0 && seat_temp != SEAT_TEMP_1 && + seat_temp != SEAT_TEMP_2 && seat_temp != SEAT_TEMP_3) { + pr_err("input value is not correct, please type correct one\n"); + return -EINVAL; + } + if (seat_temp != g_vehicle_hw->vehicle_data.seat_temp_right) { + g_vehicle_hw->vehicle_data.seat_temp_right = seat_temp; + vehicle_set_property(VEHICLE_SEAT_TEMPERATURE, SEAT_TEMP_RIGHT_INDEX, seat_temp, 0); + } + return size; +} + +static DEVICE_ATTR_RW(seat_temp_right); + +static ssize_t fan_direction_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%u\n", g_vehicle_hw->vehicle_data.fan_direction); +} + +/*echo 1/2/3/6 > /sys/devices/platform/vehicle-dummy/fan_direction*/ +static ssize_t fan_direction_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + u32 fan_direction; + + if (!size) + return -EINVAL; + fan_direction = strtoul(buf, NULL, 10); + + if (fan_direction != g_vehicle_hw->vehicle_data.fan_direction) { + g_vehicle_hw->vehicle_data.fan_direction = fan_direction; + vehicle_set_property(VEHICLE_FAN_DIRECTION, 0, fan_direction, 0); + } + return size; +} + +static DEVICE_ATTR_RW(fan_direction); + +static ssize_t fan_speed_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%u\n", g_vehicle_hw->vehicle_data.fan_speed); +} + +/*echo 1/2/3/4/5/6 > /sys/devices/platform/vehicle-dummy/fan_speed*/ +static ssize_t fan_speed_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + u32 fan_speed; + + if (!size) + return -EINVAL; + fan_speed = strtoul(buf, NULL, 10); + + if (fan_speed != g_vehicle_hw->vehicle_data.fan_speed) { + g_vehicle_hw->vehicle_data.fan_speed = fan_speed; + vehicle_set_property(VEHICLE_FAN_SPEED, 0, fan_speed, 0); + } + return size; +} + +static DEVICE_ATTR_RW(fan_speed); + +static ssize_t defrost_left_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%u\n", g_vehicle_hw->vehicle_data.defrost_left); +} + +/*echo 0/1 > /sys/devices/platform/vehicle-dummy/defrost_left*/ +static ssize_t defrost_left_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + u32 defrost; + + if (!size) + return -EINVAL; + + defrost = strtoul(buf, NULL, 10); + + if (defrost != g_vehicle_hw->vehicle_data.defrost_left) { + g_vehicle_hw->vehicle_data.defrost_left = defrost; + vehicle_set_property(VEHICLE_DEFROST, 1, defrost, 0); + } + return size; +} + +static DEVICE_ATTR_RW(defrost_left); + +static ssize_t defrost_right_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%u\n", g_vehicle_hw->vehicle_data.defrost_right); +} + +/*echo 0/1 > /sys/devices/platform/vehicle-dummy/defrost_right*/ +static ssize_t defrost_right_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + u32 defrost; + + if (!size) + return -EINVAL; + + defrost = strtoul(buf, NULL, 10); + if (defrost != DEFROST_ON && defrost != DEFROST_OFF) { + pr_err("input value is not correct, please type correct one\n"); + return -EINVAL; + } + if (defrost != g_vehicle_hw->vehicle_data.defrost_right) { + g_vehicle_hw->vehicle_data.defrost_right = defrost; + vehicle_set_property(VEHICLE_DEFROST, 2, defrost, 0); + } + return size; +} + +static DEVICE_ATTR_RW(defrost_right); + +static ssize_t ac_on_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%u\n", g_vehicle_hw->vehicle_data.ac_on); +} + +/*echo 0/1 > /sys/devices/platform/vehicle-dummy/ac_on*/ +static ssize_t ac_on_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + u32 ac_on; + + if (!size) + return -EINVAL; + + ac_on = strtoul(buf, NULL, 10); + + if (ac_on != g_vehicle_hw->vehicle_data.ac_on) { + g_vehicle_hw->vehicle_data.ac_on = ac_on; + vehicle_set_property(VEHICLE_AC, 0, ac_on, 0); + } + return size; +} + +static DEVICE_ATTR_RW(ac_on); + +static ssize_t auto_on_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%u\n", g_vehicle_hw->vehicle_data.auto_on); +} + +/*echo 0/1 > /sys/devices/platform/vehicle-dummy/auto_on*/ +static ssize_t auto_on_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + u32 auto_on; + + if (!size) + return -EINVAL; + + auto_on = strtoul(buf, NULL, 10); + + if (auto_on != g_vehicle_hw->vehicle_data.auto_on) { + g_vehicle_hw->vehicle_data.auto_on = auto_on; + vehicle_set_property(VEHICLE_AUTO_ON, VEHICLE_AREA_SEAT_ROW_1_LEFT, + auto_on, 0); + vehicle_set_property(VEHICLE_AUTO_ON, VEHICLE_AREA_SEAT_ROW_1_RIGHT, + auto_on, 0); + vehicle_set_property(VEHICLE_AUTO_ON, VEHICLE_AREA_SEAT_ROW_2_LEFT, + auto_on, 0); + vehicle_set_property(VEHICLE_AUTO_ON, VEHICLE_AREA_SEAT_ROW_2_RIGHT, + auto_on, 0); + vehicle_set_property(VEHICLE_AUTO_ON, VEHICLE_AREA_SEAT_ROW_2_CENTER, + auto_on, 0); + } + return size; +} + +static DEVICE_ATTR_RW(auto_on); + +static ssize_t hvac_on_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%u\n", g_vehicle_hw->vehicle_data.hvac_on); +} + +/*echo 0/1 > /sys/devices/platform/vehicle-dummy/hvac_on*/ +static ssize_t hvac_on_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + u32 hvac_on; + + if (!size) + return -EINVAL; + + hvac_on = strtoul(buf, NULL, 10); + + if (hvac_on != g_vehicle_hw->vehicle_data.hvac_on) { + g_vehicle_hw->vehicle_data.hvac_on = hvac_on; + vehicle_set_property(VEHICLE_HVAC_POWER_ON, VEHICLE_AREA_SEAT_ROW_1_LEFT, + hvac_on, 0); + vehicle_set_property(VEHICLE_HVAC_POWER_ON, VEHICLE_AREA_SEAT_ROW_1_RIGHT, + hvac_on, 0); + } + return size; +} + +static DEVICE_ATTR_RW(hvac_on); + +static ssize_t recirc_on_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%u\n", g_vehicle_hw->vehicle_data.recirc_on); +} + +/* echo 0/1 > /sys/devices/platform/vehicle-dummy/recirc_on*/ +static ssize_t recirc_on_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + u32 recirc_on; + + if (!size) + return -EINVAL; + + recirc_on = strtoul(buf, NULL, 10); + + if (recirc_on != g_vehicle_hw->vehicle_data.recirc_on) { + g_vehicle_hw->vehicle_data.recirc_on = recirc_on; + vehicle_set_property(VEHICLE_RECIRC_ON, 0, recirc_on, 0); + } + return size; +} + +static DEVICE_ATTR_RW(recirc_on); + +static ssize_t power_req_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%u %u\n", g_vehicle_hw->vehicle_data.power_req_state, + g_vehicle_hw->vehicle_data.power_req_param); +} + +/* echo "1 1" > /sys/devices/platform/vehicle-dummy/power_req */ +static ssize_t power_req_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + char *ret; + u32 state; + u32 param; + + if (size < 4) { + pr_err("input command format is not correct, please type command like this: \"1 1\"\n"); + return -EINVAL; + } + + state = strtoul(buf, NULL, 10); + ret = strrchr(buf, ' '); + param = strtoul(ret + 1, NULL, 10); + + if (state != POWER_REQ_STATE_ON && + state != POWER_REQ_STATE_SHUTDOWN_PREPARE && + state != POWER_REQ_STATE_CANCEL_SHUTDOWN && + state != POWER_REQ_STATE_FINISHED) { + pr_err("input power request state is not correct, please type correct one\n"); + return -EINVAL; + } + + if (param != POWER_REQ_PARAM_SHUTDOWN_IMMEDIATELY && + param != POWER_REQ_PARAM_CAN_SLEEP && + param != POWER_REQ_PARAM_SHUTDOWN_ONLY && + param != POWER_REQ_PARAM_SLEEP_IMMEDIATELY && + param != POWER_REQ_PARAM_HIBERNATE_IMMEDIATELY && + param != POWER_REQ_PARAM_CAN_HIBERNATE) { + pr_err("input power request param is not correct, please type correct one\n"); + return -EINVAL; + } + + g_vehicle_hw->vehicle_data.power_req_state = state; + g_vehicle_hw->vehicle_data.power_req_param = param; + vehicle_set_property(VEHICLE_POWER_STATE_REQ, 0, state, param); + pr_info("power control with state: %d, param: %d\n", state, param); + return size; +} + +static DEVICE_ATTR_RW(power_req); + +static const struct hw_prop_ops hw_prop_mcu_ops = { + .set_control_commands = mcu_set_control_commands, +}; + +static int vehicle_dummy_hw_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct vehicle_dummy *vehicle_dummy; + int err; + + if (IS_ERR(g_vehicle_hw)) + return PTR_ERR(g_vehicle_hw); + + vehicle_dummy = devm_kzalloc(dev, sizeof(*vehicle_dummy), GFP_KERNEL); + if (vehicle_dummy == NULL) + return -ENOMEM; + + platform_set_drvdata(pdev, vehicle_dummy); + + vehicle_dummy_class = class_create(THIS_MODULE, "vehicle_dummy_hw"); + if (IS_ERR(vehicle_dummy_class)) { + dev_err(dev, "failed to create class.\n"); + return PTR_ERR(vehicle_dummy_class); + } + + err = device_create_file(dev, &dev_attr_recirc_on) || + device_create_file(dev, &dev_attr_hvac_on) || + device_create_file(dev, &dev_attr_auto_on) || + device_create_file(dev, &dev_attr_ac_on) || + device_create_file(dev, &dev_attr_defrost_right) || + device_create_file(dev, &dev_attr_defrost_left) || + device_create_file(dev, &dev_attr_fan_speed) || + device_create_file(dev, &dev_attr_fan_direction) || + device_create_file(dev, &dev_attr_temp_left) || + device_create_file(dev, &dev_attr_temp_right) || + device_create_file(dev, &dev_attr_gear) || + device_create_file(dev, &dev_attr_power_req) || + device_create_file(dev, &dev_attr_seat_temp_left) || + device_create_file(dev, &dev_attr_seat_temp_right) || + device_create_file(dev, &dev_attr_turn); + if (err) + return err; + + vehicle_hw_prop_ops_register(&hw_prop_mcu_ops); + +#ifdef CONFIG_EXTCON + rg_edev = devm_extcon_dev_allocate(dev, rk_vehicle_dummy_extcon_register_cables); + if (IS_ERR(rg_edev)) + dev_err(dev, "failed to allocate extcon device\n"); + + err = devm_extcon_dev_register(dev, rg_edev); + if (err < 0) + dev_err(dev, "failed to register extcon device\n"); + + ev_edev = devm_extcon_dev_allocate(dev, rk_vehicle_dummy_extcon_event_cables); + if (IS_ERR(ev_edev)) + dev_err(dev, "failed to allocate extcon device\n"); + + err = devm_extcon_dev_register(dev, ev_edev); + if (err < 0) + dev_err(dev, "failed to register extcon device\n"); +#endif + + return 0; +} + +static int vehicle_dummy_hw_remove(struct platform_device *pdev) +{ + class_destroy(vehicle_dummy_class); + + return 0; +} + +static struct platform_driver vehicle_dummy_hw_driver = { + .probe = vehicle_dummy_hw_probe, + .remove = vehicle_dummy_hw_remove, + .driver = { + .name = "vehicle-dummy-hw", + } +}; + +static struct platform_device *dummy_pdev; + +static int vehicle_dummy_init(void) +{ + int err; + + dummy_pdev = platform_device_alloc("vehicle-dummy-hw", -1); + if (!dummy_pdev) { + pr_err("Failed to allocate dummy vehicle device\n"); + return -ENODEV; + } + + err = platform_device_add(dummy_pdev); + if (err != 0) { + pr_err("Failed to register dummy regulator device: %d\n", err); + platform_device_put(dummy_pdev); + return err; + } + + err = platform_driver_register(&vehicle_dummy_hw_driver); + if (err) { + pr_err("Failed to register dummy vehicle driver\n"); + platform_device_unregister(dummy_pdev); + return err; + } + return 0; +} + +static void __exit vehicle_dummy_exit(void) +{ + platform_driver_unregister(&vehicle_dummy_hw_driver); + platform_device_unregister(dummy_pdev); +} + +late_initcall(vehicle_dummy_init); +module_exit(vehicle_dummy_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("NXP Semiconductor"); +MODULE_DESCRIPTION("VEHICLE DUMMY HW"); diff --git a/drivers/misc/vehicle/vehicle_protocol_callback.c b/drivers/misc/vehicle/vehicle_protocol_callback.c new file mode 100644 index 000000000000..8a639a59656b --- /dev/null +++ b/drivers/misc/vehicle/vehicle_protocol_callback.c @@ -0,0 +1,336 @@ +/* + * Copyright 2018 NXP + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include "pb.h" +#include "pb_encode.h" +#include "pb_decode.h" +#include "vehicle_core.h" +#include "vehiclehalproto.pb.h" +#include "vehicle_protocol_callback.h" + +//VehiclePropertyType +#define VEHICLEPROPERTYTYPE_STRING 1048576 +#define VEHICLEPROPERTYTYPE_BOOLEAN 2097152 +#define VEHICLEPROPERTYTYPE_INT32 4194304 +#define VEHICLEPROPERTYTYPE_INT32_VEC 4259840 +#define VEHICLEPROPERTYTYPE_INT64 5242880 +#define VEHICLEPROPERTYTYPE_INT64_VEC 5308416 +#define VEHICLEPROPERTYTYPE_FLOAT 6291456 +#define VEHICLEPROPERTYTYPE_FLOAT_VEC 6356992 +#define VEHICLEPROPERTYTYPE_BYTES 7340032 +#define VEHICLEPROPERTYTYPE_MIXED 14680064 + + +bool decode_prop_callback(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + emulator_VehiclePropGet propget; + + if (!pb_decode(stream, emulator_VehiclePropGet_fields, &propget)) { + pr_info("%s decode failed\n", __func__); + return false; + } + + return true; +} + +bool decode_config_callback(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + emulator_VehiclePropConfig propconfigure; + + propconfigure.area_configs.funcs.decode = &decode_area_configs_callback; + propconfigure.config_array.funcs.decode = &decode_config_array_callback; + propconfigure.config_string.funcs.decode = &decode_config_string_callback; + + if (!pb_decode(stream, emulator_VehiclePropGet_fields, &propconfigure)) + return false; + + return true; +} + +bool decode_int32_values_callback(pb_istream_t *stream, const pb_field_t *field, + void **arg) +{ + uint64_t value; + + if (!pb_decode_svarint(stream, &value)) + return false; + *(u32 *)*arg = (u32)value; + return true; +} + +bool decode_int64_values_callback(pb_istream_t *stream, const pb_field_t *field, + void **arg) +{ + uint64_t value; + + if (!pb_decode_svarint(stream, &value)) + return false; + *(u64 *)*arg = (u64)value; + return true; +} + +bool decode_float_values_callback(pb_istream_t *stream, const pb_field_t *field, + void **arg) +{ + uint32_t value; + + if (!pb_decode_fixed32(stream, &value)) { + pr_info("float_values_callback failed\n"); + return false; + } + + *(u32 *)*arg = value; + return true; +} + +bool decode_value_callback(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + struct vehicle_property_set *data = (struct vehicle_property_set *)(*arg); + emulator_VehiclePropValue propvalue; + + propvalue.int32_values.funcs.decode = &decode_int32_values_callback; + propvalue.int32_values.arg = &data->value; + propvalue.float_values.funcs.decode = &decode_float_values_callback; + propvalue.float_values.arg = &data->value; + propvalue.string_value.funcs.decode = &decode_config_string_callback; + propvalue.string_value.arg = &data->value; + propvalue.int64_values.funcs.decode = &decode_int64_values_callback; + propvalue.int64_values.arg = &data->value; + // Callback list must be extended if vhal starts to + // using new property variable type. + + if (!pb_decode(stream, emulator_VehiclePropValue_fields, &propvalue)) + return false; + + data->prop = propvalue.prop; + data->area_id = propvalue.area_id; + + return true; +} + +bool decode_area_configs_callback(pb_istream_t *stream, const pb_field_t *field, + void **arg) +{ + emulator_VehicleAreaConfig vehicle_area; + + if (!pb_decode(stream, emulator_VehicleAreaConfig_fields, &vehicle_area)) + return false; + return true; +} + +bool decode_config_array_callback(pb_istream_t *stream, const pb_field_t *field, + void **arg) +{ + uint64_t value; + + if (!pb_decode_varint(stream, &value)) + return false; + return true; +} + +bool decode_config_string_callback(pb_istream_t *stream, const pb_field_t *field, + void **arg) +{ + char *buffer; + + buffer = kmalloc(1024, GFP_KERNEL); + if (!buffer) { + pr_warn("Failed allocating memory for decode\n"); + return false; + } + + if (stream->bytes_left >= 1024) { + kfree(buffer); + return false; + } + + if (!pb_read(stream, buffer, stream->bytes_left)) { + kfree(buffer); + return false; + } + + kfree(buffer); + + return true; +} + +bool encode_prop_callback(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + emulator_VehiclePropGet propget; + + //need fill arg emulator_VehiclePropGet if have this filed + if (!pb_encode(stream, emulator_VehiclePropGet_fields, &propget)) { + pr_info("%s encode failed\n", __func__); + return false; + } + + return true; +} + +bool encode_area_configs_callback(pb_ostream_t *stream, const pb_field_t *field, + void * const *arg) +{ + emulator_VehicleAreaConfig vehicle_area; + + if (!pb_encode(stream, emulator_VehicleAreaConfig_fields, &vehicle_area)) + return false; + return true; +} + +bool encode_config_array_callback(pb_ostream_t *stream, const pb_field_t *field, + void * const *arg) +{ + return pb_encode_tag_for_field(stream, field) && + pb_encode_fixed32(stream, *arg); +} + +bool encode_config_string_callback(pb_ostream_t *stream, const pb_field_t *field, + void * const *arg) +{ + return pb_encode_tag_for_field(stream, field) && + pb_encode_string(stream, *arg, strlen(*arg)); +} + +bool encode_config_callback(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + emulator_VehiclePropConfig propconfigure; + + propconfigure.area_configs.funcs.encode = &encode_area_configs_callback; + propconfigure.config_array.funcs.encode = &encode_config_array_callback; + propconfigure.config_string.funcs.encode = &encode_config_string_callback; + + if (!pb_encode(stream, emulator_VehiclePropGet_fields, &propconfigure)) + return false; + + return true; +} + +bool encode_int32_values_callback(pb_ostream_t *stream, const pb_field_t *field, + void * const *arg) +{ + u32 *value = (u32 *)*arg; + + if (!pb_encode_tag_for_field(stream, field)) + return false; + + return pb_encode_svarint(stream, *value); +} + +bool encode_power_state_value_callback(pb_ostream_t *stream, const pb_field_t *field, + void * const *arg) +{ + u32 *value = (u32 *)*arg; + + if (!pb_encode_tag_for_field(stream, field)) + return false; + if (!pb_encode_svarint(stream, *value)) + return false; + value++; + if (!pb_encode_tag_for_field(stream, field)) + return false; + if (!pb_encode_svarint(stream, *value)) + return false; + + return true; +} + +bool encode_int64_values_callback(pb_ostream_t *stream, const pb_field_t *field, + void * const *arg) +{ + u64 *value = (u64 *)*arg; + + if (!pb_encode_tag_for_field(stream, field)) + return false; + return pb_encode_svarint(stream, *value); +} + +bool encode_fix32_values_callback(pb_ostream_t *stream, const pb_field_t *field, + void * const *arg) +{ + if (!pb_encode_tag_for_field(stream, field)) + return false; + return pb_encode_fixed32(stream, (u32 *)*arg); +} + +bool encode_power_state_callback(pb_ostream_t *stream, const pb_field_t *field, + void * const *arg) +{ + struct vehicle_power_req *data = (struct vehicle_power_req *)(*arg); + emulator_VehiclePropValue propvalue = {}; + + propvalue.prop = data->prop; + propvalue.has_value_type = true; + propvalue.has_timestamp = true; + propvalue.timestamp = 0; +#ifndef CONFIG_VEHICLE_DRIVER_OREO + propvalue.has_status = true; + propvalue.status = 0; +#endif + propvalue.int32_values.funcs.encode = &encode_power_state_value_callback; + propvalue.int32_values.arg = &data->state; + propvalue.value_type = VEHICLEPROPERTYTYPE_INT32; + + pb_encode_tag_for_field(stream, field); + if (!pb_encode_submessage(stream, emulator_VehiclePropValue_fields, &propvalue)) { + pr_info("%s encode submessage failed\n", __func__); + return false; + } + return true; +} + +bool encode_value_callback(pb_ostream_t *stream, const pb_field_t *field, + void * const *arg) +{ + struct vehicle_property_set *data = (struct vehicle_property_set *)(*arg); + emulator_VehiclePropValue propvalue = {}; + + propvalue.prop = data->prop; + propvalue.has_value_type = true; + propvalue.has_timestamp = true; + propvalue.timestamp = 0; + propvalue.has_area_id = true; + propvalue.area_id = data->area_id; +#ifndef CONFIG_VEHICLE_DRIVER_OREO + propvalue.has_status = true; + propvalue.status = 0; +#endif + if (data->prop == HVAC_TEMPERATURE_SET) { + propvalue.float_values.funcs.encode = &encode_fix32_values_callback; + propvalue.float_values.arg = &data->value; + propvalue.value_type = VEHICLEPROPERTYTYPE_FLOAT; + } else if (HVAC_FAN_SPEED == data->prop || HVAC_FAN_DIRECTION == data->prop || + HVAC_AUTO_ON == data->prop || HVAC_AC_ON == data->prop || + HVAC_RECIRC_ON == data->prop || HVAC_POWER_ON == data->prop || + HVAC_DEFROSTER == data->prop || TURN_SIGNAL_STATE == data->prop || + GEAR_SELECTION == data->prop || HVAC_SEAT_TEMPERATURE == data->prop) { + propvalue.int32_values.funcs.encode = &encode_int32_values_callback; + propvalue.int32_values.arg = &data->value; + propvalue.value_type = VEHICLEPROPERTYTYPE_INT32; + } + + //propvalue.int64_values.funcs.encode = &encode_int64_values_callback; + //propvalue.string_value.funcs.encode = &encode_config_string_callback; + //propvalue.bytes_value.funcs.encode = &encode_config_string_callback; + pb_encode_tag_for_field(stream, field); + if (!pb_encode_submessage(stream, emulator_VehiclePropValue_fields, &propvalue)) { + pr_info("%s encode submessage failed\n", __func__); + return false; + } + return true; +} + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("NXP Semiconductor"); + diff --git a/drivers/misc/vehicle/vehicle_protocol_callback.h b/drivers/misc/vehicle/vehicle_protocol_callback.h new file mode 100644 index 000000000000..0610781471d8 --- /dev/null +++ b/drivers/misc/vehicle/vehicle_protocol_callback.h @@ -0,0 +1,79 @@ +/* + * Copyright 2018 NXP + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +#ifndef VEHICLE_PROTOCOL_CALLBACK_H_ +#define VEHICLE_PROTOCOL_CALLBACK_H_ + +bool decode_prop_callback(pb_istream_t *stream, + const pb_field_t *field, + void **arg); +bool decode_config_callback(pb_istream_t *stream, + const pb_field_t *field, + void **arg); +bool decode_value_callback(pb_istream_t *stream, + const pb_field_t *field, + void **arg); +bool decode_config_string_callback(pb_istream_t *stream, + const pb_field_t *field, + void **arg); +bool decode_config_array_callback(pb_istream_t *stream, + const pb_field_t *field, + void **arg); +bool decode_area_configs_callback(pb_istream_t *stream, + const pb_field_t *field, + void **arg); +bool decode_float_values_callback(pb_istream_t *stream, + const pb_field_t *field, + void **arg); +bool decode_int64_values_callback(pb_istream_t *stream, + const pb_field_t *field, + void **arg); +bool decode_int32_values_callback(pb_istream_t *stream, + const pb_field_t *field, + void **arg); +bool encode_value_callback(pb_ostream_t *stream, + const pb_field_t *field, + void * const *arg); +bool encode_prop_callback(pb_ostream_t *stream, + const pb_field_t *field, + void * const *arg); +bool encode_area_configs_callback(pb_ostream_t *stream, + const pb_field_t *field, + void * const *arg); +bool encode_config_array_callback(pb_ostream_t *stream, + const pb_field_t *field, + void * const *arg); +bool encode_config_string_callback(pb_ostream_t *stream, + const pb_field_t *field, + void * const *arg); +bool encode_config_callback(pb_ostream_t *stream, + const pb_field_t *field, + void * const *arg); +bool encode_int32_values_callback(pb_ostream_t *stream, + const pb_field_t *field, + void * const *arg); +bool encode_int64_values_callback(pb_ostream_t *stream, + const pb_field_t *field, + void * const *arg); +bool encode_fix32_values_callback(pb_ostream_t *stream, + const pb_field_t *field, + void * const *arg); +bool encode_power_state_callback(pb_ostream_t *stream, + const pb_field_t *field, + void * const *arg); +bool encode_power_state_value_callback(pb_ostream_t *stream, + const pb_field_t *field, + void * const *arg); + +#endif diff --git a/drivers/misc/vehicle/vehiclehalproto.pb.c b/drivers/misc/vehicle/vehiclehalproto.pb.c new file mode 100644 index 000000000000..3face5a2179d --- /dev/null +++ b/drivers/misc/vehicle/vehiclehalproto.pb.c @@ -0,0 +1,166 @@ +/* + * Copyright 2018 NXP + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include "vehiclehalproto.pb.h" + +const pb_field_t emulator_VehicleAreaConfig_fields[8] = { + PB_FIELD2(1, INT32, REQUIRED, STATIC, FIRST, + emulator_VehicleAreaConfig, area_id, area_id, 0), + PB_FIELD2(2, SINT32, OPTIONAL, STATIC, OTHER, + emulator_VehicleAreaConfig, min_int32_value, area_id, 0), + PB_FIELD2(3, SINT32, OPTIONAL, STATIC, OTHER, + emulator_VehicleAreaConfig, max_int32_value, min_int32_value, 0), + PB_FIELD2(4, SINT64, OPTIONAL, STATIC, OTHER, + emulator_VehicleAreaConfig, min_int64_value, max_int32_value, 0), + PB_FIELD2(5, SINT64, OPTIONAL, STATIC, OTHER, + emulator_VehicleAreaConfig, max_int64_value, min_int64_value, 0), + PB_FIELD2(6, FLOAT, OPTIONAL, STATIC, OTHER, + emulator_VehicleAreaConfig, min_float_value, max_int64_value, 0), + PB_FIELD2(7, FLOAT, OPTIONAL, STATIC, OTHER, + emulator_VehicleAreaConfig, max_float_value, min_float_value, 0), + PB_LAST_FIELD +}; + +const pb_field_t emulator_VehiclePropConfig_fields[12] = { + PB_FIELD2(1, INT32, REQUIRED, STATIC, FIRST, + emulator_VehiclePropConfig, prop, prop, 0), + PB_FIELD2(2, INT32, OPTIONAL, STATIC, OTHER, + emulator_VehiclePropConfig, access, prop, 0), + PB_FIELD2(3, INT32, OPTIONAL, STATIC, OTHER, + emulator_VehiclePropConfig, change_mode, access, 0), + PB_FIELD2(4, INT32, OPTIONAL, STATIC, OTHER, + emulator_VehiclePropConfig, value_type, change_mode, 0), + PB_FIELD2(5, INT32, OPTIONAL, STATIC, OTHER, + emulator_VehiclePropConfig, supported_areas, value_type, 0), + PB_FIELD2(6, MESSAGE, REPEATED, CALLBACK, OTHER, emulator_VehiclePropConfig, + area_configs, supported_areas, &emulator_VehicleAreaConfig_fields), + PB_FIELD2(7, INT32, OPTIONAL, STATIC, OTHER, + emulator_VehiclePropConfig, config_flags, area_configs, 0), + PB_FIELD2(8, INT32, REPEATED, CALLBACK, OTHER, + emulator_VehiclePropConfig, config_array, config_flags, 0), + PB_FIELD2(9, STRING, OPTIONAL, CALLBACK, OTHER, + emulator_VehiclePropConfig, config_string, config_array, 0), + PB_FIELD2(10, FLOAT, OPTIONAL, STATIC, OTHER, + emulator_VehiclePropConfig, min_sample_rate, config_string, 0), + PB_FIELD2(11, FLOAT, OPTIONAL, STATIC, OTHER, + emulator_VehiclePropConfig, max_sample_rate, min_sample_rate, 0), + PB_LAST_FIELD +}; + +#ifdef CONFIG_VEHICLE_DRIVER_OREO +const pb_field_t emulator_VehiclePropValue_fields[10] = { + PB_FIELD2(1, INT32, REQUIRED, STATIC, FIRST, + emulator_VehiclePropValue, prop, prop, 0), + PB_FIELD2(2, INT32, OPTIONAL, STATIC, OTHER, + emulator_VehiclePropValue, value_type, prop, 0), + PB_FIELD2(3, INT64, OPTIONAL, STATIC, OTHER, + emulator_VehiclePropValue, timestamp, value_type, 0), + PB_FIELD2(4, INT32, OPTIONAL, STATIC, OTHER, + emulator_VehiclePropValue, area_id, timestamp, 0), + PB_FIELD2(5, SINT32, REPEATED, CALLBACK, OTHER, + emulator_VehiclePropValue, int32_values, area_id, 0), + PB_FIELD2(6, SINT64, REPEATED, CALLBACK, OTHER, + emulator_VehiclePropValue, int64_values, int32_values, 0), + PB_FIELD2(7, FLOAT, REPEATED, CALLBACK, OTHER, + emulator_VehiclePropValue, float_values, int64_values, 0), + PB_FIELD2(8, STRING, OPTIONAL, CALLBACK, OTHER, + emulator_VehiclePropValue, string_value, float_values, 0), + PB_FIELD2(9, BYTES, OPTIONAL, CALLBACK, OTHER, + emulator_VehiclePropValue, bytes_value, string_value, 0), + PB_LAST_FIELD +}; +#else +const pb_field_t emulator_VehiclePropValue_fields[11] = { + PB_FIELD2(1, INT32, REQUIRED, STATIC, FIRST, emulator_VehiclePropValue, + prop, prop, 0), + PB_FIELD2(2, INT32, OPTIONAL, STATIC, OTHER, emulator_VehiclePropValue, + value_type, prop, 0), + PB_FIELD2(3, INT64, OPTIONAL, STATIC, OTHER, emulator_VehiclePropValue, + timestamp, value_type, 0), + PB_FIELD2(4, INT32, OPTIONAL, STATIC, OTHER, emulator_VehiclePropValue, + area_id, timestamp, 0), + PB_FIELD2(5, SINT32, REPEATED, CALLBACK, OTHER, emulator_VehiclePropValue, + int32_values, area_id, 0), + PB_FIELD2(6, SINT64, REPEATED, CALLBACK, OTHER, emulator_VehiclePropValue, + int64_values, int32_values, 0), + PB_FIELD2(7, FLOAT, REPEATED, CALLBACK, OTHER, emulator_VehiclePropValue, + float_values, int64_values, 0), + PB_FIELD2(8, STRING, OPTIONAL, CALLBACK, OTHER, emulator_VehiclePropValue, + string_value, float_values, 0), + PB_FIELD2(9, BYTES, OPTIONAL, CALLBACK, OTHER, emulator_VehiclePropValue, + bytes_value, string_value, 0), + PB_FIELD2(10, ENUM, OPTIONAL, STATIC, OTHER, emulator_VehiclePropValue, + status, bytes_value, 0), + PB_LAST_FIELD +}; +#endif + +const pb_field_t emulator_VehiclePropGet_fields[3] = { + PB_FIELD2(1, INT32, REQUIRED, STATIC, FIRST, emulator_VehiclePropGet, + prop, prop, 0), + PB_FIELD2(2, INT32, OPTIONAL, STATIC, OTHER, emulator_VehiclePropGet, + area_id, prop, 0), + PB_LAST_FIELD +}; + +const pb_field_t emulator_EmulatorMessage_fields[6] = { + PB_FIELD2(1, ENUM, REQUIRED, STATIC, FIRST, emulator_EmulatorMessage, msg_type, + msg_type, 0), + PB_FIELD2(2, ENUM, OPTIONAL, STATIC, OTHER, emulator_EmulatorMessage, status, + msg_type, 0), + PB_FIELD2(3, MESSAGE, REPEATED, CALLBACK, OTHER, emulator_EmulatorMessage, + prop, status, &emulator_VehiclePropGet_fields), + PB_FIELD2(4, MESSAGE, REPEATED, CALLBACK, OTHER, emulator_EmulatorMessage, + config, prop, &emulator_VehiclePropConfig_fields), + PB_FIELD2(5, MESSAGE, REPEATED, CALLBACK, OTHER, emulator_EmulatorMessage, + value, config, &emulator_VehiclePropValue_fields), + PB_LAST_FIELD +}; + +/* Check that field information fits in pb_field_t */ +#if !defined(PB_FIELD_32BIT) +/* If you get an error here, it means that you need to define PB_FIELD_32BIT + * compile-time option. You can do that in pb.h or on compiler command line. + * + * The reason you need to do this is that some of your messages contain tag + * numbers or field sizes that are larger than what can fit in 8 or 16 bit + * field descriptors. + */ +STATIC_ASSERT((pb_membersize(emulator_VehiclePropConfig, area_configs) < 65536 +&& pb_membersize(emulator_EmulatorMessage, prop) < 65536 +&& pb_membersize(emulator_EmulatorMessage, config) < 65536 +&& pb_membersize(emulator_EmulatorMessage, value) < 65536), +YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_emulator_VehicleAreaConfig_emulator_\ +VehiclePropConfig_emulator_VehiclePropValue_emulator_VehiclePropGet_emulator_EmulatorMessage) +#endif + +#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT) +/* If you get an error here, it means that you need to define PB_FIELD_16BIT + * compile-time option. You can do that in pb.h or on compiler command line. + * + * The reason you need to do this is that some of your messages contain tag + * numbers or field sizes that are larger than what can fit in the default + * 8 bit descriptors. + */ +STATIC_ASSERT((pb_membersize(emulator_VehiclePropConfig, area_configs) < 256 +&& pb_membersize(emulator_EmulatorMessage, prop) < 256 +&& pb_membersize(emulator_EmulatorMessage, config) < 256 +&& pb_membersize(emulator_EmulatorMessage, value) < 256), +YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_emulator_VehicleAreaConfig_emulator_\ +VehiclePropConfig_emulator_VehiclePropValue_emulator_VehiclePropGet_emulator_EmulatorMessage) +#endif + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("NXP Semiconductor"); diff --git a/drivers/misc/vehicle/vehiclehalproto.pb.h b/drivers/misc/vehicle/vehiclehalproto.pb.h new file mode 100644 index 000000000000..90e981bd06f7 --- /dev/null +++ b/drivers/misc/vehicle/vehiclehalproto.pb.h @@ -0,0 +1,191 @@ +/* + * Copyright 2018 NXP + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef _PB_VEHICLEHALPROTO_PB_H_ +#define _PB_VEHICLEHALPROTO_PB_H_ +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* Enum definitions */ +typedef enum _emulator_MsgType { + emulator_MsgType_GET_CONFIG_CMD = 0, + emulator_MsgType_GET_CONFIG_RESP = 1, + emulator_MsgType_GET_CONFIG_ALL_CMD = 2, + emulator_MsgType_GET_CONFIG_ALL_RESP = 3, + emulator_MsgType_GET_PROPERTY_CMD = 4, + emulator_MsgType_GET_PROPERTY_RESP = 5, + emulator_MsgType_GET_PROPERTY_ALL_CMD = 6, + emulator_MsgType_GET_PROPERTY_ALL_RESP = 7, + emulator_MsgType_SET_PROPERTY_CMD = 8, + emulator_MsgType_SET_PROPERTY_RESP = 9, + emulator_MsgType_SET_PROPERTY_ASYNC = 10 +} emulator_MsgType; + +typedef enum _emulator_Status { + emulator_Status_RESULT_OK = 0, + emulator_Status_ERROR_UNKNOWN = 1, + emulator_Status_ERROR_UNIMPLEMENTED_CMD = 2, + emulator_Status_ERROR_INVALID_PROPERTY = 3, + emulator_Status_ERROR_INVALID_AREA_ID = 4, + emulator_Status_ERROR_PROPERTY_UNINITIALIZED = 5, + emulator_Status_ERROR_WRITE_ONLY_PROPERTY = 6, + emulator_Status_ERROR_MEMORY_ALLOC_FAILED = 7, + emulator_Status_ERROR_INVALID_OPERATION = 8 +} emulator_Status; + +#ifndef CONFIG_VEHICLE_DRIVER_OREO +typedef enum _emulator_VehiclePropStatus { + emulator_VehiclePropStatus_AVAILABLE = 0, + emulator_VehiclePropStatus_UNAVAILABLE = 1, + emulator_VehiclePropStatus_ERROR = 2 +} emulator_VehiclePropStatus; +#endif + +/* Struct definitions */ +typedef struct _emulator_VehicleAreaConfig { + int32_t area_id; + bool has_min_int32_value; + int32_t min_int32_value; + bool has_max_int32_value; + int32_t max_int32_value; + bool has_min_int64_value; + int64_t min_int64_value; + bool has_max_int64_value; + int64_t max_int64_value; + bool has_min_float_value; + float min_float_value; + bool has_max_float_value; + float max_float_value; +} emulator_VehicleAreaConfig; + +typedef struct _emulator_VehiclePropGet { + int32_t prop; + bool has_area_id; + int32_t area_id; +} emulator_VehiclePropGet; + +typedef struct _emulator_VehiclePropValue { + int32_t prop; + bool has_value_type; + int32_t value_type; + bool has_timestamp; + int64_t timestamp; + bool has_area_id; + int32_t area_id; + pb_callback_t int32_values; + pb_callback_t int64_values; + pb_callback_t float_values; + pb_callback_t string_value; + pb_callback_t bytes_value; +#ifndef CONFIG_VEHICLE_DRIVER_OREO + bool has_status; + emulator_VehiclePropStatus status; +#endif +} emulator_VehiclePropValue; + +typedef struct _emulator_VehiclePropConfig { + int32_t prop; + bool has_access; + int32_t access; + bool has_change_mode; + int32_t change_mode; + bool has_value_type; + int32_t value_type; + bool has_supported_areas; + int32_t supported_areas; + pb_callback_t area_configs; + bool has_config_flags; + int32_t config_flags; + pb_callback_t config_array; + pb_callback_t config_string; + bool has_min_sample_rate; + float min_sample_rate; + bool has_max_sample_rate; + float max_sample_rate; +} emulator_VehiclePropConfig; + +typedef struct _emulator_EmulatorMessage { + emulator_MsgType msg_type; + bool has_status; + emulator_Status status; + pb_callback_t prop; + pb_callback_t config; + pb_callback_t value; +} emulator_EmulatorMessage; + +/* Default values for struct fields */ + +/* Field tags (for use in manual encoding/decoding) */ +#define emulator_VehicleAreaConfig_area_id_tag 1 +#define emulator_VehicleAreaConfig_min_int32_value_tag 2 +#define emulator_VehicleAreaConfig_max_int32_value_tag 3 +#define emulator_VehicleAreaConfig_min_int64_value_tag 4 +#define emulator_VehicleAreaConfig_max_int64_value_tag 5 +#define emulator_VehicleAreaConfig_min_float_value_tag 6 +#define emulator_VehicleAreaConfig_max_float_value_tag 7 +#define emulator_VehiclePropGet_prop_tag 1 +#define emulator_VehiclePropGet_area_id_tag 2 +#define emulator_VehiclePropValue_prop_tag 1 +#define emulator_VehiclePropValue_value_type_tag 2 +#define emulator_VehiclePropValue_timestamp_tag 3 +#ifndef CONFIG_VEHICLE_DRIVER_OREO +#define emulator_VehiclePropValue_status_tag 10 +#endif +#define emulator_VehiclePropValue_area_id_tag 4 +#define emulator_VehiclePropValue_int32_values_tag 5 +#define emulator_VehiclePropValue_int64_values_tag 6 +#define emulator_VehiclePropValue_float_values_tag 7 +#define emulator_VehiclePropValue_string_value_tag 8 +#define emulator_VehiclePropValue_bytes_value_tag 9 +#define emulator_VehiclePropConfig_prop_tag 1 +#define emulator_VehiclePropConfig_access_tag 2 +#define emulator_VehiclePropConfig_change_mode_tag 3 +#define emulator_VehiclePropConfig_value_type_tag 4 +#define emulator_VehiclePropConfig_supported_areas_tag 5 +#define emulator_VehiclePropConfig_area_configs_tag 6 +#define emulator_VehiclePropConfig_config_flags_tag 7 +#define emulator_VehiclePropConfig_config_array_tag 8 +#define emulator_VehiclePropConfig_config_string_tag 9 +#define emulator_VehiclePropConfig_min_sample_rate_tag 10 +#define emulator_VehiclePropConfig_max_sample_rate_tag 11 +#define emulator_EmulatorMessage_msg_type_tag 1 +#define emulator_EmulatorMessage_status_tag 2 +#define emulator_EmulatorMessage_prop_tag 3 +#define emulator_EmulatorMessage_config_tag 4 +#define emulator_EmulatorMessage_value_tag 5 + +/* Struct field encoding specification for nanopb */ +extern const pb_field_t emulator_VehicleAreaConfig_fields[8]; +extern const pb_field_t emulator_VehiclePropConfig_fields[12]; +#ifdef CONFIG_VEHICLE_DRIVER_OREO +extern const pb_field_t emulator_VehiclePropValue_fields[10]; +#else +extern const pb_field_t emulator_VehiclePropValue_fields[11]; +#endif +extern const pb_field_t emulator_VehiclePropGet_fields[3]; +extern const pb_field_t emulator_EmulatorMessage_fields[6]; + +/* Maximum encoded size of messages (where known) */ +#define emulator_VehicleAreaConfig_size 55 +#define emulator_VehiclePropGet_size 22 + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif diff --git a/drivers/pci/controller/dwc/pcie-dw-rockchip.c b/drivers/pci/controller/dwc/pcie-dw-rockchip.c index 9ffe4b133308..9c3bab0af051 100644 --- a/drivers/pci/controller/dwc/pcie-dw-rockchip.c +++ b/drivers/pci/controller/dwc/pcie-dw-rockchip.c @@ -1789,7 +1789,8 @@ static int rk_pcie_remove(struct platform_device *pdev) * Timeout should not happen as it's longer than regular probe actually. * But probe maybe fail, so need to double check bridge bus. */ - if (!rk_pcie || !rk_pcie->finish_probe || !rk_pcie->pci->pp.bridge->bus) { + if (!rk_pcie || !rk_pcie->pci || !rk_pcie->pci->pp.bridge || + !rk_pcie->pci->pp.bridge->bus) { dev_dbg(dev, "%s return early due to failure in threaded init\n", __func__); return 0; } diff --git a/drivers/thermal/rockchip_thermal.c b/drivers/thermal/rockchip_thermal.c index 0defdbc3ff59..7b8746404a9f 100644 --- a/drivers/thermal/rockchip_thermal.c +++ b/drivers/thermal/rockchip_thermal.c @@ -2687,7 +2687,7 @@ static int rockchip_thermal_probe(struct platform_device *pdev) if (IS_ERR(thermal->regs)) return PTR_ERR(thermal->regs); - thermal->reset = devm_reset_control_array_get(&pdev->dev, false, false); + thermal->reset = devm_reset_control_array_get_optional_exclusive(&pdev->dev); if (IS_ERR(thermal->reset)) { if (PTR_ERR(thermal->reset) != -EPROBE_DEFER) dev_err(&pdev->dev, "failed to get tsadc reset lines\n"); diff --git a/include/dt-bindings/clock/rockchip,rv1126b-cru.h b/include/dt-bindings/clock/rockchip,rv1126b-cru.h index c10caa9d25e8..09aa04325b51 100644 --- a/include/dt-bindings/clock/rockchip,rv1126b-cru.h +++ b/include/dt-bindings/clock/rockchip,rv1126b-cru.h @@ -371,6 +371,7 @@ #define CLK_MACPHY 361 #define HCLK_RKRNG_NS 362 #define HCLK_RKRNG_S_NS 363 +#define CLK_AISP_PLL_SRC 364 /* secure clks */ #define CLK_USER_OTPC_S 400 diff --git a/include/dt-bindings/display/rockchip_vop.h b/include/dt-bindings/display/rockchip_vop.h index 55d25bfd8875..f785d8a5bca4 100644 --- a/include/dt-bindings/display/rockchip_vop.h +++ b/include/dt-bindings/display/rockchip_vop.h @@ -20,4 +20,104 @@ #define ROCKCHIP_VOP2_PHY_ID_INVALID -1 +/* mcu_data[23:0] */ +#define ROCKCHIP_MCU_DATA_MAP_DATA_1x24 0 +/* + * { mcu_data[7:3], 3'b0, mcu_data[2:0], mcu_data[7:5], 2'b0, mcu_data[4:0], 3'b0 } + * That is: + * LCDC_D23 -> MCU_D7_M1 + * LCDC_D22 -> MCU_D6_M1 + * LCDC_D21 -> MCU_D5_M1 + * LCDC_D20 -> MCU_D4_M1 + * LCDC_D19 -> MCU_D3_M1 + * LCDC_D18 -> N/A + * LCDC_D17 -> N/A + * LCDC_D16 -> N/A + * LCDC_D15 -> MCU_D2_M1 + * LCDC_D14 -> MCU_D1_M1 + * LCDC_D13 -> MCU_D0_M1 + * LCDC_D12 -> MCU_D7_M0 + * LCDC_D11 -> MCU_D6_M0 + * LCDC_D10 -> MCU_D5_M0 + * LCDC_D9 -> N/A + * LCDC_D8 -> N/A + * LCDC_D7 -> MCU_D4_M0 + * LCDC_D6 -> MCU_D3_M0 + * LCDC_D5 -> MCU_D2_M0 + * LCDC_D4 -> MCU_D1_M0 + * LCDC_D3 -> MCU_D0_M0 + * LCDC_D2 -> N/A + * LCDC_D1 -> N/A + * LCDC_D0 -> N/A + */ +#define ROCKCHIP_MCU_DATA_MAP_DATA_WITH_DUMMY_0 1 +/* { 8'd0, BT1120[15:0] } */ +#define ROCKCHIP_BT1120_DATA_MAP_DATA_PADLO 0 +/* { BT1120[15:0], 8'd0 }*/ +#define ROCKCHIP_BT1120_DATA_MAP_DATA_PADHI 1 +/* + * { BT1120[15:11], 3'd0, BT1120[10:5], 2'd0, BT1120[4:0], 3'b0 } + * That is: + * LCDC_D23 -> BT1120_D15 + * LCDC_D22 -> BT1120_D14 + * LCDC_D21 -> BT1120_D13 + * LCDC_D20 -> BT1120_D12 + * LCDC_D19 -> BT1120_D11 + * LCDC_D18 -> N/A + * LCDC_D17 -> N/A + * LCDC_D16 -> N/A + * LCDC_D15 -> BT1120_D10 + * LCDC_D14 -> BT1120_D9 + * LCDC_D13 -> BT1120_D8 + * LCDC_D12 -> BT1120_D7 + * LCDC_D11 -> BT1120_D6 + * LCDC_D10 -> BT1120_D5 + * LCDC_D9 -> N/A + * LCDC_D8 -> N/A + * LCDC_D7 -> BT1120_D4 + * LCDC_D6 -> BT1120_D3 + * LCDC_D5 -> BT1120_D2 + * LCDC_D4 -> BT1120_D1 + * LCDC_D3 -> BT1120_D0 + * LCDC_D2 -> N/A + * LCDC_D1 -> N/A + * LCDC_D0 -> N/A + */ +#define ROCKCHIP_BT1120_DATA_MAP_DATA_WITH_DUMMY_0 3 +/* { 16'd0, BT656[7:0] }*/ +#define ROCKCHIP_BT656_DATA_MAP_PADLO 0 +/* { 8'd0, BT656[7:0], 8'd0 }*/ +#define ROCKCHIP_BT656_DATA_MAP_DUMMY_HI_LO 1 +/* { BT656[7:0], 16'd0 }*/ +#define ROCKCHIP_BT656_DATA_MAP_PADHI 2 +/* + * { BT656[7:3], 3'd0, BT656[2:0], BT656[7:5], 2'd0, BT656[4:0], 3'd0 } + * That is: + * LCDC_D23 -> BT656_D7_M1 + * LCDC_D22 -> BT656_D6_M1 + * LCDC_D21 -> BT656_D5_M1 + * LCDC_D20 -> BT656_D4_M1 + * LCDC_D19 -> BT656_D3_M1 + * LCDC_D18 -> N/A + * LCDC_D17 -> N/A + * LCDC_D16 -> N/A + * LCDC_D15 -> BT656_D2_M1 + * LCDC_D14 -> BT656_D1_M1 + * LCDC_D13 -> BT656_D0_M1 + * LCDC_D12 -> BT656_D7_M0 + * LCDC_D11 -> BT656_D6_M0 + * LCDC_D10 -> BT656_D5_M0 + * LCDC_D9 -> N/A + * LCDC_D8 -> N/A + * LCDC_D7 -> BT656_D4_M0 + * LCDC_D6 -> BT656_D3_M0 + * LCDC_D5 -> BT656_D2_M0 + * LCDC_D4 -> BT656_D1_M0 + * LCDC_D3 -> BT656_D0_M0 + * LCDC_D2 -> N/A + * LCDC_D1 -> N/A + * LCDC_D0 -> N/A + */ +#define ROCKCHIP_BT656_DATA_MAP_DATA_WITH_DUMMY_0 3 + #endif diff --git a/include/linux/memory_group_manager.h b/include/linux/memory_group_manager.h index ec55d74f56ad..8729e2dfb9e5 100644 --- a/include/linux/memory_group_manager.h +++ b/include/linux/memory_group_manager.h @@ -41,6 +41,11 @@ typedef int vm_fault_t; struct memory_group_manager_device; struct memory_group_manager_import_data; +/** + * enum mgm_pte_flags - Memory Group Manager PTE Flags + * @MMA_VIOLATION: Bit-number of flag used to indicate that Mismatched Memory Attributes (MMA) + * exist for a page. Specifically, this means CPU-uncached, GPU-cached. + */ enum mgm_pte_flags { MMA_VIOLATION = 0, }; @@ -138,11 +143,17 @@ struct memory_group_manager_ops { * decoded to determine the physical address and any other * properties of the mapping the manager requires. * - * This function allows the memory group manager to modify a GPU page - * table entry before it is stored by the kbase module (controller - * driver). It may set certain bits in the page table entry attributes - * or modify the physical address, based on the physical memory group ID, - * PBHA ID, PTE flags and/or additional data in struct memory_group_manager_device. + * This function allows the memory group manager to modify a GPU page table entry before it + * is stored by the kbase module (controller driver). It may set certain bits in the page + * table entry attributes or modify the physical address, based on the physical memory + * group ID, PBHA ID, PTE flags and/or additional data in + * struct memory_group_manager_device. + * + * If mgm_pte_flags has the MMA_VIOLATION bit set - ie, 1< #define KBASE_DUMMY_MODEL_COUNTER_HEADER_DWORDS (4) -#if MALI_USE_CSF #define KBASE_DUMMY_MODEL_COUNTER_PER_CORE (65) -#else /* MALI_USE_CSF */ -#define KBASE_DUMMY_MODEL_COUNTER_PER_CORE (60) -#endif /* MALI_USE_CSF */ #define KBASE_DUMMY_MODEL_COUNTERS_PER_BIT (4) #define KBASE_DUMMY_MODEL_COUNTER_ENABLED(enable_mask, ctr_idx) \ (enable_mask & (1U << (ctr_idx / KBASE_DUMMY_MODEL_COUNTERS_PER_BIT))) diff --git a/include/uapi/gpu/arm/valhall/gpu/backend/mali_kbase_gpu_regmap_jm.h b/include/uapi/gpu/arm/valhall/gpu/backend/mali_kbase_gpu_regmap_jm.h deleted file mode 100644 index d24afccf49ee..000000000000 --- a/include/uapi/gpu/arm/valhall/gpu/backend/mali_kbase_gpu_regmap_jm.h +++ /dev/null @@ -1,25 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#ifndef _UAPI_KBASE_GPU_REGMAP_JM_H_ -#define _UAPI_KBASE_GPU_REGMAP_JM_H_ - -#endif /* _UAPI_KBASE_GPU_REGMAP_JM_H_ */ diff --git a/include/uapi/gpu/arm/valhall/gpu/mali_kbase_gpu_regmap.h b/include/uapi/gpu/arm/valhall/gpu/mali_kbase_gpu_regmap.h index 8256191f331b..84c3e8c877b7 100644 --- a/include/uapi/gpu/arm/valhall/gpu/mali_kbase_gpu_regmap.h +++ b/include/uapi/gpu/arm/valhall/gpu/mali_kbase_gpu_regmap.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -22,10 +22,6 @@ #ifndef _UAPI_KBASE_GPU_REGMAP_H_ #define _UAPI_KBASE_GPU_REGMAP_H_ -#if MALI_USE_CSF #include "backend/mali_kbase_gpu_regmap_csf.h" -#else -#include "backend/mali_kbase_gpu_regmap_jm.h" -#endif /* !MALI_USE_CSF */ #endif /* _UAPI_KBASE_GPU_REGMAP_H_ */ diff --git a/include/uapi/gpu/arm/valhall/jm/mali_base_jm_kernel.h b/include/uapi/gpu/arm/valhall/jm/mali_base_jm_kernel.h deleted file mode 100644 index 48a43210d03f..000000000000 --- a/include/uapi/gpu/arm/valhall/jm/mali_base_jm_kernel.h +++ /dev/null @@ -1,892 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#ifndef _UAPI_BASE_JM_KERNEL_H_ -#define _UAPI_BASE_JM_KERNEL_H_ - -#include -#include "../mali_base_common_kernel.h" - -/* Similar to BASE_MEM_TILER_ALIGN_TOP, memory starting from the end of the - * initial commit is aligned to 'extension' pages, where 'extension' must be a power - * of 2 and no more than BASE_MEM_TILER_ALIGN_TOP_EXTENSION_MAX_PAGES - */ -#define BASE_JIT_ALLOC_MEM_TILER_ALIGN_TOP (1 << 0) - -/** - * BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE - If set, the heap info address points - * to a __u32 holding the used size in bytes; - * otherwise it points to a __u64 holding the lowest address of unused memory. - */ -#define BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE (1 << 1) - -/** - * BASE_JIT_ALLOC_VALID_FLAGS - Valid set of just-in-time memory allocation flags - * - * Note: BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE cannot be set if heap_info_gpu_addr - * in %base_jit_alloc_info is 0 (atom with BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE set - * and heap_info_gpu_addr being 0 will be rejected). - */ -#define BASE_JIT_ALLOC_VALID_FLAGS \ - (BASE_JIT_ALLOC_MEM_TILER_ALIGN_TOP | BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE) - -/* Bitpattern describing the ::base_context_create_flags that can be - * passed to base_context_init() - */ -#define BASEP_CONTEXT_CREATE_ALLOWED_FLAGS \ - (BASE_CONTEXT_CCTX_EMBEDDED | BASEP_CONTEXT_CREATE_KERNEL_FLAGS) - -/* - * Private flags used on the base context - * - * These start at bit 31, and run down to zero. - * - * They share the same space as base_context_create_flags, and so must - * not collide with them. - */ - -/* Private flag tracking whether job descriptor dumping is disabled */ -#define BASEP_CONTEXT_FLAG_JOB_DUMP_DISABLED ((base_context_create_flags)(1 << 31)) - -/* Flags for base tracepoint specific to JM */ -#define BASE_TLSTREAM_FLAGS_MASK \ - (BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS | BASE_TLSTREAM_JOB_DUMPING_ENABLED) -/* - * Dependency stuff, keep it private for now. May want to expose it if - * we decide to make the number of semaphores a configurable - * option. - */ -#define BASE_JD_ATOM_COUNT 256 - -/* Set/reset values for a software event */ -#define BASE_JD_SOFT_EVENT_SET ((unsigned char)1) -#define BASE_JD_SOFT_EVENT_RESET ((unsigned char)0) - -/** - * struct base_jd_udata - Per-job data - * - * @blob: per-job data array - * - * This structure is used to store per-job data, and is completely unused - * by the Base driver. It can be used to store things such as callback - * function pointer, data to handle job completion. It is guaranteed to be - * untouched by the Base driver. - */ -struct base_jd_udata { - __u64 blob[2]; -}; - -/** - * typedef base_jd_dep_type - Job dependency type. - * - * A flags field will be inserted into the atom structure to specify whether a - * dependency is a data or ordering dependency (by putting it before/after - * 'core_req' in the structure it should be possible to add without changing - * the structure size). - * When the flag is set for a particular dependency to signal that it is an - * ordering only dependency then errors will not be propagated. - */ -typedef __u8 base_jd_dep_type; - -#define BASE_JD_DEP_TYPE_INVALID (0) /**< Invalid dependency */ -#define BASE_JD_DEP_TYPE_DATA (1U << 0) /**< Data dependency */ -#define BASE_JD_DEP_TYPE_ORDER (1U << 1) /**< Order dependency */ - -/** - * typedef base_jd_core_req - Job chain hardware requirements. - * - * A job chain must specify what GPU features it needs to allow the - * driver to schedule the job correctly. By not specifying the - * correct settings can/will cause an early job termination. Multiple - * values can be ORed together to specify multiple requirements. - * Special case is ::BASE_JD_REQ_DEP, which is used to express complex - * dependencies, and that doesn't execute anything on the hardware. - */ -typedef __u32 base_jd_core_req; - -/* Requirements that come from the HW */ - -/* No requirement, dependency only - */ -#define BASE_JD_REQ_DEP ((base_jd_core_req)0) - -/* Requires fragment shaders - */ -#define BASE_JD_REQ_FS ((base_jd_core_req)1 << 0) - -/* Requires compute shaders - * - * This covers any of the following GPU job types: - * - Vertex Shader Job - * - Geometry Shader Job - * - An actual Compute Shader Job - * - * Compare this with BASE_JD_REQ_ONLY_COMPUTE, which specifies that the - * job is specifically just the "Compute Shader" job type, and not the "Vertex - * Shader" nor the "Geometry Shader" job type. - */ -#define BASE_JD_REQ_CS ((base_jd_core_req)1 << 1) - -/* Requires tiling */ -#define BASE_JD_REQ_T ((base_jd_core_req)1 << 2) - -/* Requires cache flushes */ -#define BASE_JD_REQ_CF ((base_jd_core_req)1 << 3) - -/* Requires value writeback */ -#define BASE_JD_REQ_V ((base_jd_core_req)1 << 4) - -/* SW-only requirements - the HW does not expose these as part of the job slot - * capabilities - */ - -/* Requires fragment job with AFBC encoding */ -#define BASE_JD_REQ_FS_AFBC ((base_jd_core_req)1 << 13) - -/* SW-only requirement: coalesce completion events. - * If this bit is set then completion of this atom will not cause an event to - * be sent to userspace, whether successful or not; completion events will be - * deferred until an atom completes which does not have this bit set. - * - * This bit may not be used in combination with BASE_JD_REQ_EXTERNAL_RESOURCES. - */ -#define BASE_JD_REQ_EVENT_COALESCE ((base_jd_core_req)1 << 5) - -/* SW Only requirement: the job chain requires a coherent core group. We don't - * mind which coherent core group is used. - */ -#define BASE_JD_REQ_COHERENT_GROUP ((base_jd_core_req)1 << 6) - -/* SW Only requirement: The performance counters should be enabled only when - * they are needed, to reduce power consumption. - */ -#define BASE_JD_REQ_PERMON ((base_jd_core_req)1 << 7) - -/* SW Only requirement: External resources are referenced by this atom. - * - * This bit may not be used in combination with BASE_JD_REQ_EVENT_COALESCE and - * BASE_JD_REQ_SOFT_EVENT_WAIT. - */ -#define BASE_JD_REQ_EXTERNAL_RESOURCES ((base_jd_core_req)1 << 8) - -/* SW Only requirement: Software defined job. Jobs with this bit set will not be - * submitted to the hardware but will cause some action to happen within the - * driver - */ -#define BASE_JD_REQ_SOFT_JOB ((base_jd_core_req)1 << 9) - -#define BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME (BASE_JD_REQ_SOFT_JOB | 0x1) -#define BASE_JD_REQ_SOFT_FENCE_TRIGGER (BASE_JD_REQ_SOFT_JOB | 0x2) -#define BASE_JD_REQ_SOFT_FENCE_WAIT (BASE_JD_REQ_SOFT_JOB | 0x3) - -/* 0x4 RESERVED for now */ - -/* SW only requirement: event wait/trigger job. - * - * - BASE_JD_REQ_SOFT_EVENT_WAIT: this job will block until the event is set. - * - BASE_JD_REQ_SOFT_EVENT_SET: this job sets the event, thus unblocks the - * other waiting jobs. It completes immediately. - * - BASE_JD_REQ_SOFT_EVENT_RESET: this job resets the event, making it - * possible for other jobs to wait upon. It completes immediately. - */ -#define BASE_JD_REQ_SOFT_EVENT_WAIT (BASE_JD_REQ_SOFT_JOB | 0x5) -#define BASE_JD_REQ_SOFT_EVENT_SET (BASE_JD_REQ_SOFT_JOB | 0x6) -#define BASE_JD_REQ_SOFT_EVENT_RESET (BASE_JD_REQ_SOFT_JOB | 0x7) - -#define BASE_JD_REQ_SOFT_DEBUG_COPY (BASE_JD_REQ_SOFT_JOB | 0x8) - -/* SW only requirement: Just In Time allocation - * - * This job requests a single or multiple just-in-time allocations through a - * list of base_jit_alloc_info structure which is passed via the jc element of - * the atom. The number of base_jit_alloc_info structures present in the - * list is passed via the nr_extres element of the atom - * - * It should be noted that the id entry in base_jit_alloc_info must not - * be reused until it has been released via BASE_JD_REQ_SOFT_JIT_FREE. - * - * Should this soft job fail it is expected that a BASE_JD_REQ_SOFT_JIT_FREE - * soft job to free the JIT allocation is still made. - * - * The job will complete immediately. - */ -#define BASE_JD_REQ_SOFT_JIT_ALLOC (BASE_JD_REQ_SOFT_JOB | 0x9) - -/* SW only requirement: Just In Time free - * - * This job requests a single or multiple just-in-time allocations created by - * BASE_JD_REQ_SOFT_JIT_ALLOC to be freed. The ID list of the just-in-time - * allocations is passed via the jc element of the atom. - * - * The job will complete immediately. - */ -#define BASE_JD_REQ_SOFT_JIT_FREE (BASE_JD_REQ_SOFT_JOB | 0xa) - -/* SW only requirement: Map external resource - * - * This job requests external resource(s) are mapped once the dependencies - * of the job have been satisfied. The list of external resources are - * passed via the jc element of the atom which is a pointer to a - * base_external_resource_list. - */ -#define BASE_JD_REQ_SOFT_EXT_RES_MAP (BASE_JD_REQ_SOFT_JOB | 0xb) - -/* SW only requirement: Unmap external resource - * - * This job requests external resource(s) are unmapped once the dependencies - * of the job has been satisfied. The list of external resources are - * passed via the jc element of the atom which is a pointer to a - * base_external_resource_list. - */ -#define BASE_JD_REQ_SOFT_EXT_RES_UNMAP (BASE_JD_REQ_SOFT_JOB | 0xc) - -/* HW Requirement: Requires Compute shaders (but not Vertex or Geometry Shaders) - * - * This indicates that the Job Chain contains GPU jobs of the 'Compute - * Shaders' type. - * - * In contrast to BASE_JD_REQ_CS, this does not indicate that the Job - * Chain contains 'Geometry Shader' or 'Vertex Shader' jobs. - */ -#define BASE_JD_REQ_ONLY_COMPUTE ((base_jd_core_req)1 << 10) - -/* HW Requirement: Use the base_jd_atom::device_nr field to specify a - * particular core group - * - * If both BASE_JD_REQ_COHERENT_GROUP and this flag are set, this flag - * takes priority - * - * This is only guaranteed to work for BASE_JD_REQ_ONLY_COMPUTE atoms. - */ -#define BASE_JD_REQ_SPECIFIC_COHERENT_GROUP ((base_jd_core_req)1 << 11) - -/* SW Flag: If this bit is set then the successful completion of this atom - * will not cause an event to be sent to userspace - */ -#define BASE_JD_REQ_EVENT_ONLY_ON_FAILURE ((base_jd_core_req)1 << 12) - -/* SW Flag: If this bit is set then completion of this atom will not cause an - * event to be sent to userspace, whether successful or not. - */ -#define BASEP_JD_REQ_EVENT_NEVER ((base_jd_core_req)1 << 14) - -/* SW Flag: Skip GPU cache clean and invalidation before starting a GPU job. - * - * If this bit is set then the GPU's cache will not be cleaned and invalidated - * until a GPU job starts which does not have this bit set or a job completes - * which does not have the BASE_JD_REQ_SKIP_CACHE_END bit set. Do not use - * if the CPU may have written to memory addressed by the job since the last job - * without this bit set was submitted. - */ -#define BASE_JD_REQ_SKIP_CACHE_START ((base_jd_core_req)1 << 15) - -/* SW Flag: Skip GPU cache clean and invalidation after a GPU job completes. - * - * If this bit is set then the GPU's cache will not be cleaned and invalidated - * until a GPU job completes which does not have this bit set or a job starts - * which does not have the BASE_JD_REQ_SKIP_CACHE_START bit set. Do not use - * if the CPU may read from or partially overwrite memory addressed by the job - * before the next job without this bit set completes. - */ -#define BASE_JD_REQ_SKIP_CACHE_END ((base_jd_core_req)1 << 16) - -/* Request the atom be executed on a specific job slot. - * - * When this flag is specified, it takes precedence over any existing job slot - * selection logic. - */ -#define BASE_JD_REQ_JOB_SLOT ((base_jd_core_req)1 << 17) - -/* SW-only requirement: The atom needs to run on a limited core mask affinity. - * - * If this bit is set then the kbase_context.limited_core_mask will be applied - * to the affinity. - */ -#define BASE_JD_REQ_LIMITED_CORE_MASK ((base_jd_core_req)1 << 20) - -/* These requirement bits are currently unused in base_jd_core_req - */ -#define BASEP_JD_REQ_RESERVED \ - (~(BASE_JD_REQ_ATOM_TYPE | BASE_JD_REQ_EXTERNAL_RESOURCES | \ - BASE_JD_REQ_EVENT_ONLY_ON_FAILURE | BASEP_JD_REQ_EVENT_NEVER | \ - BASE_JD_REQ_EVENT_COALESCE | BASE_JD_REQ_COHERENT_GROUP | \ - BASE_JD_REQ_SPECIFIC_COHERENT_GROUP | BASE_JD_REQ_FS_AFBC | BASE_JD_REQ_PERMON | \ - BASE_JD_REQ_SKIP_CACHE_START | BASE_JD_REQ_SKIP_CACHE_END | BASE_JD_REQ_JOB_SLOT | \ - BASE_JD_REQ_LIMITED_CORE_MASK)) - -/* Mask of all bits in base_jd_core_req that control the type of the atom. - * - * This allows dependency only atoms to have flags set - */ -#define BASE_JD_REQ_ATOM_TYPE \ - (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T | BASE_JD_REQ_CF | BASE_JD_REQ_V | \ - BASE_JD_REQ_SOFT_JOB | BASE_JD_REQ_ONLY_COMPUTE) - -/** - * BASE_JD_REQ_SOFT_JOB_TYPE - Mask of all bits in base_jd_core_req that - * controls the type of a soft job. - */ -#define BASE_JD_REQ_SOFT_JOB_TYPE (BASE_JD_REQ_SOFT_JOB | 0x1f) - -/* Returns non-zero value if core requirements passed define a soft job or - * a dependency only job. - */ -#define BASE_JD_REQ_SOFT_JOB_OR_DEP(core_req) \ - (((core_req)&BASE_JD_REQ_SOFT_JOB) || ((core_req)&BASE_JD_REQ_ATOM_TYPE) == BASE_JD_REQ_DEP) - -/** - * enum kbase_jd_atom_state - Atom states - * - * @KBASE_JD_ATOM_STATE_UNUSED: Atom is not used. - * @KBASE_JD_ATOM_STATE_QUEUED: Atom is queued in JD. - * @KBASE_JD_ATOM_STATE_IN_JS: Atom has been given to JS (is runnable/running). - * @KBASE_JD_ATOM_STATE_HW_COMPLETED: Atom has been completed, but not yet - * handed back to job dispatcher for - * dependency resolution. - * @KBASE_JD_ATOM_STATE_COMPLETED: Atom has been completed, but not yet handed - * back to userspace. - */ -enum kbase_jd_atom_state { - KBASE_JD_ATOM_STATE_UNUSED, - KBASE_JD_ATOM_STATE_QUEUED, - KBASE_JD_ATOM_STATE_IN_JS, - KBASE_JD_ATOM_STATE_HW_COMPLETED, - KBASE_JD_ATOM_STATE_COMPLETED -}; - -/** - * typedef base_atom_id - Type big enough to store an atom number in. - */ -typedef __u8 base_atom_id; - -/** - * struct base_dependency - base dependency - * - * @atom_id: An atom number - * @dependency_type: Dependency type - */ -struct base_dependency { - base_atom_id atom_id; - base_jd_dep_type dependency_type; -}; - -/** - * typedef base_jd_prio - Base Atom priority. - * - * Only certain priority levels are actually implemented, as specified by the - * BASE_JD_PRIO_<...> definitions below. It is undefined to use a priority - * level that is not one of those defined below. - * - * Priority levels only affect scheduling after the atoms have had dependencies - * resolved. For example, a low priority atom that has had its dependencies - * resolved might run before a higher priority atom that has not had its - * dependencies resolved. - * - * In general, fragment atoms do not affect non-fragment atoms with - * lower priorities, and vice versa. One exception is that there is only one - * priority value for each context. So a high-priority (e.g.) fragment atom - * could increase its context priority, causing its non-fragment atoms to also - * be scheduled sooner. - * - * The atoms are scheduled as follows with respect to their priorities: - * * Let atoms 'X' and 'Y' be for the same job slot who have dependencies - * resolved, and atom 'X' has a higher priority than atom 'Y' - * * If atom 'Y' is currently running on the HW, then it is interrupted to - * allow atom 'X' to run soon after - * * If instead neither atom 'Y' nor atom 'X' are running, then when choosing - * the next atom to run, atom 'X' will always be chosen instead of atom 'Y' - * * Any two atoms that have the same priority could run in any order with - * respect to each other. That is, there is no ordering constraint between - * atoms of the same priority. - * - * The sysfs file 'js_ctx_scheduling_mode' is used to control how atoms are - * scheduled between contexts. The default value, 0, will cause higher-priority - * atoms to be scheduled first, regardless of their context. The value 1 will - * use a round-robin algorithm when deciding which context's atoms to schedule - * next, so higher-priority atoms can only preempt lower priority atoms within - * the same context. See KBASE_JS_SYSTEM_PRIORITY_MODE and - * KBASE_JS_PROCESS_LOCAL_PRIORITY_MODE for more details. - */ -typedef __u8 base_jd_prio; - -/* Medium atom priority. This is a priority higher than BASE_JD_PRIO_LOW */ -#define BASE_JD_PRIO_MEDIUM ((base_jd_prio)0) -/* High atom priority. This is a priority higher than BASE_JD_PRIO_MEDIUM and - * BASE_JD_PRIO_LOW - */ -#define BASE_JD_PRIO_HIGH ((base_jd_prio)1) -/* Low atom priority. */ -#define BASE_JD_PRIO_LOW ((base_jd_prio)2) -/* Real-Time atom priority. This is a priority higher than BASE_JD_PRIO_HIGH, - * BASE_JD_PRIO_MEDIUM, and BASE_JD_PRIO_LOW - */ -#define BASE_JD_PRIO_REALTIME ((base_jd_prio)3) - -/* Invalid atom priority (max uint8_t value) */ -#define BASE_JD_PRIO_INVALID ((base_jd_prio)255) - -/* Count of the number of priority levels. This itself is not a valid - * base_jd_prio setting - */ -#define BASE_JD_NR_PRIO_LEVELS 4 - -/** - * struct base_jd_atom_v2 - Node of a dependency graph used to submit a - * GPU job chain or soft-job to the kernel driver. - * - * @jc: GPU address of a job chain. - * @udata: User data. - * @extres_list: List of external resources. - * @nr_extres: Number of external resources or JIT allocations. - * @jit_id: Zero-terminated array of IDs of just-in-time memory - * allocations written to by the atom. When the atom - * completes, the value stored at the - * &struct_base_jit_alloc_info.heap_info_gpu_addr of - * each allocation is read in order to enforce an - * overall physical memory usage limit. - * @pre_dep: Pre-dependencies. One need to use SETTER function to assign - * this field; this is done in order to reduce possibility of - * improper assignment of a dependency field. - * @atom_number: Unique number to identify the atom. - * @prio: Atom priority. Refer to base_jd_prio for more details. - * @device_nr: Core group when BASE_JD_REQ_SPECIFIC_COHERENT_GROUP - * specified. - * @jobslot: Job slot to use when BASE_JD_REQ_JOB_SLOT is specified. - * @core_req: Core requirements. - * @padding: Unused. Must be zero. - * - * This structure has changed since UK 10.2 for which base_jd_core_req was a - * __u16 value. - * - * In UK 10.3 a core_req field of a __u32 type was added to the end of the - * structure, and the place in the structure previously occupied by __u16 - * core_req was kept but renamed to compat_core_req. - * - * From UK 11.20 - compat_core_req is now occupied by __u8 jit_id[2]. - * Compatibility with UK 10.x from UK 11.y is not handled because - * the major version increase prevents this. - * - * For UK 11.20 jit_id[2] must be initialized to zero. - */ -struct base_jd_atom_v2 { - __u64 jc; - struct base_jd_udata udata; - __u64 extres_list; - __u16 nr_extres; - __u8 jit_id[2]; - struct base_dependency pre_dep[2]; - base_atom_id atom_number; - base_jd_prio prio; - __u8 device_nr; - __u8 jobslot; - base_jd_core_req core_req; - __u8 padding[8]; -}; - -/** - * struct base_jd_atom - Same as base_jd_atom_v2, but has an extra seq_nr - * at the beginning. - * - * @seq_nr: Sequence number of logical grouping of atoms. - * @jc: GPU address of a job chain. - * @udata: User data. - * @extres_list: List of external resources. - * @nr_extres: Number of external resources or JIT allocations. - * @jit_id: Zero-terminated array of IDs of just-in-time memory - * allocations written to by the atom. When the atom - * completes, the value stored at the - * &struct_base_jit_alloc_info.heap_info_gpu_addr of - * each allocation is read in order to enforce an - * overall physical memory usage limit. - * @pre_dep: Pre-dependencies. One need to use SETTER function to assign - * this field; this is done in order to reduce possibility of - * improper assignment of a dependency field. - * @atom_number: Unique number to identify the atom. - * @prio: Atom priority. Refer to base_jd_prio for more details. - * @device_nr: Core group when BASE_JD_REQ_SPECIFIC_COHERENT_GROUP - * specified. - * @jobslot: Job slot to use when BASE_JD_REQ_JOB_SLOT is specified. - * @core_req: Core requirements. - * @renderpass_id: Renderpass identifier used to associate an atom that has - * BASE_JD_REQ_START_RENDERPASS set in its core requirements - * with an atom that has BASE_JD_REQ_END_RENDERPASS set. - * @padding: Unused. Must be zero. - */ -typedef struct base_jd_atom { - __u64 seq_nr; - __u64 jc; - struct base_jd_udata udata; - __u64 extres_list; - __u16 nr_extres; - __u8 jit_id[2]; - struct base_dependency pre_dep[2]; - base_atom_id atom_number; - base_jd_prio prio; - __u8 device_nr; - __u8 jobslot; - base_jd_core_req core_req; - __u8 renderpass_id; - __u8 padding[7]; -} base_jd_atom; - -/* Job chain event code bits - * Defines the bits used to create ::base_jd_event_code - */ -enum { - BASE_JD_SW_EVENT_KERNEL = (1u << 15), /* Kernel side event */ - BASE_JD_SW_EVENT = (1u << 14), /* SW defined event */ - /* Event indicates success (SW events only) */ - BASE_JD_SW_EVENT_SUCCESS = (1u << 13), - BASE_JD_SW_EVENT_JOB = (0u << 11), /* Job related event */ - BASE_JD_SW_EVENT_BAG = (1u << 11), /* Bag related event */ - BASE_JD_SW_EVENT_INFO = (2u << 11), /* Misc/info event */ - BASE_JD_SW_EVENT_RESERVED = (3u << 11), /* Reserved event type */ - /* Mask to extract the type from an event code */ - BASE_JD_SW_EVENT_TYPE_MASK = (3u << 11) -}; - -/** - * enum base_jd_event_code - Job chain event codes - * - * @BASE_JD_EVENT_RANGE_HW_NONFAULT_START: Start of hardware non-fault status - * codes. - * Obscurely, BASE_JD_EVENT_TERMINATED - * indicates a real fault, because the - * job was hard-stopped. - * @BASE_JD_EVENT_NOT_STARTED: Can't be seen by userspace, treated as - * 'previous job done'. - * @BASE_JD_EVENT_STOPPED: Can't be seen by userspace, becomes - * TERMINATED, DONE or JOB_CANCELLED. - * @BASE_JD_EVENT_TERMINATED: This is actually a fault status code - the job - * was hard stopped. - * @BASE_JD_EVENT_ACTIVE: Can't be seen by userspace, jobs only returned on - * complete/fail/cancel. - * @BASE_JD_EVENT_RANGE_HW_NONFAULT_END: End of hardware non-fault status codes. - * Obscurely, BASE_JD_EVENT_TERMINATED - * indicates a real fault, - * because the job was hard-stopped. - * @BASE_JD_EVENT_RANGE_HW_FAULT_OR_SW_ERROR_START: Start of hardware fault and - * software error status codes. - * @BASE_JD_EVENT_RANGE_HW_FAULT_OR_SW_ERROR_END: End of hardware fault and - * software error status codes. - * @BASE_JD_EVENT_RANGE_SW_SUCCESS_START: Start of software success status - * codes. - * @BASE_JD_EVENT_RANGE_SW_SUCCESS_END: End of software success status codes. - * @BASE_JD_EVENT_RANGE_KERNEL_ONLY_START: Start of kernel-only status codes. - * Such codes are never returned to - * user-space. - * @BASE_JD_EVENT_RANGE_KERNEL_ONLY_END: End of kernel-only status codes. - * @BASE_JD_EVENT_DONE: atom has completed successfull - * @BASE_JD_EVENT_JOB_CONFIG_FAULT: Atom dependencies configuration error which - * shall result in a failed atom - * @BASE_JD_EVENT_JOB_POWER_FAULT: The job could not be executed because the - * part of the memory system required to access - * job descriptors was not powered on - * @BASE_JD_EVENT_JOB_READ_FAULT: Reading a job descriptor into the Job - * manager failed - * @BASE_JD_EVENT_JOB_WRITE_FAULT: Writing a job descriptor from the Job - * manager failed - * @BASE_JD_EVENT_JOB_AFFINITY_FAULT: The job could not be executed because the - * specified affinity mask does not intersect - * any available cores - * @BASE_JD_EVENT_JOB_BUS_FAULT: A bus access failed while executing a job - * @BASE_JD_EVENT_INSTR_INVALID_PC: A shader instruction with an illegal program - * counter was executed. - * @BASE_JD_EVENT_INSTR_INVALID_ENC: A shader instruction with an illegal - * encoding was executed. - * @BASE_JD_EVENT_INSTR_TYPE_MISMATCH: A shader instruction was executed where - * the instruction encoding did not match the - * instruction type encoded in the program - * counter. - * @BASE_JD_EVENT_INSTR_OPERAND_FAULT: A shader instruction was executed that - * contained invalid combinations of operands. - * @BASE_JD_EVENT_INSTR_TLS_FAULT: A shader instruction was executed that tried - * to access the thread local storage section - * of another thread. - * @BASE_JD_EVENT_INSTR_ALIGN_FAULT: A shader instruction was executed that - * tried to do an unsupported unaligned memory - * access. - * @BASE_JD_EVENT_INSTR_BARRIER_FAULT: A shader instruction was executed that - * failed to complete an instruction barrier. - * @BASE_JD_EVENT_DATA_INVALID_FAULT: Any data structure read as part of the job - * contains invalid combinations of data. - * @BASE_JD_EVENT_TILE_RANGE_FAULT: Tile or fragment shading was asked to - * process a tile that is entirely outside the - * bounding box of the frame. - * @BASE_JD_EVENT_STATE_FAULT: Matches ADDR_RANGE_FAULT. A virtual address - * has been found that exceeds the virtual - * address range. - * @BASE_JD_EVENT_OUT_OF_MEMORY: The tiler ran out of memory when executing a job. - * @BASE_JD_EVENT_UNKNOWN: If multiple jobs in a job chain fail, only - * the first one the reports an error will set - * and return full error information. - * Subsequent failing jobs will not update the - * error status registers, and may write an - * error status of UNKNOWN. - * @BASE_JD_EVENT_DELAYED_BUS_FAULT: The GPU received a bus fault for access to - * physical memory where the original virtual - * address is no longer available. - * @BASE_JD_EVENT_SHAREABILITY_FAULT: Matches GPU_SHAREABILITY_FAULT. A cache - * has detected that the same line has been - * accessed as both shareable and non-shareable - * memory from inside the GPU. - * @BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL1: A memory access hit an invalid table - * entry at level 1 of the translation table. - * @BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL2: A memory access hit an invalid table - * entry at level 2 of the translation table. - * @BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL3: A memory access hit an invalid table - * entry at level 3 of the translation table. - * @BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL4: A memory access hit an invalid table - * entry at level 4 of the translation table. - * @BASE_JD_EVENT_PERMISSION_FAULT: A memory access could not be allowed due to - * the permission flags set in translation - * table - * @BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL1: A bus fault occurred while reading - * level 0 of the translation tables. - * @BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL2: A bus fault occurred while reading - * level 1 of the translation tables. - * @BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL3: A bus fault occurred while reading - * level 2 of the translation tables. - * @BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL4: A bus fault occurred while reading - * level 3 of the translation tables. - * @BASE_JD_EVENT_ACCESS_FLAG: Matches ACCESS_FLAG_0. A memory access hit a - * translation table entry with the ACCESS_FLAG - * bit set to zero in level 0 of the - * page table, and the DISABLE_AF_FAULT flag - * was not set. - * @BASE_JD_EVENT_MEM_GROWTH_FAILED: raised for JIT_ALLOC atoms that failed to - * grow memory on demand - * @BASE_JD_EVENT_JOB_CANCELLED: raised when this atom was hard-stopped or its - * dependencies failed - * @BASE_JD_EVENT_JOB_INVALID: raised for many reasons, including invalid data - * in the atom which overlaps with - * BASE_JD_EVENT_JOB_CONFIG_FAULT, or if the - * platform doesn't support the feature specified in - * the atom. - * @BASE_JD_EVENT_DRV_TERMINATED: this is a special event generated to indicate - * to userspace that the KBase context has been - * destroyed and Base should stop listening for - * further events - * @BASE_JD_EVENT_REMOVED_FROM_NEXT: raised when an atom that was configured in - * the GPU has to be retried (but it has not - * started) due to e.g., GPU reset - * HW and low-level SW events are represented by event codes. - * The status of jobs which succeeded are also represented by - * an event code (see @BASE_JD_EVENT_DONE). - * Events are usually reported as part of a &struct base_jd_event. - * - * The event codes are encoded in the following way: - * * 10:0 - subtype - * * 12:11 - type - * * 13 - SW success (only valid if the SW bit is set) - * * 14 - SW event (HW event if not set) - * * 15 - Kernel event (should never be seen in userspace) - * - * Events are split up into ranges as follows: - * * BASE_JD_EVENT_RANGE__START - * * BASE_JD_EVENT_RANGE__END - * - * code is in 's range when: - * BASE_JD_EVENT_RANGE__START <= code < - * BASE_JD_EVENT_RANGE__END - * - * Ranges can be asserted for adjacency by testing that the END of the previous - * is equal to the START of the next. This is useful for optimizing some tests - * for range. - * - * A limitation is that the last member of this enum must explicitly be handled - * (with an assert-unreachable statement) in switch statements that use - * variables of this type. Otherwise, the compiler warns that we have not - * handled that enum value. - */ -enum base_jd_event_code { - /* HW defined exceptions */ - BASE_JD_EVENT_RANGE_HW_NONFAULT_START = 0, - - /* non-fatal exceptions */ - BASE_JD_EVENT_NOT_STARTED = 0x00, - BASE_JD_EVENT_DONE = 0x01, - BASE_JD_EVENT_STOPPED = 0x03, - BASE_JD_EVENT_TERMINATED = 0x04, - BASE_JD_EVENT_ACTIVE = 0x08, - - BASE_JD_EVENT_RANGE_HW_NONFAULT_END = 0x40, - BASE_JD_EVENT_RANGE_HW_FAULT_OR_SW_ERROR_START = 0x40, - - /* job exceptions */ - BASE_JD_EVENT_JOB_CONFIG_FAULT = 0x40, - BASE_JD_EVENT_JOB_POWER_FAULT = 0x41, - BASE_JD_EVENT_JOB_READ_FAULT = 0x42, - BASE_JD_EVENT_JOB_WRITE_FAULT = 0x43, - BASE_JD_EVENT_JOB_AFFINITY_FAULT = 0x44, - BASE_JD_EVENT_JOB_BUS_FAULT = 0x48, - BASE_JD_EVENT_INSTR_INVALID_PC = 0x50, - BASE_JD_EVENT_INSTR_INVALID_ENC = 0x51, - BASE_JD_EVENT_INSTR_TYPE_MISMATCH = 0x52, - BASE_JD_EVENT_INSTR_OPERAND_FAULT = 0x53, - BASE_JD_EVENT_INSTR_TLS_FAULT = 0x54, - BASE_JD_EVENT_INSTR_BARRIER_FAULT = 0x55, - BASE_JD_EVENT_INSTR_ALIGN_FAULT = 0x56, - BASE_JD_EVENT_DATA_INVALID_FAULT = 0x58, - BASE_JD_EVENT_TILE_RANGE_FAULT = 0x59, - BASE_JD_EVENT_STATE_FAULT = 0x5A, - BASE_JD_EVENT_OUT_OF_MEMORY = 0x60, - BASE_JD_EVENT_UNKNOWN = 0x7F, - - /* GPU exceptions */ - BASE_JD_EVENT_DELAYED_BUS_FAULT = 0x80, - BASE_JD_EVENT_SHAREABILITY_FAULT = 0x88, - - /* MMU exceptions */ - BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL1 = 0xC1, - BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL2 = 0xC2, - BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL3 = 0xC3, - BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL4 = 0xC4, - BASE_JD_EVENT_PERMISSION_FAULT = 0xC8, - BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL1 = 0xD1, - BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL2 = 0xD2, - BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL3 = 0xD3, - BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL4 = 0xD4, - BASE_JD_EVENT_ACCESS_FLAG = 0xD8, - - /* SW defined exceptions */ - BASE_JD_EVENT_MEM_GROWTH_FAILED = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x000, - BASE_JD_EVENT_JOB_CANCELLED = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x002, - BASE_JD_EVENT_JOB_INVALID = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x003, - - BASE_JD_EVENT_RANGE_HW_FAULT_OR_SW_ERROR_END = BASE_JD_SW_EVENT | - BASE_JD_SW_EVENT_RESERVED | 0x3FF, - - BASE_JD_EVENT_RANGE_SW_SUCCESS_START = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | 0x000, - - BASE_JD_EVENT_DRV_TERMINATED = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | - BASE_JD_SW_EVENT_INFO | 0x000, - - BASE_JD_EVENT_RANGE_SW_SUCCESS_END = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | - BASE_JD_SW_EVENT_RESERVED | 0x3FF, - - BASE_JD_EVENT_RANGE_KERNEL_ONLY_START = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | 0x000, - BASE_JD_EVENT_REMOVED_FROM_NEXT = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | - BASE_JD_SW_EVENT_JOB | 0x000, - - BASE_JD_EVENT_RANGE_KERNEL_ONLY_END = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | - BASE_JD_SW_EVENT_RESERVED | 0x3FF -}; - -/** - * struct base_jd_event_v2 - Event reporting structure - * - * @event_code: event code of type @ref base_jd_event_code. - * @atom_number: the atom number that has completed. - * @padding: padding. - * @udata: user data. - * - * This structure is used by the kernel driver to report information - * about GPU events. They can either be HW-specific events or low-level - * SW events, such as job-chain completion. - * - * The event code contains an event type field which can be extracted - * by ANDing with BASE_JD_SW_EVENT_TYPE_MASK. - */ -struct base_jd_event_v2 { - __u32 event_code; - base_atom_id atom_number; - __u8 padding[3]; - struct base_jd_udata udata; -}; - -/** - * struct base_dump_cpu_gpu_counters - Structure for - * BASE_JD_REQ_SOFT_DUMP_CPU_GPU_COUNTERS - * jobs. - * @system_time: gpu timestamp - * @cycle_counter: gpu cycle count - * @sec: cpu time(sec) - * @usec: cpu time(usec) - * @padding: padding - * - * This structure is stored into the memory pointed to by the @jc field - * of &struct base_jd_atom. - * - * It must not occupy the same CPU cache line(s) as any neighboring data. - * This is to avoid cases where access to pages containing the structure - * is shared between cached and un-cached memory regions, which would - * cause memory corruption. - */ - -struct base_dump_cpu_gpu_counters { - __u64 system_time; - __u64 cycle_counter; - __u64 sec; - __u32 usec; - __u8 padding[36]; -}; - -/** - * struct mali_base_gpu_core_props - GPU core props info - * - * @product_id: Pro specific value. - * @version_status: Status of the GPU release. No defined values, but starts at - * 0 and increases by one for each release status (alpha, beta, EAC, etc.). - * 4 bit values (0-15). - * @minor_revision: Minor release number of the GPU. "P" part of an "RnPn" - * release number. - * 8 bit values (0-255). - * @major_revision: Major release number of the GPU. "R" part of an "RnPn" - * release number. - * 4 bit values (0-15). - * @padding: padding to align to 8-byte - * @gpu_freq_khz_max: The maximum GPU frequency. Reported to applications by - * clGetDeviceInfo() - * @log2_program_counter_size: Size of the shader program counter, in bits. - * @texture_features: TEXTURE_FEATURES_x registers, as exposed by the GPU. This - * is a bitpattern where a set bit indicates that the format is supported. - * Before using a texture format, it is recommended that the corresponding - * bit be checked. - * @paddings_1: Padding bytes. - * @gpu_available_memory_size: Theoretical maximum memory available to the GPU. - * It is unlikely that a client will be able to allocate all of this memory - * for their own purposes, but this at least provides an upper bound on the - * memory available to the GPU. - * This is required for OpenCL's clGetDeviceInfo() call when - * CL_DEVICE_GLOBAL_MEM_SIZE is requested, for OpenCL GPU devices. The - * client will not be expecting to allocate anywhere near this value. - * @num_exec_engines: The number of execution engines. Only valid for tGOX - * (Bifrost) GPUs, where GPU_HAS_REG_CORE_FEATURES is defined. Otherwise, - * this is always 0. - * @paddings_2: Padding bytes. - */ -struct mali_base_gpu_core_props { - __u32 product_id; - __u16 version_status; - __u16 minor_revision; - __u16 major_revision; - __u16 padding; - __u32 gpu_freq_khz_max; - __u32 log2_program_counter_size; - __u32 texture_features[BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS]; - __u8 paddings_1[4]; - __u64 gpu_available_memory_size; - __u8 num_exec_engines; - __u8 paddings_2[7]; -}; - -#endif /* _UAPI_BASE_JM_KERNEL_H_ */ diff --git a/include/uapi/gpu/arm/valhall/jm/mali_kbase_jm_ioctl.h b/include/uapi/gpu/arm/valhall/jm/mali_kbase_jm_ioctl.h deleted file mode 100644 index be554da15ba3..000000000000 --- a/include/uapi/gpu/arm/valhall/jm/mali_kbase_jm_ioctl.h +++ /dev/null @@ -1,274 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#ifndef _UAPI_KBASE_JM_IOCTL_H_ -#define _UAPI_KBASE_JM_IOCTL_H_ - -#include -#include - -/* - * 11.1: - * - Add BASE_MEM_TILER_ALIGN_TOP under base_mem_alloc_flags - * 11.2: - * - KBASE_MEM_QUERY_FLAGS can return KBASE_REG_PF_GROW and KBASE_REG_PROTECTED, - * which some user-side clients prior to 11.2 might fault if they received - * them - * 11.3: - * - New ioctls KBASE_IOCTL_STICKY_RESOURCE_MAP and - * KBASE_IOCTL_STICKY_RESOURCE_UNMAP - * 11.4: - * - New ioctl KBASE_IOCTL_MEM_FIND_GPU_START_AND_OFFSET - * 11.5: - * - New ioctl: KBASE_IOCTL_MEM_JIT_INIT (old ioctl renamed to _OLD) - * 11.6: - * - Added flags field to base_jit_alloc_info structure, which can be used to - * specify pseudo chunked tiler alignment for JIT allocations. - * 11.7: - * - Removed UMP support - * 11.8: - * - Added BASE_MEM_UNCACHED_GPU under base_mem_alloc_flags - * 11.9: - * - Added BASE_MEM_PERMANENT_KERNEL_MAPPING and BASE_MEM_FLAGS_KERNEL_ONLY - * under base_mem_alloc_flags - * 11.10: - * - Enabled the use of nr_extres field of base_jd_atom_v2 structure for - * JIT_ALLOC and JIT_FREE type softjobs to enable multiple JIT allocations - * with one softjob. - * 11.11: - * - Added BASE_MEM_GPU_VA_SAME_4GB_PAGE under base_mem_alloc_flags - * 11.12: - * - Removed ioctl: KBASE_IOCTL_GET_PROFILING_CONTROLS - * 11.13: - * - New ioctl: KBASE_IOCTL_MEM_EXEC_INIT - * 11.14: - * - Add BASE_MEM_GROUP_ID_MASK, base_mem_group_id_get, base_mem_group_id_set - * under base_mem_alloc_flags - * 11.15: - * - Added BASEP_CONTEXT_MMU_GROUP_ID_MASK under base_context_create_flags. - * - Require KBASE_IOCTL_SET_FLAGS before BASE_MEM_MAP_TRACKING_HANDLE can be - * passed to mmap(). - * 11.16: - * - Extended ioctl KBASE_IOCTL_MEM_SYNC to accept imported dma-buf. - * - Modified (backwards compatible) ioctl KBASE_IOCTL_MEM_IMPORT behavior for - * dma-buf. Now, buffers are mapped on GPU when first imported, no longer - * requiring external resource or sticky resource tracking. UNLESS, - * CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND is enabled. - * 11.17: - * - Added BASE_JD_REQ_JOB_SLOT. - * - Reused padding field in base_jd_atom_v2 to pass job slot number. - * - New ioctl: KBASE_IOCTL_GET_CPU_GPU_TIMEINFO - * 11.18: - * - Added BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP under base_mem_alloc_flags - * 11.19: - * - Extended base_jd_atom_v2 to allow a renderpass ID to be specified. - * 11.20: - * - Added new phys_pages member to kbase_ioctl_mem_jit_init for - * KBASE_IOCTL_MEM_JIT_INIT, previous variants of this renamed to use _10_2 - * (replacing '_OLD') and _11_5 suffixes - * - Replaced compat_core_req (deprecated in 10.3) with jit_id[2] in - * base_jd_atom_v2. It must currently be initialized to zero. - * - Added heap_info_gpu_addr to base_jit_alloc_info, and - * BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE allowable in base_jit_alloc_info's - * flags member. Previous variants of this structure are kept and given _10_2 - * and _11_5 suffixes. - * - The above changes are checked for safe values in usual builds - * 11.21: - * - v2.0 of mali_trace debugfs file, which now versions the file separately - * 11.22: - * - Added base_jd_atom (v3), which is seq_nr + base_jd_atom_v2. - * KBASE_IOCTL_JOB_SUBMIT supports both in parallel. - * 11.23: - * - Modified KBASE_IOCTL_MEM_COMMIT behavior to reject requests to modify - * the physical memory backing of JIT allocations. This was not supposed - * to be a valid use case, but it was allowed by the previous implementation. - * 11.24: - * - Added a sysfs file 'serialize_jobs' inside a new sub-directory - * 'scheduling'. - * 11.25: - * - Enabled JIT pressure limit in base/kbase by default - * 11.26 - * - Added kinstr_jm API - * 11.27 - * - Backwards compatible extension to HWC ioctl. - * 11.28: - * - Added kernel side cache ops needed hint - * 11.29: - * - Reserve ioctl 52 - * 11.30: - * - Add a new priority level BASE_JD_PRIO_REALTIME - * - Add ioctl 54: This controls the priority setting. - * 11.31: - * - Added BASE_JD_REQ_LIMITED_CORE_MASK. - * - Added ioctl 55: set_limited_core_count. - * 11.32: - * - Added new HW performance counters interface to all GPUs. - * 11.33: - * - Removed Kernel legacy HWC interface - * 11.34: - * - First release of new HW performance counters interface. - * 11.35: - * - Dummy model (no mali) backend will now clear HWC values after each sample - * 11.36: - * - Remove legacy definitions: - * - base_jit_alloc_info_10_2 - * - base_jit_alloc_info_11_5 - * - kbase_ioctl_mem_jit_init_10_2 - * - kbase_ioctl_mem_jit_init_11_5 - * 11.37: - * - Fix kinstr_prfcnt issues: - * - Missing implicit sample for CMD_STOP when HWCNT buffer is full. - * - Race condition when stopping periodic sampling. - * - prfcnt_block_metadata::block_idx gaps. - * - PRFCNT_CONTROL_CMD_SAMPLE_ASYNC is removed. - * 11.38: - * - Relax the requirement to create a mapping with BASE_MEM_MAP_TRACKING_HANDLE - * before allocating GPU memory for the context. - * - CPU mappings of USER_BUFFER imported memory handles must be cached. - * 11.39: - * - Restrict child process from doing supported file operations (like mmap, ioctl, - * read, poll) on the file descriptor of mali device file that was inherited - * from the parent process. - * 11.40: - * - Remove KBASE_IOCTL_HWCNT_READER_SETUP and KBASE_HWCNT_READER_* ioctls. - * - Made the BASE_MEM_DONT_NEED memory flag queryable. - * 11.41: - * - Disallows changing the sharability on the GPU of imported dma-bufs to - * BASE_MEM_COHERENT_SYSTEM using KBASE_IOCTL_MEM_FLAGS_CHANGE. - * 11.42: - * - Implement full block state support for hardware counters. - * 11.43: - * - Made the BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP and BASE_MEM_KERNEL_SYNC memory - * flags queryable. - * 11.44: - * - Made the SAME_VA memory flag queryable. - * 11.45: - * - Re-allow child process to do supported file operations (like mmap, ioctl - * read, poll) on the file descriptor of mali device that was inherited - * from the parent process. - * 11.46: - * - Remove renderpass_id from base_jd_atom_v2 to deprecate support for JM Incremental Rendering - * 11.47: - * - Reject non-protected allocations containing the BASE_MEM_PROTECTED memory flag. - * - Reject allocations containing the BASE_MEM_DONT_NEED memory flag (it is only settable). - * - Reject allocations containing the BASE_MEM_UNUSED_BIT_xx memory flags. - * 11.48: - * - Add UNUSED_BIT_5, UNUSED_BIT_7, UNUSED_BIT_27 and UNUSED_BIT_29 previously occupied by - * kernel-only flags to kbase cap table. - * 11.49: - * - Increased KBASE_MEM_PROFILE_MAX_BUF_SIZE for more cctx memory classes. - */ - -#define BASE_UK_VERSION_MAJOR 11 -#define BASE_UK_VERSION_MINOR 49 - -/** - * struct kbase_ioctl_version_check - Check version compatibility between - * kernel and userspace - * - * @major: Major version number - * @minor: Minor version number - */ -struct kbase_ioctl_version_check { - __u16 major; - __u16 minor; -}; - -#define KBASE_IOCTL_VERSION_CHECK _IOWR(KBASE_IOCTL_TYPE, 0, struct kbase_ioctl_version_check) - -/** - * struct kbase_ioctl_job_submit - Submit jobs/atoms to the kernel - * - * @addr: Memory address of an array of struct base_jd_atom_v2 or v3 - * @nr_atoms: Number of entries in the array - * @stride: sizeof(struct base_jd_atom_v2) or sizeof(struct base_jd_atom) - */ -struct kbase_ioctl_job_submit { - __u64 addr; - __u32 nr_atoms; - __u32 stride; -}; - -#define KBASE_IOCTL_JOB_SUBMIT _IOW(KBASE_IOCTL_TYPE, 2, struct kbase_ioctl_job_submit) - -#define KBASE_IOCTL_POST_TERM _IO(KBASE_IOCTL_TYPE, 4) - -/** - * struct kbase_ioctl_soft_event_update - Update the status of a soft-event - * @event: GPU address of the event which has been updated - * @new_status: The new status to set - * @flags: Flags for future expansion - */ -struct kbase_ioctl_soft_event_update { - __u64 event; - __u32 new_status; - __u32 flags; -}; - -#define KBASE_IOCTL_SOFT_EVENT_UPDATE \ - _IOW(KBASE_IOCTL_TYPE, 28, struct kbase_ioctl_soft_event_update) - -/** - * struct kbase_kinstr_jm_fd_out - Explains the compatibility information for - * the `struct kbase_kinstr_jm_atom_state_change` structure returned from the - * kernel - * - * @size: The size of the `struct kbase_kinstr_jm_atom_state_change` - * @version: Represents a breaking change in the - * `struct kbase_kinstr_jm_atom_state_change` - * @padding: Explicit padding to get the structure up to 64bits. See - * https://www.kernel.org/doc/Documentation/ioctl/botching-up-ioctls.rst - * - * The `struct kbase_kinstr_jm_atom_state_change` may have extra members at the - * end of the structure that older user space might not understand. If the - * `version` is the same, the structure is still compatible with newer kernels. - * The `size` can be used to cast the opaque memory returned from the kernel. - */ -struct kbase_kinstr_jm_fd_out { - __u16 size; - __u8 version; - __u8 padding[5]; -}; - -/** - * struct kbase_kinstr_jm_fd_in - Options when creating the file descriptor - * - * @count: Number of atom states that can be stored in the kernel circular - * buffer. Must be a power of two - * @padding: Explicit padding to get the structure up to 64bits. See - * https://www.kernel.org/doc/Documentation/ioctl/botching-up-ioctls.rst - */ -struct kbase_kinstr_jm_fd_in { - __u16 count; - __u8 padding[6]; -}; - -union kbase_kinstr_jm_fd { - struct kbase_kinstr_jm_fd_in in; - struct kbase_kinstr_jm_fd_out out; -}; - -#define KBASE_IOCTL_KINSTR_JM_FD _IOWR(KBASE_IOCTL_TYPE, 51, union kbase_kinstr_jm_fd) - -#define KBASE_IOCTL_VERSION_CHECK_RESERVED \ - _IOWR(KBASE_IOCTL_TYPE, 52, struct kbase_ioctl_version_check) - -#endif /* _UAPI_KBASE_JM_IOCTL_H_ */ diff --git a/include/uapi/gpu/arm/valhall/jm/mali_kbase_jm_mem_flags.h b/include/uapi/gpu/arm/valhall/jm/mali_kbase_jm_mem_flags.h deleted file mode 100644 index 5818e01d1b31..000000000000 --- a/include/uapi/gpu/arm/valhall/jm/mali_kbase_jm_mem_flags.h +++ /dev/null @@ -1,65 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#ifndef _UAPI_KBASE_JM_MEM_FLAGS_H_ -#define _UAPI_KBASE_JM_MEM_FLAGS_H_ - -/* Memory allocation, access/hint flags & mask specific to JM GPU. - * - * See base_mem_alloc_flags. - */ - -/* Unused bit for JM, only used in CSF for BASE_MEM_FIXED */ -#define BASE_MEM_UNUSED_BIT_8 ((base_mem_alloc_flags)1 << 8) - -/* Unused bit for JM, only used in CSF for BASE_CSF_EVENT */ -#define BASE_MEM_UNUSED_BIT_19 ((base_mem_alloc_flags)1 << 19) - -/** - * BASE_MEM_TILER_ALIGN_TOP - Memory starting from the end of the initial commit is aligned - * to 'extension' pages, where 'extension' must be a power of 2 and no more than - * BASE_MEM_TILER_ALIGN_TOP_EXTENSION_MAX_PAGES - */ -#define BASE_MEM_TILER_ALIGN_TOP ((base_mem_alloc_flags)1 << 20) - -/* Previously BASEP_MEM_PERFORM_JIT_TRIM, can be reused in the future */ -#define BASE_MEM_UNUSED_BIT_29 ((base_mem_alloc_flags)1 << 29) - -/* A mask of flags that, when provided, cause other flags to be - * enabled but are not enabled themselves - */ -#define BASE_MEM_FLAGS_ACTION_MODIFIERS (BASE_MEM_COHERENT_SYSTEM_REQUIRED | BASE_MEM_IMPORT_SHARED) - -/* A mask of all currently reserved flags */ -#define BASE_MEM_FLAGS_RESERVED ((base_mem_alloc_flags)0) - -/* Number of bits used as flags for base memory management from kernel-side - * only (ie BASEP_MEM_* flags), located from 63 bit downwards: - * < 63 .. (64 - BASEP_MEM_FLAGS_NR_BITS) > - */ -#define BASEP_MEM_FLAGS_NR_BITS (6) - -/* A mask of all bits that are not used by a flag on JM */ -#define BASE_MEM_FLAGS_UNUSED \ - (BASE_MEM_UNUSED_BIT_5 | BASE_MEM_UNUSED_BIT_7 | BASE_MEM_UNUSED_BIT_8 | \ - BASE_MEM_UNUSED_BIT_19 | BASE_MEM_UNUSED_BIT_27 | BASE_MEM_UNUSED_BIT_29) - -#endif /* _UAPI_KBASE_JM_MEM_FLAGS_H_ */ diff --git a/include/uapi/gpu/arm/valhall/mali_base_common_kernel.h b/include/uapi/gpu/arm/valhall/mali_base_common_kernel.h index 66093c1689c1..ef09088b318c 100644 --- a/include/uapi/gpu/arm/valhall/mali_base_common_kernel.h +++ b/include/uapi/gpu/arm/valhall/mali_base_common_kernel.h @@ -25,12 +25,6 @@ #include #include "mali_kbase_mem_flags.h" -struct base_mem_handle { - struct { - __u64 handle; - } basep; -}; - #define BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS 4 /* Flags to pass to ::base_context_init. diff --git a/include/uapi/gpu/arm/valhall/mali_base_kernel.h b/include/uapi/gpu/arm/valhall/mali_base_kernel.h index 21e12a6f250a..2564860b0f12 100644 --- a/include/uapi/gpu/arm/valhall/mali_base_kernel.h +++ b/include/uapi/gpu/arm/valhall/mali_base_kernel.h @@ -167,26 +167,27 @@ struct base_fence { /** * struct base_mem_aliasing_info - Memory aliasing info * - * @handle: Handle to alias, can be BASE_MEM_WRITE_ALLOC_PAGES_HANDLE - * @offset: Offset within the handle to start aliasing from, in pages. - * Not used with BASE_MEM_WRITE_ALLOC_PAGES_HANDLE. - * @length: Length to alias, in pages. For BASE_MEM_WRITE_ALLOC_PAGES_HANDLE + * @gpu_va: GPU VA of an allocation to be aliased, can be a special value + * BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE + * @offset: Offset within the allocation to start aliasing from, in pages. + * Not used with BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE. + * @length: Length to alias, in pages. For BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE * specifies the number of times the special page is needed. * - * Describes a memory handle to be aliased. - * A subset of the handle can be chosen for aliasing, given an offset and a + * Describes a GPU memory allocation to be aliased. + * A subset of the allocation can be chosen for aliasing, given an offset and a * length. - * A special handle BASE_MEM_WRITE_ALLOC_PAGES_HANDLE is used to represent a + * A special value BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE is used to represent a * region where a special page is mapped with a write-alloc cache setup, * typically used when the write result of the GPU isn't needed, but the GPU * must write anyway. * * Offset and length are specified in pages. - * Offset must be within the size of the handle. - * Offset+length must not overrun the size of the handle. + * Offset must be within the size of the allocation. + * Offset+length must not overrun the size of the allocation. */ struct base_mem_aliasing_info { - struct base_mem_handle handle; + __u64 gpu_va; __u64 offset; __u64 length; }; @@ -487,11 +488,7 @@ struct mali_base_gpu_coherent_group_info { struct mali_base_gpu_coherent_group group[BASE_MAX_COHERENT_GROUPS]; }; -#if MALI_USE_CSF #include "csf/mali_base_csf_kernel.h" -#else -#include "jm/mali_base_jm_kernel.h" -#endif /** * struct gpu_raw_gpu_props - A complete description of the GPU's Hardware diff --git a/include/uapi/gpu/arm/valhall/mali_base_mem_priv.h b/include/uapi/gpu/arm/valhall/mali_base_mem_priv.h index 1db3ff11d0c0..5ceda9b54b17 100644 --- a/include/uapi/gpu/arm/valhall/mali_base_mem_priv.h +++ b/include/uapi/gpu/arm/valhall/mali_base_mem_priv.h @@ -33,21 +33,21 @@ * It can either be: * @li a sync from CPU to Memory: * - type = ::BASE_SYNCSET_OP_MSYNC - * - mem_handle = a handle to the memory object on which the operation + * - gpu_va = GPU VA of GPU memory object on which the operation * is taking place * - user_addr = the address of the range to be synced * - size = the amount of data to be synced, in bytes * - offset is ignored. * @li a sync from Memory to CPU: * - type = ::BASE_SYNCSET_OP_CSYNC - * - mem_handle = a handle to the memory object on which the operation + * - gpu_va = GPU VA of GPU memory object on which the operation * is taking place * - user_addr = the address of the range to be synced * - size = the amount of data to be synced, in bytes. * - offset is ignored. */ struct basep_syncset { - struct base_mem_handle mem_handle; + __u64 gpu_va; __u64 user_addr; __u64 size; __u8 type; diff --git a/include/uapi/gpu/arm/valhall/mali_gpu_props.h b/include/uapi/gpu/arm/valhall/mali_gpu_props.h index 09ce542a76c4..5b4166017329 100644 --- a/include/uapi/gpu/arm/valhall/mali_gpu_props.h +++ b/include/uapi/gpu/arm/valhall/mali_gpu_props.h @@ -31,7 +31,9 @@ /** * struct gpu_props_user_data - structure for gpu props user buffer. * @core_props: Core props. + * @unused2: Extra space for address alignment. * @l2_props: L2 props. + * @unused4: Extra space for address alignment. * @tiler_props: Tiler props. * @thread_props: Thread props. * @raw_props: Raw register values kept for backwards compatibility. Kbase @@ -47,17 +49,21 @@ struct gpu_props_user_data { __u16 version_status; __u16 minor_revision; __u16 major_revision; + __u8 unused0[2]; __u32 gpu_freq_khz_max; __u32 log2_program_counter_size; __u32 texture_features[BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS]; + __u8 unused1[4]; __u64 gpu_available_memory_size; __u8 num_exec_engines; } core_props; + __u8 unused2[7]; struct { __u8 log2_line_size; __u8 log2_cache_size; __u8 num_l2_slices; } l2_props; + __u8 unused4[2]; struct { __u32 bin_size_bytes; __u32 max_active_levels; @@ -70,6 +76,7 @@ struct gpu_props_user_data { __u8 max_task_queue; __u8 max_thread_group_split; __u8 impl_tech; + __u8 unused5; __u32 tls_alloc; } thread_props; @@ -103,6 +110,7 @@ struct gpu_props_user_data { __u32 num_groups; __u32 num_core_groups; __u32 coherency; + __u8 unused6[4]; struct { __u64 core_mask; __u32 num_cores; diff --git a/include/uapi/gpu/arm/valhall/mali_kbase_ioctl.h b/include/uapi/gpu/arm/valhall/mali_kbase_ioctl.h index 6c3f8f127fd9..46ea47f252b6 100644 --- a/include/uapi/gpu/arm/valhall/mali_kbase_ioctl.h +++ b/include/uapi/gpu/arm/valhall/mali_kbase_ioctl.h @@ -29,11 +29,7 @@ extern "C" { #include #include -#if MALI_USE_CSF #include "csf/mali_kbase_csf_ioctl.h" -#else -#include "jm/mali_kbase_jm_ioctl.h" -#endif /* MALI_USE_CSF */ #define KBASE_IOCTL_TYPE 0x80 @@ -236,9 +232,9 @@ struct kbase_ioctl_mem_jit_init { #define KBASE_IOCTL_MEM_JIT_INIT _IOW(KBASE_IOCTL_TYPE, 14, struct kbase_ioctl_mem_jit_init) /** - * struct kbase_ioctl_mem_sync - Perform cache maintenance on memory + * struct kbase_ioctl_mem_sync - Perform CPU cache maintenance on GPU memory * - * @handle: GPU memory handle (GPU VA) + * @gpu_va: GPU VA of the memory for which cache maintenance needs to be performed. * @user_addr: The address where it is mapped in user space * @size: The number of bytes to synchronise * @type: The direction to synchronise: 0 is sync to memory (clean), @@ -246,7 +242,7 @@ struct kbase_ioctl_mem_jit_init { * @padding: Padding to round up to a multiple of 8 bytes, must be zero */ struct kbase_ioctl_mem_sync { - __u64 handle; + __u64 gpu_va; __u64 user_addr; __u64 size; __u8 type; diff --git a/include/uapi/gpu/arm/valhall/mali_kbase_kinstr_jm_reader.h b/include/uapi/gpu/arm/valhall/mali_kbase_kinstr_jm_reader.h deleted file mode 100644 index 72e1b9dbf245..000000000000 --- a/include/uapi/gpu/arm/valhall/mali_kbase_kinstr_jm_reader.h +++ /dev/null @@ -1,69 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * mali_kbase_kinstr_jm_reader.h - * Provides an ioctl API to read kernel atom state changes. The flow of the - * API is: - * 1. Obtain the file descriptor with ``KBASE_IOCTL_KINSTR_JM_FD`` - * 2. Determine the buffer structure layout via the above ioctl's returned - * size and version fields in ``struct kbase_kinstr_jm_fd_out`` - * 4. Poll the file descriptor for ``POLLIN`` - * 5. Get data with read() on the fd - * 6. Use the structure version to understand how to read the data from the - * buffer - * 7. Repeat 4-6 - * 8. Close the file descriptor - */ - -#ifndef _UAPI_KBASE_KINSTR_JM_READER_H_ -#define _UAPI_KBASE_KINSTR_JM_READER_H_ - -/** - * enum kbase_kinstr_jm_reader_atom_state - Determines the work state of an atom - * @KBASE_KINSTR_JM_READER_ATOM_STATE_QUEUE: Signifies that an atom has - * entered a hardware queue - * @KBASE_KINSTR_JM_READER_ATOM_STATE_START: Signifies that work has started - * on an atom - * @KBASE_KINSTR_JM_READER_ATOM_STATE_STOP: Signifies that work has stopped - * on an atom - * @KBASE_KINSTR_JM_READER_ATOM_STATE_COMPLETE: Signifies that work has - * completed on an atom - * @KBASE_KINSTR_JM_READER_ATOM_STATE_COUNT: The number of state enumerations - * - * We can add new states to the end of this if they do not break the existing - * state machine. Old user mode code can gracefully ignore states they do not - * understand. - * - * If we need to make a breaking change to the state machine, we can do that by - * changing the version reported by KBASE_IOCTL_KINSTR_JM_FD. This will - * mean that old user mode code will fail to understand the new state field in - * the structure and gracefully not use the state change API. - */ -enum kbase_kinstr_jm_reader_atom_state { - KBASE_KINSTR_JM_READER_ATOM_STATE_QUEUE, - KBASE_KINSTR_JM_READER_ATOM_STATE_START, - KBASE_KINSTR_JM_READER_ATOM_STATE_STOP, - KBASE_KINSTR_JM_READER_ATOM_STATE_COMPLETE, - KBASE_KINSTR_JM_READER_ATOM_STATE_COUNT -}; - -#endif /* _UAPI_KBASE_KINSTR_JM_READER_H_ */ diff --git a/include/uapi/gpu/arm/valhall/mali_kbase_mem_flags.h b/include/uapi/gpu/arm/valhall/mali_kbase_mem_flags.h index 465d27a014c2..04d5ca3c3d8a 100644 --- a/include/uapi/gpu/arm/valhall/mali_kbase_mem_flags.h +++ b/include/uapi/gpu/arm/valhall/mali_kbase_mem_flags.h @@ -22,11 +22,7 @@ #ifndef _UAPI_KBASE_MEM_FLAGS_H_ #define _UAPI_KBASE_MEM_FLAGS_H_ -#if MALI_USE_CSF #include "csf/mali_kbase_csf_mem_flags.h" -#else -#include "jm/mali_kbase_jm_mem_flags.h" -#endif /* Memory allocation, access/hint flags & mask. *