MALI: rockchip: upgrade bifrost DDK to g7p1-01bet0, from g6p0-01eac0

Including modifications under drivers/base/ from the new DDK.

Resolve lots of conflicts.

Fix compilation errors when CONFIG_DEBUG_FS is disabled.

Change-Id: I69f9ac87d927441d0b92b8dac8b704922aeb6a0a
Signed-off-by: Zhen Chen <chenzhen@rock-chips.com>

Documentation/ABI/testing/sysfs-device-mali

@@ -15,10 +15,11 @@
 
 What:		/sys/class/misc/mali%u/device/core_mask
 Description:
-		This attribute is used to restrict number of shader core
-		available, is useful for debugging purposes. Reading
-		this attribute provides us mask of cores available.
-		Writing to it will set the current core mask.
+		This attribute is used to restrict the number of shader cores
+		available in this instance, is useful for debugging purposes.
+		Reading this attribute provides us mask of all cores available.
+		Writing to it will set the current core mask. Doesn't
+		allow disabling all the cores present in this instance.
 
 What:		/sys/class/misc/mali%u/device/debug_command
 Description:
@@ -43,6 +44,14 @@ Description:
 		Its a readonly attribute and on reading gives details on the
 		options used with the dummy workaround.
 
+What:		/sys/class/misc/mali%u/device/fw_timeout
+Description:
+		This attribute is available only with mali platform
+		device-driver that supports a CSF GPU. This attribute is
+		used to set the duration value in milliseconds for the
+		waiting timeout used for a GPU status change request being
+		acknowledged by the FW.
+
 What:		/sys/class/misc/mali%u/device/gpuinfo
 Description:
 		This attribute provides description of the present Mali GPU.

Documentation/devicetree/bindings/arm/mali-bifrost.txt

@@ -1,4 +1,4 @@
-# SPDX-License-Identifier: GPL-2.0
+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 #
 # (C) COPYRIGHT 2013-2021 ARM Limited. All rights reserved.
 #
@@ -97,6 +97,8 @@ for details.
 	  are used at different points so care should be taken to configure
 	  both power models in the device tree (specifically dynamic-coefficient,
 	  static-coefficient and scale) to best match the platform.
+- power_policy : Sets the GPU power policy at probe time. Available options are
+                 "coarse_demand" and "always_on". If not set, then "coarse_demand" is used.
 - system-coherency : Sets the coherency protocol to be used for coherent
 		     accesses made from the GPU.
 		     If not set then no coherency is used.

Documentation/devicetree/bindings/arm/memory_group_manager.txt

@@ -1,6 +1,6 @@
-# SPDX-License-Identifier: GPL-2.0
+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 #
-# (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
+# (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
 #
 # This program is free software and is provided to you under the terms of the
 # GNU General Public License version 2 as published by the Free Software

Documentation/devicetree/bindings/arm/priority_control_manager.txt

@@ -1,6 +1,6 @@
-# SPDX-License-Identifier: GPL-2.0
+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 #
-# (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
+# (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved.
 #
 # This program is free software and is provided to you under the terms of the
 # GNU General Public License version 2 as published by the Free Software

Documentation/devicetree/bindings/arm/protected_memory_allocator.txt

@@ -1,6 +1,6 @@
-# SPDX-License-Identifier: GPL-2.0
+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 #
-# (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
+# (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
 #
 # This program is free software and is provided to you under the terms of the
 # GNU General Public License version 2 as published by the Free Software

Documentation/devicetree/bindings/power/mali-opp.txt

@@ -1,4 +1,4 @@
-# SPDX-License-Identifier: GPL-2.0
+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 #
 # (C) COPYRIGHT 2017, 2019-2021 ARM Limited. All rights reserved.
 #

Documentation/dma-buf-test-exporter.txt

@@ -0,0 +1,42 @@
+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
+#
+# (C) COPYRIGHT 2012-2013, 2020-2021 ARM Limited. All rights reserved.
+#
+# This program is free software and is provided to you under the terms of the
+# GNU General Public License version 2 as published by the Free Software
+# Foundation, and any use by you of this program is subject to the terms
+# of such GNU license.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, you can access it online at
+# http://www.gnu.org/licenses/gpl-2.0.html.
+#
+#
+
+=====================
+dma-buf-test-exporter
+=====================
+
+Overview
+--------
+
+The dma-buf-test-exporter is a simple exporter of dma_buf objects.
+It has a private API to allocate and manipulate the buffers which are represented as dma_buf fds.
+The private API allows:
+* simple allocation of physically non-contiguous buffers
+* simple allocation of physically contiguous buffers
+* query kernel side API usage stats (number of attachments, number of mappings, mmaps)
+* failure mode configuration (fail attach, mapping, mmap)
+* kernel side memset of buffers
+
+The buffers support all of the dma_buf API, including mmap.
+
+It supports being compiled as a module both in-tree and out-of-tree.
+
+See include/linux/dma-buf-test-exporter.h for the ioctl interface.
+See Documentation/dma-buf-sharing.txt for details on dma_buf.

drivers/base/Kconfig

@@ -183,8 +183,7 @@ config SOC_BUS
 
 source "drivers/base/regmap/Kconfig"
 
-source "drivers/base/arm/memory_group_manager/Kconfig"
-source "drivers/base/arm/protected_memory_allocator/Kconfig"
+source "drivers/base/arm/Kconfig"
 
 config DMA_SHARED_BUFFER
 	bool

drivers/base/arm/Kbuild

@@ -0,0 +1,34 @@
+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
+#
+# (C) COPYRIGHT 2021 ARM Limited. All rights reserved.
+#
+# This program is free software and is provided to you under the terms of the
+# GNU General Public License version 2 as published by the Free Software
+# Foundation, and any use by you of this program is subject to the terms
+# of such GNU license.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, you can access it online at
+# http://www.gnu.org/licenses/gpl-2.0.html.
+#
+#
+
+#
+# ccflags
+#
+ccflags-y += -I$(src)/../../../include
+
+subdir-ccflags-y += $(ccflags-y)
+
+#
+# Kernel modules
+#
+obj-$(CONFIG_DMA_SHARED_BUFFER_TEST_EXPORTER) += dma_buf_test_exporter/
+obj-$(CONFIG_MALI_MEMORY_GROUP_MANAGER) += memory_group_manager/
+obj-$(CONFIG_MALI_PROTECTED_MEMORY_ALLOCATOR) += protected_memory_allocator/
+

drivers/base/arm/Kconfig

@@ -0,0 +1,64 @@
+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
+#
+# (C) COPYRIGHT 2021 ARM Limited. All rights reserved.
+#
+# This program is free software and is provided to you under the terms of the
+# GNU General Public License version 2 as published by the Free Software
+# Foundation, and any use by you of this program is subject to the terms
+# of such GNU license.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, you can access it online at
+# http://www.gnu.org/licenses/gpl-2.0.html.
+#
+#
+
+menuconfig MALI_BASE_MODULES
+	bool "Mali Base extra modules"
+	default n
+	help
+	  Enable this option to build support for a Arm Mali base modules.
+	  Those modules provide extra features or debug interfaces and,
+	  are optional for the use of the Mali GPU modules.
+
+config DMA_SHARED_BUFFER_TEST_EXPORTER
+	bool "Build dma-buf framework test exporter module"
+	depends on MALI_BASE_MODULES && DMA_SHARED_BUFFER
+	default y
+	help
+	  This option will build the dma-buf framework test exporter module.
+	  Usable to help test importers.
+
+	  Modules:
+	    - dma-buf-test-exporter.ko
+
+config MALI_MEMORY_GROUP_MANAGER
+	bool "Build Mali Memory Group Manager module"
+	depends on MALI_BASE_MODULES
+	default y
+	help
+	  This option will build the memory group manager module.
+	  This is an example implementation for allocation and release of pages
+	  for memory pools managed by Mali GPU device drivers.
+
+	  Modules:
+	    - memory_group_manager.ko
+
+config MALI_PROTECTED_MEMORY_ALLOCATOR
+	bool "Build Mali Protected Memory Allocator module"
+	depends on MALI_BASE_MODULES && MALI_CSF_SUPPORT
+	default y
+	help
+	  This option will build the protected memory allocator module.
+	  This is an example implementation for allocation and release of pages
+	  of secure memory intended to be used by the firmware
+	  of Mali GPU device drivers.
+
+	  Modules:
+	    - protected_memory_allocator.ko
+

drivers/base/arm/Makefile

@@ -0,0 +1,98 @@
+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
+#
+# (C) COPYRIGHT 2021 ARM Limited. All rights reserved.
+#
+# This program is free software and is provided to you under the terms of the
+# GNU General Public License version 2 as published by the Free Software
+# Foundation, and any use by you of this program is subject to the terms
+# of such GNU license.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, you can access it online at
+# http://www.gnu.org/licenses/gpl-2.0.html.
+#
+#
+
+#
+# Paths
+#
+KERNEL_SRC ?= /lib/modules/$(shell uname -r)/build
+KDIR ?= $(KERNEL_SRC)
+
+ifeq ($(KDIR),)
+    $(error Must specify KDIR to point to the kernel to target))
+endif
+
+vars :=
+#
+# Default configuration values
+#
+CONFIG_MALI_BASE_MODULES ?= n
+
+ifeq ($(CONFIG_MALI_BASE_MODULES),y)
+    CONFIG_MALI_CSF_SUPPORT ?= n
+
+    ifneq ($(CONFIG_DMA_SHARED_BUFFER),n)
+        CONFIG_DMA_SHARED_BUFFER_TEST_EXPORTER ?= y
+    else
+        # Prevent misuse when CONFIG_DMA_SHARED_BUFFER=n
+        CONFIG_DMA_SHARED_BUFFER_TEST_EXPORTER = n
+    endif
+
+    CONFIG_MALI_MEMORY_GROUP_MANAGER ?= y
+
+    ifneq ($(CONFIG_MALI_CSF_SUPPORT), n)
+        CONFIG_MALI_PROTECTED_MEMORY_ALLOCATOR ?= y
+    endif
+
+else
+    # Prevent misuse when CONFIG_MALI_BASE_MODULES=n
+    CONFIG_DMA_SHARED_BUFFER_TEST_EXPORTER = n
+    CONFIG_MALI_MEMORY_GROUP_MANAGER = n
+    CONFIG_MALI_PROTECTED_MEMORY_ALLOCATOR = n
+
+endif
+
+CONFIGS := \
+    CONFIG_MALI_BASE_MODULES \
+    CONFIG_MALI_CSF_SUPPORT \
+    CONFIG_DMA_SHARED_BUFFER_TEST_EXPORTER \
+    CONFIG_MALI_MEMORY_GROUP_MANAGER \
+    CONFIG_MALI_PROTECTED_MEMORY_ALLOCATOR
+
+
+#
+# MAKE_ARGS to pass the custom CONFIGs on out-of-tree build
+#
+# Generate the list of CONFIGs and values.
+# $(value config) is the name of the CONFIG option.
+# $(value $(value config)) is its value (y, m).
+# When the CONFIG is not set to y or m, it defaults to n.
+MAKE_ARGS := $(foreach config,$(CONFIGS), \
+                    $(if $(filter y m,$(value $(value config))), \
+                        $(value config)=$(value $(value config)), \
+                        $(value config)=n))
+
+#
+# EXTRA_CFLAGS to define the custom CONFIGs on out-of-tree build
+#
+# Generate the list of CONFIGs defines with values from CONFIGS.
+# $(value config) is the name of the CONFIG option.
+# When set to y or m, the CONFIG gets defined to 1.
+EXTRA_CFLAGS := $(foreach config,$(CONFIGS), \
+                    $(if $(filter y m,$(value $(value config))), \
+                        -D$(value config)=1))
+
+all:
+	$(MAKE) -C $(KDIR) M=$(CURDIR) $(MAKE_ARGS) EXTRA_CFLAGS="$(EXTRA_CFLAGS)" KBUILD_EXTRA_SYMBOLS="$(EXTRA_SYMBOLS)" modules
+
+modules_install:
+	$(MAKE) -C $(KDIR) M=$(CURDIR) $(MAKE_ARGS) modules_install
+
+clean:
+	$(MAKE) -C $(KDIR) M=$(CURDIR) $(MAKE_ARGS) clean

drivers/base/arm/Mconfig

@@ -0,0 +1,64 @@
+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
+#
+# (C) COPYRIGHT 2021 ARM Limited. All rights reserved.
+#
+# This program is free software and is provided to you under the terms of the
+# GNU General Public License version 2 as published by the Free Software
+# Foundation, and any use by you of this program is subject to the terms
+# of such GNU license.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, you can access it online at
+# http://www.gnu.org/licenses/gpl-2.0.html.
+#
+#
+
+menuconfig MALI_BASE_MODULES
+	bool "Mali Base extra modules"
+	default y if BACKEND_KERNEL
+	help
+	  Enable this option to build support for a Arm Mali base modules.
+	  Those modules provide extra features or debug interfaces and,
+	  are optional for the use of the Mali GPU modules.
+
+config DMA_SHARED_BUFFER_TEST_EXPORTER
+	bool "Build dma-buf framework test exporter module"
+	depends on MALI_BASE_MODULES
+	default y
+	help
+	  This option will build the dma-buf framework test exporter module.
+	  Usable to help test importers.
+
+	  Modules:
+	    - dma-buf-test-exporter.ko
+
+config MALI_MEMORY_GROUP_MANAGER
+	bool "Build Mali Memory Group Manager module"
+	depends on MALI_BASE_MODULES
+	default y
+	help
+	  This option will build the memory group manager module.
+	  This is an example implementation for allocation and release of pages
+	  for memory pools managed by Mali GPU device drivers.
+
+	  Modules:
+	    - memory_group_manager.ko
+
+config MALI_PROTECTED_MEMORY_ALLOCATOR
+	bool "Build Mali Protected Memory Allocator module"
+	depends on MALI_BASE_MODULES && GPU_HAS_CSF
+	default y
+	help
+	  This option will build the protected memory allocator module.
+	  This is an example implementation for allocation and release of pages
+	  of secure memory intended to be used by the firmware
+	  of Mali GPU device drivers.
+
+	  Modules:
+	    - protected_memory_allocator.ko
+

drivers/base/arm/dma_buf_lock/src/Kbuild

@@ -1,6 +1,6 @@
-# SPDX-License-Identifier: GPL-2.0
+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 #
-# (C) COPYRIGHT 2017, 2020 ARM Limited. All rights reserved.
+# (C) COPYRIGHT 2012, 2020-2021 ARM Limited. All rights reserved.
 #
 # This program is free software and is provided to you under the terms of the
 # GNU General Public License version 2 as published by the Free Software
@@ -18,9 +18,6 @@
 #
 #
 
-config MALI_KUTF
- tristate "Mali Kernel Unit Test Framework"
- default m
- help
-   Enables MALI testing framework. To compile it as a module,
-   choose M here - this will generate a single module called kutf.
+ifeq ($(CONFIG_DMA_SHARED_BUFFER), y)
+obj-m := dma_buf_lock.o
+endif

drivers/base/arm/dma_buf_lock/src/Makefile

@@ -1,6 +1,6 @@
-# SPDX-License-Identifier: GPL-2.0
+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 #
-# (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
+# (C) COPYRIGHT 2012, 2020-2021 ARM Limited. All rights reserved.
 #
 # This program is free software and is provided to you under the terms of the
 # GNU General Public License version 2 as published by the Free Software
@@ -27,8 +27,10 @@ ARCH ?= $(shell uname -m)
 KERNEL_SRC ?= /lib/modules/$(shell uname -r)/build
 KDIR ?= $(KERNEL_SRC)
 
-all:
-	$(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) EXTRA_CFLAGS="-I$(CURDIR)/../../../../include" modules CONFIG_MALI_MEMORY_GROUP_MANAGER=m
+all: dma_buf_lock
+
+dma_buf_lock:
+	$(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) EXTRA_CFLAGS="-I$(CURDIR)/../../../../../include"
 
 clean:
 	$(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean

drivers/base/arm/dma_buf_lock/src/dma_buf_lock.c

@@ -0,0 +1,898 @@
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
+/*
+ *
+ * (C) COPYRIGHT 2012-2014, 2017-2018, 2020-2021 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU license.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ */
+
+#include <linux/version.h>
+#include <linux/uaccess.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/fs.h>
+#include <linux/cdev.h>
+#include <linux/device.h>
+#include <linux/slab.h>
+#include <linux/atomic.h>
+#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE)
+#include <linux/reservation.h>
+#else
+#include <linux/dma-resv.h>
+#endif
+#include <linux/dma-buf.h>
+#include <linux/wait.h>
+#include <linux/sched.h>
+#include <linux/poll.h>
+#include <linux/anon_inodes.h>
+#include <linux/file.h>
+
+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE)
+
+#include <linux/fence.h>
+
+#define dma_fence_context_alloc(a) fence_context_alloc(a)
+#define dma_fence_init(a, b, c, d, e) fence_init(a, b, c, d, e)
+#define dma_fence_get(a) fence_get(a)
+#define dma_fence_put(a) fence_put(a)
+#define dma_fence_signal(a) fence_signal(a)
+#define dma_fence_is_signaled(a) fence_is_signaled(a)
+#define dma_fence_add_callback(a, b, c) fence_add_callback(a, b, c)
+#define dma_fence_remove_callback(a, b) fence_remove_callback(a, b)
+
+#if (KERNEL_VERSION(4, 9, 68) > LINUX_VERSION_CODE)
+#define dma_fence_get_status(a) (fence_is_signaled(a) ? (a)->status ?: 1 : 0)
+#else
+#define dma_fence_get_status(a) (fence_is_signaled(a) ? (a)->error ?: 1 : 0)
+#endif
+
+#else
+
+#include <linux/dma-fence.h>
+
+#if (KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE)
+#define dma_fence_get_status(a) (dma_fence_is_signaled(a) ? \
+	(a)->status ?: 1 \
+	: 0)
+#endif
+
+#endif /* < 4.10.0 */
+
+#include "dma_buf_lock.h"
+
+/* Maximum number of buffers that a single handle can address */
+#define DMA_BUF_LOCK_BUF_MAX 32
+
+#define DMA_BUF_LOCK_DEBUG 1
+
+#define DMA_BUF_LOCK_INIT_BIAS  0xFF
+
+static dev_t dma_buf_lock_dev;
+static struct cdev dma_buf_lock_cdev;
+static struct class *dma_buf_lock_class;
+static char dma_buf_lock_dev_name[] = "dma_buf_lock";
+
+#ifdef HAVE_UNLOCKED_IOCTL
+static long dma_buf_lock_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
+#else
+static int dma_buf_lock_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg);
+#endif
+
+static struct file_operations dma_buf_lock_fops =
+{
+	.owner   = THIS_MODULE,
+#ifdef HAVE_UNLOCKED_IOCTL
+	.unlocked_ioctl   = dma_buf_lock_ioctl,
+#else
+	.ioctl   = dma_buf_lock_ioctl,
+#endif
+	.compat_ioctl   = dma_buf_lock_ioctl,
+};
+
+typedef struct dma_buf_lock_resource
+{
+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE)
+	struct fence fence;
+#else
+	struct dma_fence fence;
+#endif
+	int *list_of_dma_buf_fds;               /* List of buffers copied from userspace */
+	atomic_t locked;                        /* Status of lock */
+	struct dma_buf **dma_bufs;
+	unsigned long exclusive;                /* Exclusive access bitmap */
+	atomic_t fence_dep_count;		/* Number of dma-fence dependencies */
+	struct list_head dma_fence_callbacks;	/* list of all callbacks set up to wait on other fences */
+	wait_queue_head_t wait;
+	struct kref refcount;
+	struct list_head link;
+	struct work_struct work;
+	int count;
+} dma_buf_lock_resource;
+
+/**
+ * struct dma_buf_lock_fence_cb - Callback data struct for dma-fence
+ * @fence_cb: Callback function
+ * @fence:    Pointer to the fence object on which this callback is waiting
+ * @res:      Pointer to dma_buf_lock_resource that is waiting on this callback
+ * @node:     List head for linking this callback to the lock resource
+ */
+struct dma_buf_lock_fence_cb {
+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE)
+	struct fence_cb fence_cb;
+	struct fence *fence;
+#else
+	struct dma_fence_cb fence_cb;
+	struct dma_fence *fence;
+#endif
+	struct dma_buf_lock_resource *res;
+	struct list_head node;
+};
+
+static LIST_HEAD(dma_buf_lock_resource_list);
+static DEFINE_MUTEX(dma_buf_lock_mutex);
+
+static inline int is_dma_buf_lock_file(struct file *);
+static void dma_buf_lock_dounlock(struct kref *ref);
+
+
+/*** dma_buf_lock fence part ***/
+
+/* Spin lock protecting all Mali fences as fence->lock. */
+static DEFINE_SPINLOCK(dma_buf_lock_fence_lock);
+
+static const char *
+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE)
+dma_buf_lock_fence_get_driver_name(struct fence *fence)
+#else
+dma_buf_lock_fence_get_driver_name(struct dma_fence *fence)
+#endif
+{
+	return "dma_buf_lock";
+}
+
+static const char *
+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE)
+dma_buf_lock_fence_get_timeline_name(struct fence *fence)
+#else
+dma_buf_lock_fence_get_timeline_name(struct dma_fence *fence)
+#endif
+{
+	return "dma_buf_lock.timeline";
+}
+
+static bool
+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE)
+dma_buf_lock_fence_enable_signaling(struct fence *fence)
+#else
+dma_buf_lock_fence_enable_signaling(struct dma_fence *fence)
+#endif
+{
+	return true;
+}
+
+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE)
+const struct fence_ops dma_buf_lock_fence_ops = {
+	.wait = fence_default_wait,
+#else
+const struct dma_fence_ops dma_buf_lock_fence_ops = {
+	.wait = dma_fence_default_wait,
+#endif
+	.get_driver_name = dma_buf_lock_fence_get_driver_name,
+	.get_timeline_name = dma_buf_lock_fence_get_timeline_name,
+	.enable_signaling = dma_buf_lock_fence_enable_signaling,
+};
+
+static void
+dma_buf_lock_fence_init(dma_buf_lock_resource *resource)
+{
+	dma_fence_init(&resource->fence,
+		       &dma_buf_lock_fence_ops,
+		       &dma_buf_lock_fence_lock,
+		       0,
+		       0);
+}
+
+static void
+dma_buf_lock_fence_free_callbacks(dma_buf_lock_resource *resource)
+{
+	struct dma_buf_lock_fence_cb *cb, *tmp;
+
+	/* Clean up and free callbacks. */
+	list_for_each_entry_safe(cb, tmp, &resource->dma_fence_callbacks, node) {
+		/* Cancel callbacks that hasn't been called yet and release the
+		 * reference taken in dma_buf_lock_fence_add_callback().
+		 */
+		dma_fence_remove_callback(cb->fence, &cb->fence_cb);
+		dma_fence_put(cb->fence);
+		list_del(&cb->node);
+		kfree(cb);
+	}
+}
+
+static void
+dma_buf_lock_fence_work(struct work_struct *pwork)
+{
+	dma_buf_lock_resource *resource =
+		container_of(pwork, dma_buf_lock_resource, work);
+
+	WARN_ON(atomic_read(&resource->fence_dep_count));
+	WARN_ON(!atomic_read(&resource->locked));
+	WARN_ON(!resource->exclusive);
+
+	mutex_lock(&dma_buf_lock_mutex);
+	kref_put(&resource->refcount, dma_buf_lock_dounlock);
+	mutex_unlock(&dma_buf_lock_mutex);
+}
+
+static void
+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE)
+dma_buf_lock_fence_callback(struct fence *fence, struct fence_cb *cb)
+#else
+dma_buf_lock_fence_callback(struct dma_fence *fence, struct dma_fence_cb *cb)
+#endif
+{
+	struct dma_buf_lock_fence_cb *dma_buf_lock_cb = container_of(cb,
+				struct dma_buf_lock_fence_cb,
+				fence_cb);
+	dma_buf_lock_resource *resource = dma_buf_lock_cb->res;
+
+#if DMA_BUF_LOCK_DEBUG
+	printk(KERN_DEBUG "dma_buf_lock_fence_callback\n");
+#endif
+
+	/* Callback function will be invoked in atomic context. */
+
+	if (atomic_dec_and_test(&resource->fence_dep_count)) {
+		atomic_set(&resource->locked, 1);
+		wake_up(&resource->wait);
+
+		if (resource->exclusive)
+			/* Warn if the work was already queued */
+			WARN_ON(!schedule_work(&resource->work));
+	}
+}
+
+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE)
+static int
+dma_buf_lock_fence_add_callback(dma_buf_lock_resource *resource,
+				struct fence *fence,
+				fence_func_t callback)
+#else
+static int
+dma_buf_lock_fence_add_callback(dma_buf_lock_resource *resource,
+				struct dma_fence *fence,
+				dma_fence_func_t callback)
+#endif
+{
+	int err = 0;
+	struct dma_buf_lock_fence_cb *fence_cb;
+
+	if (!fence)
+		return -EINVAL;
+
+	fence_cb = kmalloc(sizeof(*fence_cb), GFP_KERNEL);
+	if (!fence_cb)
+		return -ENOMEM;
+
+	fence_cb->fence = fence;
+	fence_cb->res   = resource;
+	INIT_LIST_HEAD(&fence_cb->node);
+
+	err = dma_fence_add_callback(fence, &fence_cb->fence_cb,
+				     callback);
+
+	if (err == -ENOENT) {
+		/* Fence signaled, get the completion result */
+		err = dma_fence_get_status(fence);
+
+		/* remap success completion to err code */
+		if (err == 1)
+			err = 0;
+
+		kfree(fence_cb);
+	} else if (err) {
+		kfree(fence_cb);
+	} else {
+		/*
+		 * Get reference to fence that will be kept until callback gets
+		 * cleaned up in dma_buf_lock_fence_free_callbacks().
+		 */
+		dma_fence_get(fence);
+		atomic_inc(&resource->fence_dep_count);
+		/* Add callback to resource's list of callbacks */
+		list_add(&fence_cb->node, &resource->dma_fence_callbacks);
+	}
+
+	return err;
+}
+
+#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE)
+static int
+dma_buf_lock_add_fence_reservation_callback(dma_buf_lock_resource *resource,
+					    struct reservation_object *resv,
+					    bool exclusive)
+#else
+static int
+dma_buf_lock_add_fence_reservation_callback(dma_buf_lock_resource *resource,
+					    struct dma_resv *resv,
+					    bool exclusive)
+#endif
+{
+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE)
+	struct fence *excl_fence = NULL;
+	struct fence **shared_fences = NULL;
+#else
+	struct dma_fence *excl_fence = NULL;
+	struct dma_fence **shared_fences = NULL;
+#endif
+	unsigned int shared_count = 0;
+	int err, i;
+
+#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE)
+	err = reservation_object_get_fences_rcu(
+#else
+	err = dma_resv_get_fences_rcu(
+#endif
+						resv,
+						&excl_fence,
+						&shared_count,
+						&shared_fences);
+	if (err)
+		return err;
+
+	if (excl_fence) {
+		err = dma_buf_lock_fence_add_callback(resource,
+						      excl_fence,
+						      dma_buf_lock_fence_callback);
+
+		/* Release our reference, taken by reservation_object_get_fences_rcu(),
+		 * to the fence. We have set up our callback (if that was possible),
+		 * and it's the fence's owner is responsible for singling the fence
+		 * before allowing it to disappear.
+		 */
+		dma_fence_put(excl_fence);
+
+		if (err)
+			goto out;
+	}
+
+	if (exclusive) {
+		for (i = 0; i < shared_count; i++) {
+			err = dma_buf_lock_fence_add_callback(resource,
+							      shared_fences[i],
+							      dma_buf_lock_fence_callback);
+			if (err)
+				goto out;
+		}
+	}
+
+	/* Release all our references to the shared fences, taken by
+	 * reservation_object_get_fences_rcu(). We have set up our callback (if
+	 * that was possible), and it's the fence's owner is responsible for
+	 * signaling the fence before allowing it to disappear.
+	 */
+out:
+	for (i = 0; i < shared_count; i++)
+		dma_fence_put(shared_fences[i]);
+	kfree(shared_fences);
+
+	return err;
+}
+
+static void
+dma_buf_lock_release_fence_reservation(dma_buf_lock_resource *resource,
+				       struct ww_acquire_ctx *ctx)
+{
+	unsigned int r;
+
+	for (r = 0; r < resource->count; r++)
+		ww_mutex_unlock(&resource->dma_bufs[r]->resv->lock);
+	ww_acquire_fini(ctx);
+}
+
+static int
+dma_buf_lock_acquire_fence_reservation(dma_buf_lock_resource *resource,
+				       struct ww_acquire_ctx *ctx)
+{
+#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE)
+	struct reservation_object *content_resv = NULL;
+#else
+	struct dma_resv *content_resv = NULL;
+#endif
+	unsigned int content_resv_idx = 0;
+	unsigned int r;
+	int err = 0;
+
+	ww_acquire_init(ctx, &reservation_ww_class);
+
+retry:
+	for (r = 0; r < resource->count; r++) {
+		if (resource->dma_bufs[r]->resv == content_resv) {
+			content_resv = NULL;
+			continue;
+		}
+
+		err = ww_mutex_lock(&resource->dma_bufs[r]->resv->lock, ctx);
+		if (err)
+			goto error;
+	}
+
+	ww_acquire_done(ctx);
+	return err;
+
+error:
+	content_resv_idx = r;
+
+	/* Unlock the locked one ones */
+	while (r--)
+		ww_mutex_unlock(&resource->dma_bufs[r]->resv->lock);
+
+	if (content_resv)
+		ww_mutex_unlock(&content_resv->lock);
+
+	/* If we deadlock try with lock_slow and retry */
+	if (err == -EDEADLK) {
+#if DMA_BUF_LOCK_DEBUG
+		printk(KERN_DEBUG "deadlock at dma_buf fd %i\n",
+		       resource->list_of_dma_buf_fds[content_resv_idx]);
+#endif
+		content_resv = resource->dma_bufs[content_resv_idx]->resv;
+		ww_mutex_lock_slow(&content_resv->lock, ctx);
+		goto retry;
+	}
+
+	/* If we are here the function failed */
+	ww_acquire_fini(ctx);
+	return err;
+}
+
+static int dma_buf_lock_handle_release(struct inode *inode, struct file *file)
+{
+	dma_buf_lock_resource *resource;
+
+	if (!is_dma_buf_lock_file(file))
+		return -EINVAL;
+
+	resource = file->private_data;
+#if DMA_BUF_LOCK_DEBUG
+	printk("dma_buf_lock_handle_release\n");
+#endif
+	mutex_lock(&dma_buf_lock_mutex);
+	kref_put(&resource->refcount, dma_buf_lock_dounlock);
+	mutex_unlock(&dma_buf_lock_mutex);
+
+	return 0;
+}
+
+static unsigned int dma_buf_lock_handle_poll(struct file *file,
+                                             struct poll_table_struct *wait)
+{
+	dma_buf_lock_resource *resource;
+	unsigned int ret = 0;
+
+	if (!is_dma_buf_lock_file(file))
+		return POLLERR;
+
+	resource = file->private_data;
+#if DMA_BUF_LOCK_DEBUG
+	printk("dma_buf_lock_handle_poll\n");
+#endif
+	if (atomic_read(&resource->locked) == 1) {
+		/* Resources have been locked */
+		ret = POLLIN | POLLRDNORM;
+		if (resource->exclusive)
+			ret |=  POLLOUT | POLLWRNORM;
+	}
+	else
+	{
+		if (!poll_does_not_wait(wait))
+			poll_wait(file, &resource->wait, wait);
+	}
+#if DMA_BUF_LOCK_DEBUG
+	printk("dma_buf_lock_handle_poll : return %i\n", ret);
+#endif
+	return ret;
+}
+
+static const struct file_operations dma_buf_lock_handle_fops = {
+	.owner		= THIS_MODULE,
+	.release	= dma_buf_lock_handle_release,
+	.poll		= dma_buf_lock_handle_poll,
+};
+
+/*
+ * is_dma_buf_lock_file - Check if struct file* is associated with dma_buf_lock
+ */
+static inline int is_dma_buf_lock_file(struct file *file)
+{
+	return file->f_op == &dma_buf_lock_handle_fops;
+}
+
+
+
+/*
+ * Start requested lock.
+ *
+ * Allocates required memory, copies dma_buf_fd list from userspace,
+ * acquires related reservation objects, and starts the lock.
+ */
+static int dma_buf_lock_dolock(dma_buf_lock_k_request *request)
+{
+	dma_buf_lock_resource *resource;
+	struct ww_acquire_ctx ww_ctx;
+	int size;
+	int fd;
+	int i;
+	int ret;
+
+	if (request->list_of_dma_buf_fds == NULL)
+		return -EINVAL;
+	if (request->count <= 0)
+		return -EINVAL;
+	if (request->count > DMA_BUF_LOCK_BUF_MAX)
+		return -EINVAL;
+	if (request->exclusive != DMA_BUF_LOCK_NONEXCLUSIVE &&
+	    request->exclusive != DMA_BUF_LOCK_EXCLUSIVE)
+		return -EINVAL;
+
+	resource = kzalloc(sizeof(dma_buf_lock_resource), GFP_KERNEL);
+	if (resource == NULL)
+		return -ENOMEM;
+
+	atomic_set(&resource->locked, 0);
+	kref_init(&resource->refcount);
+	INIT_LIST_HEAD(&resource->link);
+	INIT_WORK(&resource->work, dma_buf_lock_fence_work);
+	resource->count = request->count;
+
+	/* Allocate space to store dma_buf_fds received from user space */
+	size = request->count * sizeof(int);
+	resource->list_of_dma_buf_fds = kmalloc(size, GFP_KERNEL);
+
+	if (resource->list_of_dma_buf_fds == NULL) {
+		kfree(resource);
+		return -ENOMEM;
+	}
+
+	/* Allocate space to store dma_buf pointers associated with dma_buf_fds */
+	size = sizeof(struct dma_buf *) * request->count;
+	resource->dma_bufs = kmalloc(size, GFP_KERNEL);
+
+	if (resource->dma_bufs == NULL) {
+		kfree(resource->list_of_dma_buf_fds);
+		kfree(resource);
+		return -ENOMEM;
+	}
+
+	/* Copy requested list of dma_buf_fds from user space */
+	size = request->count * sizeof(int);
+	if (copy_from_user(resource->list_of_dma_buf_fds,
+			   (void __user *)request->list_of_dma_buf_fds,
+			   size) != 0) {
+		kfree(resource->list_of_dma_buf_fds);
+		kfree(resource->dma_bufs);
+		kfree(resource);
+		return -ENOMEM;
+	}
+#if DMA_BUF_LOCK_DEBUG
+	for (i = 0; i < request->count; i++)
+		printk("dma_buf %i = %X\n", i, resource->list_of_dma_buf_fds[i]);
+#endif
+
+	/* Initialize the fence associated with dma_buf_lock resource */
+	dma_buf_lock_fence_init(resource);
+
+	INIT_LIST_HEAD(&resource->dma_fence_callbacks);
+
+	atomic_set(&resource->fence_dep_count, DMA_BUF_LOCK_INIT_BIAS);
+
+	/* Add resource to global list */
+	mutex_lock(&dma_buf_lock_mutex);
+
+	list_add(&resource->link, &dma_buf_lock_resource_list);
+
+	mutex_unlock(&dma_buf_lock_mutex);
+
+	for (i = 0; i < request->count; i++)
+	{
+		/* Convert fd into dma_buf structure */
+		resource->dma_bufs[i] = dma_buf_get(resource->list_of_dma_buf_fds[i]);
+
+		if (IS_ERR_VALUE(PTR_ERR(resource->dma_bufs[i])))
+		{
+			mutex_lock(&dma_buf_lock_mutex);
+			kref_put(&resource->refcount, dma_buf_lock_dounlock);
+			mutex_unlock(&dma_buf_lock_mutex);
+			return -EINVAL;
+		}
+
+		/*Check the reservation object associated with dma_buf */
+		if (resource->dma_bufs[i]->resv == NULL) {
+			mutex_lock(&dma_buf_lock_mutex);
+			kref_put(&resource->refcount, dma_buf_lock_dounlock);
+			mutex_unlock(&dma_buf_lock_mutex);
+			return -EINVAL;
+		}
+#if DMA_BUF_LOCK_DEBUG
+		printk(KERN_DEBUG "dma_buf_lock_dolock : dma_buf_fd %i dma_buf %p dma_fence reservation %p\n",
+		       resource->list_of_dma_buf_fds[i], resource->dma_bufs[i], resource->dma_bufs[i]->resv);
+#endif
+	}
+
+	init_waitqueue_head(&resource->wait);
+
+	kref_get(&resource->refcount);
+
+	/* Create file descriptor associated with lock request */
+	fd = anon_inode_getfd("dma_buf_lock", &dma_buf_lock_handle_fops,
+	                      (void *)resource, 0);
+	if (fd < 0)
+	{
+		mutex_lock(&dma_buf_lock_mutex);
+		kref_put(&resource->refcount, dma_buf_lock_dounlock);
+		kref_put(&resource->refcount, dma_buf_lock_dounlock);
+		mutex_unlock(&dma_buf_lock_mutex);
+		return fd;
+	}
+
+	resource->exclusive = request->exclusive;
+
+	/* Start locking process */
+	ret = dma_buf_lock_acquire_fence_reservation(resource, &ww_ctx);
+	if (ret) {
+#if DMA_BUF_LOCK_DEBUG
+		printk(KERN_DEBUG "dma_buf_lock_dolock : Error %d locking reservations.\n", ret);
+#endif
+		put_unused_fd(fd);
+		mutex_lock(&dma_buf_lock_mutex);
+		kref_put(&resource->refcount, dma_buf_lock_dounlock);
+		kref_put(&resource->refcount, dma_buf_lock_dounlock);
+		mutex_unlock(&dma_buf_lock_mutex);
+		return ret;
+	}
+
+	/* Take an extra reference for exclusive access, which will be dropped
+	 * once the pre-existing fences attached to dma-buf resources, for which
+	 * we have commited for exclusive access, are signaled.
+	 * At a given time there can be only one exclusive fence attached to a
+	 * reservation object, so the new exclusive fence replaces the original
+	 * fence and the future sync is done against the new fence which is
+	 * supposed to be signaled only after the original fence was signaled.
+	 * If the new exclusive fence is signaled prematurely then the resources
+	 * would become available for new access while they are already being
+	 * written to by the original owner.
+	 */
+	if (resource->exclusive)
+		kref_get(&resource->refcount);
+
+	for (i = 0; i < request->count; i++) {
+#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE)
+		struct reservation_object *resv = resource->dma_bufs[i]->resv;
+#else
+		struct dma_resv *resv = resource->dma_bufs[i]->resv;
+#endif
+		if (!test_bit(i, &resource->exclusive)) {
+
+#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE)
+			ret = reservation_object_reserve_shared(resv);
+#else
+			ret = dma_resv_reserve_shared(resv, 0);
+#endif
+			if (ret) {
+#if DMA_BUF_LOCK_DEBUG
+				printk(KERN_DEBUG "dma_buf_lock_dolock : Error %d reserving space for shared fence.\n", ret);
+#endif
+				break;
+			}
+
+			ret = dma_buf_lock_add_fence_reservation_callback(resource,
+									  resv,
+									  false);
+			if (ret) {
+#if DMA_BUF_LOCK_DEBUG
+				printk(KERN_DEBUG "dma_buf_lock_dolock : Error %d adding reservation to callback.\n", ret);
+#endif
+				break;
+			}
+
+#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE)
+			reservation_object_add_shared_fence(resv, &resource->fence);
+#else
+			dma_resv_add_shared_fence(resv, &resource->fence);
+#endif
+		} else {
+			ret = dma_buf_lock_add_fence_reservation_callback(resource,
+									  resv,
+									  true);
+			if (ret) {
+#if DMA_BUF_LOCK_DEBUG
+				printk(KERN_DEBUG "dma_buf_lock_dolock : Error %d adding reservation to callback.\n", ret);
+#endif
+				break;
+			}
+
+#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE)
+			reservation_object_add_excl_fence(resv, &resource->fence);
+#else
+			dma_resv_add_excl_fence(resv, &resource->fence);
+#endif
+		}
+	}
+
+	dma_buf_lock_release_fence_reservation(resource, &ww_ctx);
+
+	/* Test if the callbacks were already triggered */
+	if (!atomic_sub_return(DMA_BUF_LOCK_INIT_BIAS, &resource->fence_dep_count)) {
+		atomic_set(&resource->locked, 1);
+
+		/* Drop the extra reference taken for exclusive access */
+		if (resource->exclusive)
+			dma_buf_lock_fence_work(&resource->work);
+	}
+
+	if (IS_ERR_VALUE((unsigned long)ret))
+	{
+		put_unused_fd(fd);
+
+		mutex_lock(&dma_buf_lock_mutex);
+		kref_put(&resource->refcount, dma_buf_lock_dounlock);
+		kref_put(&resource->refcount, dma_buf_lock_dounlock);
+		mutex_unlock(&dma_buf_lock_mutex);
+
+		return ret;
+	}
+
+#if DMA_BUF_LOCK_DEBUG
+	printk("dma_buf_lock_dolock : complete\n");
+#endif
+	mutex_lock(&dma_buf_lock_mutex);
+	kref_put(&resource->refcount, dma_buf_lock_dounlock);
+	mutex_unlock(&dma_buf_lock_mutex);
+
+	return fd;
+}
+
+static void dma_buf_lock_dounlock(struct kref *ref)
+{
+	int i;
+	dma_buf_lock_resource *resource = container_of(ref, dma_buf_lock_resource, refcount);
+
+	atomic_set(&resource->locked, 0);
+
+	/* Signal the resource's fence. */
+	dma_fence_signal(&resource->fence);
+
+	dma_buf_lock_fence_free_callbacks(resource);
+
+	list_del(&resource->link);
+
+	for (i = 0; i < resource->count; i++)
+	{
+		if (resource->dma_bufs[i])
+			dma_buf_put(resource->dma_bufs[i]);
+	}
+
+	kfree(resource->dma_bufs);
+	kfree(resource->list_of_dma_buf_fds);
+	dma_fence_put(&resource->fence);
+}
+
+static int __init dma_buf_lock_init(void)
+{
+	int err;
+#if DMA_BUF_LOCK_DEBUG
+	printk("dma_buf_lock_init\n");
+#endif
+	err = alloc_chrdev_region(&dma_buf_lock_dev, 0, 1, dma_buf_lock_dev_name);
+
+	if (err == 0) {
+		cdev_init(&dma_buf_lock_cdev, &dma_buf_lock_fops);
+
+		err = cdev_add(&dma_buf_lock_cdev, dma_buf_lock_dev, 1);
+
+		if (err == 0) {
+			dma_buf_lock_class = class_create(THIS_MODULE, dma_buf_lock_dev_name);
+			if (IS_ERR(dma_buf_lock_class))
+				err = PTR_ERR(dma_buf_lock_class);
+			else
+			{
+				struct device *mdev;
+				mdev = device_create(dma_buf_lock_class, NULL, dma_buf_lock_dev, NULL, dma_buf_lock_dev_name);
+				if (!IS_ERR(mdev))
+					return 0;
+
+				err = PTR_ERR(mdev);
+				class_destroy(dma_buf_lock_class);
+			}
+			cdev_del(&dma_buf_lock_cdev);
+		}
+
+		unregister_chrdev_region(dma_buf_lock_dev, 1);
+	}
+#if DMA_BUF_LOCK_DEBUG
+	printk("dma_buf_lock_init failed\n");
+#endif
+	return err;
+}
+
+static void __exit dma_buf_lock_exit(void)
+{
+#if DMA_BUF_LOCK_DEBUG
+	printk("dma_buf_lock_exit\n");
+#endif
+
+	/* Unlock all outstanding references */
+	while (1)
+	{
+		mutex_lock(&dma_buf_lock_mutex);
+		if (list_empty(&dma_buf_lock_resource_list))
+		{
+			mutex_unlock(&dma_buf_lock_mutex);
+			break;
+		}
+		else
+		{
+			dma_buf_lock_resource *resource = list_entry(dma_buf_lock_resource_list.next,
+			                                             dma_buf_lock_resource, link);
+			kref_put(&resource->refcount, dma_buf_lock_dounlock);
+			mutex_unlock(&dma_buf_lock_mutex);
+		}
+	}
+
+	device_destroy(dma_buf_lock_class, dma_buf_lock_dev);
+
+	class_destroy(dma_buf_lock_class);
+
+	cdev_del(&dma_buf_lock_cdev);
+
+	unregister_chrdev_region(dma_buf_lock_dev, 1);
+}
+
+#ifdef HAVE_UNLOCKED_IOCTL
+static long dma_buf_lock_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
+#else
+static int dma_buf_lock_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
+#endif
+{
+	dma_buf_lock_k_request request;
+	int size = _IOC_SIZE(cmd);
+
+	if (_IOC_TYPE(cmd) != DMA_BUF_LOCK_IOC_MAGIC)
+		return -ENOTTY;
+	if ((_IOC_NR(cmd) < DMA_BUF_LOCK_IOC_MINNR) || (_IOC_NR(cmd) > DMA_BUF_LOCK_IOC_MAXNR))
+		return -ENOTTY;
+
+	switch (cmd)
+	{
+		case DMA_BUF_LOCK_FUNC_LOCK_ASYNC:
+			if (size != sizeof(dma_buf_lock_k_request))
+				return -ENOTTY;
+			if (copy_from_user(&request, (void __user *)arg, size))
+				return -EFAULT;
+#if DMA_BUF_LOCK_DEBUG
+			printk("DMA_BUF_LOCK_FUNC_LOCK_ASYNC - %i\n", request.count);
+#endif
+			return dma_buf_lock_dolock(&request);
+	}
+
+	return -ENOTTY;
+}
+
+module_init(dma_buf_lock_init);
+module_exit(dma_buf_lock_exit);
+
+MODULE_LICENSE("GPL");
+

drivers/base/arm/dma_buf_lock/src/dma_buf_lock.h

@@ -0,0 +1,46 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+/*
+ *
+ * (C) COPYRIGHT 2012, 2020-2021 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU license.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ */
+
+#ifndef _DMA_BUF_LOCK_H
+#define _DMA_BUF_LOCK_H
+
+typedef enum dma_buf_lock_exclusive
+{
+	DMA_BUF_LOCK_NONEXCLUSIVE = 0,
+	DMA_BUF_LOCK_EXCLUSIVE = -1
+} dma_buf_lock_exclusive;
+
+typedef struct dma_buf_lock_k_request
+{
+	int count;
+	int *list_of_dma_buf_fds;
+	int timeout;
+	dma_buf_lock_exclusive exclusive;
+} dma_buf_lock_k_request;
+
+#define DMA_BUF_LOCK_IOC_MAGIC '~'
+
+#define DMA_BUF_LOCK_FUNC_LOCK_ASYNC       _IOW(DMA_BUF_LOCK_IOC_MAGIC, 11, dma_buf_lock_k_request)
+
+#define DMA_BUF_LOCK_IOC_MINNR 11
+#define DMA_BUF_LOCK_IOC_MAXNR 11
+
+#endif /* _DMA_BUF_LOCK_H */

drivers/base/arm/dma_buf_test_exporter/Kbuild

@@ -0,0 +1,23 @@
+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
+#
+# (C) COPYRIGHT 2012, 2020-2021 ARM Limited. All rights reserved.
+#
+# This program is free software and is provided to you under the terms of the
+# GNU General Public License version 2 as published by the Free Software
+# Foundation, and any use by you of this program is subject to the terms
+# of such GNU license.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, you can access it online at
+# http://www.gnu.org/licenses/gpl-2.0.html.
+#
+#
+
+ifeq ($(CONFIG_DMA_SHARED_BUFFER_TEST_EXPORTER), y)
+obj-m += dma-buf-test-exporter.o
+endif

drivers/base/arm/dma_buf_test_exporter/build.bp

@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+/*
+ *
+ * (C) COPYRIGHT 2017, 2020-2021 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU license.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ */
+
+bob_kernel_module {
+    name: "dma-buf-test-exporter",
+    defaults: [
+        "kernel_defaults"
+    ],
+    srcs: [
+        "Kbuild",
+        "dma-buf-test-exporter.c",
+    ],
+    enabled: false,
+    dma_shared_buffer_test_exporter: {
+        kbuild_options: ["CONFIG_DMA_SHARED_BUFFER_TEST_EXPORTER=y"],
+        enabled: true,
+    },
+}

drivers/base/arm/dma_buf_test_exporter/dma-buf-test-exporter.c

@@ -0,0 +1,824 @@
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
+/*
+ *
+ * (C) COPYRIGHT 2012-2021 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU license.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ */
+
+#include <linux/dma-buf-test-exporter.h>
+#include <linux/dma-buf.h>
+#include <linux/miscdevice.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include <linux/version.h>
+#include <linux/module.h>
+#include <linux/fs.h>
+#include <linux/atomic.h>
+#include <linux/mm.h>
+#include <linux/highmem.h>
+#if (KERNEL_VERSION(4, 8, 0) > LINUX_VERSION_CODE)
+#include <linux/dma-attrs.h>
+#endif
+#include <linux/dma-mapping.h>
+
+/* Maximum size allowed in a single DMA_BUF_TE_ALLOC call */
+#define DMA_BUF_TE_ALLOC_MAX_SIZE ((8ull << 30) >> PAGE_SHIFT) /* 8 GB */
+
+/* Since kernel version 5.0 CONFIG_ARCH_NO_SG_CHAIN replaced CONFIG_ARCH_HAS_SG_CHAIN */
+#if KERNEL_VERSION(5, 0, 0) > LINUX_VERSION_CODE
+#if (!defined(ARCH_HAS_SG_CHAIN) && !defined(CONFIG_ARCH_HAS_SG_CHAIN))
+#define NO_SG_CHAIN
+#endif
+#elif defined(CONFIG_ARCH_NO_SG_CHAIN)
+#define NO_SG_CHAIN
+#endif
+
+struct dma_buf_te_alloc {
+	/* the real alloc */
+	size_t nr_pages;
+	struct page **pages;
+
+	/* the debug usage tracking */
+	int nr_attached_devices;
+	int nr_device_mappings;
+	int nr_cpu_mappings;
+
+	/* failure simulation */
+	int fail_attach;
+	int fail_map;
+	int fail_mmap;
+
+	bool contiguous;
+	dma_addr_t contig_dma_addr;
+	void *contig_cpu_addr;
+};
+
+struct dma_buf_te_attachment {
+	struct sg_table *sg;
+	bool attachment_mapped;
+};
+
+static struct miscdevice te_device;
+
+#if (KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE)
+static int dma_buf_te_attach(struct dma_buf *buf, struct device *dev, struct dma_buf_attachment *attachment)
+#else
+static int dma_buf_te_attach(struct dma_buf *buf, struct dma_buf_attachment *attachment)
+#endif
+{
+	struct dma_buf_te_alloc	*alloc;
+	alloc = buf->priv;
+
+	if (alloc->fail_attach)
+		return -EFAULT;
+
+	attachment->priv = kzalloc(sizeof(struct dma_buf_te_attachment), GFP_KERNEL);
+	if (!attachment->priv)
+		return -ENOMEM;
+
+	/* dma_buf is externally locked during call */
+	alloc->nr_attached_devices++;
+	return 0;
+}
+
+static void dma_buf_te_detach(struct dma_buf *buf, struct dma_buf_attachment *attachment)
+{
+	struct dma_buf_te_alloc *alloc = buf->priv;
+	struct dma_buf_te_attachment *pa = attachment->priv;
+
+	/* dma_buf is externally locked during call */
+
+	WARN(pa->attachment_mapped, "WARNING: dma-buf-test-exporter detected detach with open device mappings");
+
+	alloc->nr_attached_devices--;
+
+	kfree(pa);
+}
+
+static struct sg_table *dma_buf_te_map(struct dma_buf_attachment *attachment, enum dma_data_direction direction)
+{
+	struct sg_table *sg;
+	struct scatterlist *iter;
+	struct dma_buf_te_alloc	*alloc;
+	struct dma_buf_te_attachment *pa = attachment->priv;
+	size_t i;
+	int ret;
+
+	alloc = attachment->dmabuf->priv;
+
+	if (alloc->fail_map)
+		return ERR_PTR(-ENOMEM);
+
+	if (WARN(pa->attachment_mapped,
+	    "WARNING: Attempted to map already mapped attachment."))
+		return ERR_PTR(-EBUSY);
+
+#ifdef NO_SG_CHAIN
+	/* if the ARCH can't chain we can't have allocs larger than a single sg can hold */
+	if (alloc->nr_pages > SG_MAX_SINGLE_ALLOC)
+		return ERR_PTR(-EINVAL);
+#endif /* NO_SG_CHAIN */
+
+	sg = kmalloc(sizeof(struct sg_table), GFP_KERNEL);
+	if (!sg)
+		return ERR_PTR(-ENOMEM);
+
+	/* from here we access the allocation object, so lock the dmabuf pointing to it */
+	mutex_lock(&attachment->dmabuf->lock);
+
+	if (alloc->contiguous)
+		ret = sg_alloc_table(sg, 1, GFP_KERNEL);
+	else
+		ret = sg_alloc_table(sg, alloc->nr_pages, GFP_KERNEL);
+	if (ret) {
+		mutex_unlock(&attachment->dmabuf->lock);
+		kfree(sg);
+		return ERR_PTR(ret);
+	}
+
+	if (alloc->contiguous) {
+		sg_dma_len(sg->sgl) = alloc->nr_pages * PAGE_SIZE;
+		sg_set_page(sg->sgl, pfn_to_page(PFN_DOWN(alloc->contig_dma_addr)), alloc->nr_pages * PAGE_SIZE, 0);
+		sg_dma_address(sg->sgl) = alloc->contig_dma_addr;
+	} else {
+		for_each_sg(sg->sgl, iter, alloc->nr_pages, i)
+			sg_set_page(iter, alloc->pages[i], PAGE_SIZE, 0);
+	}
+
+	if (!dma_map_sg(attachment->dev, sg->sgl, sg->nents, direction)) {
+		mutex_unlock(&attachment->dmabuf->lock);
+		sg_free_table(sg);
+		kfree(sg);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	alloc->nr_device_mappings++;
+	pa->attachment_mapped = true;
+	pa->sg = sg;
+	mutex_unlock(&attachment->dmabuf->lock);
+	return sg;
+}
+
+static void dma_buf_te_unmap(struct dma_buf_attachment *attachment,
+							 struct sg_table *sg, enum dma_data_direction direction)
+{
+	struct dma_buf_te_alloc *alloc;
+	struct dma_buf_te_attachment *pa = attachment->priv;
+
+	alloc = attachment->dmabuf->priv;
+
+	mutex_lock(&attachment->dmabuf->lock);
+
+	WARN(!pa->attachment_mapped, "WARNING: Unmatched unmap of attachment.");
+
+	alloc->nr_device_mappings--;
+	pa->attachment_mapped = false;
+	pa->sg = NULL;
+	mutex_unlock(&attachment->dmabuf->lock);
+
+	dma_unmap_sg(attachment->dev, sg->sgl, sg->nents, direction);
+	sg_free_table(sg);
+	kfree(sg);
+}
+
+static void dma_buf_te_release(struct dma_buf *buf)
+{
+	size_t i;
+	struct dma_buf_te_alloc *alloc;
+	alloc = buf->priv;
+	/* no need for locking */
+
+	if (alloc->contiguous) {
+#if (KERNEL_VERSION(4, 8, 0) <= LINUX_VERSION_CODE)
+		dma_free_attrs(te_device.this_device,
+						alloc->nr_pages * PAGE_SIZE,
+						alloc->contig_cpu_addr,
+						alloc->contig_dma_addr,
+						DMA_ATTR_WRITE_COMBINE);
+#else
+		DEFINE_DMA_ATTRS(attrs);
+
+		dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs);
+		dma_free_attrs(te_device.this_device,
+						alloc->nr_pages * PAGE_SIZE,
+						alloc->contig_cpu_addr, alloc->contig_dma_addr, &attrs);
+#endif
+	} else {
+		for (i = 0; i < alloc->nr_pages; i++)
+			__free_page(alloc->pages[i]);
+	}
+#if (KERNEL_VERSION(4, 12, 0) <= LINUX_VERSION_CODE)
+	kvfree(alloc->pages);
+#else
+	kfree(alloc->pages);
+#endif
+	kfree(alloc);
+}
+
+static int dma_buf_te_sync(struct dma_buf *dmabuf,
+			enum dma_data_direction direction,
+			bool start_cpu_access)
+{
+	struct dma_buf_attachment *attachment;
+
+	mutex_lock(&dmabuf->lock);
+
+	list_for_each_entry(attachment, &dmabuf->attachments, node) {
+		struct dma_buf_te_attachment *pa = attachment->priv;
+		struct sg_table *sg = pa->sg;
+		if (!sg) {
+			dev_dbg(te_device.this_device, "no mapping for device %s\n", dev_name(attachment->dev));
+			continue;
+		}
+
+		if (start_cpu_access) {
+			dev_dbg(te_device.this_device, "sync cpu with device %s\n", dev_name(attachment->dev));
+
+			dma_sync_sg_for_cpu(attachment->dev, sg->sgl, sg->nents, direction);
+		} else {
+			dev_dbg(te_device.this_device, "sync device %s with cpu\n", dev_name(attachment->dev));
+
+			dma_sync_sg_for_device(attachment->dev, sg->sgl, sg->nents, direction);
+		}
+	}
+
+	mutex_unlock(&dmabuf->lock);
+	return 0;
+}
+
+#if (KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE)
+static int dma_buf_te_begin_cpu_access(struct dma_buf *dmabuf,
+					enum dma_data_direction direction)
+#else
+static int dma_buf_te_begin_cpu_access(struct dma_buf *dmabuf, size_t start,
+					size_t len,
+					enum dma_data_direction direction)
+#endif
+{
+	return dma_buf_te_sync(dmabuf, direction, true);
+}
+
+#if (KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE)
+static int dma_buf_te_end_cpu_access(struct dma_buf *dmabuf,
+				enum dma_data_direction direction)
+{
+	return dma_buf_te_sync(dmabuf, direction, false);
+}
+#else
+static void dma_buf_te_end_cpu_access(struct dma_buf *dmabuf, size_t start,
+				size_t len,
+				enum dma_data_direction direction)
+{
+	dma_buf_te_sync(dmabuf, direction, false);
+}
+#endif
+
+static void dma_buf_te_mmap_open(struct vm_area_struct *vma)
+{
+	struct dma_buf *dma_buf;
+	struct dma_buf_te_alloc *alloc;
+	dma_buf = vma->vm_private_data;
+	alloc = dma_buf->priv;
+
+	mutex_lock(&dma_buf->lock);
+	alloc->nr_cpu_mappings++;
+	mutex_unlock(&dma_buf->lock);
+}
+
+static void dma_buf_te_mmap_close(struct vm_area_struct *vma)
+{
+	struct dma_buf *dma_buf;
+	struct dma_buf_te_alloc *alloc;
+	dma_buf = vma->vm_private_data;
+	alloc = dma_buf->priv;
+
+	BUG_ON(alloc->nr_cpu_mappings <= 0);
+	mutex_lock(&dma_buf->lock);
+	alloc->nr_cpu_mappings--;
+	mutex_unlock(&dma_buf->lock);
+}
+
+#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
+static int dma_buf_te_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
+#elif KERNEL_VERSION(5, 1, 0) > LINUX_VERSION_CODE
+static int dma_buf_te_mmap_fault(struct vm_fault *vmf)
+#else
+static vm_fault_t dma_buf_te_mmap_fault(struct vm_fault *vmf)
+#endif
+{
+	struct dma_buf_te_alloc *alloc;
+	struct dma_buf *dmabuf;
+	struct page *pageptr;
+
+#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
+	dmabuf = vma->vm_private_data;
+#else
+	dmabuf = vmf->vma->vm_private_data;
+#endif
+	alloc = dmabuf->priv;
+
+	if (vmf->pgoff > alloc->nr_pages)
+		return VM_FAULT_SIGBUS;
+
+	pageptr = alloc->pages[vmf->pgoff];
+
+	BUG_ON(!pageptr);
+
+	get_page(pageptr);
+	vmf->page = pageptr;
+
+	return 0;
+}
+
+struct vm_operations_struct dma_buf_te_vm_ops = {
+	.open = dma_buf_te_mmap_open,
+	.close = dma_buf_te_mmap_close,
+	.fault = dma_buf_te_mmap_fault
+};
+
+static int dma_buf_te_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
+{
+	struct dma_buf_te_alloc *alloc;
+	alloc = dmabuf->priv;
+
+	if (alloc->fail_mmap)
+		return -ENOMEM;
+
+	vma->vm_flags |= VM_IO | VM_DONTEXPAND | VM_DONTDUMP;
+	vma->vm_ops = &dma_buf_te_vm_ops;
+	vma->vm_private_data = dmabuf;
+
+	/*  we fault in the pages on access */
+
+	/* call open to do the ref-counting */
+	dma_buf_te_vm_ops.open(vma);
+
+	return 0;
+}
+
+#if KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE
+static void *dma_buf_te_kmap_atomic(struct dma_buf *buf, unsigned long page_num)
+{
+	/* IGNORE */
+	return NULL;
+}
+#endif
+
+static void *dma_buf_te_kmap(struct dma_buf *buf, unsigned long page_num)
+{
+	struct dma_buf_te_alloc *alloc;
+
+	alloc = buf->priv;
+	if (page_num >= alloc->nr_pages)
+		return NULL;
+
+	return kmap(alloc->pages[page_num]);
+}
+static void dma_buf_te_kunmap(struct dma_buf *buf,
+		unsigned long page_num, void *addr)
+{
+	struct dma_buf_te_alloc *alloc;
+
+	alloc = buf->priv;
+	if (page_num >= alloc->nr_pages)
+		return;
+
+	kunmap(alloc->pages[page_num]);
+	return;
+}
+
+static struct dma_buf_ops dma_buf_te_ops = {
+	/* real handlers */
+	.attach = dma_buf_te_attach,
+	.detach = dma_buf_te_detach,
+	.map_dma_buf = dma_buf_te_map,
+	.unmap_dma_buf = dma_buf_te_unmap,
+	.release = dma_buf_te_release,
+	.mmap = dma_buf_te_mmap,
+	.begin_cpu_access = dma_buf_te_begin_cpu_access,
+	.end_cpu_access = dma_buf_te_end_cpu_access,
+#if KERNEL_VERSION(4, 12, 0) > LINUX_VERSION_CODE
+	.kmap = dma_buf_te_kmap,
+	.kunmap = dma_buf_te_kunmap,
+
+	/* nop handlers for mandatory functions we ignore */
+	.kmap_atomic = dma_buf_te_kmap_atomic
+#else
+#if KERNEL_VERSION(5, 6, 0) > LINUX_VERSION_CODE
+	.map = dma_buf_te_kmap,
+	.unmap = dma_buf_te_kunmap,
+#endif
+
+#if KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE
+	/* nop handlers for mandatory functions we ignore */
+	.map_atomic = dma_buf_te_kmap_atomic
+#endif
+#endif
+};
+
+static int do_dma_buf_te_ioctl_version(struct dma_buf_te_ioctl_version __user *buf)
+{
+	struct dma_buf_te_ioctl_version v;
+
+	if (copy_from_user(&v, buf, sizeof(v)))
+		return -EFAULT;
+
+	if (v.op != DMA_BUF_TE_ENQ)
+		return -EFAULT;
+
+	v.op = DMA_BUF_TE_ACK;
+	v.major = DMA_BUF_TE_VER_MAJOR;
+	v.minor = DMA_BUF_TE_VER_MINOR;
+
+	if (copy_to_user(buf, &v, sizeof(v)))
+		return -EFAULT;
+	else
+		return 0;
+}
+
+static int do_dma_buf_te_ioctl_alloc(struct dma_buf_te_ioctl_alloc __user *buf, bool contiguous)
+{
+	struct dma_buf_te_ioctl_alloc alloc_req;
+	struct dma_buf_te_alloc *alloc;
+	struct dma_buf *dma_buf;
+	size_t i = 0;
+	size_t max_nr_pages = DMA_BUF_TE_ALLOC_MAX_SIZE;
+	int fd;
+
+	if (copy_from_user(&alloc_req, buf, sizeof(alloc_req))) {
+		dev_err(te_device.this_device, "%s: couldn't get user data", __func__);
+		goto no_input;
+	}
+
+	if (!alloc_req.size) {
+		dev_err(te_device.this_device, "%s: no size specified", __func__);
+		goto invalid_size;
+	}
+
+#ifdef NO_SG_CHAIN
+	/* Whilst it is possible to allocate larger buffer, we won't be able to
+	 * map it during actual usage (mmap() still succeeds). We fail here so
+	 * userspace code can deal with it early than having driver failure
+	 * later on.
+	 */
+	if (max_nr_pages > SG_MAX_SINGLE_ALLOC)
+		max_nr_pages = SG_MAX_SINGLE_ALLOC;
+#endif /* NO_SG_CHAIN */
+
+	if (alloc_req.size > max_nr_pages) {
+		dev_err(te_device.this_device, "%s: buffer size of %llu pages exceeded the mapping limit of %zu pages",
+				__func__, alloc_req.size, max_nr_pages);
+		goto invalid_size;
+	}
+
+	alloc = kzalloc(sizeof(struct dma_buf_te_alloc), GFP_KERNEL);
+	if (alloc == NULL) {
+		dev_err(te_device.this_device, "%s: couldn't alloc object", __func__);
+		goto no_alloc_object;
+	}
+
+	alloc->nr_pages = alloc_req.size;
+	alloc->contiguous = contiguous;
+
+#if (KERNEL_VERSION(4, 12, 0) <= LINUX_VERSION_CODE)
+	alloc->pages = kvzalloc(sizeof(struct page *) * alloc->nr_pages, GFP_KERNEL);
+#else
+	alloc->pages = kzalloc(sizeof(struct page *) * alloc->nr_pages, GFP_KERNEL);
+#endif
+
+	if (!alloc->pages) {
+		dev_err(te_device.this_device,
+				"%s: couldn't alloc %zu page structures",
+				__func__, alloc->nr_pages);
+		goto free_alloc_object;
+	}
+
+	if (contiguous) {
+		dma_addr_t dma_aux;
+
+#if (KERNEL_VERSION(4, 8, 0) <= LINUX_VERSION_CODE)
+		alloc->contig_cpu_addr = dma_alloc_attrs(te_device.this_device,
+				alloc->nr_pages * PAGE_SIZE,
+				&alloc->contig_dma_addr,
+				GFP_KERNEL | __GFP_ZERO,
+				DMA_ATTR_WRITE_COMBINE);
+#else
+		DEFINE_DMA_ATTRS(attrs);
+
+		dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs);
+		alloc->contig_cpu_addr = dma_alloc_attrs(te_device.this_device,
+				alloc->nr_pages * PAGE_SIZE,
+				&alloc->contig_dma_addr,
+				GFP_KERNEL | __GFP_ZERO, &attrs);
+#endif
+		if (!alloc->contig_cpu_addr) {
+			dev_err(te_device.this_device, "%s: couldn't alloc contiguous buffer %zu pages",
+				__func__, alloc->nr_pages);
+			goto free_page_struct;
+		}
+		dma_aux = alloc->contig_dma_addr;
+		for (i = 0; i < alloc->nr_pages; i++) {
+			alloc->pages[i] = pfn_to_page(PFN_DOWN(dma_aux));
+			dma_aux += PAGE_SIZE;
+		}
+	} else {
+		for (i = 0; i < alloc->nr_pages; i++) {
+			alloc->pages[i] = alloc_page(GFP_KERNEL | __GFP_ZERO);
+			if (alloc->pages[i] == NULL) {
+				dev_err(te_device.this_device, "%s: couldn't alloc page", __func__);
+				goto no_page;
+			}
+		}
+	}
+
+	/* alloc ready, let's export it */
+	{
+		struct dma_buf_export_info export_info = {
+			.exp_name = "dma_buf_te",
+			.owner = THIS_MODULE,
+			.ops = &dma_buf_te_ops,
+			.size = alloc->nr_pages << PAGE_SHIFT,
+			.flags = O_CLOEXEC | O_RDWR,
+			.priv = alloc,
+		};
+
+		dma_buf = dma_buf_export(&export_info);
+	}
+
+	if (IS_ERR_OR_NULL(dma_buf)) {
+		dev_err(te_device.this_device, "%s: couldn't export dma_buf", __func__);
+		goto no_export;
+	}
+
+	/* get fd for buf */
+	fd = dma_buf_fd(dma_buf, O_CLOEXEC);
+
+	if (fd < 0) {
+		dev_err(te_device.this_device, "%s: couldn't get fd from dma_buf", __func__);
+		goto no_fd;
+	}
+
+	return fd;
+
+no_fd:
+	dma_buf_put(dma_buf);
+no_export:
+	/* i still valid */
+no_page:
+	if (contiguous) {
+#if (KERNEL_VERSION(4, 8, 0) <= LINUX_VERSION_CODE)
+		dma_free_attrs(te_device.this_device,
+						alloc->nr_pages * PAGE_SIZE,
+						alloc->contig_cpu_addr,
+						alloc->contig_dma_addr,
+						DMA_ATTR_WRITE_COMBINE);
+#else
+		DEFINE_DMA_ATTRS(attrs);
+
+		dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs);
+		dma_free_attrs(te_device.this_device,
+						alloc->nr_pages * PAGE_SIZE,
+						alloc->contig_cpu_addr, alloc->contig_dma_addr, &attrs);
+#endif
+	} else {
+		while (i-- > 0)
+			__free_page(alloc->pages[i]);
+	}
+free_page_struct:
+#if (KERNEL_VERSION(4, 12, 0) <= LINUX_VERSION_CODE)
+	kvfree(alloc->pages);
+#else
+	kfree(alloc->pages);
+#endif
+free_alloc_object:
+	kfree(alloc);
+no_alloc_object:
+invalid_size:
+no_input:
+	return -EFAULT;
+}
+
+static int do_dma_buf_te_ioctl_status(struct dma_buf_te_ioctl_status __user *arg)
+{
+	struct dma_buf_te_ioctl_status status;
+	struct dma_buf *dmabuf;
+	struct dma_buf_te_alloc *alloc;
+	int res = -EINVAL;
+
+	if (copy_from_user(&status, arg, sizeof(status)))
+		return -EFAULT;
+
+	dmabuf = dma_buf_get(status.fd);
+	if (IS_ERR_OR_NULL(dmabuf))
+		return -EINVAL;
+
+	/* verify it's one of ours */
+	if (dmabuf->ops != &dma_buf_te_ops)
+		goto err_have_dmabuf;
+
+	/* ours, get the current status */
+	alloc = dmabuf->priv;
+
+	/* lock while reading status to take a snapshot */
+	mutex_lock(&dmabuf->lock);
+	status.attached_devices = alloc->nr_attached_devices;
+	status.device_mappings = alloc->nr_device_mappings;
+	status.cpu_mappings = alloc->nr_cpu_mappings;
+	mutex_unlock(&dmabuf->lock);
+
+	if (copy_to_user(arg, &status, sizeof(status)))
+		goto err_have_dmabuf;
+
+	/* All OK */
+	res = 0;
+
+err_have_dmabuf:
+	dma_buf_put(dmabuf);
+	return res;
+}
+
+static int do_dma_buf_te_ioctl_set_failing(struct dma_buf_te_ioctl_set_failing __user *arg)
+{
+	struct dma_buf *dmabuf;
+	struct dma_buf_te_ioctl_set_failing f;
+	struct dma_buf_te_alloc *alloc;
+	int res = -EINVAL;
+
+	if (copy_from_user(&f, arg, sizeof(f)))
+		return -EFAULT;
+
+	dmabuf = dma_buf_get(f.fd);
+	if (IS_ERR_OR_NULL(dmabuf))
+		return -EINVAL;
+
+	/* verify it's one of ours */
+	if (dmabuf->ops != &dma_buf_te_ops)
+		goto err_have_dmabuf;
+
+	/* ours, set the fail modes */
+	alloc = dmabuf->priv;
+	/* lock to set the fail modes atomically */
+	mutex_lock(&dmabuf->lock);
+	alloc->fail_attach = f.fail_attach;
+	alloc->fail_map    = f.fail_map;
+	alloc->fail_mmap   = f.fail_mmap;
+	mutex_unlock(&dmabuf->lock);
+
+	/* success */
+	res = 0;
+
+err_have_dmabuf:
+	dma_buf_put(dmabuf);
+	return res;
+}
+
+static u32 dma_te_buf_fill(struct dma_buf *dma_buf, unsigned int value)
+{
+	struct dma_buf_attachment *attachment;
+	struct sg_table *sgt;
+	struct scatterlist *sg;
+	unsigned int count;
+	unsigned int offset = 0;
+	int ret = 0;
+	size_t i;
+
+	attachment = dma_buf_attach(dma_buf, te_device.this_device);
+	if (IS_ERR_OR_NULL(attachment))
+		return -EBUSY;
+
+	sgt = dma_buf_map_attachment(attachment, DMA_BIDIRECTIONAL);
+	if (IS_ERR_OR_NULL(sgt)) {
+		ret = PTR_ERR(sgt);
+		goto no_import;
+	}
+
+	ret = dma_buf_begin_cpu_access(dma_buf,
+#if KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE
+				       0, dma_buf->size,
+#endif
+				       DMA_BIDIRECTIONAL);
+	if (ret)
+		goto no_cpu_access;
+
+	for_each_sg(sgt->sgl, sg, sgt->nents, count) {
+		for (i = 0; i < sg_dma_len(sg); i = i + PAGE_SIZE) {
+			void *addr = NULL;
+#if KERNEL_VERSION(5, 6, 0) <= LINUX_VERSION_CODE
+			addr = dma_buf_te_kmap(dma_buf, i >> PAGE_SHIFT);
+#else
+			addr = dma_buf_kmap(dma_buf, i >> PAGE_SHIFT);
+#endif
+			if (!addr) {
+				ret = -EPERM;
+				goto no_kmap;
+			}
+			memset(addr, value, PAGE_SIZE);
+#if KERNEL_VERSION(5, 6, 0) <= LINUX_VERSION_CODE
+			dma_buf_te_kunmap(dma_buf, i >> PAGE_SHIFT, addr);
+#else
+			dma_buf_kunmap(dma_buf, i >> PAGE_SHIFT, addr);
+#endif
+		}
+		offset += sg_dma_len(sg);
+	}
+
+no_kmap:
+	dma_buf_end_cpu_access(dma_buf,
+#if KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE
+			       0, dma_buf->size,
+#endif
+			       DMA_BIDIRECTIONAL);
+no_cpu_access:
+	dma_buf_unmap_attachment(attachment, sgt, DMA_BIDIRECTIONAL);
+no_import:
+	dma_buf_detach(dma_buf, attachment);
+	return ret;
+}
+
+static int do_dma_buf_te_ioctl_fill(struct dma_buf_te_ioctl_fill __user *arg)
+{
+
+	struct dma_buf *dmabuf;
+	struct dma_buf_te_ioctl_fill f;
+	int ret;
+
+	if (copy_from_user(&f, arg, sizeof(f)))
+		return -EFAULT;
+
+	dmabuf = dma_buf_get(f.fd);
+	if (IS_ERR_OR_NULL(dmabuf))
+		return -EINVAL;
+
+	ret = dma_te_buf_fill(dmabuf, f.value);
+	dma_buf_put(dmabuf);
+
+	return ret;
+}
+
+static long dma_buf_te_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	switch (cmd) {
+	case DMA_BUF_TE_VERSION:
+		return do_dma_buf_te_ioctl_version((struct dma_buf_te_ioctl_version __user *)arg);
+	case DMA_BUF_TE_ALLOC:
+		return do_dma_buf_te_ioctl_alloc((struct dma_buf_te_ioctl_alloc __user *)arg, false);
+	case DMA_BUF_TE_ALLOC_CONT:
+		return do_dma_buf_te_ioctl_alloc((struct dma_buf_te_ioctl_alloc __user *)arg, true);
+	case DMA_BUF_TE_QUERY:
+		return do_dma_buf_te_ioctl_status((struct dma_buf_te_ioctl_status __user *)arg);
+	case DMA_BUF_TE_SET_FAILING:
+		return do_dma_buf_te_ioctl_set_failing((struct dma_buf_te_ioctl_set_failing __user *)arg);
+	case DMA_BUF_TE_FILL:
+		return do_dma_buf_te_ioctl_fill((struct dma_buf_te_ioctl_fill __user *)arg);
+	default:
+		return -ENOTTY;
+	}
+}
+
+static const struct file_operations dma_buf_te_fops = {
+	.owner = THIS_MODULE,
+	.unlocked_ioctl = dma_buf_te_ioctl,
+	.compat_ioctl = dma_buf_te_ioctl,
+};
+
+static int __init dma_buf_te_init(void)
+{
+	int res;
+	te_device.minor = MISC_DYNAMIC_MINOR;
+	te_device.name = "dma_buf_te";
+	te_device.fops = &dma_buf_te_fops;
+
+	res = misc_register(&te_device);
+	if (res) {
+		printk(KERN_WARNING"Misc device registration failed of 'dma_buf_te'\n");
+		return res;
+	}
+	te_device.this_device->coherent_dma_mask = DMA_BIT_MASK(32);
+
+	dev_info(te_device.this_device, "dma_buf_te ready\n");
+	return 0;
+
+}
+
+static void __exit dma_buf_te_exit(void)
+{
+	misc_deregister(&te_device);
+}
+
+module_init(dma_buf_te_init);
+module_exit(dma_buf_te_exit);
+MODULE_LICENSE("GPL");

drivers/base/arm/memory_group_manager/Kbuild

@@ -1,4 +1,4 @@
-# SPDX-License-Identifier: GPL-2.0
+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 #
 # (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
 #
@@ -18,4 +18,6 @@
 #
 #
 
-obj-$(CONFIG_MALI_MEMORY_GROUP_MANAGER) := memory_group_manager.o
+ifeq ($(CONFIG_MALI_MEMORY_GROUP_MANAGER), y)
+obj-m := memory_group_manager.o
+endif

drivers/base/arm/memory_group_manager/build.bp

@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
  * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
@@ -21,10 +21,16 @@
 
 bob_kernel_module {
     name: "memory_group_manager",
+    defaults: [
+        "kernel_defaults"
+    ],
     srcs: [
         "Kbuild",
         "memory_group_manager.c",
     ],
-    kbuild_options: ["CONFIG_MALI_MEMORY_GROUP_MANAGER=m"],
-    defaults: ["kernel_defaults"],
+    enabled: false,
+    mali_memory_group_manager: {
+        kbuild_options: ["CONFIG_MALI_MEMORY_GROUP_MANAGER=y"],
+        enabled: true,
+    },
 }

drivers/base/arm/memory_group_manager/memory_group_manager.c

@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
  * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
@@ -25,7 +25,7 @@
 #include <linux/platform_device.h>
 #include <linux/version.h>
 #include <linux/module.h>
-#ifdef CONFIG_DEBUG_FS
+#if IS_ENABLED(CONFIG_DEBUG_FS)
 #include <linux/debugfs.h>
 #endif
 #include <linux/mm.h>
@@ -97,12 +97,12 @@ struct mgm_group {
 struct mgm_groups {
 	struct mgm_group groups[MEMORY_GROUP_MANAGER_NR_GROUPS];
 	struct device *dev;
-#ifdef CONFIG_DEBUG_FS
+#if IS_ENABLED(CONFIG_DEBUG_FS)
 	struct dentry *mgm_debugfs_root;
 #endif
 };
 
-#ifdef CONFIG_DEBUG_FS
+#if IS_ENABLED(CONFIG_DEBUG_FS)
 
 static int mgm_size_get(void *data, u64 *val)
 {

drivers/base/arm/protected_memory_allocator/Kbuild

@@ -1,4 +1,4 @@
-# SPDX-License-Identifier: GPL-2.0
+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 #
 # (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
 #
@@ -18,4 +18,6 @@
 #
 #
 
-obj-$(CONFIG_MALI_PROTECTED_MEMORY_ALLOCATOR) := protected_memory_allocator.o
+ifeq ($(CONFIG_MALI_PROTECTED_MEMORY_ALLOCATOR), y)
+obj-m := protected_memory_allocator.o
+endif

drivers/base/arm/protected_memory_allocator/Makefile

@@ -1,37 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-#
-# (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
-#
-# This program is free software and is provided to you under the terms of the
-# GNU General Public License version 2 as published by the Free Software
-# Foundation, and any use by you of this program is subject to the terms
-# of such GNU license.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, you can access it online at
-# http://www.gnu.org/licenses/gpl-2.0.html.
-#
-#
-
-# linux build system bootstrap for out-of-tree module
-
-# default to building for the host
-ARCH ?= $(shell uname -m)
-
-# Handle Android Common Kernel source naming
-KERNEL_SRC ?= /lib/modules/$(shell uname -r)/build
-KDIR ?= $(KERNEL_SRC)
-
-all:
-	$(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) EXTRA_CFLAGS="-I$(CURDIR)/../../../../include" modules CONFIG_MALI_PROTECTED_MEMORY_ALLOCATOR=m
-
-clean:
-	$(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean
-
-modules_install:
-	$(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) modules_install

drivers/base/arm/protected_memory_allocator/build.bp

@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
  * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
@@ -21,14 +21,16 @@
 
 bob_kernel_module {
     name: "protected_memory_allocator",
+    defaults: [
+        "kernel_defaults"
+    ],
     srcs: [
         "Kbuild",
         "protected_memory_allocator.c",
     ],
-    kbuild_options: ["CONFIG_MALI_PROTECTED_MEMORY_ALLOCATOR=m"],
-    defaults: ["kernel_defaults"],
     enabled: false,
-    build_csf_only_module: {
+    mali_protected_memory_allocator: {
+        kbuild_options: ["CONFIG_MALI_PROTECTED_MEMORY_ALLOCATOR=y"],
         enabled: true,
     },
 }

drivers/base/arm/protected_memory_allocator/protected_memory_allocator.c

@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
  * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.

drivers/gpu/arm/bifrost/Kbuild

@@ -1,4 +1,4 @@
-# SPDX-License-Identifier: GPL-2.0
+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 #
 # (C) COPYRIGHT 2012-2021 ARM Limited. All rights reserved.
 #
@@ -18,221 +18,237 @@
 #
 #
 
-# Driver version string which is returned to userspace via an ioctl
-MALI_RELEASE_NAME ?= '"g6p0-01eac0"'
-
-# Paths required for build
-
 # make $(src) as absolute path if it is not already, by prefixing $(srctree)
+# This is to prevent any build issue due to wrong path.
 src:=$(if $(patsubst /%,,$(src)),$(srctree)/$(src),$(src))
-KBASE_PATH = $(src)
-KBASE_PLATFORM_PATH = $(KBASE_PATH)/platform_dummy
-UMP_PATH = $(src)/../../../base
 
+#
+# Prevent misuse when Kernel configurations are not present by default
+# in out-of-tree builds
+#
+ifneq ($(CONFIG_ANDROID),n)
+ifeq ($(CONFIG_GPU_TRACEPOINTS),n)
+    $(error CONFIG_GPU_TRACEPOINTS must be set in Kernel configuration)
+endif
+endif
+
+ifeq ($(CONFIG_DMA_SHARED_BUFFER),n)
+    $(error CONFIG_DMA_SHARED_BUFFER must be set in Kernel configuration)
+endif
+
+ifeq ($(CONFIG_PM_DEVFREQ),n)
+    $(error CONFIG_PM_DEVFREQ must be set in Kernel configuration)
+endif
+
+ifeq ($(CONFIG_DEVFREQ_THERMAL),n)
+    $(error CONFIG_DEVFREQ_THERMAL must be set in Kernel configuration)
+endif
+
+ifeq ($(CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND),n)
+    $(error CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND must be set in Kernel configuration)
+endif
+
+ifeq ($(CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS), y)
+    ifneq ($(CONFIG_DEBUG_FS), y)
+        $(error CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS depends on CONFIG_DEBUG_FS to be set in Kernel configuration)
+    endif
+endif
+
+ifeq ($(CONFIG_MALI_BIFROST_FENCE_DEBUG), y)
+    ifneq ($(CONFIG_SYNC), y)
+        ifneq ($(CONFIG_SYNC_FILE), y)
+            $(error CONFIG_MALI_BIFROST_FENCE_DEBUG depends on CONFIG_SYNC || CONFIG_SYNC_FILE to be set in Kernel configuration)
+        endif
+    endif
+endif
+
+#
+# Configurations
+#
+
+# Driver version string which is returned to userspace via an ioctl
+MALI_RELEASE_NAME ?= '"g7p1-01bet0"'
 # Set up defaults if not defined by build system
-MALI_CUSTOMER_RELEASE ?= 1
-MALI_UNIT_TEST ?= 0
+ifeq ($(CONFIG_MALI_BIFROST_DEBUG), y)
+    MALI_UNIT_TEST = 1
+    MALI_CUSTOMER_RELEASE ?= 0
+else
+    MALI_UNIT_TEST ?= 0
+    MALI_CUSTOMER_RELEASE ?= 1
+endif
 MALI_COVERAGE ?= 0
+
 CONFIG_MALI_PLATFORM_NAME ?= "devicetree"
+
+# Kconfig passes in the name with quotes for in-tree builds - remove them.
+MALI_PLATFORM_DIR := $(shell echo $(CONFIG_MALI_PLATFORM_NAME))
+
+ifeq ($(CONFIG_MALI_CSF_SUPPORT),y)
+    MALI_JIT_PRESSURE_LIMIT_BASE = 0
+    MALI_USE_CSF = 1
+else
+    MALI_JIT_PRESSURE_LIMIT_BASE ?= 1
+    MALI_USE_CSF ?= 0
+endif
+
+ifneq ($(CONFIG_MALI_KUTF), n)
+    MALI_KERNEL_TEST_API ?= 1
+else
+    MALI_KERNEL_TEST_API ?= 0
+endif
+
 # Experimental features (corresponding -D definition should be appended to
-# DEFINES below, e.g. for MALI_EXPERIMENTAL_FEATURE,
+# ccflags-y below, e.g. for MALI_EXPERIMENTAL_FEATURE,
 # -DMALI_EXPERIMENTAL_FEATURE=$(MALI_EXPERIMENTAL_FEATURE) should be appended)
 #
 # Experimental features must default to disabled, e.g.:
 # MALI_EXPERIMENTAL_FEATURE ?= 0
 MALI_INCREMENTAL_RENDERING ?= 0
 
-ifeq ($(CONFIG_MALI_CSF_SUPPORT),y)
-MALI_JIT_PRESSURE_LIMIT_BASE = 0
-MALI_USE_CSF = 1
-else
-MALI_JIT_PRESSURE_LIMIT_BASE ?= 1
-MALI_USE_CSF ?= 0
-endif
+#
+# ccflags
+#
+ccflags-y = \
+    -DMALI_CUSTOMER_RELEASE=$(MALI_CUSTOMER_RELEASE) \
+    -DMALI_USE_CSF=$(MALI_USE_CSF) \
+    -DMALI_KERNEL_TEST_API=$(MALI_KERNEL_TEST_API) \
+    -DMALI_UNIT_TEST=$(MALI_UNIT_TEST) \
+    -DMALI_COVERAGE=$(MALI_COVERAGE) \
+    -DMALI_RELEASE_NAME=$(MALI_RELEASE_NAME) \
+    -DMALI_JIT_PRESSURE_LIMIT_BASE=$(MALI_JIT_PRESSURE_LIMIT_BASE) \
+    -DMALI_INCREMENTAL_RENDERING=$(MALI_INCREMENTAL_RENDERING) \
+    -DMALI_KBASE_BUILD \
+    -DMALI_PLATFORM_DIR=$(MALI_PLATFORM_DIR)
 
-ifneq ($(CONFIG_MALI_KUTF), n)
-MALI_KERNEL_TEST_API ?= 1
-else
-MALI_KERNEL_TEST_API ?= 0
-endif
-
-# Set up our defines, which will be passed to gcc
-DEFINES = \
-	-DMALI_CUSTOMER_RELEASE=$(MALI_CUSTOMER_RELEASE) \
-	-DMALI_USE_CSF=$(MALI_USE_CSF) \
-	-DMALI_KERNEL_TEST_API=$(MALI_KERNEL_TEST_API) \
-	-DMALI_UNIT_TEST=$(MALI_UNIT_TEST) \
-	-DMALI_COVERAGE=$(MALI_COVERAGE) \
-	-DMALI_RELEASE_NAME=$(MALI_RELEASE_NAME) \
-	-DMALI_JIT_PRESSURE_LIMIT_BASE=$(MALI_JIT_PRESSURE_LIMIT_BASE) \
-	-DMALI_INCREMENTAL_RENDERING=$(MALI_INCREMENTAL_RENDERING)
 
 ifeq ($(KBUILD_EXTMOD),)
 # in-tree
-DEFINES +=-DMALI_KBASE_PLATFORM_PATH=../../$(src)/platform/$(CONFIG_MALI_PLATFORM_NAME)
+    ccflags-y +=-DMALI_KBASE_PLATFORM_PATH=../../$(src)/platform/$(CONFIG_MALI_PLATFORM_NAME)
 else
 # out-of-tree
-DEFINES +=-DMALI_KBASE_PLATFORM_PATH=$(src)/platform/$(CONFIG_MALI_PLATFORM_NAME)
+    ccflags-y +=-DMALI_KBASE_PLATFORM_PATH=$(src)/platform/$(CONFIG_MALI_PLATFORM_NAME)
 endif
 
-DEFINES += -I$(srctree)/drivers/staging/android
+ccflags-y += \
+    -I$(srctree)/include/linux \
+    -I$(srctree)/drivers/staging/android \
+    -I$(src) \
+    -I$(src)/platform/$(MALI_PLATFORM_DIR) \
+    -I$(src)/../../../base \
+    -I$(src)/../../../../include
 
-DEFINES += -DMALI_KBASE_BUILD
+subdir-ccflags-y += $(ccflags-y)
 
-# Use our defines when compiling
-ccflags-y += $(DEFINES) -I$(KBASE_PATH)   -I$(KBASE_PLATFORM_PATH) -I$(UMP_PATH) -I$(srctree)/include/linux
-subdir-ccflags-y += $(DEFINES) -I$(KBASE_PATH)   -I$(KBASE_PLATFORM_PATH) -I$(UMP_PATH) -I$(srctree)/include/linux
-
-SRC := \
-	context/mali_kbase_context.c \
-	debug/mali_kbase_debug_ktrace.c \
-	device/mali_kbase_device.c \
-	device/mali_kbase_device_hw.c \
-	mali_kbase_cache_policy.c \
-	mali_kbase_ccswe.c \
-	mali_kbase_mem.c \
-	mali_kbase_mem_pool_group.c \
-	mali_kbase_native_mgm.c \
-	mali_kbase_ctx_sched.c \
-	mali_kbase_jm.c \
-	mali_kbase_gpuprops.c \
-	mali_kbase_pm.c \
-	mali_kbase_config.c \
-	mali_kbase_vinstr.c \
-	mali_kbase_hwcnt.c \
-	mali_kbase_hwcnt_gpu.c \
-	mali_kbase_hwcnt_legacy.c \
-	mali_kbase_hwcnt_types.c \
-	mali_kbase_hwcnt_virtualizer.c \
-	mali_kbase_softjobs.c \
-	mali_kbase_hw.c \
-	mali_kbase_debug.c \
-	mali_kbase_gpu_memory_debugfs.c \
-	mali_kbase_mem_linux.c \
-	mali_kbase_core_linux.c \
-	mali_kbase_mem_profile_debugfs.c \
-	mmu/mali_kbase_mmu.c \
-	mmu/mali_kbase_mmu_hw_direct.c \
-	mmu/mali_kbase_mmu_mode_aarch64.c \
-	mali_kbase_disjoint_events.c \
-	mali_kbase_debug_mem_view.c \
-	mali_kbase_smc.c \
-	mali_kbase_mem_pool.c \
-	mali_kbase_mem_pool_debugfs.c \
-	mali_kbase_debugfs_helper.c \
-	mali_kbase_strings.c \
-	mali_kbase_as_fault_debugfs.c \
-	mali_kbase_regs_history_debugfs.c \
-	mali_kbase_dvfs_debugfs.c \
-	mali_power_gpu_frequency_trace.c \
-	mali_kbase_trace_gpu_mem.c \
-	thirdparty/mali_kbase_mmap.c \
-	tl/mali_kbase_timeline.c \
-	tl/mali_kbase_timeline_io.c \
-	tl/mali_kbase_tlstream.c \
-	tl/mali_kbase_tracepoints.c \
-	gpu/mali_kbase_gpu.c
-
-ifeq ($(MALI_USE_CSF),1)
-	SRC += \
-		mali_kbase_hwcnt_backend_csf.c \
-		mali_kbase_hwcnt_backend_csf_if_fw.c \
-		debug/backend/mali_kbase_debug_ktrace_csf.c \
-		device/backend/mali_kbase_device_csf.c \
-		device/backend/mali_kbase_device_hw_csf.c \
-		gpu/backend/mali_kbase_gpu_fault_csf.c \
-		tl/backend/mali_kbase_timeline_csf.c \
-		mmu/backend/mali_kbase_mmu_csf.c \
-		context/backend/mali_kbase_context_csf.c
-else
-	SRC += \
-		mali_kbase_hwcnt_backend_jm.c \
-		mali_kbase_dummy_job_wa.c \
-		mali_kbase_debug_job_fault.c \
-		mali_kbase_event.c \
-		mali_kbase_jd.c \
-		mali_kbase_jd_debugfs.c \
-		mali_kbase_js.c \
-		mali_kbase_js_ctx_attr.c \
-		mali_kbase_kinstr_jm.c \
-		debug/backend/mali_kbase_debug_ktrace_jm.c \
-		device/backend/mali_kbase_device_jm.c \
-		device/backend/mali_kbase_device_hw_jm.c \
-		gpu/backend/mali_kbase_gpu_fault_jm.c \
-		tl/backend/mali_kbase_timeline_jm.c \
-		mmu/backend/mali_kbase_mmu_jm.c \
-		context/backend/mali_kbase_context_jm.c
-endif
-
-ifeq ($(CONFIG_MALI_CINSTR_GWT),y)
-	SRC += mali_kbase_gwt.c
-endif
-
-
-ifeq ($(MALI_CUSTOMER_RELEASE),0)
-	SRC += mali_kbase_regs_dump_debugfs.c
-endif
-
-
-ccflags-y += -I$(KBASE_PATH) -I$(KBASE_PATH)/debug \
-	-I$(KBASE_PATH)/debug/backend
-
-# Tell the Linux build system from which .o file to create the kernel module
+#
+# Kernel Modules
+#
 obj-$(CONFIG_MALI_BIFROST) += bifrost_kbase.o
+obj-$(CONFIG_MALI_KUTF)    += tests/
 
-# Tell the Linux build system to enable building of our .c files
-bifrost_kbase-y := $(SRC:.c=.o)
+bifrost_kbase-y := \
+    mali_kbase_cache_policy.o \
+    mali_kbase_ccswe.o \
+    mali_kbase_mem.o \
+    mali_kbase_mem_pool_group.o \
+    mali_kbase_native_mgm.o \
+    mali_kbase_ctx_sched.o \
+    mali_kbase_gpuprops.o \
+    mali_kbase_pm.o \
+    mali_kbase_config.o \
+    mali_kbase_vinstr.o \
+    mali_kbase_hwcnt.o \
+    mali_kbase_hwcnt_gpu.o \
+    mali_kbase_hwcnt_legacy.o \
+    mali_kbase_hwcnt_types.o \
+    mali_kbase_hwcnt_virtualizer.o \
+    mali_kbase_softjobs.o \
+    mali_kbase_hw.o \
+    mali_kbase_debug.o \
+    mali_kbase_gpu_memory_debugfs.o \
+    mali_kbase_mem_linux.o \
+    mali_kbase_core_linux.o \
+    mali_kbase_mem_profile_debugfs.o \
+    mali_kbase_disjoint_events.o \
+    mali_kbase_debug_mem_view.o \
+    mali_kbase_smc.o \
+    mali_kbase_mem_pool.o \
+    mali_kbase_mem_pool_debugfs.o \
+    mali_kbase_debugfs_helper.o \
+    mali_kbase_strings.o \
+    mali_kbase_as_fault_debugfs.o \
+    mali_kbase_regs_history_debugfs.o \
+    mali_kbase_dvfs_debugfs.o \
+    mali_power_gpu_frequency_trace.o \
+    mali_kbase_trace_gpu_mem.o
 
-# Kconfig passes in the name with quotes for in-tree builds - remove them.
-platform_name := $(shell echo $(CONFIG_MALI_PLATFORM_NAME))
-MALI_PLATFORM_DIR := platform/$(platform_name)
-ccflags-y += -I$(src)/$(MALI_PLATFORM_DIR)
-include $(src)/$(MALI_PLATFORM_DIR)/Kbuild
+bifrost_kbase-$(CONFIG_MALI_CINSTR_GWT) += mali_kbase_gwt.o
 
-ifeq ($(CONFIG_MALI_BIFROST_DEVFREQ),y)
-  ifeq ($(CONFIG_DEVFREQ_THERMAL),y)
-    include $(src)/ipa/Kbuild
-  endif
+bifrost_kbase-$(CONFIG_SYNC) += \
+    mali_kbase_sync_android.o \
+    mali_kbase_sync_common.o
+
+bifrost_kbase-$(CONFIG_SYNC_FILE) += \
+    mali_kbase_fence_ops.o \
+    mali_kbase_sync_file.o \
+    mali_kbase_sync_common.o
+
+ifeq ($(CONFIG_MALI_CSF_SUPPORT),y)
+    bifrost_kbase-y += \
+        mali_kbase_hwcnt_backend_csf.o \
+        mali_kbase_hwcnt_backend_csf_if_fw.o
+else
+    bifrost_kbase-y += \
+        mali_kbase_jm.o \
+        mali_kbase_hwcnt_backend_jm.o \
+        mali_kbase_dummy_job_wa.o \
+        mali_kbase_debug_job_fault.o \
+        mali_kbase_event.o \
+        mali_kbase_jd.o \
+        mali_kbase_jd_debugfs.o \
+        mali_kbase_js.o \
+        mali_kbase_js_ctx_attr.o \
+        mali_kbase_kinstr_jm.o
+
+    bifrost_kbase-$(CONFIG_MALI_BIFROST_DMA_FENCE) += \
+        mali_kbase_fence_ops.o \
+        mali_kbase_dma_fence.o \
+        mali_kbase_fence.o
+
+    bifrost_kbase-$(CONFIG_SYNC_FILE) += \
+        mali_kbase_fence_ops.o \
+        mali_kbase_fence.o
 endif
 
-ifeq ($(MALI_USE_CSF),1)
-	include $(src)/csf/Kbuild
-else
-# empty
+
+INCLUDE_SUBDIR = \
+    $(src)/context/Kbuild \
+    $(src)/debug/Kbuild \
+    $(src)/device/Kbuild \
+    $(src)/backend/gpu/Kbuild \
+    $(src)/mmu/Kbuild \
+    $(src)/tl/Kbuild \
+    $(src)/gpu/Kbuild \
+    $(src)/thirdparty/Kbuild \
+    $(src)/platform/$(MALI_PLATFORM_DIR)/Kbuild
+
+ifeq ($(CONFIG_MALI_CSF_SUPPORT),y)
+    INCLUDE_SUBDIR += $(src)/csf/Kbuild
 endif
 
 ifeq ($(CONFIG_MALI_ARBITER_SUPPORT),y)
-	include $(src)/arbiter/Kbuild
+    INCLUDE_SUBDIR += $(src)/arbiter/Kbuild
+endif
+
+ifeq ($(CONFIG_MALI_BIFROST_DEVFREQ),y)
+    ifeq ($(CONFIG_DEVFREQ_THERMAL),y)
+        INCLUDE_SUBDIR += $(src)/ipa/Kbuild
+    endif
+endif
+
+ifeq ($(KBUILD_EXTMOD),)
+# in-tree
+    -include $(INCLUDE_SUBDIR)
 else
-# empty
+# out-of-tree
+    include $(INCLUDE_SUBDIR)
 endif
-
-ifeq ($(MALI_USE_CSF),0)
-	bifrost_kbase-$(CONFIG_MALI_BIFROST_DMA_FENCE) += \
-		mali_kbase_fence_ops.o \
-		mali_kbase_dma_fence.o \
-		mali_kbase_fence.o
-
-	bifrost_kbase-$(CONFIG_SYNC_FILE) += \
-		mali_kbase_fence_ops.o \
-		mali_kbase_fence.o
-endif
-
-bifrost_kbase-$(CONFIG_SYNC) += \
-	mali_kbase_sync_android.o \
-	mali_kbase_sync_common.o
-
-bifrost_kbase-$(CONFIG_SYNC_FILE) += \
-	mali_kbase_fence_ops.o \
-	mali_kbase_sync_file.o \
-	mali_kbase_sync_common.o
-
-include  $(src)/backend/gpu/Kbuild
-bifrost_kbase-y += $(BACKEND:.c=.o)
-
-
-ccflags-y += -I$(src)/backend/gpu
-subdir-ccflags-y += -I$(src)/backend/gpu
-
-# For kutf and mali_kutf_irq_latency_test
-obj-$(CONFIG_MALI_KUTF) += tests/

drivers/gpu/arm/bifrost/Kconfig

@@ -1,4 +1,4 @@
-# SPDX-License-Identifier: GPL-2.0
+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 #
 # (C) COPYRIGHT 2012-2021 ARM Limited. All rights reserved.
 #
@@ -18,11 +18,12 @@
 #
 #
 
-
 menuconfig MALI_BIFROST
 	tristate "Mali Bifrost series support"
 	select GPU_TRACEPOINTS if ANDROID
 	select DMA_SHARED_BUFFER
+	select PM_DEVFREQ
+	select DEVFREQ_THERMAL
 	default n
 	help
 	  Enable this option to build support for a ARM Mali Bifrost GPU.
@@ -30,20 +31,43 @@ menuconfig MALI_BIFROST
 	  To compile this driver as a module, choose M here:
 	  this will generate a single module, called mali_kbase.
 
+if MALI_BIFROST
+
+config MALI_PLATFORM_NAME
+	depends on MALI_BIFROST
+	string "Platform name"
+	default "devicetree"
+	help
+	  Enter the name of the desired platform configuration directory to
+	  include in the build. 'platform/$(MALI_PLATFORM_NAME)/Makefile' must
+	  exist.
+
+config MALI_REAL_HW
+	depends on MALI_BIFROST
+	def_bool !MALI_BIFROST_NO_MALI
+
+menu "Platform specific options"
+source "drivers/gpu/arm/midgard/platform/Kconfig"
+endmenu
+
 config MALI_CSF_SUPPORT
-	bool "Mali CSF based GPU support"
+	bool "Enable Mali CSF based GPU support"
 	depends on MALI_BIFROST=m
 	default n
 	help
 	  Enables support for CSF based GPUs.
 
-config MALI_BIFROST_GATOR_SUPPORT
-	bool "Enable Streamline tracing support"
-	depends on MALI_BIFROST
-	default n
+config MALI_BIFROST_DEVFREQ
+	bool "Enable devfreq support for Mali"
+	depends on MALI_BIFROST && PM_DEVFREQ
+	select DEVFREQ_GOV_SIMPLE_ONDEMAND
+	default y
 	help
-	  Enables kbase tracing used by the Arm Streamline Performance Analyzer.
-	  The tracepoints are used to derive GPU activity charts in Streamline.
+	  Support devfreq for Mali.
+
+	  Using the devfreq framework and, by default, the simple on-demand
+	  governor, the frequency of Mali will be dynamically selected from the
+	  available OPPs.
 
 config MALI_BIFROST_DVFS
 	bool "Enable legacy DVFS"
@@ -52,28 +76,25 @@ config MALI_BIFROST_DVFS
 	help
 	  Choose this option to enable legacy DVFS in the Mali Midgard DDK.
 
+config MALI_BIFROST_GATOR_SUPPORT
+	bool "Enable Streamline tracing support"
+	depends on MALI_BIFROST
+	default y
+	help
+	  Enables kbase tracing used by the Arm Streamline Performance Analyzer.
+	  The tracepoints are used to derive GPU activity charts in Streamline.
+
 config MALI_BIFROST_ENABLE_TRACE
 	bool "Enable kbase tracing"
 	depends on MALI_BIFROST
 	default y if MALI_BIFROST_DEBUG
 	default n
 	help
-	  Enables tracing in kbase.  Trace log available through
+	  Enables tracing in kbase. Trace log available through
 	  the "mali_trace" debugfs file, when the CONFIG_DEBUG_FS is enabled
 
-config MALI_BIFROST_DEVFREQ
-	bool "devfreq support for Mali"
-	depends on MALI_BIFROST && PM_DEVFREQ
-	select DEVFREQ_GOV_SIMPLE_ONDEMAND
-	help
-	  Support devfreq for Mali.
-
-	  Using the devfreq framework and, by default, the simpleondemand
-	  governor, the frequency of Mali will be dynamically selected from the
-	  available OPPs.
-
 config MALI_BIFROST_DMA_FENCE
-	bool "DMA_BUF fence support for Mali"
+	bool "Enable DMA_BUF fence support for Mali"
 	depends on MALI_BIFROST
 	default n
 	help
@@ -82,18 +103,9 @@ config MALI_BIFROST_DMA_FENCE
 	  This option should only be enabled if the Linux Kernel has built in
 	  support for DMA_BUF fences.
 
-config MALI_PLATFORM_NAME
-	depends on MALI_BIFROST
-	string "Platform name"
-	default "devicetree"
-	help
-	  Enter the name of the desired platform configuration directory to
-	  include in the build. 'platform/$(MALI_PLATFORM_NAME)/Kbuild' must
-	  exist.
-
 config MALI_ARBITER_SUPPORT
 	bool "Enable arbiter support for Mali"
-	depends on MALI_BIFROST
+	depends on MALI_BIFROST && !MALI_CSF_SUPPORT
 	default n
 	help
 	  Enable support for the arbiter interface in the driver.
@@ -102,126 +114,13 @@ config MALI_ARBITER_SUPPORT
 
 	  If unsure, say N.
 
-# MALI_BIFROST_EXPERT configuration options
-
-menuconfig MALI_BIFROST_EXPERT
-	depends on MALI_BIFROST
-	bool "Enable Expert Settings"
-	default n
-	help
-	  Enabling this option and modifying the default settings may produce a driver with performance or
-	  other limitations.
-
-config MALI_CORESTACK
-	bool "Support controlling power to the GPU core stack"
-	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
-	default n
-	help
-	  Enabling this feature on supported GPUs will let the driver powering
-	  on/off the GPU core stack independently without involving the Power
-	  Domain Controller. This should only be enabled on platforms which
-	  integration of the PDC to the Mali GPU is known to be problematic.
-	  This feature is currently only supported on t-Six and t-HEx GPUs.
-
-	  If unsure, say N.
-
-config MALI_BIFROST_DEBUG
-	bool "Debug build"
-	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
-	default n
-	help
-	  Select this option for increased checking and reporting of errors.
-
-config MALI_BIFROST_FENCE_DEBUG
-	bool "Debug sync fence usage"
-	depends on MALI_BIFROST && MALI_BIFROST_EXPERT && (SYNC || SYNC_FILE)
-	default y if MALI_BIFROST_DEBUG
-	help
-	  Select this option to enable additional checking and reporting on the
-	  use of sync fences in the Mali driver.
-
-	  This will add a 3s timeout to all sync fence waits in the Mali
-	  driver, so that when work for Mali has been waiting on a sync fence
-	  for a long time a debug message will be printed, detailing what fence
-	  is causing the block, and which dependent Mali atoms are blocked as a
-	  result of this.
-
-	  The timeout can be changed at runtime through the js_soft_timeout
-	  device attribute, where the timeout is specified in milliseconds.
-
-config MALI_BIFROST_NO_MALI
-	bool "No Mali"
-	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
-	default n
-	help
-	  This can be used to test the driver in a simulated environment
-	  whereby the hardware is not physically present. If the hardware is physically
-	  present it will not be used. This can be used to test the majority of the
-	  driver without needing actual hardware or for software benchmarking.
-	  All calls to the simulated hardware will complete immediately as if the hardware
-	  completed the task.
-
-config MALI_REAL_HW
-	def_bool !MALI_BIFROST_NO_MALI
-
-config MALI_BIFROST_ERROR_INJECT
-	bool "Error injection"
-	depends on MALI_BIFROST && MALI_BIFROST_EXPERT && MALI_BIFROST_NO_MALI
-	default n
-	help
-	  Enables insertion of errors to test module failure and recovery mechanisms.
-
-config MALI_BIFROST_SYSTEM_TRACE
-	bool "Enable system event tracing support"
-	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
-	default y if MALI_BIFROST_DEBUG
-	default n
-	help
-	  Choose this option to enable system trace events for each
-	  kbase event. This is typically used for debugging but has
-	  minimal overhead when not in use. Enable only if you know what
-	  you are doing.
-
-config MALI_2MB_ALLOC
-	bool "Attempt to allocate 2MB pages"
-	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
-	default n
-	help
-	  Rather than allocating all GPU memory page-by-page, attempt to
-	  allocate 2MB pages from the kernel. This reduces TLB pressure and
-	  helps to prevent memory fragmentation.
-
-	  If in doubt, say N
-
-config MALI_PWRSOFT_765
-	bool "PWRSOFT-765 ticket"
-	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
-	default n
-	help
-	  PWRSOFT-765 fixes devfreq cooling devices issues. The fix was merged
-	  in kernel v4.10, however if backported into the kernel then this
-	  option must be manually selected.
-
-	  If using kernel >= v4.10 then say N, otherwise if devfreq cooling
-	  changes have been backported say Y to avoid compilation errors.
-
-config MALI_MEMORY_FULLY_BACKED
-	bool "Memory fully physically-backed"
-	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
-	default n
-	help
-	  This option enables full physical backing of all virtual
-	  memory allocations in the kernel. Notice that this build
-	  option only affects allocations of grow-on-GPU-page-fault
-	  memory.
-
 config MALI_DMA_BUF_MAP_ON_DEMAND
-	bool "Map imported dma-bufs on demand"
+	bool "Enable map imported dma-bufs on demand"
 	depends on MALI_BIFROST
 	default n
 	help
 	  This option caused kbase to set up the GPU mapping of imported
-	  dma-buf when needed to run atoms.  This is the legacy behaviour.
+	  dma-buf when needed to run atoms. This is the legacy behavior.
 
 	  This is intended for testing and the option will get removed in the
 	  future.
@@ -240,51 +139,123 @@ config MALI_DMA_BUF_LEGACY_COMPAT
 	  flushes in other drivers. This only has an effect for clients using
 	  UK 11.18 or older. For later UK versions it is not possible.
 
-config MALI_HW_ERRATA_1485982_NOT_AFFECTED
-	bool "Disable workaround for BASE_HW_ISSUE_GPU2017_1336"
+menuconfig MALI_BIFROST_EXPERT
+	depends on MALI_BIFROST
+	bool "Enable Expert Settings"
+	default n
+	help
+	  Enabling this option and modifying the default settings may produce
+	  a driver with performance or other limitations.
+
+if MALI_BIFROST_EXPERT
+
+config MALI_2MB_ALLOC
+	bool "Attempt to allocate 2MB pages"
 	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
 	default n
 	help
-	  This option disables the default workaround for GPU2017-1336. The
-	  workaround keeps the L2 cache powered up except for powerdown and reset.
+	  Rather than allocating all GPU memory page-by-page, attempt to
+	  allocate 2MB pages from the kernel. This reduces TLB pressure and
+	  helps to prevent memory fragmentation.
 
-	  The workaround introduces a limitation that will prevent the running of
-	  protected mode content on fully coherent platforms, as the switch to IO
-	  coherency mode requires the L2 to be turned off.
+	  If in doubt, say N
 
-config MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE
-	bool "Use alternative workaround for BASE_HW_ISSUE_GPU2017_1336"
-	depends on MALI_BIFROST && MALI_BIFROST_EXPERT && !MALI_HW_ERRATA_1485982_NOT_AFFECTED
+config MALI_MEMORY_FULLY_BACKED
+	bool "Enable memory fully physically-backed"
+	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
 	default n
 	help
-	  This option uses an alternative workaround for GPU2017-1336. Lowering
-	  the GPU clock to a, platform specific, known good frequeuncy before
-	  powering down the L2 cache. The clock can be specified in the device
-	  tree using the property, opp-mali-errata-1485982. Otherwise the
-	  slowest clock will be selected.
+	  This option enables full physical backing of all virtual
+	  memory allocations in the kernel. Notice that this build
+	  option only affects allocations of grow-on-GPU-page-fault
+	  memory.
+
+config MALI_CORESTACK
+	bool "Enable support of GPU core stack power control"
+	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
+	default n
+	help
+	  Enabling this feature on supported GPUs will let the driver powering
+	  on/off the GPU core stack independently without involving the Power
+	  Domain Controller. This should only be enabled on platforms which
+	  integration of the PDC to the Mali GPU is known to be problematic.
+	  This feature is currently only supported on t-Six and t-HEx GPUs.
+
+	  If unsure, say N.
+
+comment "Platform options"
+	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
+
+config MALI_BIFROST_NO_MALI
+	bool "Enable No Mali"
+	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
+	default n
+	help
+	  This can be used to test the driver in a simulated environment
+	  whereby the hardware is not physically present. If the hardware is physically
+	  present it will not be used. This can be used to test the majority of the
+	  driver without needing actual hardware or for software benchmarking.
+	  All calls to the simulated hardware will complete immediately as if the hardware
+	  completed the task.
+
+config MALI_BIFROST_ERROR_INJECT
+	bool "Enable No Mali error injection"
+	depends on MALI_BIFROST && MALI_BIFROST_EXPERT && MALI_BIFROST_NO_MALI
+	default n
+	help
+	  Enables insertion of errors to test module failure and recovery mechanisms.
 
 config MALI_GEM5_BUILD
 	bool "Enable build of Mali kernel driver for GEM5"
-	depends on MALI_BIFROST
+	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
 	default n
 	help
 	  This option is to do a Mali GEM5 build.
 	  If unsure, say N.
 
-# Instrumentation options.
+comment "Debug options"
+	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
 
-config MALI_JOB_DUMP
-	bool "Enable system level support needed for job dumping"
+config MALI_BIFROST_DEBUG
+	bool "Enable debug build"
 	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
 	default n
 	help
-	  Choose this option to enable system level support needed for
-	  job dumping. This is typically used for instrumentation but has
+	  Select this option for increased checking and reporting of errors.
+
+config MALI_BIFROST_FENCE_DEBUG
+	bool "Enable debug sync fence usage"
+	depends on MALI_BIFROST && MALI_BIFROST_EXPERT && (SYNC || SYNC_FILE)
+	default y if MALI_BIFROST_DEBUG
+	help
+	  Select this option to enable additional checking and reporting on the
+	  use of sync fences in the Mali driver.
+
+	  This will add a 3s timeout to all sync fence waits in the Mali
+	  driver, so that when work for Mali has been waiting on a sync fence
+	  for a long time a debug message will be printed, detailing what fence
+	  is causing the block, and which dependent Mali atoms are blocked as a
+	  result of this.
+
+	  The timeout can be changed at runtime through the js_soft_timeout
+	  device attribute, where the timeout is specified in milliseconds.
+
+config MALI_BIFROST_SYSTEM_TRACE
+	bool "Enable system event tracing support"
+	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
+	default y if MALI_BIFROST_DEBUG
+	default n
+	help
+	  Choose this option to enable system trace events for each
+	  kbase event. This is typically used for debugging but has
 	  minimal overhead when not in use. Enable only if you know what
 	  you are doing.
 
+comment "Instrumentation options"
+	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
+
 choice
-	prompt "Performance counters set"
+	prompt "Select Performance counters set"
 	default MALI_PRFCNT_SET_PRIMARY
 	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
 
@@ -321,7 +292,7 @@ config MALI_PRFCNT_SET_TERTIARY
 endchoice
 
 config MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS
-	bool "Allow runtime selection of performance counters set via debugfs"
+	bool "Enable runtime selection of performance counters set via debugfs"
 	depends on MALI_BIFROST && MALI_BIFROST_EXPERT && DEBUG_FS
 	default n
 	help
@@ -343,5 +314,65 @@ config MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS
 
 	  If unsure, say N.
 
-source "drivers/gpu/arm/midgard/platform/Kconfig"
-# source "drivers/gpu/arm/midgard/tests/Kconfig"
+config MALI_JOB_DUMP
+	bool "Enable system level support needed for job dumping"
+	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
+	default n
+	help
+	  Choose this option to enable system level support needed for
+	  job dumping. This is typically used for instrumentation but has
+	  minimal overhead when not in use. Enable only if you know what
+	  you are doing.
+
+comment "Workarounds"
+	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
+
+config MALI_PWRSOFT_765
+	bool "Enable workaround for PWRSOFT-765"
+	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
+	default n
+	help
+	  PWRSOFT-765 fixes devfreq cooling devices issues. The fix was merged
+	  in kernel v4.10, however if backported into the kernel then this
+	  option must be manually selected.
+
+	  If using kernel >= v4.10 then say N, otherwise if devfreq cooling
+	  changes have been backported say Y to avoid compilation errors.
+
+config MALI_HW_ERRATA_1485982_NOT_AFFECTED
+	bool "Disable workaround for BASE_HW_ISSUE_GPU2017_1336"
+	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
+	default n
+	help
+	  This option disables the default workaround for GPU2017-1336. The
+	  workaround keeps the L2 cache powered up except for powerdown and reset.
+
+	  The workaround introduces a limitation that will prevent the running of
+	  protected mode content on fully coherent platforms, as the switch to IO
+	  coherency mode requires the L2 to be turned off.
+
+config MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE
+	bool "Use alternative workaround for BASE_HW_ISSUE_GPU2017_1336"
+	depends on MALI_BIFROST && MALI_BIFROST_EXPERT && !MALI_HW_ERRATA_1485982_NOT_AFFECTED
+	default n
+	help
+	  This option uses an alternative workaround for GPU2017-1336. Lowering
+	  the GPU clock to a, platform specific, known good frequency before
+	  powering down the L2 cache. The clock can be specified in the device
+	  tree using the property, opp-mali-errata-1485982. Otherwise the
+	  slowest clock will be selected.
+
+endif
+
+config MALI_ARBITRATION
+	bool "Enable Virtualization reference code"
+	depends on MALI_BIFROST
+	default n
+	help
+	  Enables the build of several reference modules used in the reference
+	  virtualization setup for Mali
+	  If unsure, say N.
+
+source "drivers/gpu/arm/midgard/tests/Kconfig"
+
+endif

drivers/gpu/arm/bifrost/Makefile

@@ -1,4 +1,4 @@
-# SPDX-License-Identifier: GPL-2.0
+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 #
 # (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved.
 #
@@ -18,47 +18,198 @@
 #
 #
 
-# Handle Android Common Kernel source naming
 KERNEL_SRC ?= /lib/modules/$(shell uname -r)/build
 KDIR ?= $(KERNEL_SRC)
 
-# out-of-tree
-ifeq ($(KBUILD_EXTMOD),)
-export CONFIG_MALI_MIDGARD?=m
-
-ifneq ($(CONFIG_MALI_MIDGARD),n)
-export CONFIG_MALI_CSF_SUPPORT?=n
-export CONFIG_MALI_KUTF?=m
-export CONFIG_MALI_REAL_HW?=y
-
-# Handle default y/m in Kconfig
-export CONFIG_MALI_BIFROST_GATOR_SUPPORT?=y
-export CONFIG_MALI_BIFROST_DEVFREQ?=n
-ifneq ($(CONFIG_PM_DEVFREQ),n)
-export CONFIG_MALI_BIFROST_DEVFREQ?=y
+ifeq ($(KDIR),)
+    $(error Must specify KDIR to point to the kernel to target))
 endif
 
-DEFINES += -DCONFIG_MALI_MIDGARD=$(CONFIG_MALI_MIDGARD) \
-	-DCONFIG_MALI_CSF_SUPPORT=$(CONFIG_MALI_CSF_SUPPORT) \
-	-DCONFIG_MALI_KUTF=$(CONFIG_MALI_KUTF) \
-	-DCONFIG_MALI_REAL_HW=$(CONFIG_MALI_REAL_HW) \
-	-DCONFIG_MALI_GATOR_SUPPORT=$(CONFIG_MALI_BIFROST_GATOR_SUPPORT) \
-	-DCONFIG_MALI_DEVFREQ=$(CONFIG_MALI_BIFROST_DEVFREQ)
+#
+# Default configuration values
+#
+# Dependency resolution is done through statements as Kconfig
+# is not supported for out-of-tree builds.
+#
 
-export DEFINES
+CONFIG_MALI_BIFROST ?= m
+ifeq ($(CONFIG_MALI_BIFROST),m)
+    CONFIG_MALI_BIFROST_GATOR_SUPPORT ?= y
+    CONFIG_MALI_ARBITRATION ?= n
+    CONFIG_MALI_PARTITION_MANAGER ?= n
 
-endif
+    ifneq ($(CONFIG_MALI_BIFROST_NO_MALI),y)
+        # Prevent misuse when CONFIG_MALI_BIFROST_NO_MALI=y
+        CONFIG_MALI_REAL_HW ?= y
+    endif
+
+    ifeq ($(CONFIG_MALI_BIFROST_DVFS),y)
+        # Prevent misuse when CONFIG_MALI_BIFROST_DVFS=y
+        CONFIG_MALI_BIFROST_DEVFREQ ?= n
+    else
+        CONFIG_MALI_BIFROST_DEVFREQ ?= y
+    endif
+
+    ifeq ($(CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND), y)
+        # Prevent misuse when CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND=y
+        CONFIG_MALI_DMA_BUF_LEGACY_COMPAT = n
+    endif
+
+    ifeq ($(CONFIG_BSP_HAS_HYPERVISOR),y)
+        ifneq ($(CONFIG_MALI_ARBITRATION), n)
+            CONFIG_MALI_XEN ?= m
+        endif
+    endif
+
+    #
+    # Expert/Debug/Test released configurations
+    #
+    ifeq ($(CONFIG_MALI_BIFROST_EXPERT), y)
+        ifeq ($(CONFIG_MALI_BIFROST_NO_MALI), y)
+            CONFIG_MALI_REAL_HW = n
+        else
+            # Prevent misuse when CONFIG_MALI_BIFROST_NO_MALI=n
+            CONFIG_MALI_REAL_HW = y
+            CONFIG_MALI_BIFROST_ERROR_INJECT = n
+        endif
+
+        ifeq ($(CONFIG_MALI_HW_ERRATA_1485982_NOT_AFFECTED), y)
+            # Prevent misuse when CONFIG_MALI_HW_ERRATA_1485982_NOT_AFFECTED=y
+            CONFIG_MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE = n
+        endif
+
+        ifeq ($(CONFIG_MALI_BIFROST_DEBUG), y)
+            CONFIG_MALI_BIFROST_ENABLE_TRACE ?= y
+            CONFIG_MALI_BIFROST_SYSTEM_TRACE ?= y
+
+            ifeq ($(CONFIG_SYNC), y)
+                CONFIG_MALI_BIFROST_FENCE_DEBUG ?= y
+            else
+                ifeq ($(CONFIG_SYNC_FILE), y)
+                    CONFIG_MALI_BIFROST_FENCE_DEBUG ?= y
+                else
+                    CONFIG_MALI_BIFROST_FENCE_DEBUG = n
+                endif
+            endif
+        else
+            # Prevent misuse when CONFIG_MALI_BIFROST_DEBUG=n
+            CONFIG_MALI_BIFROST_ENABLE_TRACE = n
+            CONFIG_MALI_BIFROST_SYSTEM_TRACE = n
+            CONFIG_MALI_BIFROST_FENCE_DEBUG = n
+        endif
+    else
+        # Prevent misuse when CONFIG_MALI_BIFROST_EXPERT=n
+        CONFIG_MALI_CORESTACK = n
+        CONFIG_MALI_2MB_ALLOC = n
+        CONFIG_MALI_PWRSOFT_765 = n
+        CONFIG_MALI_MEMORY_FULLY_BACKED = n
+        CONFIG_MALI_JOB_DUMP = n
+        CONFIG_MALI_BIFROST_NO_MALI = n
+        CONFIG_MALI_REAL_HW = y
+        CONFIG_MALI_BIFROST_ERROR_INJECT = n
+        CONFIG_MALI_HW_ERRATA_1485982_NOT_AFFECTED = n
+        CONFIG_MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE = n
+        CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS = n
+        CONFIG_MALI_BIFROST_DEBUG = n
+        CONFIG_MALI_BIFROST_ENABLE_TRACE = n
+        CONFIG_MALI_BIFROST_SYSTEM_TRACE = n
+        CONFIG_MALI_BIFROST_FENCE_DEBUG = n
+    endif
+
+    ifeq ($(CONFIG_MALI_BIFROST_DEBUG), y)
+        CONFIG_MALI_KUTF ?= y
+        ifeq ($(CONFIG_MALI_KUTF), y)
+            CONFIG_MALI_KUTF_IRQ_TEST ?= y
+            CONFIG_MALI_KUTF_CLK_RATE_TRACE ?= y
+        else
+            # Prevent misuse when CONFIG_MALI_KUTF=n
+            CONFIG_MALI_KUTF_IRQ_TEST = n
+            CONFIG_MALI_KUTF_CLK_RATE_TRACE = n
+        endif
+    else
+        # Prevent misuse when CONFIG_MALI_BIFROST_DEBUG=n
+        CONFIG_MALI_KUTF = n
+        CONFIG_MALI_KUTF_IRQ_TEST = n
+        CONFIG_MALI_KUTF_CLK_RATE_TRACE = n
+    endif
+else
+    # Prevent misuse when CONFIG_MALI_BIFROST=n
+    CONFIG_MALI_ARBITRATION = n
+    CONFIG_MALI_KUTF = n
+    CONFIG_MALI_KUTF_IRQ_TEST = n
+    CONFIG_MALI_KUTF_CLK_RATE_TRACE = n
 endif
 
-KBASE_PATH_RELATIVE = $(CURDIR)
+# All Mali CONFIG should be listed here
+CONFIGS := \
+    CONFIG_MALI_BIFROST \
+    CONFIG_MALI_CSF_SUPPORT \
+    CONFIG_MALI_BIFROST_GATOR_SUPPORT \
+    CONFIG_MALI_BIFROST_DMA_FENCE \
+    CONFIG_MALI_ARBITER_SUPPORT \
+    CONFIG_MALI_ARBITRATION \
+    CONFIG_MALI_PARTITION_MANAGER \
+    CONFIG_MALI_REAL_HW \
+    CONFIG_MALI_GEM5_BUILD \
+    CONFIG_MALI_BIFROST_DEVFREQ \
+    CONFIG_MALI_BIFROST_DVFS \
+    CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND \
+    CONFIG_MALI_DMA_BUF_LEGACY_COMPAT \
+    CONFIG_MALI_BIFROST_EXPERT \
+    CONFIG_MALI_CORESTACK \
+    CONFIG_MALI_2MB_ALLOC \
+    CONFIG_MALI_PWRSOFT_765 \
+    CONFIG_MALI_MEMORY_FULLY_BACKED \
+    CONFIG_MALI_JOB_DUMP \
+    CONFIG_MALI_BIFROST_NO_MALI \
+    CONFIG_MALI_BIFROST_ERROR_INJECT \
+    CONFIG_MALI_HW_ERRATA_1485982_NOT_AFFECTED \
+    CONFIG_MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE \
+    CONFIG_MALI_PRFCNT_SET_PRIMARY \
+    CONFIG_MALI_BIFROST_PRFCNT_SET_SECONDARY \
+    CONFIG_MALI_PRFCNT_SET_TERTIARY \
+    CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS \
+    CONFIG_MALI_BIFROST_DEBUG \
+    CONFIG_MALI_BIFROST_ENABLE_TRACE \
+    CONFIG_MALI_BIFROST_SYSTEM_TRACE \
+    CONFIG_MALI_BIFROST_FENCE_DEBUG \
+    CONFIG_MALI_KUTF \
+    CONFIG_MALI_KUTF_IRQ_TEST \
+    CONFIG_MALI_KUTF_CLK_RATE_TRACE \
+    CONFIG_MALI_XEN
 
 
-# we get the symbols from modules using KBUILD_EXTRA_SYMBOLS to prevent warnings about unknown functions
+#
+# MAKE_ARGS to pass the custom CONFIGs on out-of-tree build
+#
+# Generate the list of CONFIGs and values.
+# $(value config) is the name of the CONFIG option.
+# $(value $(value config)) is its value (y, m).
+# When the CONFIG is not set to y or m, it defaults to n.
+MAKE_ARGS := $(foreach config,$(CONFIGS), \
+                    $(if $(filter y m,$(value $(value config))), \
+                        $(value config)=$(value $(value config)), \
+                        $(value config)=n))
+
+#
+# EXTRA_CFLAGS to define the custom CONFIGs on out-of-tree build
+#
+# Generate the list of CONFIGs defines with values from CONFIGS.
+# $(value config) is the name of the CONFIG option.
+# When set to y or m, the CONFIG gets defined to 1.
+EXTRA_CFLAGS := $(foreach config,$(CONFIGS), \
+                    $(if $(filter y m,$(value $(value config))), \
+                        -D$(value config)=1))
+
+#
+# KBUILD_EXTRA_SYMBOLS to prevent warnings about unknown functions
+#
+
 all:
-	$(MAKE) -C $(KDIR) M=$(CURDIR) EXTRA_CFLAGS="-I$(CURDIR)/../../../../include -I$(CURDIR)/../../../../tests/include $(SCONS_CFLAGS)" $(SCONS_CONFIGS) KBUILD_EXTRA_SYMBOLS="$(EXTRA_SYMBOLS)" modules
+	$(MAKE) -C $(KDIR) M=$(CURDIR) $(MAKE_ARGS) EXTRA_CFLAGS="$(EXTRA_CFLAGS)" KBUILD_EXTRA_SYMBOLS="$(EXTRA_SYMBOLS)" modules
 
 modules_install:
-	$(MAKE) -C $(KDIR) M=$(CURDIR) modules_install
+	$(MAKE) -C $(KDIR) M=$(CURDIR) $(MAKE_ARGS) modules_install
 
 clean:
-	$(MAKE) -C $(KDIR) M=$(CURDIR) clean
+	$(MAKE) -C $(KDIR) M=$(CURDIR) $(MAKE_ARGS) clean

drivers/gpu/arm/bifrost/Mconfig

@@ -1,4 +1,4 @@
-# SPDX-License-Identifier: GPL-2.0
+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 #
 # (C) COPYRIGHT 2012-2021 ARM Limited. All rights reserved.
 #
@@ -27,13 +27,52 @@ menuconfig MALI_BIFROST
 	  To compile this driver as a module, choose M here:
 	  this will generate a single module, called mali_kbase.
 
-config MALI_CSF_SUPPORT
-	bool "Mali CSF based GPU support"
+config MALI_PLATFORM_NAME
 	depends on MALI_BIFROST
-	default n
+	string "Platform name"
+	default "hisilicon" if PLATFORM_HIKEY960
+	default "hisilicon" if PLATFORM_HIKEY970
+	default "devicetree"
+	help
+	  Enter the name of the desired platform configuration directory to
+	  include in the build. 'platform/$(MALI_PLATFORM_NAME)/Makefile' must
+	  exist.
+
+	  When PLATFORM_CUSTOM is set, this needs to be set manually to
+	  pick up the desired platform files.
+
+config MALI_REAL_HW
+	bool
+	depends on MALI_BIFROST
+	default y
+	default n if NO_MALI
+
+config MALI_CSF_SUPPORT
+	bool "Enable Mali CSF based GPU support"
+	depends on MALI_BIFROST
+	default y if GPU_HAS_CSF
 	help
 	  Enables support for CSF based GPUs.
 
+config MALI_BIFROST_DEVFREQ
+	bool "Enable devfreq support for Mali"
+	depends on MALI_BIFROST
+	default y if PLATFORM_JUNO
+	default y if PLATFORM_CUSTOM
+	help
+	  Support devfreq for Mali.
+
+	  Using the devfreq framework and, by default, the simple on-demand
+	  governor, the frequency of Mali will be dynamically selected from the
+	  available OPPs.
+
+config MALI_BIFROST_DVFS
+	bool "Enable legacy DVFS"
+	depends on MALI_BIFROST && !MALI_BIFROST_DEVFREQ
+	default n
+	help
+	  Choose this option to enable legacy DVFS in the Mali Midgard DDK.
+
 config MALI_BIFROST_GATOR_SUPPORT
 	bool "Enable Streamline tracing support"
 	depends on MALI_BIFROST && !BACKEND_USER
@@ -42,35 +81,17 @@ config MALI_BIFROST_GATOR_SUPPORT
 	  Enables kbase tracing used by the Arm Streamline Performance Analyzer.
 	  The tracepoints are used to derive GPU activity charts in Streamline.
 
-config MALI_BIFROST_DVFS
-	bool "Enable legacy DVFS"
-	depends on MALI_BIFROST && !MALI_BIFROST_DEVFREQ
-	default n
-	help
-	  Choose this option to enable legacy DVFS in the Mali Midgard DDK.
-
 config MALI_BIFROST_ENABLE_TRACE
 	bool "Enable kbase tracing"
+	depends on MALI_BIFROST
 	default y if MALI_BIFROST_DEBUG
 	default n
 	help
-	  Enables tracing in kbase.  Trace log available through
+	  Enables tracing in kbase. Trace log available through
 	  the "mali_trace" debugfs file, when the CONFIG_DEBUG_FS is enabled
 
-config MALI_BIFROST_DEVFREQ
-	bool "devfreq support for Mali"
-	depends on MALI_BIFROST
-	default y if PLATFORM_JUNO
-	default y if PLATFORM_CUSTOM
-	help
-	  Support devfreq for Mali.
-
-	  Using the devfreq framework and, by default, the simpleondemand
-	  governor, the frequency of Mali will be dynamically selected from the
-	  available OPPs.
-
 config MALI_BIFROST_DMA_FENCE
-	bool "DMA_BUF fence support for Mali"
+	bool "Enable DMA_BUF fence support for Mali"
 	depends on MALI_BIFROST
 	default n
 	help
@@ -79,23 +100,9 @@ config MALI_BIFROST_DMA_FENCE
 	  This option should only be enabled if the Linux Kernel has built in
 	  support for DMA_BUF fences.
 
-config MALI_PLATFORM_NAME
-	depends on MALI_BIFROST
-	string "Platform name"
-	default "hisilicon" if PLATFORM_HIKEY960
-	default "hisilicon" if PLATFORM_HIKEY970
-	default "devicetree"
-	help
-	  Enter the name of the desired platform configuration directory to
-	  include in the build. 'platform/$(MALI_PLATFORM_NAME)/Kbuild' must
-	  exist.
-
-	  When PLATFORM_CUSTOM is set, this needs to be set manually to
-	  pick up the desired platform files.
-
 config MALI_ARBITER_SUPPORT
 	bool "Enable arbiter support for Mali"
-	depends on MALI_BIFROST && !GPU_HAS_CSF
+	depends on MALI_BIFROST && !MALI_CSF_SUPPORT
 	default n
 	help
 	  Enable support for the arbiter interface in the driver.
@@ -104,18 +111,68 @@ config MALI_ARBITER_SUPPORT
 
 	  If unsure, say N.
 
-# MALI_BIFROST_EXPERT configuration options
+config DMA_BUF_SYNC_IOCTL_SUPPORTED
+	bool "Enable Kernel DMA buffers support DMA_BUF_IOCTL_SYNC"
+	depends on MALI_BIFROST && BACKEND_KERNEL
+	default y
+
+config MALI_DMA_BUF_MAP_ON_DEMAND
+	bool "Enable map imported dma-bufs on demand"
+	depends on MALI_BIFROST
+	default n
+	default y if !DMA_BUF_SYNC_IOCTL_SUPPORTED
+	help
+	  This option caused kbase to set up the GPU mapping of imported
+	  dma-buf when needed to run atoms. This is the legacy behavior.
+
+	  This is intended for testing and the option will get removed in the
+	  future.
+
+config MALI_DMA_BUF_LEGACY_COMPAT
+	bool "Enable legacy compatibility cache flush on dma-buf map"
+	depends on MALI_BIFROST && !MALI_DMA_BUF_MAP_ON_DEMAND
+	default n
+	help
+	  This option enables compatibility with legacy dma-buf mapping
+	  behavior, then the dma-buf is mapped on import, by adding cache
+	  maintenance where MALI_DMA_BUF_MAP_ON_DEMAND would do the mapping,
+	  including a cache flush.
+
+	  This option might work-around issues related to missing cache
+	  flushes in other drivers. This only has an effect for clients using
+	  UK 11.18 or older. For later UK versions it is not possible.
 
 menuconfig MALI_BIFROST_EXPERT
 	depends on MALI_BIFROST
 	bool "Enable Expert Settings"
 	default y
 	help
-	  Enabling this option and modifying the default settings may produce a driver with performance or
-	  other limitations.
+	  Enabling this option and modifying the default settings may produce
+	  a driver with performance or other limitations.
+
+config MALI_2MB_ALLOC
+	bool "Attempt to allocate 2MB pages"
+	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
+	default n
+	help
+	  Rather than allocating all GPU memory page-by-page, attempt to
+	  allocate 2MB pages from the kernel. This reduces TLB pressure and
+	  helps to prevent memory fragmentation.
+
+	  If in doubt, say N
+
+config MALI_MEMORY_FULLY_BACKED
+	bool "Enable memory fully physically-backed"
+	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
+	default n
+	help
+	  This option enables full physical backing of all virtual
+	  memory allocations in the kernel. Notice that this build
+	  option only affects allocations of grow-on-GPU-page-fault
+	  memory.
 
 config MALI_CORESTACK
-	bool "Support controlling power to the GPU core stack"
+	bool "Enable support of GPU core stack power control"
 	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
 	default n
 	help
@@ -127,39 +184,16 @@ config MALI_CORESTACK
 
 	  If unsure, say N.
 
-config MALI_BIFROST_DEBUG
-	bool "Debug build"
-	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
-	default y if DEBUG
-	default n
-	help
-	  Select this option for increased checking and reporting of errors.
-
-config MALI_BIFROST_FENCE_DEBUG
-	bool "Debug sync fence usage"
-	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
-	default y if MALI_BIFROST_DEBUG
-	help
-	  Select this option to enable additional checking and reporting on the
-	  use of sync fences in the Mali driver.
-
-	  This will add a 3s timeout to all sync fence waits in the Mali
-	  driver, so that when work for Mali has been waiting on a sync fence
-	  for a long time a debug message will be printed, detailing what fence
-	  is causing the block, and which dependent Mali atoms are blocked as a
-	  result of this.
-
-	  The timeout can be changed at runtime through the js_soft_timeout
-	  device attribute, where the timeout is specified in milliseconds.
-
 choice
 	prompt "Error injection level"
+	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
 	default MALI_ERROR_INJECT_NONE
 	help
 	  Enables insertion of errors to test module failure and recovery mechanisms.
 
 config MALI_ERROR_INJECT_NONE
 	bool "disabled"
+	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
 	help
 	  Error injection is disabled.
 
@@ -179,14 +213,49 @@ endchoice
 
 config MALI_ERROR_INJECT_ON
 	string
+	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
 	default "0" if MALI_ERROR_INJECT_NONE
 	default "1" if MALI_ERROR_INJECT_TRACK_LIST
 	default "2" if MALI_ERROR_INJECT_RANDOM
 
 config MALI_BIFROST_ERROR_INJECT
 	bool
+	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
 	default y if !MALI_ERROR_INJECT_NONE
 
+config MALI_GEM5_BUILD
+	bool "Enable build of Mali kernel driver for GEM5"
+	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
+	default n
+	help
+	  This option is to do a Mali GEM5 build.
+	  If unsure, say N.
+
+config MALI_BIFROST_DEBUG
+	bool "Enable debug build"
+	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
+	default y if DEBUG
+	default n
+	help
+	  Select this option for increased checking and reporting of errors.
+
+config MALI_BIFROST_FENCE_DEBUG
+	bool "Enable debug sync fence usage"
+	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
+	default y if MALI_BIFROST_DEBUG
+	help
+	  Select this option to enable additional checking and reporting on the
+	  use of sync fences in the Mali driver.
+
+	  This will add a 3s timeout to all sync fence waits in the Mali
+	  driver, so that when work for Mali has been waiting on a sync fence
+	  for a long time a debug message will be printed, detailing what fence
+	  is causing the block, and which dependent Mali atoms are blocked as a
+	  result of this.
+
+	  The timeout can be changed at runtime through the js_soft_timeout
+	  device attribute, where the timeout is specified in milliseconds.
+
 config MALI_BIFROST_SYSTEM_TRACE
 	bool "Enable system event tracing support"
 	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
@@ -198,56 +267,35 @@ config MALI_BIFROST_SYSTEM_TRACE
 	  minimal overhead when not in use. Enable only if you know what
 	  you are doing.
 
-config MALI_2MB_ALLOC
-	bool "Attempt to allocate 2MB pages"
+# Instrumentation options.
+
+# config MALI_PRFCNT_SET_PRIMARY exists in the Kernel Kconfig but is configured using CINSTR_PRIMARY_HWC in Mconfig.
+# config MALI_BIFROST_PRFCNT_SET_SECONDARY exists in the Kernel Kconfig but is configured using CINSTR_SECONDARY_HWC in Mconfig.
+# config MALI_PRFCNT_SET_TERTIARY exists in the Kernel Kconfig but is configured using CINSTR_TERTIARY_HWC in Mconfig.
+# config MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS exists in the Kernel Kconfig but is configured using CINSTR_HWC_SET_SELECT_VIA_DEBUG_FS in Mconfig.
+
+config MALI_JOB_DUMP
+	bool "Enable system level support needed for job dumping"
 	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
 	default n
 	help
-	  Rather than allocating all GPU memory page-by-page, attempt to
-	  allocate 2MB pages from the kernel. This reduces TLB pressure and
-	  helps to prevent memory fragmentation.
-
-	  If in doubt, say N
+	  Choose this option to enable system level support needed for
+	  job dumping. This is typically used for instrumentation but has
+	  minimal overhead when not in use. Enable only if you know what
+	  you are doing.
 
 config MALI_PWRSOFT_765
-	bool "PWRSOFT-765 ticket"
+	bool "Enable workaround for PWRSOFT-765"
 	depends on MALI_BIFROST && MALI_BIFROST_EXPERT
 	default n
 	help
-	  PWRSOFT-765 fixes devfreq cooling devices issues. However, they are
-	  not merged in mainline kernel yet. So this define helps to guard those
-	  parts of the code.
+	  PWRSOFT-765 fixes devfreq cooling devices issues. The fix was merged
+	  in kernel v4.10, however if backported into the kernel then this
+	  option must be manually selected.
 
-config MALI_MEMORY_FULLY_BACKED
-	bool "Memory fully physically-backed"
-	default n
-	help
-	  This option enables full backing of all virtual memory allocations
-	  for the kernel. This only affects grow-on-GPU-page-fault memory.
+	  If using kernel >= v4.10 then say N, otherwise if devfreq cooling
+	  changes have been backported say Y to avoid compilation errors.
 
-config MALI_DMA_BUF_MAP_ON_DEMAND
-	bool "Map imported dma-bufs on demand"
-	depends on MALI_BIFROST
-	default n
-	default y if !DMA_BUF_SYNC_IOCTL_SUPPORTED
-	help
-	  This option caused kbase to set up the GPU mapping of imported
-	  dma-buf when needed to run atoms.  This is the legacy behaviour.
-
-config MALI_DMA_BUF_LEGACY_COMPAT
-	bool "Enable legacy compatibility cache flush on dma-buf map"
-	depends on MALI_BIFROST && !MALI_DMA_BUF_MAP_ON_DEMAND
-	default n
-	help
-	  This option enables compatibility with legacy dma-buf mapping
-	  behavior, then the dma-buf is mapped on import, by adding cache
-	  maintenance where MALI_DMA_BUF_MAP_ON_DEMAND would do the mapping,
-	  including a cache flush.
-
-config MALI_REAL_HW
-	bool
-	default y
-	default n if NO_MALI
 
 config MALI_HW_ERRATA_1485982_NOT_AFFECTED
 	bool "Disable workaround for BASE_HW_ISSUE_GPU2017_1336"
@@ -273,20 +321,6 @@ config MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE
 	  tree using the property, opp-mali-errata-1485982. Otherwise the
 	  slowest clock will be selected.
 
-config MALI_GEM5_BUILD
-	bool "Enable build of Mali kernel driver for GEM5"
-	depends on MALI_BIFROST
-	default n
-	help
-	  This option is to do a Mali GEM5 build.
-	  If unsure, say N.
-
-# Instrumentation options.
-
-# config MALI_JOB_DUMP exists in the Kernel Kconfig but is configured using CINSTR_JOB_DUMP in Mconfig.
-# config MALI_PRFCNT_SET_PRIMARY exists in the Kernel Kconfig but is configured using CINSTR_PRIMARY_HWC in Mconfig.
-# config MALI_BIFROST_PRFCNT_SET_SECONDARY exists in the Kernel Kconfig but is configured using CINSTR_SECONDARY_HWC in Mconfig.
-# config MALI_PRFCNT_SET_TERTIARY exists in the Kernel Kconfig but is configured using CINSTR_TERTIARY_HWC in Mconfig.
-# config MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS exists in the Kernel Kconfig but is configured using CINSTR_HWC_SET_SELECT_VIA_DEBUG_FS in Mconfig.
 
+source "kernel/drivers/gpu/arm/midgard/arbitration/Mconfig"
 source "kernel/drivers/gpu/arm/midgard/tests/Mconfig"

drivers/gpu/arm/bifrost/arbiter/Kbuild

@@ -1,6 +1,6 @@
-# SPDX-License-Identifier: GPL-2.0
+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 #
-# (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
+# (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
 #
 # This program is free software and is provided to you under the terms of the
 # GNU General Public License version 2 as published by the Free Software
@@ -19,5 +19,5 @@
 #
 
 bifrost_kbase-y += \
-	arbiter/mali_kbase_arbif.o \
-	arbiter/mali_kbase_arbiter_pm.o
+    arbiter/mali_kbase_arbif.o \
+    arbiter/mali_kbase_arbiter_pm.o

drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.c

@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
  * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
@@ -170,11 +170,15 @@ static void on_gpu_lost(struct device *dev)
  *
  * Initialise Kbase Arbiter interface and assign callback functions.
  *
- * Return: 0 on success else a Linux error code
+ * Return:
+ * * 0			- the interface was initialized or was not specified
+ * *			in the device tree.
+ * * -EFAULT		- the interface was specified but failed to initialize.
+ * * -EPROBE_DEFER	- module dependencies are not yet available.
  */
 int kbase_arbif_init(struct kbase_device *kbdev)
 {
-#ifdef CONFIG_OF
+#if IS_ENABLED(CONFIG_OF)
 	struct arbiter_if_arb_vm_ops ops;
 	struct arbiter_if_dev *arb_if;
 	struct device_node *arbiter_if_node;
@@ -218,8 +222,8 @@ int kbase_arbif_init(struct kbase_device *kbdev)
 	ops.arb_vm_max_config = on_max_config;
 	ops.arb_vm_update_freq = on_update_freq;
 
-
 	kbdev->arb.arb_freq.arb_freq = 0;
+	kbdev->arb.arb_freq.freq_updated = false;
 	mutex_init(&kbdev->arb.arb_freq.arb_freq_lock);
 
 	/* register kbase arbiter_if callbacks */
@@ -229,6 +233,8 @@ int kbase_arbif_init(struct kbase_device *kbdev)
 		if (err) {
 			dev_err(&pdev->dev, "Failed to register with arbiter\n");
 			module_put(pdev->dev.driver->owner);
+			if (err != -EPROBE_DEFER)
+				err = -EFAULT;
 			return err;
 		}
 	}
@@ -307,6 +313,8 @@ void kbase_arbif_gpu_stopped(struct kbase_device *kbdev, u8 gpu_required)
 	if (arb_if && arb_if->vm_ops.vm_arb_gpu_stopped) {
 		dev_dbg(kbdev->dev, "%s\n", __func__);
 		KBASE_TLSTREAM_TL_ARBITER_STOPPED(kbdev, kbdev);
+		if (gpu_required)
+			KBASE_TLSTREAM_TL_ARBITER_REQUESTED(kbdev, kbdev);
 		arb_if->vm_ops.vm_arb_gpu_stopped(arb_if, gpu_required);
 	}
 }

drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.h

@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
  * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
@@ -57,8 +57,11 @@ enum kbase_arbif_evt {
  * Initialize the arbiter interface and also determines
  * if Arbiter functionality is required.
  *
- * Return: 0 if the Arbiter interface was successfully initialized or the
- *           Arbiter was not required.
+ * Return:
+ * * 0			- the interface was initialized or was not specified
+ * *			in the device tree.
+ * * -EFAULT		- the interface was specified but failed to initialize.
+ * * -EPROBE_DEFER	- module dependencies are not yet available.
  */
 int kbase_arbif_init(struct kbase_device *kbdev);
 

drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_defs.h

@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
  * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.

drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_interface.h

@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
  * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
@@ -132,6 +132,11 @@ struct arbiter_if_vm_arb_ops {
 	 * @dev: The device structure to supply in the callbacks.
 	 * @ops: The callbacks that the device driver supports
 	 *       (none are optional).
+	 *
+	 * Return:
+	 * * 0			- successful.
+	 * * -EINVAL		- invalid argument.
+	 * * -EPROBE_DEFER	- module dependencies are not yet available.
 	 */
 	int (*vm_arb_register_dev)(struct arbiter_if_dev *arbif_dev,
 		struct device *dev, struct arbiter_if_arb_vm_ops *ops);

drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.c

@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
  * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
@@ -27,9 +27,9 @@
 #include <mali_kbase.h>
 #include <mali_kbase_pm.h>
 #include <mali_kbase_hwaccess_jm.h>
-#include <mali_kbase_irq_internal.h>
+#include <backend/gpu/mali_kbase_irq_internal.h>
 #include <mali_kbase_hwcnt_context.h>
-#include <mali_kbase_pm_internal.h>
+#include <backend/gpu/mali_kbase_pm_internal.h>
 #include <tl/mali_kbase_tracepoints.h>
 #include <mali_kbase_gpuprops.h>
 
@@ -37,6 +37,7 @@
  * after the following time (in milliseconds) has ellapsed.
  */
 #define GPU_REQUEST_TIMEOUT 1000
+#define KHZ_TO_HZ 1000
 
 #define MAX_L2_SLICES_MASK		0xFF
 
@@ -528,8 +529,16 @@ int kbase_arbiter_pm_gpu_assigned(struct kbase_device *kbdev)
 static void kbase_arbiter_pm_vm_gpu_start(struct kbase_device *kbdev)
 {
 	struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
+	bool freq_updated = false;
 
 	lockdep_assert_held(&arb_vm_state->vm_state_lock);
+	mutex_lock(&kbdev->arb.arb_freq.arb_freq_lock);
+	if (kbdev->arb.arb_freq.freq_updated) {
+		kbdev->arb.arb_freq.freq_updated = false;
+		freq_updated = true;
+	}
+	mutex_unlock(&kbdev->arb.arb_freq.arb_freq_lock);
+
 	cancel_request_timer(kbdev);
 	switch (arb_vm_state->vm_state) {
 	case KBASE_VM_STATE_INITIALIZING:
@@ -555,9 +564,16 @@ static void kbase_arbiter_pm_vm_gpu_start(struct kbase_device *kbdev)
 		kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_SUSPENDED);
 		break;
 	default:
-		dev_warn(kbdev->dev,
-			"GPU_GRANTED when not expected - state %s\n",
-			kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state));
+		/*
+		 * GPU_GRANTED can be received when there is a frequency update
+		 * Only show a warning if received in an unexpected state
+		 * without a frequency update
+		 */
+		if (!freq_updated)
+			dev_warn(kbdev->dev,
+				"GPU_GRANTED when not expected - state %s\n",
+				kbase_arbiter_pm_vm_state_str(
+					arb_vm_state->vm_state));
 		break;
 	}
 }
@@ -1005,13 +1021,25 @@ int kbase_arbiter_pm_ctx_active_handle_suspend(struct kbase_device *kbdev,
  * kbase_arbiter_pm_update_gpu_freq() - Updates GPU clock frequency received
  * from arbiter.
  * @arb_freq - Pointer to struchture holding GPU clock frequenecy data
- * @freq - New frequency value
+ * @freq - New frequency value in KHz
  */
 void kbase_arbiter_pm_update_gpu_freq(struct kbase_arbiter_freq *arb_freq,
-		uint32_t freq)
+	uint32_t freq)
 {
+	struct kbase_gpu_clk_notifier_data ndata;
+
 	mutex_lock(&arb_freq->arb_freq_lock);
-	arb_freq->arb_freq = freq;
+	if (arb_freq->arb_freq != freq) {
+		ndata.new_rate = freq * KHZ_TO_HZ;
+		ndata.old_rate = arb_freq->arb_freq * KHZ_TO_HZ;
+		ndata.gpu_clk_handle = arb_freq;
+		arb_freq->arb_freq = freq;
+		arb_freq->freq_updated = true;
+		if (arb_freq->nb)
+			arb_freq->nb->notifier_call(arb_freq->nb,
+						    POST_RATE_CHANGE, &ndata);
+	}
+
 	mutex_unlock(&arb_freq->arb_freq_lock);
 }
 
@@ -1046,14 +1074,64 @@ static unsigned long get_arb_gpu_clk_rate(struct kbase_device *kbdev,
 			(struct kbase_arbiter_freq *) gpu_clk_handle;
 
 	mutex_lock(&arb_dev_freq->arb_freq_lock);
-	freq = arb_dev_freq->arb_freq;
+	/* Convert from KHz to Hz */
+	freq = arb_dev_freq->arb_freq * KHZ_TO_HZ;
 	mutex_unlock(&arb_dev_freq->arb_freq_lock);
 	return freq;
 }
 
+/**
+ * arb_gpu_clk_notifier_register() - Register a clock rate change notifier.
+ * @kbdev          - kbase_device pointer
+ * @gpu_clk_handle - Handle unique to the enumerated GPU clock
+ * @nb             - notifier block containing the callback function pointer
+ *
+ * Returns 0 on success, negative error code otherwise.
+ *
+ * This function registers a callback function that is invoked whenever the
+ * frequency of the clock corresponding to @gpu_clk_handle changes.
+ */
+static int arb_gpu_clk_notifier_register(struct kbase_device *kbdev,
+	void *gpu_clk_handle, struct notifier_block *nb)
+{
+	int ret = 0;
+	struct kbase_arbiter_freq *arb_dev_freq =
+		(struct kbase_arbiter_freq *)gpu_clk_handle;
+
+	if (!arb_dev_freq->nb)
+		arb_dev_freq->nb = nb;
+	else
+		ret = -EBUSY;
+
+	return ret;
+}
+
+/**
+ * gpu_clk_notifier_unregister() - Unregister clock rate change notifier
+ * @kbdev          - kbase_device pointer
+ * @gpu_clk_handle - Handle unique to the enumerated GPU clock
+ * @nb             - notifier block containing the callback function pointer
+ *
+ * This function pointer is used to unregister a callback function that
+ * was previously registered to get notified of a frequency change of the
+ * clock corresponding to @gpu_clk_handle.
+ */
+static void arb_gpu_clk_notifier_unregister(struct kbase_device *kbdev,
+	void *gpu_clk_handle, struct notifier_block *nb)
+{
+	struct kbase_arbiter_freq *arb_dev_freq =
+		(struct kbase_arbiter_freq *)gpu_clk_handle;
+	if (arb_dev_freq->nb == nb) {
+		arb_dev_freq->nb = NULL;
+	} else {
+		dev_err(kbdev->dev, "%s - notifier did not match\n",
+			 __func__);
+	}
+}
+
 struct kbase_clk_rate_trace_op_conf arb_clk_rate_trace_ops = {
 	.get_gpu_clk_rate = get_arb_gpu_clk_rate,
 	.enumerate_gpu_clk = enumerate_arb_gpu_clk,
-	.gpu_clk_notifier_register = NULL,
-	.gpu_clk_notifier_unregister = NULL
+	.gpu_clk_notifier_register = arb_gpu_clk_notifier_register,
+	.gpu_clk_notifier_unregister = arb_gpu_clk_notifier_unregister
 };

drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.h

@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
  * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
@@ -168,15 +168,27 @@ extern struct kbase_clk_rate_trace_op_conf arb_clk_rate_trace_ops;
 /**
  * struct kbase_arbiter_freq - Holding the GPU clock frequency data retrieved
  * from arbiter
- * @arb_freq:                 GPU clock frequency value
- * @arb_freq_lock:            Mutex protecting access to arbfreq value
+ * @arb_freq:      GPU clock frequency value
+ * @arb_freq_lock: Mutex protecting access to arbfreq value
+ * @nb:            Notifier block to receive rate change callbacks
+ * @freq_updated:  Flag to indicate whether a frequency changed has just been
+ *                 communicated to avoid "GPU_GRANTED when not expected" warning
  */
 struct kbase_arbiter_freq {
 	uint32_t arb_freq;
 	struct mutex arb_freq_lock;
+	struct notifier_block *nb;
+	bool freq_updated;
 };
 
+/**
+ * kbase_arbiter_pm_update_gpu_freq() - Update GPU frequency
+ * @arb_freq: Pointer to GPU clock frequency data
+ * @freq:     The new frequency
+ *
+ * Updates the GPU frequency and triggers any notifications
+ */
 void kbase_arbiter_pm_update_gpu_freq(struct kbase_arbiter_freq *arb_freq,
-		uint32_t freq);
+	uint32_t freq);
 
 #endif /*_MALI_KBASE_ARBITER_PM_H_ */

drivers/gpu/arm/bifrost/backend/gpu/Kbuild

@@ -1,4 +1,4 @@
-# SPDX-License-Identifier: GPL-2.0
+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 #
 # (C) COPYRIGHT 2014-2021 ARM Limited. All rights reserved.
 #
@@ -18,47 +18,32 @@
 #
 #
 
-BACKEND += \
-	backend/gpu/mali_kbase_cache_policy_backend.c \
-	backend/gpu/mali_kbase_gpuprops_backend.c \
-	backend/gpu/mali_kbase_irq_linux.c \
-	backend/gpu/mali_kbase_js_backend.c \
-	backend/gpu/mali_kbase_pm_backend.c \
-	backend/gpu/mali_kbase_pm_driver.c \
-	backend/gpu/mali_kbase_pm_metrics.c \
-	backend/gpu/mali_kbase_pm_ca.c \
-	backend/gpu/mali_kbase_pm_always_on.c \
-	backend/gpu/mali_kbase_pm_coarse_demand.c \
-	backend/gpu/mali_kbase_pm_policy.c \
-	backend/gpu/mali_kbase_time.c \
-	backend/gpu/mali_kbase_l2_mmu_config.c \
-	backend/gpu/mali_kbase_clk_rate_trace_mgr.c
+bifrost_kbase-y += \
+    backend/gpu/mali_kbase_cache_policy_backend.o \
+    backend/gpu/mali_kbase_gpuprops_backend.o \
+    backend/gpu/mali_kbase_irq_linux.o \
+    backend/gpu/mali_kbase_js_backend.o \
+    backend/gpu/mali_kbase_pm_backend.o \
+    backend/gpu/mali_kbase_pm_driver.o \
+    backend/gpu/mali_kbase_pm_metrics.o \
+    backend/gpu/mali_kbase_pm_ca.o \
+    backend/gpu/mali_kbase_pm_always_on.o \
+    backend/gpu/mali_kbase_pm_coarse_demand.o \
+    backend/gpu/mali_kbase_pm_policy.o \
+    backend/gpu/mali_kbase_time.o \
+    backend/gpu/mali_kbase_l2_mmu_config.o \
+    backend/gpu/mali_kbase_clk_rate_trace_mgr.o
 
-ifeq ($(MALI_USE_CSF),1)
-# empty
-else
-	BACKEND += \
-		backend/gpu/mali_kbase_instr_backend.c \
-		backend/gpu/mali_kbase_jm_as.c \
-		backend/gpu/mali_kbase_debug_job_fault_backend.c \
-		backend/gpu/mali_kbase_jm_hw.c \
-		backend/gpu/mali_kbase_jm_rb.c
+ifeq ($(MALI_USE_CSF),0)
+    bifrost_kbase-y += \
+        backend/gpu/mali_kbase_instr_backend.o \
+        backend/gpu/mali_kbase_jm_as.o \
+        backend/gpu/mali_kbase_debug_job_fault_backend.o \
+        backend/gpu/mali_kbase_jm_hw.o \
+        backend/gpu/mali_kbase_jm_rb.o
 endif
 
-ifeq ($(MALI_CUSTOMER_RELEASE),0)
-BACKEND += \
-	backend/gpu/mali_kbase_pm_always_on_demand.c
-endif
 
-ifeq ($(CONFIG_MALI_BIFROST_DEVFREQ),y)
-BACKEND += \
-	backend/gpu/mali_kbase_devfreq.c
-endif
+bifrost_kbase-$(CONFIG_MALI_BIFROST_DEVFREQ) += \
+    backend/gpu/mali_kbase_devfreq.o
 
-ifeq ($(CONFIG_MALI_BIFROST_NO_MALI),y)
-	# Dummy model
-	BACKEND += backend/gpu/mali_kbase_model_dummy.c
-	BACKEND += backend/gpu/mali_kbase_model_linux.c
-	# HW error simulation
-	BACKEND += backend/gpu/mali_kbase_model_error_generator.c
-endif

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_backend_config.h

@@ -1,7 +1,7 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
- * (C) COPYRIGHT 2014-2018, 2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2018, 2020-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_cache_policy_backend.c

@@ -1,7 +1,7 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
- * (C) COPYRIGHT 2014-2016, 2018, 2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2016, 2018, 2020-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_cache_policy_backend.h

@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
  * (C) COPYRIGHT 2014-2016, 2020-2021 ARM Limited. All rights reserved.

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_clk_rate_trace_mgr.c

@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
  * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved.
@@ -27,7 +27,7 @@
 #include <mali_kbase_config_defaults.h>
 #include <linux/clk.h>
 #include <asm/div64.h>
-#include "mali_kbase_clk_rate_trace_mgr.h"
+#include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h"
 
 #ifdef CONFIG_TRACE_POWER_GPU_FREQUENCY
 #include <trace/events/power_gpu_frequency.h>
@@ -299,8 +299,8 @@ void kbase_clk_rate_trace_manager_notify_all(
 
 	kbdev = container_of(clk_rtm, struct kbase_device, pm.clk_rtm);
 
-	dev_dbg(kbdev->dev, "GPU clock %u rate changed to %lu",
-		clk_index, new_rate);
+	dev_dbg(kbdev->dev, "%s - GPU clock %u rate changed to %lu, pid: %d",
+		__func__, clk_index, new_rate, current->pid);
 
 	/* Raise standard `power/gpu_frequency` ftrace event */
 	{

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_clk_rate_trace_mgr.h

@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
  * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved.

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_debug_job_fault_backend.c

@@ -1,7 +1,7 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
- * (C) COPYRIGHT 2012-2015, 2018-2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2015, 2018-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software
@@ -23,7 +23,7 @@
 #include <device/mali_kbase_device.h>
 #include "mali_kbase_debug_job_fault.h"
 
-#ifdef CONFIG_DEBUG_FS
+#if IS_ENABLED(CONFIG_DEBUG_FS)
 
 /*GPU_CONTROL_REG(r)*/
 static int gpu_control_reg_snapshot[] = {

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.c

@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
  * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved.
@@ -28,7 +28,7 @@
 #include <linux/of.h>
 #include <linux/clk.h>
 #include <linux/devfreq.h>
-#ifdef CONFIG_DEVFREQ_THERMAL
+#if IS_ENABLED(CONFIG_DEVFREQ_THERMAL)
 #include <linux/devfreq_cooling.h>
 #endif
 
@@ -190,7 +190,7 @@ kbase_devfreq_target(struct device *dev, unsigned long *target_freq, u32 flags)
 
 	dev_dbg(dev, "%lu-->%lu\n", kbdev->current_nominal_freq, nominal_freq);
 
-#ifdef CONFIG_REGULATOR
+#if IS_ENABLED(CONFIG_REGULATOR)
 	/* Regulators and clocks work in pairs: every clock has a regulator,
 	 * and we never expect to have more regulators than clocks.
 	 *
@@ -242,7 +242,7 @@ kbase_devfreq_target(struct device *dev, unsigned long *target_freq, u32 flags)
 		}
 	}
 
-#ifdef CONFIG_REGULATOR
+#if IS_ENABLED(CONFIG_REGULATOR)
 	for (i = 0; i < kbdev->nr_clocks; i++) {
 		if (kbdev->regulators[i] &&
 				kbdev->current_voltages[i] != volts[i] &&
@@ -381,18 +381,21 @@ static void kbase_devfreq_term_freq_table(struct kbase_device *kbdev)
 	struct devfreq_dev_profile *dp = &kbdev->devfreq_profile;
 
 	kfree(dp->freq_table);
+	dp->freq_table = NULL;
 }
 
 static void kbase_devfreq_term_core_mask_table(struct kbase_device *kbdev)
 {
 	kfree(kbdev->devfreq_table);
+	kbdev->devfreq_table = NULL;
 }
 
 static void kbase_devfreq_exit(struct device *dev)
 {
 	struct kbase_device *kbdev = dev_get_drvdata(dev);
 
-	kbase_devfreq_term_freq_table(kbdev);
+	if (kbdev)
+		kbase_devfreq_term_freq_table(kbdev);
 }
 
 static void kbasep_devfreq_read_suspend_clock(struct kbase_device *kbdev,
@@ -464,7 +467,7 @@ static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev)
 		u64 core_mask, opp_freq,
 			real_freqs[BASE_MAX_NR_CLOCKS_REGULATORS];
 		int err;
-#ifdef CONFIG_REGULATOR
+#if IS_ENABLED(CONFIG_REGULATOR)
 		u32 opp_volts[BASE_MAX_NR_CLOCKS_REGULATORS];
 #endif
 
@@ -492,7 +495,7 @@ static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev)
 					err);
 			continue;
 		}
-#ifdef CONFIG_REGULATOR
+#if IS_ENABLED(CONFIG_REGULATOR)
 		err = of_property_read_u32_array(node,
 			"opp-microvolt", opp_volts, kbdev->nr_regulators);
 		if (err < 0) {
@@ -546,7 +549,7 @@ static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev)
 				kbdev->devfreq_table[i].real_freqs[j] =
 					real_freqs[j];
 		}
-#ifdef CONFIG_REGULATOR
+#if IS_ENABLED(CONFIG_REGULATOR)
 		if (kbdev->nr_regulators > 0) {
 			int j;
 
@@ -631,8 +634,12 @@ void kbase_devfreq_enqueue_work(struct kbase_device *kbdev,
 
 	WARN_ON(work_type == DEVFREQ_WORK_NONE);
 	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
-	kbdev->devfreq_queue.req_type = work_type;
-	queue_work(kbdev->devfreq_queue.workq, &kbdev->devfreq_queue.work);
+	/* Skip enqueuing a work if workqueue has already been terminated. */
+	if (likely(kbdev->devfreq_queue.workq)) {
+		kbdev->devfreq_queue.req_type = work_type;
+		queue_work(kbdev->devfreq_queue.workq,
+			   &kbdev->devfreq_queue.work);
+	}
 	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
 	dev_dbg(kbdev->dev, "Enqueuing devfreq req: %s\n",
 		kbase_devfreq_req_type_name(work_type));
@@ -654,11 +661,19 @@ static int kbase_devfreq_work_init(struct kbase_device *kbdev)
 
 static void kbase_devfreq_work_term(struct kbase_device *kbdev)
 {
-	destroy_workqueue(kbdev->devfreq_queue.workq);
+	unsigned long flags;
+	struct workqueue_struct *workq;
+
+	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
+	workq = kbdev->devfreq_queue.workq;
+	kbdev->devfreq_queue.workq = NULL;
+	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
+
+	destroy_workqueue(workq);
 }
 
 static unsigned long kbase_devfreq_get_static_power(struct devfreq *devfreq,
-						    unsigned long voltage)
+		unsigned long voltage)
 {
 	struct device *dev = devfreq->dev.parent;
 	struct kbase_device *kbdev = dev_get_drvdata(dev);
@@ -718,14 +733,6 @@ int kbase_devfreq_init(struct kbase_device *kbdev)
 		return err;
 	}
 
-	/* Initialise devfreq suspend/resume workqueue */
-	err = kbase_devfreq_work_init(kbdev);
-	if (err) {
-		kbase_devfreq_term_freq_table(kbdev);
-		dev_err(kbdev->dev, "Devfreq initialization failed");
-		return err;
-	}
-
 	of_property_read_u32(np, "upthreshold",
 			     &ondemand_data.upthreshold);
 	of_property_read_u32(np, "downdifferential",
@@ -734,8 +741,21 @@ int kbase_devfreq_init(struct kbase_device *kbdev)
 				"simple_ondemand", &ondemand_data);
 	if (IS_ERR(kbdev->devfreq)) {
 		err = PTR_ERR(kbdev->devfreq);
-		kbase_devfreq_work_term(kbdev);
+		kbdev->devfreq = NULL;
+		kbase_devfreq_term_core_mask_table(kbdev);
 		kbase_devfreq_term_freq_table(kbdev);
+		dev_err(kbdev->dev, "Fail to add devfreq device(%d)\n", err);
+		return err;
+	}
+
+	/* Initialize devfreq suspend/resume workqueue */
+	err = kbase_devfreq_work_init(kbdev);
+	if (err) {
+		if (devfreq_remove_device(kbdev->devfreq))
+			dev_err(kbdev->dev, "Fail to rm devfreq\n");
+		kbdev->devfreq = NULL;
+		kbase_devfreq_term_core_mask_table(kbdev);
+		dev_err(kbdev->dev, "Fail to init devfreq workqueue\n");
 		return err;
 	}
 
@@ -759,12 +779,12 @@ int kbase_devfreq_init(struct kbase_device *kbdev)
 
 	mali_mdevp.data = kbdev->devfreq;
 	kbdev->mdev_info = rockchip_system_monitor_register(kbdev->dev,
-							    &mali_mdevp);
+			&mali_mdevp);
 	if (IS_ERR(kbdev->mdev_info)) {
 		dev_dbg(kbdev->dev, "without system monitor\n");
-		kbdev->mdev_info = NULL;
+               kbdev->mdev_info = NULL;
 	}
-#ifdef CONFIG_DEVFREQ_THERMAL
+#if IS_ENABLED(CONFIG_DEVFREQ_THERMAL)
 	if (of_find_compatible_node(kbdev->dev->of_node, NULL,
 				    "simple-power-model")) {
 		of_property_read_u32(kbdev->dev->of_node,
@@ -782,23 +802,23 @@ int kbase_devfreq_init(struct kbase_device *kbdev)
 		if (!kbase_dcp->dyn_power_coeff) {
 			err = -EINVAL;
 			dev_err(kbdev->dev, "failed to get dynamic-coefficient\n");
-			goto cooling_failed;
+			goto ipa_init_failed;
 		}
 
 		kbdev->devfreq_cooling =
 			of_devfreq_cooling_register_power(kbdev->dev->of_node,
-							  kbdev->devfreq,
-							  kbase_dcp);
+					kbdev->devfreq,
+					kbase_dcp);
 		if (IS_ERR(kbdev->devfreq_cooling)) {
 			err = PTR_ERR(kbdev->devfreq_cooling);
 			dev_err(kbdev->dev, "failed to register cooling device\n");
-			goto cooling_failed;
+			goto cooling_reg_failed;
 		}
 	} else {
 		err = kbase_ipa_init(kbdev);
 		if (err) {
 			dev_err(kbdev->dev, "IPA initialization failed\n");
-			goto cooling_failed;
+			goto ipa_init_failed;
 		}
 
 		kbdev->devfreq_cooling = of_devfreq_cooling_register_power(
@@ -808,26 +828,32 @@ int kbase_devfreq_init(struct kbase_device *kbdev)
 		if (IS_ERR(kbdev->devfreq_cooling)) {
 			err = PTR_ERR(kbdev->devfreq_cooling);
 			dev_err(kbdev->dev,
-				"Failed to register cooling device (%d)\n",
-				err);
-			goto cooling_failed;
-		}
+					"Failed to register cooling device (%d)\n",
+					err);
+			goto cooling_reg_failed;
+               }
+
 	}
 #endif
 
 	return 0;
 
-#ifdef CONFIG_DEVFREQ_THERMAL
-cooling_failed:
+#if IS_ENABLED(CONFIG_DEVFREQ_THERMAL)
+cooling_reg_failed:
+	kbase_ipa_term(kbdev);
+ipa_init_failed:
 	devfreq_unregister_opp_notifier(kbdev->dev, kbdev->devfreq);
 #endif /* CONFIG_DEVFREQ_THERMAL */
+
 opp_notifier_failed:
+	kbase_devfreq_work_term(kbdev);
+
 	if (devfreq_remove_device(kbdev->devfreq))
 		dev_err(kbdev->dev, "Failed to terminate devfreq (%d)\n", err);
-	else
-		kbdev->devfreq = NULL;
 
-	kbase_devfreq_work_term(kbdev);
+	kbdev->devfreq = NULL;
+
+	kbase_devfreq_term_core_mask_table(kbdev);
 
 	return err;
 }
@@ -838,8 +864,7 @@ void kbase_devfreq_term(struct kbase_device *kbdev)
 
 	dev_dbg(kbdev->dev, "Term Mali devfreq\n");
 
-	rockchip_system_monitor_unregister(kbdev->mdev_info);
-#ifdef CONFIG_DEVFREQ_THERMAL
+#if IS_ENABLED(CONFIG_DEVFREQ_THERMAL)
 	if (kbdev->devfreq_cooling)
 		devfreq_cooling_unregister(kbdev->devfreq_cooling);
 
@@ -850,6 +875,8 @@ void kbase_devfreq_term(struct kbase_device *kbdev)
 
 	devfreq_unregister_opp_notifier(kbdev->dev, kbdev->devfreq);
 
+	kbase_devfreq_work_term(kbdev);
+
 	err = devfreq_remove_device(kbdev->devfreq);
 	if (err)
 		dev_err(kbdev->dev, "Failed to terminate devfreq (%d)\n", err);
@@ -857,6 +884,4 @@ void kbase_devfreq_term(struct kbase_device *kbdev)
 		kbdev->devfreq = NULL;
 
 	kbase_devfreq_term_core_mask_table(kbdev);
-
-	kbase_devfreq_work_term(kbdev);
 }

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.h

@@ -1,7 +1,7 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
- * (C) COPYRIGHT 2014, 2019-2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014, 2019-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_gpuprops_backend.c

@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
  * (C) COPYRIGHT 2014-2021 ARM Limited. All rights reserved.
@@ -45,6 +45,12 @@ int kbase_backend_gpuprops_get(struct kbase_device *kbdev,
 	registers.core_features = kbase_reg_read(kbdev,
 				GPU_CONTROL_REG(CORE_FEATURES));
 #else /* !MALI_USE_CSF */
+	if (((registers.gpu_id & GPU_ID2_PRODUCT_MODEL) ==
+	     GPU_ID2_PRODUCT_TGRX) ||
+	    ((registers.gpu_id & GPU_ID2_PRODUCT_MODEL) ==
+	     GPU_ID2_PRODUCT_TVAX))
+		registers.core_features =
+			kbase_reg_read(kbdev, GPU_CONTROL_REG(CORE_FEATURES));
 #endif /* MALI_USE_CSF */
 	registers.tiler_features = kbase_reg_read(kbdev,
 				GPU_CONTROL_REG(TILER_FEATURES));

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_backend.c

@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
  * (C) COPYRIGHT 2014-2021 ARM Limited. All rights reserved.

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_defs.h

@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
  * (C) COPYRIGHT 2014, 2016, 2018-2021 ARM Limited. All rights reserved.
@@ -26,7 +26,7 @@
 #ifndef _KBASE_INSTR_DEFS_H_
 #define _KBASE_INSTR_DEFS_H_
 
-#include "../../mali_kbase_hwcnt_gpu.h"
+#include <mali_kbase_hwcnt_gpu.h>
 
 /*
  * Instrumentation State Machine States

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_internal.h

@@ -1,7 +1,7 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
- * (C) COPYRIGHT 2014, 2018, 2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014, 2018, 2020-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_internal.h

@@ -1,7 +1,7 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
- * (C) COPYRIGHT 2014-2015, 2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2015, 2020-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_linux.c

@@ -1,7 +1,7 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
- * (C) COPYRIGHT 2014-2016,2018-2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2016, 2018-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software
@@ -25,7 +25,7 @@
 
 #include <linux/interrupt.h>
 
-#if !defined(CONFIG_MALI_BIFROST_NO_MALI)
+#if !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI)
 
 /* GPU IRQ Tags */
 #define	JOB_IRQ_TAG	0
@@ -232,7 +232,7 @@ int kbase_set_custom_irq_handler(struct kbase_device *kbdev,
 		result = -EINVAL;
 		dev_err(kbdev->dev, "Can't request interrupt %d (index %d)\n",
 					kbdev->irqs[irq_type].irq, irq_type);
-#ifdef CONFIG_SPARSE_IRQ
+#if IS_ENABLED(CONFIG_SPARSE_IRQ)
 		dev_err(kbdev->dev, "You have CONFIG_SPARSE_IRQ support enabled - is the interrupt number correct for this configuration?\n");
 #endif /* CONFIG_SPARSE_IRQ */
 	}
@@ -461,7 +461,7 @@ int kbase_install_interrupts(struct kbase_device *kbdev)
 		if (err) {
 			dev_err(kbdev->dev, "Can't request interrupt %d (index %d)\n",
 							kbdev->irqs[i].irq, i);
-#ifdef CONFIG_SPARSE_IRQ
+#if IS_ENABLED(CONFIG_SPARSE_IRQ)
 			dev_err(kbdev->dev, "You have CONFIG_SPARSE_IRQ support enabled - is the interrupt number correct for this configuration?\n");
 #endif /* CONFIG_SPARSE_IRQ */
 			goto release;
@@ -501,4 +501,4 @@ void kbase_synchronize_irqs(struct kbase_device *kbdev)
 
 KBASE_EXPORT_TEST_API(kbase_synchronize_irqs);
 
-#endif /* !defined(CONFIG_MALI_BIFROST_NO_MALI) */
+#endif /* !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) */

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_as.c

@@ -1,7 +1,7 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
- * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_defs.h

@@ -1,7 +1,7 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
- * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c

@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
  * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved.
@@ -300,7 +300,7 @@ void kbase_job_hw_submit(struct kbase_device *kbdev,
 			&kbdev->gpu_props.props.raw_props.js_features[js],
 			"ctx_nr,atom_nr");
 	kbase_kinstr_jm_atom_hw_submit(katom);
-#ifdef CONFIG_GPU_TRACEPOINTS
+#if IS_ENABLED(CONFIG_GPU_TRACEPOINTS)
 	if (!kbase_backend_nr_atoms_submitted(kbdev, js)) {
 		/* If this is the only job on the slot, trace it as starting */
 		char js_string[16];

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_internal.h

@@ -1,7 +1,7 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
- * (C) COPYRIGHT 2011-2016, 2018-2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2016, 2018-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software
@@ -59,7 +59,7 @@ void kbase_job_submit_nolock(struct kbase_device *kbdev,
 void kbase_job_done_slot(struct kbase_device *kbdev, int s, u32 completion_code,
 					u64 job_tail, ktime_t *end_timestamp);
 
-#ifdef CONFIG_GPU_TRACEPOINTS
+#if IS_ENABLED(CONFIG_GPU_TRACEPOINTS)
 static inline char *kbasep_make_job_slot_string(int js, char *js_string,
 						size_t js_size)
 {

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c

@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
  * (C) COPYRIGHT 2014-2021 ARM Limited. All rights reserved.
@@ -289,9 +289,11 @@ static void kbase_gpu_release_atom(struct kbase_device *kbdev,
 		katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY;
 		kbase_pm_metrics_update(kbdev, end_timestamp);
 
+		/* Inform platform at start/finish of atom */
+		kbasep_platform_event_atom_complete(katom);
+
 		if (katom->core_req & BASE_JD_REQ_PERMON)
 			kbase_pm_release_gpu_cycle_counter_nolock(kbdev);
-		/* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
 
 		KBASE_TLSTREAM_TL_NRET_ATOM_LPU(kbdev, katom,
 			&kbdev->gpu_props.props.raw_props.js_features
@@ -301,6 +303,8 @@ static void kbase_gpu_release_atom(struct kbase_device *kbdev,
 			&kbdev->gpu_props.props.raw_props.js_features
 				[katom->slot_nr]);
 
+		/* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
+
 	case KBASE_ATOM_GPU_RB_READY:
 		/* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
 
@@ -847,7 +851,7 @@ void kbase_backend_slot_update(struct kbase_device *kbdev)
 					break;
 
 				katom[idx]->gpu_rb_state =
-				KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV;
+					KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV;
 
 				/* ***TRANSITION TO HIGHER STATE*** */
 				/* fallthrough */
@@ -987,7 +991,11 @@ void kbase_backend_slot_update(struct kbase_device *kbdev)
 
 				kbase_job_hw_submit(kbdev, katom[idx], js);
 				katom[idx]->gpu_rb_state =
-						KBASE_ATOM_GPU_RB_SUBMITTED;
+					KBASE_ATOM_GPU_RB_SUBMITTED;
+
+				/* ***TRANSITION TO HIGHER STATE*** */
+				/* fallthrough */
+			case KBASE_ATOM_GPU_RB_SUBMITTED:
 
 				/* Inform power management at start/finish of
 				 * atom so it can update its GPU utilisation
@@ -996,10 +1004,9 @@ void kbase_backend_slot_update(struct kbase_device *kbdev)
 				kbase_pm_metrics_update(kbdev,
 						&katom[idx]->start_timestamp);
 
-				/* ***TRANSITION TO HIGHER STATE*** */
-				/* fallthrough */
-			case KBASE_ATOM_GPU_RB_SUBMITTED:
-				/* Atom submitted to HW, nothing else to do */
+				/* Inform platform at start/finish of atom */
+				kbasep_platform_event_atom_submit(katom[idx]);
+
 				break;
 
 			case KBASE_ATOM_GPU_RB_RETURN_TO_JS:
@@ -1225,7 +1232,7 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js,
 	 * - Schedule out the parent context if necessary, and schedule a new
 	 *   one in.
 	 */
-#ifdef CONFIG_GPU_TRACEPOINTS
+#if IS_ENABLED(CONFIG_GPU_TRACEPOINTS)
 	{
 		/* The atom in the HEAD */
 		struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js,

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.h

@@ -1,7 +1,7 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
- * (C) COPYRIGHT 2014-2018, 2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2018, 2020-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_backend.c

@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
  * (C) COPYRIGHT 2014-2021 ARM Limited. All rights reserved.

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_internal.h

@@ -1,7 +1,7 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
- * (C) COPYRIGHT 2014-2015, 2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2015, 2020-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_l2_mmu_config.c

@@ -1,7 +1,7 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
- * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_l2_mmu_config.h

@@ -1,7 +1,7 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
- * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_always_on.c

@@ -1,7 +1,7 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
- * (C) COPYRIGHT 2010-2015, 2018-2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2015, 2018-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software
@@ -58,6 +58,7 @@ const struct kbase_pm_policy kbase_pm_always_on_policy_ops = {
 	always_on_term,			/* term */
 	always_on_shaders_needed,	/* shaders_needed */
 	always_on_get_core_active,	/* get_core_active */
+	NULL,				/* handle_event */
 	KBASE_PM_POLICY_ID_ALWAYS_ON,	/* id */
 #if MALI_USE_CSF
 	ALWAYS_ON_PM_SCHED_FLAGS,	/* pm_sched_flags */

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_always_on.h

@@ -1,7 +1,7 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
- * (C) COPYRIGHT 2011-2015, 2018, 2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2015, 2018, 2020-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c

@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
  * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved.
@@ -37,7 +37,7 @@
 #include <backend/gpu/mali_kbase_pm_internal.h>
 #include <backend/gpu/mali_kbase_devfreq.h>
 #include <mali_kbase_dummy_job_wa.h>
-#include <mali_kbase_irq_internal.h>
+#include <backend/gpu/mali_kbase_irq_internal.h>
 
 static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data);
 static void kbase_pm_hwcnt_disable_worker(struct work_struct *data);

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.c

@@ -1,7 +1,7 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
- * (C) COPYRIGHT 2013-2018, 2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2013-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software
@@ -26,9 +26,6 @@
 #include <mali_kbase.h>
 #include <mali_kbase_pm.h>
 #include <backend/gpu/mali_kbase_pm_internal.h>
-#ifdef MALI_BIFROST_NO_MALI
-#include <backend/gpu/mali_kbase_model_dummy.h>
-#endif
 #include <mali_kbase_dummy_job_wa.h>
 
 int kbase_pm_ca_init(struct kbase_device *kbdev)
@@ -123,9 +120,7 @@ u64 kbase_pm_ca_get_instr_core_mask(struct kbase_device *kbdev)
 {
 	lockdep_assert_held(&kbdev->hwaccess_lock);
 
-#ifdef CONFIG_MALI_BIFROST_NO_MALI
-	return (((1ull) << KBASE_DUMMY_MODEL_MAX_SHADER_CORES) - 1);
-#elif MALI_USE_CSF
+#if   MALI_USE_CSF
 	return kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER);
 #else
 	return kbdev->pm.backend.pm_shaders_core_mask;

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.h

@@ -1,7 +1,7 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
- * (C) COPYRIGHT 2011-2018, 2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2018, 2020-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca_devfreq.h

@@ -1,7 +1,7 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
- * (C) COPYRIGHT 2017, 2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2017, 2020-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_coarse_demand.c

@@ -1,7 +1,7 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
- * (C) COPYRIGHT 2012-2016, 2018-2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2016, 2018-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software
@@ -57,6 +57,7 @@ const struct kbase_pm_policy kbase_pm_coarse_demand_policy_ops = {
 	coarse_demand_term,			/* term */
 	coarse_demand_shaders_needed,		/* shaders_needed */
 	coarse_demand_get_core_active,		/* get_core_active */
+	NULL,					/* handle_event */
 	KBASE_PM_POLICY_ID_COARSE_DEMAND,	/* id */
 #if MALI_USE_CSF
 	COARSE_ON_DEMAND_PM_SCHED_FLAGS,	/* pm_sched_flags */

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_coarse_demand.h

@@ -1,7 +1,7 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
- * (C) COPYRIGHT 2012-2015, 2018, 2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2015, 2018, 2020-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h

@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
  * (C) COPYRIGHT 2014-2021 ARM Limited. All rights reserved.
@@ -28,9 +28,6 @@
 
 #include "mali_kbase_pm_always_on.h"
 #include "mali_kbase_pm_coarse_demand.h"
-#if !MALI_CUSTOMER_RELEASE
-#include "mali_kbase_pm_always_on_demand.h"
-#endif
 
 /* Forward definition - see mali_kbase.h */
 struct kbase_device;
@@ -179,8 +176,12 @@ struct kbasep_pm_metrics_state {
  * @work: Work item which cancels the timer
  * @timer: Timer for powering off the shader cores
  * @configured_interval: Period of GPU poweroff timer
- * @configured_ticks: User-configured number of ticks to wait after the shader
- *                    power down request is received before turning off the cores
+ * @default_ticks: User-configured number of ticks to wait after the shader
+ *                 power down request is received before turning off the cores
+ * @configured_ticks: Power-policy configured number of ticks to wait after the
+ *                    shader power down request is received before turning off
+ *                    the cores. For simple power policies, this is equivalent
+ *                    to @default_ticks.
  * @remaining_ticks: Number of remaining timer ticks until shaders are powered off
  * @cancel_queued: True if the cancellation work item has been queued. This is
  *                 required to ensure that it is not queued twice, e.g. after
@@ -194,6 +195,7 @@ struct kbasep_pm_tick_timer_state {
 	struct hrtimer timer;
 
 	ktime_t configured_interval;
+	unsigned int default_ticks;
 	unsigned int configured_ticks;
 	unsigned int remaining_ticks;
 
@@ -204,9 +206,6 @@ struct kbasep_pm_tick_timer_state {
 union kbase_pm_policy_data {
 	struct kbasep_pm_policy_always_on always_on;
 	struct kbasep_pm_policy_coarse_demand coarse_demand;
-#if !MALI_CUSTOMER_RELEASE
-	struct kbasep_pm_policy_always_on_demand always_on_demand;
-#endif
 };
 
 /**
@@ -217,7 +216,8 @@ union kbase_pm_policy_data {
  *
  * @pm_current_policy: The policy that is currently actively controlling the
  *                     power state.
- * @pm_policy_data:    Private data for current PM policy
+ * @pm_policy_data:    Private data for current PM policy. This is automatically
+ *                     zeroed when a policy change occurs.
  * @reset_done:        Flag when a reset is complete
  * @reset_done_wait:   Wait queue to wait for changes to @reset_done
  * @gpu_cycle_counter_requests: The reference count of active gpu cycle counter
@@ -464,6 +464,33 @@ enum kbase_pm_policy_id {
 	KBASE_PM_POLICY_ID_ALWAYS_ON
 };
 
+/**
+ * enum kbase_pm_policy_event - PM Policy event ID
+ */
+enum kbase_pm_policy_event {
+	/**
+	 * @KBASE_PM_POLICY_EVENT_IDLE: Indicates that the GPU power state
+	 * model has determined that the GPU has gone idle.
+	 */
+	KBASE_PM_POLICY_EVENT_IDLE,
+	/**
+	 * @KBASE_PM_POLICY_EVENT_POWER_ON: Indicates that the GPU state model
+	 * is preparing to power on the GPU.
+	 */
+	KBASE_PM_POLICY_EVENT_POWER_ON,
+	/**
+	 * @KBASE_PM_POLICY_EVENT_TIMER_HIT: Indicates that the GPU became
+	 * active while the Shader Tick Timer was holding the GPU in a powered
+	 * on state.
+	 */
+	KBASE_PM_POLICY_EVENT_TIMER_HIT,
+	/**
+	 * @KBASE_PM_POLICY_EVENT_TIMER_MISS: Indicates that the GPU did not
+	 * become active before the Shader Tick Timer timeout occurred.
+	 */
+	KBASE_PM_POLICY_EVENT_TIMER_MISS,
+};
+
 /**
  * struct kbase_pm_policy - Power policy structure.
  *
@@ -476,6 +503,9 @@ enum kbase_pm_policy_id {
  * @shaders_needed:     Function called to find out if shader cores are needed
  * @get_core_active:    Function called to get the current overall GPU power
  *                      state
+ * @handle_event:       Function called when a PM policy event occurs. Should be
+ *                      set to NULL if the power policy doesn't require any
+ *                      event notifications.
  * @id:                 Field indicating an ID for this policy. This is not
  *                      necessarily the same as its index in the list returned
  *                      by kbase_pm_list_policies().
@@ -536,6 +566,16 @@ struct kbase_pm_policy {
 	 */
 	bool (*get_core_active)(struct kbase_device *kbdev);
 
+	/**
+	 * Function called when a power event occurs
+	 *
+	 * @kbdev: The kbase device structure for the device (must be a
+	 *         valid pointer)
+	 * @event: The id of the power event that has occurred
+	 */
+	void (*handle_event)(struct kbase_device *kbdev,
+			     enum kbase_pm_policy_event event);
+
 	enum kbase_pm_policy_id id;
 
 #if MALI_USE_CSF

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c

@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
  * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved.
@@ -244,7 +244,7 @@ static u32 core_type_to_reg(enum kbase_pm_core_type core_type,
 	return (u32)core_type + (u32)action;
 }
 
-#ifdef CONFIG_ARM64
+#if IS_ENABLED(CONFIG_ARM64)
 static void mali_cci_flush_l2(struct kbase_device *kbdev)
 {
 	const u32 mask = CLEAN_CACHES_COMPLETED | RESET_COMPLETED;
@@ -1343,6 +1343,12 @@ static int kbase_pm_shaders_update_state(struct kbase_device *kbdev)
 				kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER,
 						backend->shaders_avail, ACTION_PWRON);
 
+				if (backend->pm_current_policy &&
+				    backend->pm_current_policy->handle_event)
+					backend->pm_current_policy->handle_event(
+						kbdev,
+						KBASE_PM_POLICY_EVENT_POWER_ON);
+
 				backend->shaders_state = KBASE_SHADERS_PEND_ON_CORESTACK_ON;
 			}
 			break;
@@ -1395,6 +1401,12 @@ static int kbase_pm_shaders_update_state(struct kbase_device *kbdev)
 				/* Wait for being disabled */
 				;
 			} else if (!backend->shaders_desired) {
+				if (backend->pm_current_policy &&
+				    backend->pm_current_policy->handle_event)
+					backend->pm_current_policy->handle_event(
+						kbdev,
+						KBASE_PM_POLICY_EVENT_IDLE);
+
 				if (kbdev->pm.backend.protected_transition_override ||
 #ifdef CONFIG_MALI_ARBITER_SUPPORT
 						kbase_pm_is_suspending(kbdev) ||
@@ -1455,9 +1467,21 @@ static int kbase_pm_shaders_update_state(struct kbase_device *kbdev)
 			}
 
 			if (backend->shaders_desired) {
+				if (backend->pm_current_policy &&
+				    backend->pm_current_policy->handle_event)
+					backend->pm_current_policy->handle_event(
+						kbdev,
+						KBASE_PM_POLICY_EVENT_TIMER_HIT);
+
 				stt->remaining_ticks = 0;
 				backend->shaders_state = KBASE_SHADERS_ON_CORESTACK_ON_RECHECK;
 			} else if (stt->remaining_ticks == 0) {
+				if (backend->pm_current_policy &&
+				    backend->pm_current_policy->handle_event)
+					backend->pm_current_policy->handle_event(
+						kbdev,
+						KBASE_PM_POLICY_EVENT_TIMER_MISS);
+
 				backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON;
 #ifdef CONFIG_MALI_ARBITER_SUPPORT
 			} else if (kbase_pm_is_suspending(kbdev) ||
@@ -1776,7 +1800,8 @@ int kbase_pm_state_machine_init(struct kbase_device *kbdev)
 	hrtimer_init(&stt->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 	stt->timer.function = shader_tick_timer_callback;
 	stt->configured_interval = HR_TIMER_DELAY_NSEC(DEFAULT_PM_GPU_POWEROFF_TICK_NS);
-	stt->configured_ticks = DEFAULT_PM_POWEROFF_TICK_SHADER;
+	stt->default_ticks = DEFAULT_PM_POWEROFF_TICK_SHADER;
+	stt->configured_ticks = stt->default_ticks;
 
 	return 0;
 }
@@ -1912,24 +1937,43 @@ static void kbase_pm_timed_out(struct kbase_device *kbdev)
 		kbase_reset_gpu(kbdev);
 }
 
-void kbase_pm_wait_for_l2_powered(struct kbase_device *kbdev)
+int kbase_pm_wait_for_l2_powered(struct kbase_device *kbdev)
 {
 	unsigned long flags;
 	unsigned long timeout;
-	int err;
+	long remaining;
+	int err = 0;
 
 	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
 	kbase_pm_update_state(kbdev);
 	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
 
-	timeout = jiffies + msecs_to_jiffies(PM_TIMEOUT_MS);
+#if MALI_USE_CSF
+	timeout = kbase_csf_timeout_in_jiffies(PM_TIMEOUT_MS);
+#else
+	timeout = msecs_to_jiffies(PM_TIMEOUT_MS);
+#endif
 
 	/* Wait for cores */
-	err = wait_event_killable(kbdev->pm.backend.gpu_in_desired_state_wait,
-			kbase_pm_is_in_desired_state_with_l2_powered(kbdev));
+#if KERNEL_VERSION(4, 13, 1) <= LINUX_VERSION_CODE
+	remaining = wait_event_killable_timeout(
+#else
+	remaining = wait_event_timeout(
+#endif
+		kbdev->pm.backend.gpu_in_desired_state_wait,
+		kbase_pm_is_in_desired_state_with_l2_powered(kbdev), timeout);
 
-	if (err < 0 && time_after(jiffies, timeout))
+	if (!remaining) {
 		kbase_pm_timed_out(kbdev);
+		err = -ETIMEDOUT;
+	} else if (remaining < 0) {
+		dev_info(
+			kbdev->dev,
+			"Wait for desired PM state with L2 powered got interrupted");
+		err = (int)remaining;
+	}
+
+	return err;
 }
 
 int kbase_pm_wait_for_desired_state(struct kbase_device *kbdev)
@@ -2045,6 +2089,7 @@ static void update_user_reg_page_mapping(struct kbase_device *kbdev)
 }
 #endif
 
+
 /*
  * pmu layout:
  * 0x0000: PMU TAG (RO) (0xCAFECAFE)
@@ -2462,7 +2507,7 @@ void kbase_pm_cache_snoop_enable(struct kbase_device *kbdev)
 {
 	if ((kbdev->current_gpu_coherency_mode == COHERENCY_ACE) &&
 		!kbdev->cci_snoop_enabled) {
-#ifdef CONFIG_ARM64
+#if IS_ENABLED(CONFIG_ARM64)
 		if (kbdev->snoop_enable_smc != 0)
 			kbase_invoke_smc_fid(kbdev->snoop_enable_smc, 0, 0, 0);
 #endif /* CONFIG_ARM64 */
@@ -2474,7 +2519,7 @@ void kbase_pm_cache_snoop_enable(struct kbase_device *kbdev)
 void kbase_pm_cache_snoop_disable(struct kbase_device *kbdev)
 {
 	if (kbdev->cci_snoop_enabled) {
-#ifdef CONFIG_ARM64
+#if IS_ENABLED(CONFIG_ARM64)
 		if (kbdev->snoop_disable_smc != 0) {
 			mali_cci_flush_l2(kbdev);
 			kbase_invoke_smc_fid(kbdev->snoop_disable_smc, 0, 0, 0);
@@ -2754,9 +2799,8 @@ kbase_pm_request_gpu_cycle_counter_do_request(struct kbase_device *kbdev)
 		/* This might happen after GPU reset.
 		 * Then counter needs to be kicked.
 		 */
-		if (!IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) &&
-		    (!(kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS)) &
-		       GPU_STATUS_CYCLE_COUNT_ACTIVE))) {
+		if (!(kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS)) &
+		      GPU_STATUS_CYCLE_COUNT_ACTIVE)) {
 			kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
 					GPU_COMMAND_CYCLE_COUNT_START);
 		}

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h

@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
  * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved.
@@ -28,7 +28,7 @@
 
 #include <mali_kbase_hwaccess_pm.h>
 
-#include "mali_kbase_pm_ca.h"
+#include "backend/gpu/mali_kbase_pm_ca.h"
 #include "mali_kbase_pm_policy.h"
 
 
@@ -263,8 +263,10 @@ int kbase_pm_wait_for_desired_state(struct kbase_device *kbdev);
  * because this function will take that lock itself.
  *
  * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ *
+ * Return: 0 on success, error code on error
  */
-void kbase_pm_wait_for_l2_powered(struct kbase_device *kbdev);
+int kbase_pm_wait_for_l2_powered(struct kbase_device *kbdev);
 
 /**
  * kbase_pm_update_dynamic_cores_onoff - Update the L2 and shader power state

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_l2_states.h

@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
  * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved.

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_mcu_states.h

@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
  * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved.

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_metrics.c

@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
  * (C) COPYRIGHT 2011-2021 ARM Limited. All rights reserved.
@@ -28,7 +28,7 @@
 #include <backend/gpu/mali_kbase_pm_internal.h>
 
 #if MALI_USE_CSF
-#include "mali_kbase_clk_rate_trace_mgr.h"
+#include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h"
 #include <csf/ipa_control/mali_kbase_csf_ipa_control.h>
 #else
 #include <backend/gpu/mali_kbase_jm_rb.h>
@@ -206,7 +206,7 @@ static void kbase_pm_get_dvfs_utilisation_calc(struct kbase_device *kbdev)
 			"Failed to query the increment of GPU_ACTIVE counter: err=%d",
 			err);
 	} else {
-		u64 diff_ns, margin_ns;
+		u64 diff_ns;
 		s64 diff_ns_signed;
 		u32 ns_time;
 		ktime_t diff = ktime_sub(
@@ -219,15 +219,7 @@ static void kbase_pm_get_dvfs_utilisation_calc(struct kbase_device *kbdev)
 
 		diff_ns = (u64)diff_ns_signed;
 
-		/* Use a margin value that is approximately 1% of the time
-		 * difference.
-		 */
-		margin_ns = diff_ns >> 6;
-
-		/* Calculate time difference in units of 256ns */
-		ns_time = (u32)(diff_ns >> KBASE_PM_TIME_SHIFT);
-
-#ifndef CONFIG_MALI_BIFROST_NO_MALI
+#if !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI)
 		/* The GPU_ACTIVE counter shouldn't clock-up more time than has
 		 * actually elapsed - but still some margin needs to be given
 		 * when doing the comparison. There could be some drift between
@@ -239,6 +231,10 @@ static void kbase_pm_get_dvfs_utilisation_calc(struct kbase_device *kbdev)
 		 * time.
 		 */
 		if (!kbdev->pm.backend.metrics.skip_gpu_active_sanity_check) {
+			/* Use a margin value that is approximately 1% of the time
+			 * difference.
+			 */
+			u64 margin_ns = diff_ns >> 6;
 			if (gpu_active_counter > (diff_ns + margin_ns)) {
 				dev_info(
 					kbdev->dev,
@@ -247,9 +243,9 @@ static void kbase_pm_get_dvfs_utilisation_calc(struct kbase_device *kbdev)
 					(unsigned long long)diff_ns);
 			}
 		}
-#else
-		CSTD_UNUSED(margin_ns);
 #endif
+		/* Calculate time difference in units of 256ns */
+		ns_time = (u32)(diff_ns >> KBASE_PM_TIME_SHIFT);
 
 		/* Add protected_time to gpu_active_counter so that time in
 		 * protected mode is included in the apparent GPU active time,

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c

@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
  * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved.
@@ -33,29 +33,30 @@
 #include <csf/mali_kbase_csf_firmware.h>
 #endif
 
+#include <linux/of.h>
+
 static const struct kbase_pm_policy *const all_policy_list[] = {
-#ifdef CONFIG_MALI_BIFROST_NO_MALI
-	&kbase_pm_always_on_policy_ops,
 	&kbase_pm_coarse_demand_policy_ops,
-#if !MALI_CUSTOMER_RELEASE
-	&kbase_pm_always_on_demand_policy_ops,
-#endif
-#else				/* CONFIG_MALI_BIFROST_NO_MALI */
-	&kbase_pm_coarse_demand_policy_ops,
-#if !MALI_CUSTOMER_RELEASE
-	&kbase_pm_always_on_demand_policy_ops,
-#endif
 	&kbase_pm_always_on_policy_ops
-#endif /* CONFIG_MALI_BIFROST_NO_MALI */
 };
 
-#if MALI_USE_CSF
 void kbase_pm_policy_init(struct kbase_device *kbdev)
 {
-	unsigned long flags;
 	const struct kbase_pm_policy *default_policy = all_policy_list[0];
+	struct device_node *np = kbdev->dev->of_node;
+	const char *power_policy_name;
+	unsigned long flags;
+	int i;
 
-#if defined CONFIG_MALI_BIFROST_DEBUG
+	if (of_property_read_string(np, "power_policy", &power_policy_name) == 0) {
+		for (i = 0; i < ARRAY_SIZE(all_policy_list); i++)
+			if (sysfs_streq(all_policy_list[i]->name, power_policy_name)) {
+				default_policy = all_policy_list[i];
+				break;
+			}
+	}
+
+#if MALI_USE_CSF && defined(CONFIG_MALI_BIFROST_DEBUG)
 	/* Use always_on policy if module param fw_debug=1 is
 	 * passed, to aid firmware debugging.
 	 */
@@ -63,31 +64,18 @@ void kbase_pm_policy_init(struct kbase_device *kbdev)
 		default_policy = &kbase_pm_always_on_policy_ops;
 #endif
 
-
 	default_policy->init(kbdev);
 
+#if MALI_USE_CSF
 	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
 	kbdev->pm.backend.pm_current_policy = default_policy;
-	kbdev->pm.backend.csf_pm_sched_flags =
-				default_policy->pm_sched_flags;
+	kbdev->pm.backend.csf_pm_sched_flags = default_policy->pm_sched_flags;
 	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
-}
-#else /* MALI_USE_CSF */
-void kbase_pm_policy_init(struct kbase_device *kbdev)
-{
-	kbdev->pm.backend.pm_current_policy = all_policy_list[0];
-
-#if MALI_USE_CSF && defined CONFIG_MALI_BIFROST_DEBUG
-	/* Use always_on policy if module param fw_debug=1 is
-	 * passed, to aid firmware debugging.
-	 */
-	if (fw_debug)
-		kbdev->pm.backend.pm_current_policy =
-			&kbase_pm_always_on_policy_ops;
+#else
+	CSTD_UNUSED(flags);
+	kbdev->pm.backend.pm_current_policy = default_policy;
 #endif
-	kbdev->pm.backend.pm_current_policy->init(kbdev);
 }
-#endif /* MALI_USE_CSF */
 
 void kbase_pm_policy_term(struct kbase_device *kbdev)
 {
@@ -373,6 +361,9 @@ void kbase_pm_set_policy(struct kbase_device *kbdev,
 	if (old_policy->term)
 		old_policy->term(kbdev);
 
+	memset(&kbdev->pm.backend.pm_policy_data, 0,
+	       sizeof(union kbase_pm_policy_data));
+
 	KBASE_KTRACE_ADD(kbdev, PM_CURRENT_POLICY_INIT, NULL, new_policy->id);
 	if (new_policy->init)
 		new_policy->init(kbdev);

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.h

@@ -1,7 +1,7 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
- * (C) COPYRIGHT 2010-2015, 2018-2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2015, 2018-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_shader_states.h

@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
  * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved.

drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c

@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
  * (C) COPYRIGHT 2014-2016, 2018-2021 ARM Limited. All rights reserved.
@@ -76,9 +76,6 @@ void kbase_backend_get_gpu_time_norequest(struct kbase_device *kbdev,
  */
 static bool timedwait_cycle_count_active(struct kbase_device *kbdev)
 {
-#ifdef CONFIG_MALI_BIFROST_NO_MALI
-	return true;
-#else
 	bool success = false;
 	const unsigned int timeout = 100;
 	const unsigned long remaining = jiffies + msecs_to_jiffies(timeout);
@@ -91,7 +88,6 @@ static bool timedwait_cycle_count_active(struct kbase_device *kbdev)
 		}
 	}
 	return success;
-#endif
 }
 #endif
 

drivers/gpu/arm/bifrost/build.bp

@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
  * (C) COPYRIGHT 2017-2021 ARM Limited. All rights reserved.
@@ -25,14 +25,17 @@
  * both mali_kbase and the test modules. */
 bob_defaults {
     name: "mali_kbase_shared_config_defaults",
+    defaults: [
+        "kernel_defaults",
+    ],
     no_mali: {
-        kbuild_options: ["CONFIG_MALI_BIFROST_NO_MALI=y"],
+        kbuild_options: [
+            "CONFIG_MALI_BIFROST_NO_MALI=y",
+            "CONFIG_MALI_NO_MALI_DEFAULT_GPU={{.gpu}}",
+        ],
     },
-    mali_real_hw: {
-        kbuild_options: ["CONFIG_MALI_REAL_HW=y"],
-    },
-    mali_dma_fence: {
-        kbuild_options: ["CONFIG_MALI_BIFROST_DMA_FENCE=y"],
+    gpu_has_csf: {
+        kbuild_options: ["CONFIG_MALI_CSF_SUPPORT=y"],
     },
     mali_devfreq: {
         kbuild_options: ["CONFIG_MALI_BIFROST_DEVFREQ=y"],
@@ -40,8 +43,62 @@ bob_defaults {
     mali_midgard_dvfs: {
         kbuild_options: ["CONFIG_MALI_BIFROST_DVFS=y"],
     },
+    mali_gator_support: {
+        kbuild_options: ["CONFIG_MALI_BIFROST_GATOR_SUPPORT=y"],
+    },
+    mali_midgard_enable_trace: {
+        kbuild_options: ["CONFIG_MALI_BIFROST_ENABLE_TRACE=y"],
+    },
+    mali_dma_fence: {
+        kbuild_options: ["CONFIG_MALI_BIFROST_DMA_FENCE=y"],
+    },
+    mali_arbiter_support: {
+        kbuild_options: ["CONFIG_MALI_ARBITER_SUPPORT=y"],
+    },
+    mali_dma_buf_map_on_demand: {
+        kbuild_options: ["CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND=y"],
+    },
+    mali_dma_buf_legacy_compat: {
+        kbuild_options: ["CONFIG_MALI_DMA_BUF_LEGACY_COMPAT=y"],
+    },
+    mali_2mb_alloc: {
+        kbuild_options: ["CONFIG_MALI_2MB_ALLOC=y"],
+    },
+    mali_memory_fully_backed: {
+        kbuild_options: ["CONFIG_MALI_MEMORY_FULLY_BACKED=y"],
+    },
+    mali_corestack: {
+        kbuild_options: ["CONFIG_MALI_CORESTACK=y"],
+    },
+    mali_real_hw: {
+        kbuild_options: ["CONFIG_MALI_REAL_HW=y"],
+    },
+    mali_error_inject_none: {
+        kbuild_options: ["CONFIG_MALI_ERROR_INJECT_NONE=y"],
+    },
+    mali_error_inject_track_list: {
+        kbuild_options: ["CONFIG_MALI_ERROR_INJECT_TRACK_LIST=y"],
+    },
+    mali_error_inject_random: {
+        kbuild_options: ["CONFIG_MALI_ERROR_INJECT_RANDOM=y"],
+    },
+    mali_error_inject: {
+        kbuild_options: ["CONFIG_MALI_BIFROST_ERROR_INJECT=y"],
+    },
+    mali_gem5_build: {
+       kbuild_options: ["CONFIG_MALI_GEM5_BUILD=y"],
+    },
     mali_debug: {
-        kbuild_options: ["CONFIG_MALI_BIFROST_DEBUG=y"],
+        kbuild_options: [
+            "CONFIG_MALI_BIFROST_DEBUG=y",
+            "MALI_KERNEL_TEST_API={{.debug}}",
+        ],
+    },
+    mali_fence_debug: {
+        kbuild_options: ["CONFIG_MALI_BIFROST_FENCE_DEBUG=y"],
+    },
+    mali_system_trace: {
+        kbuild_options: ["CONFIG_MALI_BIFROST_SYSTEM_TRACE=y"],
     },
     buslog: {
         kbuild_options: ["CONFIG_MALI_BUSLOG=y"],
@@ -52,91 +109,8 @@ bob_defaults {
     cinstr_gwt: {
         kbuild_options: ["CONFIG_MALI_CINSTR_GWT=y"],
     },
-    mali_gator_support: {
-        kbuild_options: ["CONFIG_MALI_BIFROST_GATOR_SUPPORT=y"],
-    },
-    mali_midgard_enable_trace: {
-        kbuild_options: ["CONFIG_MALI_BIFROST_ENABLE_TRACE=y"],
-    },
-    mali_system_trace: {
-        kbuild_options: ["CONFIG_MALI_BIFROST_SYSTEM_TRACE=y"],
-    },
-    mali_pwrsoft_765: {
-        kbuild_options: ["CONFIG_MALI_PWRSOFT_765=y"],
-    },
-    mali_memory_fully_backed: {
-        kbuild_options: ["CONFIG_MALI_MEMORY_FULLY_BACKED=y"],
-    },
-    mali_dma_buf_map_on_demand: {
-        kbuild_options: ["CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND=y"],
-    },
-    mali_dma_buf_legacy_compat: {
-        kbuild_options: ["CONFIG_MALI_DMA_BUF_LEGACY_COMPAT=y"],
-    },
-    mali_arbiter_support: {
-        kbuild_options: ["CONFIG_MALI_ARBITER_SUPPORT=y"],
-    },
-    mali_gem5_build: {
-       kbuild_options: ["CONFIG_MALI_GEM5_BUILD=y"],
-    },
-    kbuild_options: [
-        "MALI_UNIT_TEST={{.unit_test_code}}",
-        "MALI_CUSTOMER_RELEASE={{.release}}",
-        "MALI_USE_CSF={{.gpu_has_csf}}",
-        "MALI_KERNEL_TEST_API={{.debug}}",
-    ],
-    defaults: ["kernel_defaults"],
-}
-
-bob_kernel_module {
-    name: "mali_kbase",
-    srcs: [
-        "*.c",
-        "*.h",
-        "Kbuild",
-        "backend/gpu/*.c",
-        "backend/gpu/*.h",
-        "backend/gpu/Kbuild",
-        "context/*.c",
-        "context/*.h",
-        "ipa/*.c",
-        "ipa/*.h",
-        "ipa/Kbuild",
-        "platform/*.h",
-        "platform/*/*.c",
-        "platform/*/*.h",
-        "platform/*/Kbuild",
-        "thirdparty/*.c",
-        "debug/*.c",
-        "debug/*.h",
-        "device/*.c",
-        "device/*.h",
-        "gpu/*.c",
-        "gpu/*.h",
-        "tl/*.c",
-        "tl/*.h",
-        "mmu/*.c",
-        "mmu/*.h",
-    ],
-    kbuild_options: [
-        "CONFIG_MALI_KUTF=n",
-        "CONFIG_MALI_MIDGARD=m",
-        "CONFIG_MALI_NO_MALI_DEFAULT_GPU={{.gpu}}",
-        "CONFIG_MALI_PLATFORM_NAME={{.mali_platform_name}}",
-    ],
-    buslog: {
-        extra_symbols: [
-            "bus_logger",
-        ],
-    },
-    mali_corestack: {
-        kbuild_options: ["CONFIG_MALI_CORESTACK=y"],
-    },
-    mali_error_inject: {
-        kbuild_options: ["CONFIG_MALI_BIFROST_ERROR_INJECT=y"],
-    },
-    mali_error_inject_random: {
-        kbuild_options: ["CONFIG_MALI_ERROR_INJECT_RANDOM=y"],
+    cinstr_primary_hwc: {
+        kbuild_options: ["CONFIG_MALI_PRFCNT_SET_PRIMARY=y"],
     },
     cinstr_secondary_hwc: {
         kbuild_options: ["CONFIG_MALI_BIFROST_PRFCNT_SET_SECONDARY=y"],
@@ -147,8 +121,11 @@ bob_kernel_module {
     cinstr_hwc_set_select_via_debug_fs: {
         kbuild_options: ["CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS=y"],
     },
-    mali_2mb_alloc: {
-        kbuild_options: ["CONFIG_MALI_2MB_ALLOC=y"],
+    mali_job_dump: {
+        kbuild_options: ["CONFIG_MALI_JOB_DUMP"],
+    },
+    mali_pwrsoft_765: {
+        kbuild_options: ["CONFIG_MALI_PWRSOFT_765=y"],
     },
     mali_hw_errata_1485982_not_affected: {
         kbuild_options: ["CONFIG_MALI_HW_ERRATA_1485982_NOT_AFFECTED=y"],
@@ -156,6 +133,71 @@ bob_kernel_module {
     mali_hw_errata_1485982_use_clock_alternative: {
         kbuild_options: ["CONFIG_MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE=y"],
     },
+    kbuild_options: [
+        "CONFIG_MALI_PLATFORM_NAME={{.mali_platform_name}}",
+        "MALI_CUSTOMER_RELEASE={{.release}}",
+        "MALI_UNIT_TEST={{.unit_test_code}}",
+        "MALI_USE_CSF={{.gpu_has_csf}}",
+        "MALI_JIT_PRESSURE_LIMIT_BASE={{.jit_pressure_limit_base}}",
+
+        // Start of CS experimental features definitions.
+        // If there is nothing below, definition should be added as follows:
+        // "MALI_EXPERIMENTAL_FEATURE={{.experimental_feature}}"
+        // experimental_feature above comes from Mconfig in
+        // <ddk_root>/product/base/
+        // However, in Mconfig, experimental_feature should be looked up (for
+        // similar explanation to this one) as ALLCAPS, i.e.
+        // EXPERIMENTAL_FEATURE.
+        //
+        // IMPORTANT: MALI_CS_EXPERIMENTAL should NEVER be defined below as it
+        // is an umbrella feature that would be open for inappropriate use
+        // (catch-all for experimental CS code without separating it into
+        // different features).
+        "MALI_INCREMENTAL_RENDERING={{.incremental_rendering}}",
+        "GPU_TIMESTAMP_CORRECTION={{.gpu_timestamp_correction}}",
+    ],
+}
+
+bob_kernel_module {
+    name: "mali_kbase",
+    defaults: [
+        "mali_kbase_shared_config_defaults",
+    ],
+    srcs: [
+        "*.c",
+        "*.h",
+        "Kbuild",
+        "backend/gpu/*.c",
+        "backend/gpu/*.h",
+        "backend/gpu/Kbuild",
+        "context/*.c",
+        "context/*.h",
+        "context/Kbuild",
+        "ipa/*.c",
+        "ipa/*.h",
+        "ipa/Kbuild",
+        "platform/*.h",
+        "platform/*/*.c",
+        "platform/*/*.h",
+        "platform/*/Kbuild",
+        "thirdparty/*.c",
+        "thirdparty/Kbuild",
+        "debug/*.c",
+        "debug/*.h",
+        "debug/Kbuild",
+        "device/*.c",
+        "device/*.h",
+        "device/Kbuild",
+        "gpu/*.c",
+        "gpu/*.h",
+        "gpu/Kbuild",
+        "tl/*.c",
+        "tl/*.h",
+        "tl/Kbuild",
+        "mmu/*.c",
+        "mmu/*.h",
+        "mmu/Kbuild",
+    ],
     gpu_has_job_manager: {
         srcs: [
             "context/backend/*_jm.c",
@@ -172,7 +214,6 @@ bob_kernel_module {
         ],
     },
     gpu_has_csf: {
-        kbuild_options: ["CONFIG_MALI_CSF_SUPPORT=y"],
         srcs: [
             "context/backend/*_csf.c",
             "csf/*.c",
@@ -199,5 +240,13 @@ bob_kernel_module {
             "arbiter/Kbuild",
         ],
     },
-    defaults: ["mali_kbase_shared_config_defaults"],
+    kbuild_options: [
+        "CONFIG_MALI_BIFROST=m",
+        "CONFIG_MALI_KUTF=n",
+    ],
+    buslog: {
+        extra_symbols: [
+            "bus_logger",
+        ],
+    },
 }

drivers/gpu/arm/bifrost/context/Kbuild

@@ -1,6 +1,6 @@
-# SPDX-License-Identifier: GPL-2.0
+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 #
-# (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
+# (C) COPYRIGHT 2012-2013, 2016-2017, 2020-2021 ARM Limited. All rights reserved.
 #
 # This program is free software and is provided to you under the terms of the
 # GNU General Public License version 2 as published by the Free Software
@@ -18,10 +18,10 @@
 #
 #
 
+bifrost_kbase-y += context/mali_kbase_context.o
 
-config MALI_PROTECTED_MEMORY_ALLOCATOR
-	tristate "MALI_PROTECTED_MEMORY_ALLOCATOR"
-	help
-	  This option enables an example implementation of a protected memory allocator
-	  for allocation and release of pages of secure memory intended to be used
-	  by the firmware of Mali GPU device drivers.
+ifeq ($(CONFIG_MALI_CSF_SUPPORT),y)
+    bifrost_kbase-y += context/backend/mali_kbase_context_csf.o
+else
+    bifrost_kbase-y += context/backend/mali_kbase_context_jm.o
+endif

drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_csf.c

@@ -1,7 +1,7 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
- * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software
@@ -32,7 +32,7 @@
 #include <mmu/mali_kbase_mmu.h>
 #include <tl/mali_kbase_timeline.h>
 
-#ifdef CONFIG_DEBUG_FS
+#if IS_ENABLED(CONFIG_DEBUG_FS)
 #include <csf/mali_kbase_csf_csg_debugfs.h>
 #include <csf/mali_kbase_csf_kcpu_debugfs.h>
 #include <csf/mali_kbase_csf_tiler_heap_debugfs.h>

drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_jm.c

@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
  * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
@@ -34,7 +34,7 @@
 #include <mmu/mali_kbase_mmu.h>
 #include <tl/mali_kbase_timeline.h>
 
-#ifdef CONFIG_DEBUG_FS
+#if IS_ENABLED(CONFIG_DEBUG_FS)
 #include <mali_kbase_debug_mem_view.h>
 #include <mali_kbase_mem_pool_debugfs.h>
 
@@ -147,7 +147,7 @@ static const struct kbase_context_init context_init[] = {
 	  "JS kctx initialization failed" },
 	{ kbase_jd_init, kbase_jd_exit, "JD initialization failed" },
 	{ kbase_context_submit_check, NULL, "Enabling job submission failed" },
-#ifdef CONFIG_DEBUG_FS
+#if IS_ENABLED(CONFIG_DEBUG_FS)
 	{ kbase_debug_job_fault_context_init,
 	  kbase_debug_job_fault_context_term,
 	  "Job fault context initialization failed" },
@@ -155,6 +155,8 @@ static const struct kbase_context_init context_init[] = {
 	{ NULL, kbase_context_flush_jobs, NULL },
 	{ kbase_context_add_to_dev_list, kbase_context_remove_from_dev_list,
 	  "Adding kctx to device failed" },
+	{ kbasep_platform_context_init, kbasep_platform_context_term,
+	  "Platform callback for kctx initialization failed" },
 };
 
 static void kbase_context_term_partial(

drivers/gpu/arm/bifrost/context/mali_kbase_context.c

@@ -1,7 +1,7 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
- * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software
@@ -154,7 +154,7 @@ int kbase_context_common_init(struct kbase_context *kctx)
 	atomic_set(&kctx->event_count, 0);
 #if !MALI_USE_CSF
 	atomic_set(&kctx->event_closed, false);
-#ifdef CONFIG_GPU_TRACEPOINTS
+#if IS_ENABLED(CONFIG_GPU_TRACEPOINTS)
 	atomic_set(&kctx->jctx.work_id, 0);
 #endif
 #endif

drivers/gpu/arm/bifrost/context/mali_kbase_context.h

@@ -1,7 +1,7 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
- * (C) COPYRIGHT 2011-2017, 2019-2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2017, 2019-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software

drivers/gpu/arm/bifrost/context/mali_kbase_context_internal.h

@@ -1,7 +1,7 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
- * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software

drivers/gpu/arm/bifrost/csf/Kbuild

@@ -1,6 +1,6 @@
-# SPDX-License-Identifier: GPL-2.0
+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 #
-# (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved.
+# (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved.
 #
 # This program is free software and is provided to you under the terms of the
 # GNU General Public License version 2 as published by the Free Software
@@ -19,24 +19,29 @@
 #
 
 bifrost_kbase-y += \
-	csf/mali_kbase_csf_firmware_cfg.o \
-	csf/mali_kbase_csf_trace_buffer.o \
-	csf/mali_kbase_csf.o \
-	csf/mali_kbase_csf_scheduler.o \
-	csf/mali_kbase_csf_kcpu.o \
-	csf/mali_kbase_csf_tiler_heap.o \
-	csf/mali_kbase_csf_timeout.o \
-	csf/mali_kbase_csf_tl_reader.o \
-	csf/mali_kbase_csf_heap_context_alloc.o \
-	csf/mali_kbase_csf_reset_gpu.o \
-	csf/mali_kbase_csf_csg_debugfs.o \
-	csf/mali_kbase_csf_kcpu_debugfs.o \
-	csf/mali_kbase_csf_protected_memory.o \
-	csf/mali_kbase_csf_tiler_heap_debugfs.o \
-	csf/mali_kbase_csf_cpu_queue_debugfs.o
+    csf/mali_kbase_csf_firmware_cfg.o \
+    csf/mali_kbase_csf_trace_buffer.o \
+    csf/mali_kbase_csf.o \
+    csf/mali_kbase_csf_scheduler.o \
+    csf/mali_kbase_csf_kcpu.o \
+    csf/mali_kbase_csf_tiler_heap.o \
+    csf/mali_kbase_csf_timeout.o \
+    csf/mali_kbase_csf_tl_reader.o \
+    csf/mali_kbase_csf_heap_context_alloc.o \
+    csf/mali_kbase_csf_reset_gpu.o \
+    csf/mali_kbase_csf_csg_debugfs.o \
+    csf/mali_kbase_csf_kcpu_debugfs.o \
+    csf/mali_kbase_csf_protected_memory.o \
+    csf/mali_kbase_csf_tiler_heap_debugfs.o \
+    csf/mali_kbase_csf_cpu_queue_debugfs.o
 
 bifrost_kbase-$(CONFIG_MALI_REAL_HW) += csf/mali_kbase_csf_firmware.o
 
-bifrost_kbase-$(CONFIG_MALI_BIFROST_NO_MALI) += csf/mali_kbase_csf_firmware_no_mali.o
 
-include $(src)/csf/ipa_control/Kbuild
+ifeq ($(KBUILD_EXTMOD),)
+# in-tree
+    -include $(src)/csf/ipa_control/Kbuild
+else
+# out-of-tree
+    include $(src)/csf/ipa_control/Kbuild
+endif

drivers/gpu/arm/bifrost/csf/ipa_control/Kbuild

@@ -1,6 +1,6 @@
-# SPDX-License-Identifier: GPL-2.0
+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 #
-# (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
+# (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved.
 #
 # This program is free software and is provided to you under the terms of the
 # GNU General Public License version 2 as published by the Free Software
@@ -19,4 +19,4 @@
 #
 
 bifrost_kbase-y += \
-	csf/ipa_control/mali_kbase_csf_ipa_control.o
+    csf/ipa_control/mali_kbase_csf_ipa_control.o

drivers/gpu/arm/bifrost/csf/ipa_control/mali_kbase_csf_ipa_control.c

@@ -1,7 +1,7 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
- * (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software
@@ -20,7 +20,7 @@
  */
 
 #include <mali_kbase.h>
-#include "mali_kbase_clk_rate_trace_mgr.h"
+#include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h"
 #include "mali_kbase_csf_ipa_control.h"
 
 /*

drivers/gpu/arm/bifrost/csf/ipa_control/mali_kbase_csf_ipa_control.h

@@ -1,7 +1,7 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
- * (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software

drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c

@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
  * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved.
@@ -144,14 +144,21 @@ static void gpu_munmap_user_io_pages(struct kbase_context *kctx,
 	mutex_unlock(&kctx->kbdev->csf.reg_lock);
 }
 
-static void init_user_output_page(struct kbase_queue *queue)
+static void init_user_io_pages(struct kbase_queue *queue)
 {
-	u32 *addr = (u32 *)(queue->user_io_addr + PAGE_SIZE);
+	u32 *input_addr = (u32 *)(queue->user_io_addr);
+	u32 *output_addr = (u32 *)(queue->user_io_addr + PAGE_SIZE);
 
-	addr[CS_EXTRACT_LO/4] = 0;
-	addr[CS_EXTRACT_HI/4] = 0;
+	input_addr[CS_INSERT_LO/4] = 0;
+	input_addr[CS_INSERT_HI/4] = 0;
 
-	addr[CS_ACTIVE/4] = 0;
+	input_addr[CS_EXTRACT_INIT_LO/4] = 0;
+	input_addr[CS_EXTRACT_INIT_HI/4] = 0;
+
+	output_addr[CS_EXTRACT_LO/4] = 0;
+	output_addr[CS_EXTRACT_HI/4] = 0;
+
+	output_addr[CS_ACTIVE/4] = 0;
 }
 
 /* Map the input/output pages in the shared interface segment of MCU firmware
@@ -350,7 +357,7 @@ int kbase_csf_alloc_command_stream_user_pages(struct kbase_context *kctx,
 	if (ret)
 		goto kernel_map_failed;
 
-	init_user_output_page(queue);
+	init_user_io_pages(queue);
 
 	ret = gpu_mmap_user_io_pages(kctx->kbdev, queue->phys, reg);
 	if (ret)
@@ -455,19 +462,39 @@ static void release_queue(struct kbase_queue *queue)
 static void oom_event_worker(struct work_struct *data);
 static void fatal_event_worker(struct work_struct *data);
 
-int kbase_csf_queue_register(struct kbase_context *kctx,
-			     struct kbase_ioctl_cs_queue_register *reg)
+/* Between reg and reg_ex, one and only one must be null */
+static int csf_queue_register_internal(struct kbase_context *kctx,
+			struct kbase_ioctl_cs_queue_register *reg,
+			struct kbase_ioctl_cs_queue_register_ex *reg_ex)
 {
 	struct kbase_queue *queue;
 	int ret = 0;
 	struct kbase_va_region *region;
-	u64 queue_addr = reg->buffer_gpu_addr;
-	size_t queue_size = reg->buffer_size >> PAGE_SHIFT;
+	u64 queue_addr;
+	size_t queue_size;
+
+	/* Only one pointer expected, otherwise coding error */
+	if ((reg == NULL && reg_ex == NULL) || (reg && reg_ex)) {
+		dev_err(kctx->kbdev->dev,
+			"Error, one and only one param-ptr expected!");
+		return -EINVAL;
+	}
+
+	/* struct kbase_ioctl_cs_queue_register_ex contains a full
+	 * struct kbase_ioctl_cs_queue_register at the start address. So
+	 * the pointer can be safely cast to pointing to a
+	 * kbase_ioctl_cs_queue_register object.
+	 */
+	if (reg_ex)
+		reg = (struct kbase_ioctl_cs_queue_register *)reg_ex;
 
 	/* Validate the queue priority */
 	if (reg->priority > BASE_QUEUE_MAX_PRIORITY)
 		return -EINVAL;
 
+	queue_addr = reg->buffer_gpu_addr;
+	queue_size = reg->buffer_size >> PAGE_SHIFT;
+
 	mutex_lock(&kctx->csf.lock);
 
 	/* Check if queue is already registered */
@@ -492,6 +519,35 @@ int kbase_csf_queue_register(struct kbase_context *kctx,
 		goto out_unlock_vm;
 	}
 
+	/* Check address validity on cs_trace buffer etc. Don't care
+	 * if not enabled (i.e. when size is 0).
+	 */
+	if (reg_ex && reg_ex->ex_buffer_size) {
+		int buf_pages = (reg_ex->ex_buffer_size +
+				 (1 << PAGE_SHIFT) - 1) >> PAGE_SHIFT;
+
+		region = kbase_region_tracker_find_region_enclosing_address(
+				kctx, reg_ex->ex_buffer_base);
+		if (kbase_is_region_invalid_or_free(region)) {
+			ret = -ENOENT;
+			goto out_unlock_vm;
+		}
+
+		if (buf_pages > (region->nr_pages -
+				 ((reg_ex->ex_buffer_base >> PAGE_SHIFT) -
+				 region->start_pfn))) {
+			ret = -EINVAL;
+			goto out_unlock_vm;
+		}
+
+		region = kbase_region_tracker_find_region_enclosing_address(
+				kctx, reg_ex->ex_offset_var_addr);
+		if (kbase_is_region_invalid_or_free(region)) {
+			ret = -ENOENT;
+			goto out_unlock_vm;
+		}
+	}
+
 	queue = kzalloc(sizeof(struct kbase_queue), GFP_KERNEL);
 
 	if (!queue) {
@@ -529,6 +585,22 @@ int kbase_csf_queue_register(struct kbase_context *kctx,
 
 	region->flags |= KBASE_REG_NO_USER_FREE;
 
+	/* Initialize the cs_trace configuration parameters, When buffer_size
+	 * is 0, trace is disabled. Here we only update the fields when
+	 * enabled, otherwise leave them as default zeros.
+	 */
+	if (reg_ex && reg_ex->ex_buffer_size) {
+		u32 cfg = CS_INSTR_CONFIG_EVENT_SIZE_SET(
+					0, reg_ex->ex_event_size);
+		cfg = CS_INSTR_CONFIG_EVENT_STATE_SET(
+					cfg, reg_ex->ex_event_state);
+
+		queue->trace_cfg = cfg;
+		queue->trace_buffer_size = reg_ex->ex_buffer_size;
+		queue->trace_buffer_base = reg_ex->ex_buffer_base;
+		queue->trace_offset_ptr = reg_ex->ex_offset_var_addr;
+	}
+
 out_unlock_vm:
 	kbase_gpu_vm_unlock(kctx);
 out:
@@ -537,6 +609,37 @@ out:
 	return ret;
 }
 
+int kbase_csf_queue_register(struct kbase_context *kctx,
+			     struct kbase_ioctl_cs_queue_register *reg)
+{
+	return csf_queue_register_internal(kctx, reg, NULL);
+}
+
+int kbase_csf_queue_register_ex(struct kbase_context *kctx,
+				struct kbase_ioctl_cs_queue_register_ex *reg)
+{
+	struct kbase_csf_global_iface const *const iface =
+						&kctx->kbdev->csf.global_iface;
+	u32 const glb_version = iface->version;
+	u32 instr = iface->instr_features;
+	u8 max_size = GLB_INSTR_FEATURES_EVENT_SIZE_MAX_GET(instr);
+	u32 min_buf_size = (1u << reg->ex_event_size) *
+			GLB_INSTR_FEATURES_OFFSET_UPDATE_RATE_GET(instr);
+
+	/* If cs_trace_command not supported, the call fails */
+	if (glb_version < kbase_csf_interface_version(1, 1, 0))
+		return -EINVAL;
+
+	/* Validate the cs_trace configuration parameters */
+        if (reg->ex_buffer_size &&
+            ((reg->ex_event_size > max_size) ||
+             (reg->ex_buffer_size & (reg->ex_buffer_size - 1)) ||
+             (reg->ex_buffer_size < min_buf_size)))
+          return -EINVAL;
+
+        return csf_queue_register_internal(kctx, NULL, reg);
+}
+
 static void unbind_queue(struct kbase_context *kctx,
 		struct kbase_queue *queue);
 
@@ -787,6 +890,8 @@ static void unbind_stopped_queue(struct kbase_context *kctx,
 		kbase_csf_scheduler_spin_lock(kctx->kbdev, &flags);
 		bitmap_clear(queue->group->protm_pending_bitmap,
 				queue->csi_index, 1);
+		KBASE_KTRACE_ADD_CSF_GRP_Q(kctx->kbdev, PROTM_PENDING_CLEAR,
+			 queue->group, queue, queue->group->protm_pending_bitmap[0]);
 		queue->group->bound_queues[queue->csi_index] = NULL;
 		queue->group = NULL;
 		kbase_csf_scheduler_spin_unlock(kctx->kbdev, flags);
@@ -1913,6 +2018,7 @@ void kbase_csf_event_signal(struct kbase_context *kctx, bool notify_gpu)
 		spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags);
 		if (kctx->kbdev->pm.backend.gpu_powered)
 			kbase_csf_ring_doorbell(kctx->kbdev, CSF_KERNEL_DOORBELL_NR);
+		KBASE_KTRACE_ADD(kctx->kbdev, SYNC_UPDATE_EVENT_NOTIFY_GPU, kctx, 0u);
 		spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags);
 	}
 
@@ -2251,7 +2357,11 @@ static void protm_event_worker(struct work_struct *data)
 	struct kbase_queue_group *const group =
 		container_of(data, struct kbase_queue_group, protm_event_work);
 
+	KBASE_KTRACE_ADD_CSF_GRP(group->kctx->kbdev, PROTM_EVENT_WORKER_BEGIN,
+				 group, 0u);
 	kbase_csf_scheduler_group_protm_enter(group);
+	KBASE_KTRACE_ADD_CSF_GRP(group->kctx->kbdev, PROTM_EVENT_WORKER_END,
+				 group, 0u);
 }
 
 static void report_queue_fatal_error(struct kbase_queue *const queue,
@@ -2308,13 +2418,16 @@ handle_fault_event(struct kbase_queue *const queue,
 
 	kbase_csf_scheduler_spin_lock_assert_held(kbdev);
 
-	dev_warn(kbdev->dev, "CSI: %d\n"
-			"CS_FAULT.EXCEPTION_TYPE: 0x%x (%s)\n"
-			"CS_FAULT.EXCEPTION_DATA: 0x%x\n"
-			"CS_FAULT_INFO.EXCEPTION_DATA: 0x%llx\n",
-			queue->csi_index, cs_fault_exception_type,
-			kbase_gpu_exception_name(cs_fault_exception_type),
-			cs_fault_exception_data, cs_fault_info_exception_data);
+	dev_warn(kbdev->dev,
+		 "Ctx %d_%d Group %d CSG %d CSI: %d\n"
+		 "CS_FAULT.EXCEPTION_TYPE: 0x%x (%s)\n"
+		 "CS_FAULT.EXCEPTION_DATA: 0x%x\n"
+		 "CS_FAULT_INFO.EXCEPTION_DATA: 0x%llx\n",
+		 queue->kctx->tgid, queue->kctx->id, queue->group->handle,
+		 queue->group->csg_nr, queue->csi_index,
+		 cs_fault_exception_type,
+		 kbase_gpu_exception_name(cs_fault_exception_type),
+		 cs_fault_exception_data, cs_fault_info_exception_data);
 
 	if (cs_fault_exception_type ==
 	    CS_FAULT_EXCEPTION_TYPE_RESOURCE_EVICTION_TIMEOUT)
@@ -2398,11 +2511,12 @@ handle_fatal_event(struct kbase_queue *const queue,
 	kbase_csf_scheduler_spin_lock_assert_held(kbdev);
 
 	dev_warn(kbdev->dev,
-		 "CSG: %d, CSI: %d\n"
+		 "Ctx %d_%d Group %d CSG %d CSI: %d\n"
 		 "CS_FATAL.EXCEPTION_TYPE: 0x%x (%s)\n"
 		 "CS_FATAL.EXCEPTION_DATA: 0x%x\n"
 		 "CS_FATAL_INFO.EXCEPTION_DATA: 0x%llx\n",
-		 queue->group->handle, queue->csi_index,
+		 queue->kctx->tgid, queue->kctx->id, queue->group->handle,
+		 queue->group->csg_nr, queue->csi_index,
 		 cs_fatal_exception_type,
 		 kbase_gpu_exception_name(cs_fatal_exception_type),
 		 cs_fatal_exception_data, cs_fatal_info_exception_data);
@@ -2505,23 +2619,28 @@ static void process_cs_interrupts(struct kbase_queue_group *const group,
 			if ((cs_req & CS_REQ_EXCEPTION_MASK) ^
 			    (cs_ack & CS_ACK_EXCEPTION_MASK)) {
 				KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_FAULT_INTERRUPT, group, queue, cs_req ^ cs_ack);
-				handle_queue_exception_event(queue, cs_req,
-							     cs_ack);
+				handle_queue_exception_event(queue, cs_req, cs_ack);
 			}
 
 			/* PROTM_PEND and TILER_OOM can be safely ignored
 			 * because they will be raised again if the group
 			 * is assigned a CSG slot in future.
 			 */
-			if (group_suspending)
+			if (group_suspending) {
+				u32 const cs_req_remain = cs_req & ~CS_REQ_EXCEPTION_MASK;
+				u32 const cs_ack_remain = cs_ack & ~CS_ACK_EXCEPTION_MASK;
+
+				KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_IGNORED_INTERRUPTS_GROUP_SUSPEND,
+							   group, queue, cs_req_remain ^ cs_ack_remain);
 				continue;
+			}
 
 			if (((cs_req & CS_REQ_TILER_OOM_MASK) ^
 			     (cs_ack & CS_ACK_TILER_OOM_MASK))) {
 				get_queue(queue);
-				KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_TILER_OOM_INTERRUPT, group, queue, cs_req ^ cs_ack);
-				if (WARN_ON(!queue_work(
-					    wq, &queue->oom_event_work))) {
+				KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_TILER_OOM_INTERRUPT, group, queue,
+							   cs_req ^ cs_ack);
+				if (WARN_ON(!queue_work(wq, &queue->oom_event_work))) {
 					/* The work item shall not have been
 					 * already queued, there can be only
 					 * one pending OoM event for a
@@ -2533,12 +2652,17 @@ static void process_cs_interrupts(struct kbase_queue_group *const group,
 
 			if ((cs_req & CS_REQ_PROTM_PEND_MASK) ^
 			    (cs_ack & CS_ACK_PROTM_PEND_MASK)) {
+				KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_PROTM_PEND_INTERRUPT, group, queue,
+							   cs_req ^ cs_ack);
+
 				dev_dbg(kbdev->dev,
 					"Protected mode entry request for queue on csi %d bound to group-%d on slot %d",
 					queue->csi_index, group->handle,
 					group->csg_nr);
 
 				bitmap_set(group->protm_pending_bitmap, i, 1);
+				KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, PROTM_PENDING_SET, group, queue,
+							   group->protm_pending_bitmap[0]);
 				protm_pend = true;
 			}
 		}
@@ -2567,7 +2691,7 @@ static void process_csg_interrupts(struct kbase_device *const kbdev,
 	int const csg_nr)
 {
 	struct kbase_csf_cmd_stream_group_info *ginfo;
-	struct kbase_queue_group *group;
+	struct kbase_queue_group *group = NULL;
 	u32 req, ack, irqreq, irqack;
 
 	kbase_csf_scheduler_spin_lock_assert_held(kbdev);
@@ -2575,6 +2699,8 @@ static void process_csg_interrupts(struct kbase_device *const kbdev,
 	if (WARN_ON(csg_nr >= kbdev->csf.global_iface.group_num))
 		return;
 
+	KBASE_KTRACE_ADD(kbdev, CSG_INTERRUPT_PROCESS, NULL, csg_nr);
+
 	ginfo = &kbdev->csf.global_iface.groups[csg_nr];
 	req = kbase_csf_firmware_csg_input_read(ginfo, CSG_REQ);
 	ack = kbase_csf_firmware_csg_output(ginfo, CSG_ACK);
@@ -2583,7 +2709,7 @@ static void process_csg_interrupts(struct kbase_device *const kbdev,
 
 	/* There may not be any pending CSG/CS interrupts to process */
 	if ((req == ack) && (irqreq == irqack))
-		return;
+		goto out;
 
 	/* Immediately set IRQ_ACK bits to be same as the IRQ_REQ bits before
 	 * examining the CS_ACK & CS_REQ bits. This would ensure that Host
@@ -2604,10 +2730,10 @@ static void process_csg_interrupts(struct kbase_device *const kbdev,
 	 * slot scheduler spinlock is required.
 	 */
 	if (!group)
-		return;
+		goto out;
 
 	if (WARN_ON(kbase_csf_scheduler_group_get_slot_locked(group) != csg_nr))
-		return;
+		goto out;
 
 	if ((req ^ ack) & CSG_REQ_SYNC_UPDATE_MASK) {
 		kbase_csf_firmware_csg_input_mask(ginfo,
@@ -2624,7 +2750,8 @@ static void process_csg_interrupts(struct kbase_device *const kbdev,
 			CSG_REQ_IDLE_MASK);
 
 		set_bit(csg_nr, scheduler->csg_slots_idle_mask);
-
+		KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_IDLE_SET, group,
+					 scheduler->csg_slots_idle_mask[0]);
 		KBASE_KTRACE_ADD_CSF_GRP(kbdev,  CSG_IDLE_INTERRUPT, group, req ^ ack);
 		dev_dbg(kbdev->dev, "Idle notification received for Group %u on slot %d\n",
 			 group->handle, csg_nr);
@@ -2640,6 +2767,8 @@ static void process_csg_interrupts(struct kbase_device *const kbdev,
 		kbase_csf_firmware_csg_input_mask(ginfo, CSG_REQ, ack,
 			CSG_REQ_PROGRESS_TIMER_EVENT_MASK);
 
+		KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_PROGRESS_TIMER_INTERRUPT,
+					 group, req ^ ack);
 		dev_info(kbdev->dev,
 			"Timeout notification received for group %u of ctx %d_%d on slot %d\n",
 			group->handle, group->kctx->tgid, group->kctx->id, csg_nr);
@@ -2648,6 +2777,11 @@ static void process_csg_interrupts(struct kbase_device *const kbdev,
 	}
 
 	process_cs_interrupts(group, ginfo, irqreq, irqack);
+
+out:
+	/* group may still be NULL here */
+	KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_INTERRUPT_PROCESS_END, group,
+				 ((u64)req ^ ack) | (((u64)irqreq ^ irqack) << 32));
 }
 
 /**
@@ -2741,6 +2875,7 @@ void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val)
 
 	lockdep_assert_held(&kbdev->hwaccess_lock);
 
+	KBASE_KTRACE_ADD(kbdev, CSF_INTERRUPT, NULL, val);
 	kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), val);
 
 	if (val & JOB_IRQ_GLOBAL_IF) {
@@ -2761,6 +2896,7 @@ void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val)
 					global_iface, GLB_REQ);
 			glb_ack = kbase_csf_firmware_global_output(
 					global_iface, GLB_ACK);
+			KBASE_KTRACE_ADD(kbdev, GLB_REQ_ACQ, NULL, glb_req ^ glb_ack);
 
 			if ((glb_req ^ glb_ack) & GLB_REQ_PROTM_EXIT_MASK) {
 				dev_dbg(kbdev->dev, "Protected mode exit interrupt received");
@@ -2768,8 +2904,8 @@ void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val)
 						global_iface, GLB_REQ, glb_ack,
 						GLB_REQ_PROTM_EXIT_MASK);
 				WARN_ON(!kbase_csf_scheduler_protected_mode_in_use(kbdev));
+				KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_EXIT_PROTM, scheduler->active_protm_grp, 0u);
 				scheduler->active_protm_grp = NULL;
-				KBASE_KTRACE_ADD(kbdev, SCHEDULER_EXIT_PROTM, NULL, 0u);
 				kbdev->protected_mode = false;
 				kbase_ipa_control_protm_exited(kbdev);
 				kbase_hwcnt_backend_csf_protm_exited(
@@ -2778,13 +2914,20 @@ void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val)
 
 			/* Handle IDLE Hysteresis notification event */
 			if ((glb_req ^ glb_ack) & GLB_REQ_IDLE_EVENT_MASK) {
+				int non_idle_offslot_grps;
+				bool can_suspend_on_idle;
 				dev_dbg(kbdev->dev, "Idle-hysteresis event flagged");
 				kbase_csf_firmware_global_input_mask(
 						global_iface, GLB_REQ, glb_ack,
 						GLB_REQ_IDLE_EVENT_MASK);
 
-				if (!atomic_read(&scheduler->non_idle_offslot_grps)) {
-					if (kbase_pm_idle_groups_sched_suspendable(kbdev))
+				non_idle_offslot_grps = atomic_read(&scheduler->non_idle_offslot_grps);
+				can_suspend_on_idle = kbase_pm_idle_groups_sched_suspendable(kbdev);
+				KBASE_KTRACE_ADD(kbdev, SCHEDULER_CAN_IDLE, NULL,
+					((u64)(u32)non_idle_offslot_grps) | (((u64)can_suspend_on_idle) << 32));
+
+				if (!non_idle_offslot_grps) {
+					if (can_suspend_on_idle)
 						queue_work(system_highpri_wq,
 							   &scheduler->gpu_idle_work);
 				} else {
@@ -2809,6 +2952,7 @@ void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val)
 
 		if (!remaining) {
 			wake_up_all(&kbdev->csf.event_wait);
+			KBASE_KTRACE_ADD(kbdev, CSF_INTERRUPT_END, NULL, val);
 			return;
 		}
 	}
@@ -2823,6 +2967,7 @@ void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val)
 	kbase_csf_scheduler_spin_unlock(kbdev, flags);
 
 	wake_up_all(&kbdev->csf.event_wait);
+	KBASE_KTRACE_ADD(kbdev, CSF_INTERRUPT_END, NULL, val);
 }
 
 void kbase_csf_doorbell_mapping_term(struct kbase_device *kbdev)

drivers/gpu/arm/bifrost/csf/mali_kbase_csf.h

@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
  * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved.
@@ -39,7 +39,7 @@
  */
 #define KBASEP_USER_DB_NR_INVALID ((s8)-1)
 
-#define FIRMWARE_PING_INTERVAL_MS (2000) /* 2 seconds */
+#define FIRMWARE_PING_INTERVAL_MS (4000) /* 4 seconds */
 
 #define FIRMWARE_IDLE_HYSTERESIS_TIME_MS (10) /* Default 10 milliseconds */
 
@@ -212,6 +212,22 @@ void kbase_csf_ctx_term(struct kbase_context *kctx);
 int kbase_csf_queue_register(struct kbase_context *kctx,
 			     struct kbase_ioctl_cs_queue_register *reg);
 
+/**
+ * kbase_csf_queue_register_ex - Register a GPU command queue with
+ *                               extended format.
+ *
+ * @kctx:	Pointer to the kbase context within which the
+ *		queue is to be registered.
+ * @reg:	Pointer to the structure which contains details of the
+ *		queue to be registered within the provided
+ *		context, together with the extended parameter fields
+ *              for supporting cs trace command.
+ *
+ * Return:	0 on success, or negative on failure.
+ */
+int kbase_csf_queue_register_ex(struct kbase_context *kctx,
+			     struct kbase_ioctl_cs_queue_register_ex *reg);
+
 /**
  * kbase_csf_queue_terminate - Terminate a GPU command queue.
  *

drivers/gpu/arm/bifrost/csf/mali_kbase_csf_cpu_queue_debugfs.c

@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
  * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved.
@@ -23,7 +23,7 @@
 #include <mali_kbase.h>
 #include <linux/seq_file.h>
 
-#ifdef CONFIG_DEBUG_FS
+#if IS_ENABLED(CONFIG_DEBUG_FS)
 
 bool kbase_csf_cpu_queue_read_dump_req(struct kbase_context *kctx,
 					struct base_csf_notification *req)

drivers/gpu/arm/bifrost/csf/mali_kbase_csf_cpu_queue_debugfs.h

@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
  * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved.
@@ -68,7 +68,7 @@ bool kbase_csf_cpu_queue_read_dump_req(struct kbase_context *kctx,
  */
 static inline bool kbase_csf_cpu_queue_dump_needed(struct kbase_context *kctx)
 {
-#ifdef CONFIG_DEBUG_FS
+#if IS_ENABLED(CONFIG_DEBUG_FS)
 	return (atomic_read(&kctx->csf.cpu_queue.dump_req_status) ==
 		BASE_CSF_CPU_QUEUE_DUMP_ISSUED);
 #else

drivers/gpu/arm/bifrost/csf/mali_kbase_csf_csg_debugfs.c

@@ -1,7 +1,7 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
- * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software
@@ -25,7 +25,7 @@
 #include <linux/delay.h>
 #include <csf/mali_kbase_csf_trace_buffer.h>
 
-#ifdef CONFIG_DEBUG_FS
+#if IS_ENABLED(CONFIG_DEBUG_FS)
 #include "mali_kbase_csf_tl_reader.h"
 
 /**
@@ -87,6 +87,32 @@ static void kbasep_csf_scheduler_dump_active_queue_cs_status_wait(
 			   blocked_reason)));
 }
 
+static void kbasep_csf_scheduler_dump_active_cs_trace(struct seq_file *file,
+			struct kbase_csf_cmd_stream_info const *const stream)
+{
+	u32 val = kbase_csf_firmware_cs_input_read(stream,
+			CS_INSTR_BUFFER_BASE_LO);
+	u64 addr = ((u64)kbase_csf_firmware_cs_input_read(stream,
+				CS_INSTR_BUFFER_BASE_HI) << 32) | val;
+	val = kbase_csf_firmware_cs_input_read(stream,
+				CS_INSTR_BUFFER_SIZE);
+
+	seq_printf(file, "CS_TRACE_BUF_ADDR: 0x%16llx, SIZE: %u\n", addr, val);
+
+	/* Write offset variable address (pointer) */
+	val = kbase_csf_firmware_cs_input_read(stream,
+			CS_INSTR_BUFFER_OFFSET_POINTER_LO);
+	addr = ((u64)kbase_csf_firmware_cs_input_read(stream,
+			CS_INSTR_BUFFER_OFFSET_POINTER_HI) << 32) | val;
+	seq_printf(file, "CS_TRACE_BUF_OFFSET_PTR: 0x%16llx\n", addr);
+
+	/* EVENT_SIZE and EVENT_STATEs */
+	val = kbase_csf_firmware_cs_input_read(stream, CS_INSTR_CONFIG);
+	seq_printf(file, "TRACE_EVENT_SIZE: 0x%x, TRACE_EVENT_STAES 0x%x\n",
+			CS_INSTR_CONFIG_EVENT_SIZE_GET(val),
+			CS_INSTR_CONFIG_EVENT_STATE_GET(val));
+}
+
 /**
  * kbasep_csf_scheduler_dump_active_queue() - Print GPU command queue
  *                                            debug information
@@ -134,7 +160,9 @@ static void kbasep_csf_scheduler_dump_active_queue(struct seq_file *file,
 			queue->csi_index, queue->base_addr, queue->priority,
 			cs_insert, cs_extract, cs_active, queue->doorbell_nr);
 
-	/* Print status information for blocked group waiting for sync object */
+	/* Print status information for blocked group waiting for sync object. For on-slot queues,
+	 * if cs_trace is enabled, dump the interface's cs_trace configuration.
+	 */
 	if (kbase_csf_scheduler_group_get_slot(queue->group) < 0) {
 		if (CS_STATUS_WAIT_SYNC_WAIT_GET(queue->status_wait)) {
 			wait_status = queue->status_wait;
@@ -212,6 +240,11 @@ static void kbasep_csf_scheduler_dump_active_queue(struct seq_file *file,
 			file, wait_status, wait_sync_value,
 			wait_sync_live_value, wait_sync_pointer, sb_status,
 			blocked_reason);
+		/* Dealing with cs_trace */
+		if (kbase_csf_scheduler_queue_has_trace(queue))
+			kbasep_csf_scheduler_dump_active_cs_trace(file, stream);
+		else
+			seq_puts(file, "NO CS_TRACE\n");
 	}
 
 	seq_puts(file, "\n");

drivers/gpu/arm/bifrost/csf/mali_kbase_csf_csg_debugfs.h

@@ -1,7 +1,7 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
- * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software

drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h

@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
  * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved.
@@ -314,6 +314,10 @@ struct kbase_csf_notification {
  *                  are non-zero
  * @blocked_reason: Value shows if the queue is blocked, and if so,
  *                  the reason why it is blocked
+ * @trace_buffer_base: CS trace buffer base address.
+ * @trace_offset_ptr:  Pointer to the CS trace buffer offset variable.
+ * @trace_buffer_size: CS trace buffer size for the queue.
+ * @trace_cfg:         CS trace configuration parameters.
  * @error:          GPU command queue fatal information to pass to user space.
  * @fatal_event_work: Work item to handle the CS fatal event reported for this
  *                    queue.
@@ -344,6 +348,10 @@ struct kbase_queue {
 	u32 sync_value;
 	u32 sb_status;
 	u32 blocked_reason;
+	u64 trace_buffer_base;
+	u64 trace_offset_ptr;
+	u32 trace_buffer_size;
+	u32 trace_cfg;
 	struct kbase_csf_notification error;
 	struct work_struct fatal_event_work;
 	u64 cs_fatal_info;
@@ -667,7 +675,7 @@ struct kbase_csf_context {
 	struct vm_area_struct *user_reg_vma;
 	struct kbase_csf_scheduler_context sched;
 	struct list_head error_list;
-#ifdef CONFIG_DEBUG_FS
+#if IS_ENABLED(CONFIG_DEBUG_FS)
 	struct kbase_csf_cpu_queue_context cpu_queue;
 #endif
 };

drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c

@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
  * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved.
@@ -993,7 +993,12 @@ static int parse_capabilities(struct kbase_device *kbdev)
 
 	iface->group_stride = shared_info[GLB_GROUP_STRIDE/4];
 	iface->prfcnt_size = shared_info[GLB_PRFCNT_SIZE/4];
-	iface->instr_features = shared_info[GLB_INSTR_FEATURES / 4];
+
+	if (iface->version >= kbase_csf_interface_version(1, 1, 0)) {
+		iface->instr_features = shared_info[GLB_INSTR_FEATURES / 4];
+	} else {
+		iface->instr_features = 0;
+	}
 
 	if ((GROUP_CONTROL_0 +
 		(unsigned long)iface->group_num * iface->group_stride) >
@@ -1671,29 +1676,8 @@ u32 kbase_csf_firmware_set_mcu_core_pwroff_time(struct kbase_device *kbdev, u32
 }
 
 
-int kbase_csf_firmware_init(struct kbase_device *kbdev)
+int kbase_csf_firmware_early_init(struct kbase_device *kbdev)
 {
-	const struct firmware *firmware;
-	const u32 magic = FIRMWARE_HEADER_MAGIC;
-	u8 version_major, version_minor;
-	u32 version_hash;
-	u32 entry_end_offset;
-	u32 entry_offset;
-	int ret;
-
-	if (WARN_ON((kbdev->as_free & MCU_AS_BITMASK) == 0))
-		return -EINVAL;
-	kbdev->as_free &= ~MCU_AS_BITMASK;
-
-	ret = kbase_mmu_init(kbdev, &kbdev->csf.mcu_mmu, NULL,
-		BASE_MEM_GROUP_DEFAULT);
-
-	if (ret != 0) {
-		/* Release the address space */
-		kbdev->as_free |= MCU_AS_BITMASK;
-		return ret;
-	}
-
 	init_waitqueue_head(&kbdev->csf.event_wait);
 	kbdev->csf.interrupt_received = false;
 	kbdev->csf.fw_timeout_ms = CSF_FIRMWARE_TIMEOUT_MS;
@@ -1708,17 +1692,46 @@ int kbase_csf_firmware_init(struct kbase_device *kbdev)
 
 	mutex_init(&kbdev->csf.reg_lock);
 
+	return 0;
+}
+
+int kbase_csf_firmware_init(struct kbase_device *kbdev)
+{
+	const struct firmware *firmware;
+	const u32 magic = FIRMWARE_HEADER_MAGIC;
+	u8 version_major, version_minor;
+	u32 version_hash;
+	u32 entry_end_offset;
+	u32 entry_offset;
+	int ret;
+
+	lockdep_assert_held(&kbdev->fw_load_lock);
+
+	if (WARN_ON((kbdev->as_free & MCU_AS_BITMASK) == 0))
+		return -EINVAL;
+	kbdev->as_free &= ~MCU_AS_BITMASK;
+
+	ret = kbase_mmu_init(kbdev, &kbdev->csf.mcu_mmu, NULL,
+		BASE_MEM_GROUP_DEFAULT);
+
+	if (ret != 0) {
+		/* Release the address space */
+		kbdev->as_free |= MCU_AS_BITMASK;
+		return ret;
+	}
+
 	kbdev->csf.gpu_idle_hysteresis_ms = FIRMWARE_IDLE_HYSTERESIS_TIME_MS;
-	kbdev->csf.gpu_idle_dur_count = convert_dur_to_idle_count(kbdev,
-						FIRMWARE_IDLE_HYSTERESIS_TIME_MS);
+	kbdev->csf.gpu_idle_dur_count = convert_dur_to_idle_count(
+		kbdev, FIRMWARE_IDLE_HYSTERESIS_TIME_MS);
 
 	kbdev->csf.mcu_core_pwroff_dur_us = DEFAULT_GLB_PWROFF_TIMEOUT_US;
-	kbdev->csf.mcu_core_pwroff_dur_count =
-		convert_dur_to_core_pwroff_count(kbdev, DEFAULT_GLB_PWROFF_TIMEOUT_US);
+	kbdev->csf.mcu_core_pwroff_dur_count = convert_dur_to_core_pwroff_count(
+		kbdev, DEFAULT_GLB_PWROFF_TIMEOUT_US);
 
 	ret = kbase_mcu_shared_interface_region_tracker_init(kbdev);
 	if (ret != 0) {
-		dev_err(kbdev->dev, "Failed to setup the rb tree for managing shared interface segment\n");
+		dev_err(kbdev->dev,
+			"Failed to setup the rb tree for managing shared interface segment\n");
 		goto error;
 	}
 
@@ -2081,7 +2094,7 @@ int kbase_csf_trigger_firmware_config_update(struct kbase_device *kbdev)
 	int err = 0;
 
 	/* Ensure GPU is powered-up until we complete config update.*/
-	kbase_pm_context_active(kbdev);
+	kbase_csf_scheduler_pm_active(kbdev);
 
 	/* The 'reg_lock' is also taken and is held till the update is
 	 * complete, to ensure the config update gets serialized.
@@ -2098,7 +2111,7 @@ int kbase_csf_trigger_firmware_config_update(struct kbase_device *kbdev)
 				      GLB_REQ_FIRMWARE_CONFIG_UPDATE_MASK);
 	mutex_unlock(&kbdev->csf.reg_lock);
 
-	kbase_pm_context_idle(kbdev);
+	kbase_csf_scheduler_pm_idle(kbdev);
 	return err;
 }
 

drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.h

@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 /*
  *
  * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved.
@@ -79,7 +79,7 @@
 #define MAX_SUPPORTED_STREAMS_PER_GROUP 32
 
 /* Waiting timeout for status change acknowledgment, in milliseconds */
-#define CSF_FIRMWARE_TIMEOUT_MS (800) /* Relaxed to 800ms from 100ms */
+#define CSF_FIRMWARE_TIMEOUT_MS (3000) /* Relaxed to 3000ms from 800ms due to Android */
 
 struct kbase_device;
 
@@ -266,7 +266,7 @@ u32 kbase_csf_firmware_csg_output(
  * @group_stride: Stride in bytes in JASID0 virtual address between
  *                CSG capability structures.
  * @prfcnt_size: Performance counters size.
- * @instr_features: Instrumentation features.
+ * @instr_features: Instrumentation features. (csf >= 1.1.0)
  * @groups: Address of an array of CSG capability structures.
  */
 struct kbase_csf_global_iface {
@@ -376,23 +376,31 @@ void kbase_csf_read_firmware_memory(struct kbase_device *kbdev,
 void kbase_csf_update_firmware_memory(struct kbase_device *kbdev,
 	u32 gpu_addr, u32 value);
 
+/**
+ * kbase_csf_firmware_early_init() - Early initializatin for the firmware.
+ * @kbdev: Kbase device
+ *
+ * Initialize resources related to the firmware. Must be called at kbase probe.
+ *
+ * Return: 0 if successful, negative error code on failure
+ */
+int kbase_csf_firmware_early_init(struct kbase_device *kbdev);
+
 /**
  * kbase_csf_firmware_init() - Load the firmware for the CSF MCU
+ * @kbdev: Kbase device
  *
  * Request the firmware from user space and load it into memory.
  *
  * Return: 0 if successful, negative error code on failure
- *
- * @kbdev: Kbase device
  */
 int kbase_csf_firmware_init(struct kbase_device *kbdev);
 
 /**
  * kbase_csf_firmware_term() - Unload the firmware
+ * @kbdev: Kbase device
  *
  * Frees the memory allocated by kbase_csf_firmware_init()
- *
- * @kbdev: Kbase device
  */
 void kbase_csf_firmware_term(struct kbase_device *kbdev);
 
@@ -443,12 +451,8 @@ void kbase_csf_enter_protected_mode(struct kbase_device *kbdev);
 
 static inline bool kbase_csf_firmware_mcu_halted(struct kbase_device *kbdev)
 {
-#ifndef CONFIG_MALI_BIFROST_NO_MALI
 	return (kbase_reg_read(kbdev, GPU_CONTROL_REG(MCU_STATUS)) ==
 		MCU_STATUS_HALTED);
-#else
-	return true;
-#endif
 }
 
 /**
@@ -584,6 +588,7 @@ bool kbase_csf_firmware_core_attr_updated(struct kbase_device *kbdev);
  *                         hardware performance counter data.
  * @instr_features:        Instrumentation features. Bits 7:4 hold the max size
  *                         of events. Bits 3:0 hold the offset update rate.
+ *                         (csf >= 1,1,0)
  */
 u32 kbase_csf_firmware_get_glb_iface(
 	struct kbase_device *kbdev, struct basep_cs_group_control *group_data,